o i(@sddlZddlmZddlmZmZmZmZmZmZm Z ddl m Z ddl m Z mZddgZd d ZGd ddeZd ed e d ede_   d#de ede ede ede ede ede edeedededededededefddZde ede ede ede ede ede edededededededefdd Zde ede ede ede ede ede edededededededefd!d"ZdS)$N)Tensor) Optimizer_use_grad_for_differentiable _get_value_default_to_fused_or_foreach_differentiable_doc _foreach_doc _maximize_doc) is_compiling)ListOptionalASGDasgdcCt|tjs t|S|SN isinstancetorchrtensorxr_/var/www/html/eduruby.in/lip-sync/lip-sync-env/lib/python3.10/site-packages/torch/optim/asgd.py _to_tensor s  rcsbeZdZ        ddeed ed effd d Zfd dZddZedddZ Z S)r{Gz?-C6??.ArNFforeachmaximizedifferentiablec sVd|ks td|d|kstd|t|||||||| d} t|| dS)NzInvalid learning rate: zInvalid weight_decay value: )lrlambdalphat0 weight_decayrr r!) ValueErrordictsuper__init__) selfparamsr#r$r%r&r'rr r!defaults __class__rrr+s  z ASGD.__init__cst||jD]}|dd|dd|ddq t|j}t|dko3t |dd}|sF|D] }t t |d|d<q8t|dkoTt |dd}|se|D] }t |d|d<qYt|dkost |dd}|s|D]}t t |d|d<qxdSdS) Nrr Fr!rstepetamu) r* __setstate__ param_groups setdefaultliststatevalueslenrZ is_tensorrfloat)r,r8groupZ state_valuesZstep_is_tensorsZ eta_is_tensorZ mu_is_tensorr/rrr4/s4       zASGD.__setstate__c Cs|dD]b}|jdurf|||jjrtd||j|j|} t| dkrJtd| d<t|d| d<td| d <tj|tj d | d <|| d || d || d|| dqdS) Nr-z&ASGD does not support sparse gradientsrr"r1r#r2g?r3)Z memory_formatax) gradappendZ is_sparse RuntimeErrorr8r:rrZ zeros_likeZpreserve_format) r,r<params_with_gradgradsmusaxsetas state_stepspr8rrr _init_groupIs(       zASGD._init_groupc Csd}|durt |}Wdn1swY|jD];}g}g}g}g}g}g} |||||||| t|||||| |d|d|d|d|d|d|d|d d q |S) zPerforms a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nr$r#r&r%r'rr r!)r$r#r&r%r'rr r!)rZ enable_gradr5rIr) r,closureZlossr<rBrCrDrErFrGrrrr1`s<  z ASGD.step)rrrrrNFFr) __name__ __module__ __qualname__r boolr+r4rIrr1 __classcell__rrr/rrs(   ahImplements Averaged Stochastic Gradient Descent. It has been proposed in `Acceleration of stochastic approximation by averaging`_. Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-2) lambd (float, optional): decay term (default: 1e-4) alpha (float, optional): power for eta update (default: 0.75) t0 (float, optional): point at which to start averaging (default: 1e6) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) z zx .. _Acceleration of stochastic approximation by averaging: https://dl.acm.org/citation.cfm?id=131098 Fr-rCrErDrFrGrr r!r$r#r&r%r'c Csn|dur t||dd\}}|rtjrtd|r"tjs"t}nt}|||||||| | | | | ||d dS)znFunctional API that performs asgd algorithm computation. See :class:`~torch.optim.ASGD` for details. NF)Z use_fusedz6torch.jit.script not supported with foreach optimizers)r$r#r&r%r'r r!)rrZjitZ is_scriptingrA_multi_tensor_asgd_single_tensor_asgd)r-rCrErDrFrGrr r!r$r#r&r%r'_funcrrrrs, cCsFdd} t|D]\}}||}| s|n| }||}||}||}||}t|r;t|}t|}t|}|d7}t|}| dkrN|j|| d}t|}|d|||j|| dtsl| dkrx|| | |n| || |d|||| }| || dt d||}| |qdS)NcSrrrrrrrrs  z'_single_tensor_asgd.._to_tensorrrr%) enumerater is_complex view_as_realraddZmul_add_r itemsubmulcopy_max)r-rCrErDrFrGr$r#r&r%r'r r!riparamr?r3r>r2Zstep_tr1Z eta_valuenew_etanew_murrrrQs6       rQc Cst|dkrdS| rJdt||||||g} | D]\\}}}}}}}| r.t|}dd}||}||}||}t|d| dkr[| rStj||| dntj||| d}t|d}t |d||tj||| dt t|D]*}t s|| dkr|| ||||||qz||||qzt t|D]1}t|d||t||| }|||tdtdt|||}|||qqdS)Nrz#_foreach ops don't support autogradcSsdd|DS)NcSs$g|]}t|rt|n|qSr)rrVrW).0trrr /szE_multi_tensor_asgd.._view_complex_as_real..r)Z tensor_listrrr_view_complex_as_real.sz1_multi_tensor_asgd.._view_complex_as_realrrT)r:rZ"_group_tensors_by_device_and_dtyper9rZ _foreach_negZ _foreach_add_Z _foreach_addrZ _foreach_mul_ranger rZrYr[r\r]rr^)r-rCrErDrFrGr$r#r&r%r'r r!Zgrouped_tensorsZgrouped_paramsZ grouped_gradsZ grouped_axsZ grouped_musZ grouped_etasZgrouped_state_stepsrRrfr2r_rarbrrrrPsD     (rP)NFF)rrZ optimizerrrrrrr r Z torch._utilsr typingr r __all__rr__doc__rNr;rrQrPrrrrs $ z        4      <