Ë
    ùÿæiv@  ã            $       ó&  — d dl mZ d dlZd dlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ ddgZ G d„ de«      Zd	e› d
e	› de› de› de› d�e_        dee   dee   dee   dee   dee   dee   dedededededededededdf d„Zdee   dee   dee   dee   dee   dee   dedededededededededdf d„Z ee¬«      	 	 	 	 	 d"dee   dee   dee   dee   dee   dee   d edz  dedededededededededdf"d!„«       Zy)#é    )ÚcastN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_params_docÚ
_to_scalarÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚASGDÚasgdc                   óŽ   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 ddedeez  dedededededz  d	ed
ededdfˆ fd„Zˆ fd„Zd„ Z	e
dd„«       Zˆ xZS )r   NÚparamsÚlrÚlambdÚalphaÚt0Úweight_decayÚforeachÚmaximizeÚdifferentiableÚ
capturableÚreturnc           
      óæ   •— t        |t        «      r|j                  «       dk7  rt        d«      ‚d|k  st        d|› �«      ‚d|k  st        d|› �«      ‚||||||||	|
dœ	}t        ‰| �  ||«       y )Nr   zTensor lr must be 1-elementg        zInvalid learning rate: zInvalid weight_decay value: )	r   r   r   r   r   r   r   r    r!   )Ú
isinstancer   ÚnumelÚ
ValueErrorÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r    r!   ÚdefaultsÚ	__class__s               €úe/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/torch/optim/asgd.pyr(   zASGD.__init__   s‘   ø€ ô �bœ&Ô! b§h¡h£j°A¢oÜÐ:Ó;Ð;Ø�bŠyÜÐ6°r°dÐ;Ó<Ð<Ø�lÒ"ÜÐ;¸L¸>ÐJÓKÐKð ØØØØ(ØØ Ø,Ø$ñ

ˆô 	‰Ñ˜ Õ*ó    c                 ó  •— t         ‰| �  |«       | j                  D �]f  }|j                  dd «       |j                  dd«       |j                  dd«       |j                  dd«       |d   D �]  }| j                  j                  |g «      }t        |«      dk7  sŒ/t        j                  |d   «      s;t        |d   «      }t        j                  |t        «       |j                  ¬	«      |d<   t        j                  |d
   «      s0t        j                  |d
   t        «       |j                  ¬	«      |d
<   t        j                  |d   «      rŒãt        j                  |d   t        «       |j                  ¬	«      |d<   �Œ �Œi y )Nr   r   Fr    r!   r   r   Ústep)ÚdtypeÚdeviceÚetaÚmu)r'   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r1   )r)   r7   ÚgroupÚpÚp_stateÚstep_valr+   s         €r,   r4   zASGD.__setstate__?   s<  ø€ Ü‰Ñ˜UÔ#Ø×&Õ&ˆEØ×Ñ˜Y¨Ô-Ø×Ñ˜Z¨Ô/Ø×ÑÐ-¨uÔ5Ø×Ñ˜\¨5Ô1Ø˜8•_�ØŸ*™*Ÿ.™.¨¨BÓ/�Ü�w“< 1Ó$Ü Ÿ?™?¨7°6©?Ô;Ü#(¨°©Ó#9˜Ü*/¯,©,Ø$Ô,=Ó,?ÈÏÉô+˜ ™ô !Ÿ?™?¨7°5©>Ô:Ü).¯©Ø# E™NÔ2CÓ2EÈaÏhÉhô*˜ ™ô !Ÿ?™?¨7°4©=Õ9Ü(-¯©Ø# D™MÔ1BÓ1DÈQÏXÉXô)˜ ›ò %ñ 'r-   c                 óŽ  — d}|d   D �]¹  }	|	j                   €Œ|t        j                  |	«      z  }|j                  |	«       |	j                   j                  rt        d«      ‚|j                  |	j                   «       | j                  |	   }
t        |
«      dk(  r×t        j                  d|	j                  t        «       ¬«      |
d<   t        j                  t        |d   «      |	j                  t        «       ¬«      j                  «       j                  «       |
d	<   t        j                  d|	j                  t        «       ¬«      |
d
<   t        j                   |	t        j"                  ¬«      |
d<   |j                  |
d
   «       |j                  |
d   «       |j                  |
d	   «       |j                  |
d   «       �Œ¼ |S )NFr   z&ASGD does not support sparse gradientsr   © )r1   r0   r/   r   r2   r3   )Úmemory_formatÚax)Úgradr:   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr7   r9   Úzerosr1   r   Ú	as_tensorr   ÚcloneÚdetachÚonesÚ
zeros_likeÚpreserve_format)r)   r>   Úparams_with_gradÚgradsÚmusÚaxsÚetasÚstate_stepsÚhas_complexr?   r7   s              r,   Ú_init_groupzASGD._init_groupW   sv  € ØˆØ�x•ˆAØ�v‰vÑ!Øœu×/Ñ/°Ó2Ñ2�Ø ×'Ñ'¨Ô*Ø—6‘6×#Ò#Ü&Ð'OÓPÐPØ—‘˜QŸV™VÔ$àŸ
™
 1™�ä�u“: ’?Ü$)§K¡KØ 1§8¡8Ô3DÓ3Fô%�E˜&‘Mô Ÿ™Ü& u¨T¡{Ó3Ø#$§8¡8Ü"3Ó"5ô÷
 ™›ß™›ð ˜%‘Lô #(§*¡*Ø 1§8¡8Ô3DÓ3Fô#�E˜$‘Kô #(×"2Ñ"2Ø¬×)>Ñ)>ô#�E˜$‘Kð —
‘
˜5 ™;Ô'Ø—
‘
˜5 ™;Ô'Ø—‘˜E %™LÔ)Ø×"Ñ" 5¨¡=Ö1ðC !ðD Ðr-   c                 ób  — | j                  «        d}|�$t        j                  «       5   |«       }ddd«       | j                  D ][  }g }g }g }g }g }g }	| j	                  |||||||	«      }
t        ||||||	|d   |d   |d   |d   |d   |d   |d   |d	   |d
   |
¬«       Œ] |S # 1 sw Y   ŒuxY w)z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r    r!   )
r   r   r   r   r   r   r   r    r!   rX   )Ú'_accelerator_graph_capture_health_checkr:   Úenable_gradr5   rY   r   )r)   ÚclosureÚlossr>   rR   rS   rT   rU   rV   rW   rX   s              r,   r/   z	ASGD.step}   sù   € ð 	×4Ñ4Ô6àˆØÐÜ×"Ñ"Õ$Ù“y�÷ %ð ×&Ô&ˆEØ-/ÐØ"$ˆEØ "ˆCØ "ˆCØ!#ˆDØ(*ˆKà×*Ñ*ØÐ'¨°°S¸$ÀóˆKô Ø ØØØØØØ˜G‘nØ˜‘;Ø˜‘;Ø˜G‘nØ" >Ñ2Ø˜iÑ(Ø˜zÑ*Ø$Ð%5Ñ6Ø  Ñ.Ø'ö!ð 'ð> ˆ÷E %Ð$ús   ©B%Â%B.)	g{®Gáz„?g-Cëâ6?g      è?g    €„.Ar   NFFF©N)Ú__name__Ú
__module__Ú__qualname__r   r<   r   Úboolr(   r4   rY   r   r/   Ú__classcell__)r+   s   @r,   r   r      s¹   ø„ ð "ØØØØØ#ØØ$Ø ñ+àð+ð �F‰Nð+ð ð	+ð
 ð+ð ð+ð ð+ð ˜‘ð+ð ð+ð ð+ð ð+ð 
õ+ôBò0$ðL "ò-ó "ô-r-   zŸImplements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        am  
        lr (float, Tensor, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        z¿

    .. _Acceleration of stochastic approximation by averaging:
        https://meyn.ece.ufl.edu/wp-content/uploads/sites/77/archive/spm_files/Courses/ECE555-2011/555media/poljud92.pdf

    r   rS   rU   rT   rV   rW   r   r   r   r   r   r   r    r!   rX   r"   c       	   
      ó0  — t         j                  j                  «       st        |«      }t	        | «      D �]Þ  \  }}||   }|s|n| }||   }||   }||   }||   }t         j
                  j                  «       s—|r•t        «       }|j                  j                  |j                  j                  cxk(  r3|j                  j                  cxk(  r|j                  j                  k(  rn n|j                  j                  |v st        d|› d�«      ‚t        j                  |«      r?t        j                  |«      }t        j                  |«      }t        j                  |«      }|dz  }|
dk7  r|j                  ||
¬«      }|r,|j                  d||z  z
  «       |j                  ||d¬«       n6t!        |«      }|j                  d||z  z
  «       |j#                  || ¬«       |s|j%                  «       dk7  r0|j#                  |j'                  |«      j                  |«      «       n|j)                  |«       |r`|j)                  |d||z  |z  z   |	z  z  «       |j)                  dt        j*                  ||z
  t        j,                  |«      «      z  «       �Œit!        |«      }t        j.                  |d||z  |z  z   |	z  z  «      }|j)                  |«       t        j.                  dt1        d||z
  «      z  «      }|j)                  |«       �Œá y )NúUIf capturable=True, params, mus, etas, and state_steps must be on supported devices: Ú.r   r   ©r   éÿÿÿÿ©Úvalue)r:   ÚjitÚis_scriptingr   Ú	enumerateÚcompilerÚis_compilingr   r1   ÚtypeÚAssertionErrorrG   Úview_as_realÚaddÚmul_Úaddcmul_r   Úadd_ÚitemÚsubÚcopy_ÚmaximumÚ	ones_likerL   Úmax)r   rS   rU   rT   rV   rW   r   r   r   r   r   r   r    r!   rX   ÚiÚparamrF   r3   rE   r2   Ústep_tÚcapturable_supported_devicesÚ	eta_valuer/   Únew_etaÚnew_mus                              r,   Ú_single_tensor_asgdr…   Å   s¬  € ô$ �9‰9×!Ñ!Ô#Ü˜‹^ˆä˜f×%‰ˆˆ5Ø�Q‰xˆÙ#‰t¨$¨ˆØ�‰VˆØ�‰VˆØ�1‰gˆØ˜Q‘ˆô �~‰~×*Ñ*Ô,±Ü+LÓ+NÐ(à—‘×!Ñ!Ø—9‘9—>‘>ô&à—:‘:—?‘?ô&ð —=‘=×%Ñ%õ&ð —L‘L×%Ñ%Ð)EÑEä$ð-Ø-IÐ,JÈ!ðMóð ô
 ×Ñ˜EÔ"Ü×%Ñ% dÓ+ˆDÜ×&Ñ& uÓ-ˆEÜ×#Ñ# BÓ'ˆBð 	�!‰ˆà˜1ÒØ—8‘8˜E¨�8Ó6ˆDáØ�J‰J�q˜5 3™;‘Ô'Ø�N‰N˜4 ¨BˆNÕ/ä" 3›ˆIØ�J‰J�q˜5 9Ñ,Ñ,Ô-Ø�J‰J�t I :ˆJÔ.ñ ˜Ÿ™› ašØ�G‰G�E—I‘I˜b“M×&Ñ& rÓ*Õ+à�H‰H�UŒOáà�I‰I�b˜Q ¨¡¨fÑ!4Ñ4¸Ñ>Ñ?Ô@Ø�H‰H�QœŸ™ v°¡{´E·O±OÀFÓ4KÓLÑLÖMä˜fÓ%ˆDÜ—o‘o b¨Q°¸±¸dÑ1BÑ-BÀuÑ,LÑ&MÓNˆGØ�I‰I�gÔÜ—_‘_ Q¬¨Q°°r±	Ó):Ñ%:Ó;ˆFØ�H‰H�VÖñs &r-   c       	         óp  ‡"— t        | «      dk(  ry |rt        d«      ‚t        j                  j	                  «       s?|r=t        d¬«      Š"t        ˆ"fd„t        | |||d¬«      D «       «      st        d‰"› d	�«      ‚t        |«      }t        j                  | |||||g«      }|j                  «       D �]x  \  \  }}\  \  }}}}}}}t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }|rt        |||«       |rt        j                   |«      }t        j                  j	                  «       s=|d   j"                  r.t        j$                  |t        j&                  d
d¬«      d
¬«       nt        j$                  |d«       |
dk7  rN|rt        j$                  |||
¬«       |}nt        j(                  |||
¬«      }t        j$                  |||¬«       nt        j(                  |||¬«      }t        j*                  |||d¬«       ~t        j,                  ||«      }t        j*                  |||«       ~|rót        j,                  ||«      }t        j.                  |d
«       t        j0                  |«       t        j2                  ||«       ~t        j4                  ||«      } t        j6                  | |«       t        j$                  | d«       t        j8                  | |	«       t        j0                  | «       t        j6                  | |«       t        j2                  || «       �ŒÝ|D �!cg c](  }!t        j:                  |d||z  |!z  z   |	z  z  |¬«      ‘Œ* } }!|D �!cg c]2  }!t        j:                  dt=        dt?        |!«      |z
  «      z  |¬«      ‘Œ4 }}!t        j2                  || «       t        j2                  ||«       �Œ{ y c c}!w c c}!w )Nr   z#_foreach ops don't support autogradF)Úsupports_xlac              3   ó,  •K  — | ]‹  \  }}}}|j                   j                  |j                   j                  cxk(  xr5 |j                   j                  cxk(  xr |j                   j                  k(  nc xr |j                   j                  ‰v –— Œ� y ­wr_   )r1   rq   )Ú.0r?   r3   r2   r/   r�   s        €r,   Ú	<genexpr>z%_multi_tensor_asgd.<locals>.<genexpr>3  sq   øè ø€ ð 
ñ %UÑ ��2�s˜Dð �H‰H�M‰M˜RŸY™YŸ^™^ÖR¨s¯z©z¯©ÖRÀ$Ç+Á+×BRÑBRÔRò >Ø—‘—‘Ð!=Ð=ó>á$Tùs   ƒBBT)Ústrictrf   rg   g      ð?Úcpu)r1   rh   r   ri   rj   ) r9   rr   r:   ro   rp   r   ÚallÚzipr   r   Ú"_group_tensors_by_device_and_dtypeÚitemsr   Úlistr   r   Ú_foreach_negÚis_cpuÚ_foreach_add_r=   Ú_foreach_addÚ_foreach_addcmul_Ú_foreach_subÚ_foreach_maximum_Ú_foreach_reciprocal_Ú_foreach_copy_Ú_foreach_mulÚ_foreach_mul_Ú_foreach_pow_rL   r}   r   )#r   rS   rU   rT   rV   rW   r   r   r   r   r   r   r    r!   rX   Úgrouped_tensorsr1   Ú_Úgrouped_params_Úgrouped_grads_Úgrouped_axs_Úgrouped_mus_Úgrouped_etas_Úgrouped_state_steps_Úgrouped_paramsÚgrouped_gradsÚgrouped_axsÚgrouped_musÚgrouped_etasÚgrouped_state_stepsÚintermediateÚnew_musÚnew_etasr/   r�   s#                                     @r,   Ú_multi_tensor_asgdr¯     sè  ø€ ô$ ˆ6ƒ{�aÒØáÜÐBÓCÐCô �>‰>×&Ñ&Ô(©ZÜ'HØô(
Ð$ô ó 
ô %(¨°°T¸;ÈtÕ$Tó
ô 
ô
 !ð&Ø&BÐ%CÀ1ðFóð ô
 
�B‹€Bä×BÑBØ	�˜˜S $¨Ð4ó€Oð 
×	Ñ	×	 ñ
	‰ˆ�ñ 
ñ	
ØØØØØØ à	äœd¤6™l¨OÓ<ˆÜœT¤&™\¨>Ó:ˆÜœ4¤™<¨Ó6ˆÜœ4¤™<¨Ó6ˆÜœD¤™L¨-Ó8ˆÜ"¤4¬¡<Ð1EÓFÐáÜ˜.¨-¸ÔEáÜ!×.Ñ.¨}Ó=ˆMô �~‰~×*Ñ*Ô,Ð1DÀQÑ1G×1NÒ1NÜ×ÑØ#¤U§\¡\°#¸eÔ%DÈCöô ×ÑÐ 3°QÔ7ð ˜1ÒÙÜ×#Ñ# M°>ÈÕVØ,‘ä$×1Ñ1Ø! >¸ô �ô ×Ñ ¨nÀEÖJä ×-Ñ-Ø˜~°UôˆLô 	×Ñ °¸lÐRTÕUØô ×)Ñ)¨.¸+ÓFˆÜ×Ñ ¨\¸;ÔGØñ ä×(Ñ(Ð)<¸bÓAˆGÜ×#Ñ# G¨SÔ1Ü×&Ñ& wÔ/Ü× Ñ  ¨gÔ6Øô ×)Ñ)Ð*=¸uÓEˆHÜ×Ñ ¨"Ô-Ü×Ñ ¨!Ô,Ü×Ñ ¨%Ô0Ü×&Ñ& xÔ0Ü×Ñ ¨"Ô-Ü× Ñ  ¨xÖ8ñ 0óá/�Dô —‘  q¨5°2©:¸Ñ+<Ñ'<ÀÑ&FÑ GÐPVÖWØ/ð ð ñ 0óá/�Dô —‘ ¤C¨¬:°dÓ+;¸bÑ+@Ó$AÑ AÈ&ÖQØ/ð ð ô × Ñ  ¨xÔ8Ü× Ñ  ¨gÖ6ñs 
!ùò`ùòs   Î-P.Ï7P3)Úsingle_tensor_fnr   c                ó  — |€t        | |d¬«      \  }}|r)t        j                  j                  «       rt	        d«      ‚|r%t        j                  j                  «       st
        }nt        } || |||||||||||||	|
¬«       y)znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   r   r   r   r   r   r    r!   rX   )r   r:   rl   rm   rJ   r¯   r…   )r   rS   rU   rT   rV   rW   r   r   r    r!   rX   r   r   r   r   r   rŸ   Úfuncs                     r,   r   r   ¨  s�   € ð4 €Ü1Ø�N¨eô
‰
ˆˆ7ñ ”5—9‘9×)Ñ)Ô+ÜÐSÓTÐTá”u—y‘y×-Ñ-Ô/Ü!‰ä"ˆáØØØØØØØØØØØ!ØØ%ØØör-   )NFFFF)Útypingr   r:   r   Ú	optimizerr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r‘   r<   rc   r…   r¯   r   rC   r-   r,   Ú<module>r¸      s  ðå ã Ý ÷÷ ÷ ÷ ñ ð& �6Ð
€ôNˆ9ô Nðb	ð 
ˆð 	ð 
ˆð 	Ø	ˆð 	Ø	Ðð 	Ø	Ðð ð€„ð.NØ�‰LðNà�‰<ðNð 
ˆf‰ðNð 
ˆf‰ð	Nð
 ˆv‰,ðNð �f‘ðNð ðNð 	ðNð 	ðNð ðNð ðNð ðNð ðNð ðNð  ð!Nð" 
ó#NðbO7Ø�‰LðO7à�‰<ðO7ð 
ˆf‰ðO7ð 
ˆf‰ð	O7ð
 ˆv‰,ðO7ð �f‘ðO7ð ðO7ð 	ðO7ð 	ðO7ð ðO7ð ðO7ð ðO7ð ðO7ð ðO7ð  ð!O7ð" 
ó#O7ñd  Ð1DÔEð  ØØ ØØñ6Ø�‰Lð6à�‰<ð6ð 
ˆf‰ð6ð 
ˆf‰ð	6ð
 ˆv‰,ð6ð �f‘ð6ð �D‰[ð6ð ð6ð ð6ð ð6ð ð6ð ð6ð  	ð!6ð" 	ð#6ð$ ð%6ð& ð'6ð( 
ò)6ó Fñ6r-   