Ë
    ùÿæiE  ã                    ó  — d Z ddlmZ ddlZddlmZ ddlmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ ddgZ G d	„ de«      Zd
de› de› de
› de› de› d�z   e_         dee   dee   dee   dee   dee   dededededededededdfd„Zdee   dee   dee   dee   dee   dededededededededdfd„Z e	e¬«      	 	 	 	 	 d"dee   dee   dee   dee   dee   d edz  dededededededededdfd!„«       Zy)#z1Implementation for the Resilient backpropagation.é    )ÚcastN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ_maximize_docÚ_params_docÚ
_to_scalarÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚRpropÚrpropc                   óš   ‡ — e Zd Z	 	 	 ddddddœdedeez  deeef   deeef   ded	edz  d
ededdfˆ fd„Zˆ fd„Z	d„ Z
edd„«       Zˆ xZS )r   FN)Ú
capturableÚforeachÚmaximizeÚdifferentiableÚparamsÚlrÚetasÚ
step_sizesr   r   r   r   Úreturnc                ó  •— t        |t        «      r|j                  «       dk7  rt        d«      ‚d|k  st        d|› �«      ‚d|d   cxk  rdcxk  r|d   k  sn t        d|d   › d|d   › �«      ‚|||||||d	œ}	t        ‰
| �  ||	«       y )
Nr   zTensor lr must be 1-elementg        zInvalid learning rate: r   ç      ð?zInvalid eta values: z, )r   r   r   r   r   r   r   )Ú
isinstancer   ÚnumelÚ
ValueErrorÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s             €úf/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/torch/optim/rprop.pyr&   zRprop.__init__   s«   ø€ ô �bœ&Ô! b§h¡h£j°A¢oÜÐ:Ó;Ð;Ø�bŠyÜÐ6°r°dÐ;Ó<Ð<Ø�T˜!‘WÔ,˜sÔ, T¨!¡WÔ,ÜÐ3°D¸±G°9¸B¸tÀA¹w¸iÐHÓIÐIð ØØ$ØØ Ø,Ø$ñ
ˆô 	‰Ñ˜ Õ*ó    c                 ó0  •— t         ‰| �  |«       | j                  D ]÷  }|j                  dd «       |j                  dd«       |j                  dd«       |j                  dd«       |d   D ]¥  }| j                  j                  |g «      }t        |«      dk7  sŒ.t        j                  |d   «      rŒGt        |d   «      }|d   r*t        j                  |t        «       |j                  ¬	«      nt        j                  |t        «       ¬
«      |d<   Œ§ Œù y )Nr   r   Fr   r   r   r   Ústep©ÚdtypeÚdevice©r/   )r%   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r0   )r'   r5   ÚgroupÚpÚp_stateÚstep_valr)   s         €r*   r2   zRprop.__setstate__=   sð   ø€ Ü‰Ñ˜UÔ#Ø×&Ô&ˆEØ×Ñ˜Y¨Ô-Ø×Ñ˜Z¨Ô/Ø×ÑÐ-¨uÔ5Ø×Ñ˜\¨5Ô1Ø˜8”_�ØŸ*™*Ÿ.™.¨¨BÓ/�Ü�w“< 1Ó$¬U¯_©_¸WÀV¹_Õ-MÜ$ W¨V¡_Ó5�Hð
 ! Ò.ô Ÿ™Ø$Ô,=Ó,?ÈÏÉõô #Ÿ\™\¨(Ô:KÓ:MÔNð ˜F’Oñ	 %ñ 'r+   c           	      ód  — d}|d   D �]¤  }|j                   €Œ|t        j                  |«      z  }|j                  |«       |j                   }	|	j                  rt        d«      ‚|j                  |	«       | j                  |   }
t        |
«      dk(  rÞ|d   r*t        j                  dt        «       |j                  ¬«      nt        j                  dt        «       ¬«      |
d	<   t        j                  |t        j                  ¬
«      |
d<   |j                  j                  r*t        j                  |	t        |d   |d   «      «      |
d<   n%t        j                  |	t!        |d   «      «      |
d<   |j                  |
d   «       |j                  |
d   «       |j                  |
d	   «       �Œ§ |S )NFr   z'Rprop does not support sparse gradientsr   r   © r.   r1   r-   ©Úmemory_formatÚprevr   Ú	step_size)Úgradr8   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr5   r7   Úzerosr   r0   Ú
zeros_likeÚpreserve_formatr/   Ú	full_likeÚcomplexr   )r'   r<   r   ÚgradsÚprevsr   Ústate_stepsÚhas_complexr=   rF   r5   s              r*   Ú_init_groupzRprop._init_groupP   sw  € ØˆØ�x•ˆAØ�v‰vˆ~ØØœ5×+Ñ+¨AÓ.Ñ.ˆKØ�M‰M˜!ÔØ—6‘6ˆDØ�~Š~Ü"Ð#LÓMÐMà�L‰L˜ÔØ—J‘J˜q‘MˆEô �5‹z˜QŠð ˜\Ò*ô —K‘K Ô*;Ó*=ÀaÇhÁhÕOäŸ™ RÔ/@Ó/BÔCð �f‘ô !&× 0Ñ 0°Ä%×BWÑBWÔ X��f‘Ø—7‘7×%Ò%ô */¯©Øœg e¨D¡k°5¸±;Ó?ó*�E˜+Ò&ô */¯©¸¼zÈ%ÐPTÉ+Ó?VÓ)W�E˜+Ñ&à�L‰L˜˜v™Ô'Ø×Ñ˜e KÑ0Ô1Ø×Ñ˜u V™}Ö-ðA !ðD Ðr+   c                 ób  — | j                  «        d}|�$t        j                  «       5   |«       }ddd«       | j                  D ][  }g }g }g }g }g }|d   \  }	}
|d   \  }}|d   }|d   }| j	                  ||||||«      }t        ||||||||	|
|||d   |d   |¬«       Œ] |S # 1 sw Y   ŒuxY w)	z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   )	Ústep_size_minÚstep_size_maxÚetaminusÚetaplusr   r   r   r   rS   )Ú'_accelerator_graph_capture_health_checkr8   Úenable_gradr3   rT   r   )r'   ÚclosureÚlossr<   r   rP   rQ   r   rR   rX   rY   rV   rW   r   r   rS   s                   r*   r-   z
Rprop.stepv   sý   € ð 	×4Ñ4Ô6àˆØÐÜ×"Ñ"Õ$Ù“y�÷ %ð ×&Ô&ˆEØ#%ˆFØ"$ˆEØ"$ˆEØ')ˆJØ(*ˆKà % f¡ÑˆH�gØ+0°Ñ+>Ñ(ˆM˜=Ø˜IÑ&ˆGØ˜ZÑ(ˆHà×*Ñ*Ø�v˜u e¨Z¸óˆKô ØØØØØØ+Ø+Ø!ØØØ!Ø$Ð%5Ñ6Ø  Ñ.Ø'öð! 'ðB ˆ÷I %Ð$ús   ©B%Â%B.)g{®Gáz„?)g      à?g333333ó?)g�íµ ÷Æ°>é2   ©N)Ú__name__Ú
__module__Ú__qualname__r   r:   r   ÚtupleÚboolr&   r2   rT   r   r-   Ú__classcell__)r)   s   @r*   r   r      s·   ø„ ð "Ø$.Ø*4ð+ð !Ø#ØØ$ò+àð+ð �F‰Nð+ð �E˜5�LÑ!ð	+ð
 ˜% ˜,Ñ'ð+ð ð+ð ˜‘ð+ð ð+ð ð+ð 
õ+ô<ò&$ðL "ò/ó "ô/r+   a¼
  Implements the resilient backpropagation algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \theta_0 \in \mathbf{R}^d \text{ (params)},f(\theta)
                \text{ (objective)},                                                             \\
            &\hspace{13mm}      \eta_{+/-} \text{ (etaplus, etaminus)}, \Gamma_{max/min}
                \text{ (step sizes)}                                                             \\
            &\textbf{initialize} :   g^0_{prev} \leftarrow 0,
                \: \eta_0 \leftarrow \text{lr (learning rate)}                                   \\
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \textbf{for} \text{  } i = 0, 1, \ldots, d-1 \: \mathbf{do}            \\
            &\hspace{10mm}  \textbf{if} \:   g^i_{prev} g^i_t  > 0                               \\
            &\hspace{15mm}  \eta^i_t \leftarrow \mathrm{min}(\eta^i_{t-1} \eta_{+},
                \Gamma_{max})                                                                    \\
            &\hspace{10mm}  \textbf{else if}  \:  g^i_{prev} g^i_t < 0                           \\
            &\hspace{15mm}  \eta^i_t \leftarrow \mathrm{max}(\eta^i_{t-1} \eta_{-},
                \Gamma_{min})                                                                    \\
            &\hspace{15mm}  g^i_t \leftarrow 0                                                   \\
            &\hspace{10mm}  \textbf{else}  \:                                                    \\
            &\hspace{15mm}  \eta^i_t \leftarrow \eta^i_{t-1}                                     \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1}- \eta_t \mathrm{sign}(g_t)             \\
            &\hspace{5mm}g_{prev} \leftarrow  g_t                                                \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to the paper
    `A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm
    <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.21.1417>`_.z

    Args:
        a{  
        lr (float, optional): learning rate (default: 1e-2)
        etas (Tuple[float, float], optional): pair of (etaminus, etaplus), that
            are multiplicative increase and decrease factors
            (default: (0.5, 1.2))
        step_sizes (Tuple[float, float], optional): a pair of minimal and
            maximal allowed step sizes (default: (1e-6, 50))
        z	
        z

    r   rP   rQ   r   rR   rV   rW   rX   rY   r   r   r   rS   r   c                ó  — t        | «      D �]ú  \  }}||   }|	s|n| }||   }||   }||   }t        j                  j                  «       s`|
r^t	        «       }|j
                  j                  |j
                  j                  k(  r|j
                  j                  |v st        d|› d�«      ‚|dz  }t        j                  |«      rTt        j                  |«      }t        j                  |«      }t        j                  |«      }t        j                  |«      }|r.|j                  |j                  «       «      j                  «       }n|j                  |«      j                  «       }|
r |j                  t        j                  |j                  d«      ||«      «       |j                  t        j                  |j!                  d«      ||«      «       |j                  t        j                  |j#                  d«      d|«      «       n<|||j                  d«      <   |||j!                  d«      <   d||j#                  d«      <   |j%                  |«      j'                  ||«       |j                  t        j(                  ¬«      }|
r6|j                  t        j                  |j#                  |«      d|«      «       nd||j#                  |«      <   |j+                  |j                  «       |d¬«       |j                  |«       �Œý y )NúIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   rB   éÿÿÿÿ©Úvalue)Ú	enumerater8   ÚcompilerÚis_compilingr   r0   ÚtypeÚAssertionErrorrG   Úview_as_realÚmulÚcloneÚsignÚcopy_ÚwhereÚgtÚltÚeqÚmul_Úclamp_rM   Úaddcmul_)r   rP   rQ   r   rR   rV   rW   rX   rY   r   r   r   rS   ÚiÚparamrF   rD   rE   r-   Úcapturable_supported_devicesrt   s                        r*   Ú_single_tensor_rpropr€   ß   sk  € ô  ˜f×%‰ˆˆ5Ø�Q‰xˆÙ#‰t¨$¨ˆØ�Q‰xˆØ˜q‘Mˆ	Ø˜1‰~ˆô �~‰~×*Ñ*Ô,±Ü+LÓ+NÐ(à—‘×!Ñ! T§[¡[×%5Ñ%5Ò5Ø—L‘L×%Ñ%Ð)EÑEä$Ø_Ð`|Ð_}Ð}~Ðóð ð 	�‰	ˆä×Ñ˜EÔ"Ü×%Ñ% dÓ+ˆDÜ×%Ñ% dÓ+ˆDÜ×&Ñ& uÓ-ˆEÜ×*Ñ*¨9Ó5ˆIÙØ—8‘8˜DŸJ™J›LÓ)×.Ñ.Ó0‰Dà—8‘8˜D“>×&Ñ&Ó(ˆDáØ�J‰J”u—{‘{ 4§7¡7¨1£:¨w¸Ó=Ô>Ø�J‰J”u—{‘{ 4§7¡7¨1£:¨x¸Ó>Ô?Ø�J‰J”u—{‘{ 4§7¡7¨1£:¨q°$Ó7Õ8à&ˆD�—‘˜“ÑØ'ˆD�—‘˜“ÑØ ˆD�—‘˜“Ñð 	�‰�tÓ×#Ñ# M°=ÔAð �z‰z¬×(=Ñ(=ˆzÓ>ˆÙØ�J‰J”u—{‘{ 4§7¡7¨8Ó#4°a¸Ó>Õ?à&'ˆD�—‘˜Ó"Ñ#ð 	�‰�t—y‘y“{ I°RˆÔ8Ø�
‰
�4Öñi &r+   c          
      ó   ‡— t        | «      dk(  ry |rt        d«      ‚t        j                  j	                  «       s;|
r9t        «       Št        ˆfd„t        | |d¬«      D «       «      st        d‰› d�«      ‚t        j                  | ||||g«      }|j                  «       D �]%  \  \  }}}}}}t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        j                  j	                  «       s=|d   j                  r.t        j                  |t        j                   dd	¬
«      d¬«       nt        j                  |d«       |rt#        ||||«       t        j$                  ||«      }|	rt        j&                  |«       t        j(                  ||«       |	rt        j&                  |«       |}t        j*                  |«       |
r§|D ]¡  }|j-                  t        j.                  |j1                  d«      ||«      «       |j-                  t        j.                  |j3                  d«      ||«      «       |j-                  t        j.                  |j5                  d«      d|«      «       Œ£ nC|D ]>  }|||j1                  d«      <   |||j3                  d«      <   d||j5                  d«      <   Œ@ t        j6                  ||«       |D ]  }|j9                  ||«       Œ t        |«      }t;        t        |«      «      D ]@  }||   j-                  t        j.                  ||   j5                  |«      d||   «      «       ŒB ~|D �cg c]  }|j=                  «       ‘Œ }}t        j>                  |||d¬«       �Œ( y c c}w )Nr   z#_foreach ops don't support autogradc              3   ó²   •K  — | ]N  \  }}|j                   j                  |j                   j                  k(  xr |j                   j                  ‰v –— ŒP y ­wr_   )r0   ro   )Ú.0r=   r-   r   s      €r*   Ú	<genexpr>z&_multi_tensor_rprop.<locals>.<genexpr>?  sR   øè ø€ ð 
ñ A‘��4ð �H‰H�M‰M˜TŸ[™[×-Ñ-Ñ-ò >Ø—‘—‘Ð!=Ð=ó>á@ùs   ƒAAT)Ústrictrg   rh   r!   Úcpu)r0   )Úalphar   ri   rj   ) r7   rp   r8   rm   rn   r   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Úlistr   Úis_cpuÚ_foreach_add_r;   r   Ú_foreach_mulÚ_foreach_neg_Ú_foreach_copy_Ú_foreach_sign_ru   rv   rw   rx   ry   Ú_foreach_mul_r{   Úrangert   Ú_foreach_addcmul_) r   rP   rQ   r   rR   rV   rW   rX   rY   r   r   r   rS   Úgrouped_tensorsÚgrouped_params_Úgrouped_grads_Úgrouped_prevs_Úgrouped_step_sizes_Úgrouped_state_steps_Ú_Úgrouped_paramsÚgrouped_gradsÚgrouped_prevsÚgrouped_step_sizesÚgrouped_state_stepsÚsignsrt   rE   r}   rF   Ú
grad_signsr   s                                   @r*   Ú_multi_tensor_rpropr¤   &  sZ  ø€ ô  ˆ6ƒ{�aÒØáÜÐBÓCÐCô �>‰>×&Ñ&Ô(©ZÜ'HÓ'JÐ$Üó 
ô ˜v {¸4Õ@ó
ô 
ô
 !Ø[Ð\xÐ[yÐyzÐ{óð ô  ×BÑBØ	�˜˜z¨;Ð7ó€Oð ×"Ñ"×$ñ		ñ 	ØØØØØØÜœd¤6™l¨OÓ<ˆÜœT¤&™\¨>Ó:ˆÜœT¤&™\¨>Ó:ˆÜ!¤$¤v¡,Ð0CÓDÐÜ"¤4¬¡<Ð1EÓFÐô �~‰~×*Ñ*Ô,Ð1DÀQÑ1G×1NÒ1NÜ×ÑØ#¤U§\¡\°#¸eÔ%DÈCöô ×ÑÐ 3°QÔ7ñ ÜØ ¨}Ð>Pôô ×"Ñ" =°-Ó@ˆÙÜ×Ñ Ô&ô
 	×Ñ˜]¨MÔ:ÙÜ×Ñ Ô.Ø%ˆä×Ñ˜UÔ#ÙÛ�Ø—
‘
œ5Ÿ;™; t§w¡w¨q£z°7¸DÓAÔBØ—
‘
œ5Ÿ;™; t§w¡w¨q£z°8¸TÓBÔCØ—
‘
œ5Ÿ;™; t§w¡w¨q£z°1°dÓ;Õ<ñ ó
 �Ø#*��T—W‘W˜Q“ZÑ Ø#+��T—W‘W˜Q“ZÑ Ø#$��T—W‘W˜Q“ZÒ ð ô 	×ÑÐ.°Ô6Û+ˆIØ×Ñ˜]¨MÕ:ð ,ô
 ˜]Ó+ˆÜ”s˜=Ó)Ö*ˆAØ˜!Ñ×"Ñ"Ü—‘˜E !™HŸK™K¨Ó1°1°mÀAÑ6FÓGõð +ð ñ /<Ó<©m d�d—i‘i•k¨mˆ
Ð<Ü×ÑØ˜JÐ(:À"÷	
ñE %ùòB =s   ÎO)Úsingle_tensor_fnr   c
                óz  — t         j                  j                  «       st        d„ |D «       «      st	        d«      ‚|€t        | |d¬«      \  }}|r)t         j                  j                  «       rt	        d«      ‚|r%t         j                  j                  «       st        }nt        } || |||||
|||||||	¬«       y)zpFunctional API that performs rprop algorithm computation.

    See :class:`~torch.optim.Rprop` for details.
    c              3   óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wr_   )r"   r8   r   )rƒ   Úts     r*   r„   zrprop.<locals>.<genexpr>¶  s   è ø€ ð 5Ù-8¨Œ
�1”e—l‘l×#©[ùs   ‚$&zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)rV   rW   rX   rY   r   r   r   rS   )
r8   rm   rn   rˆ   rJ   r   ÚjitÚis_scriptingr¤   r€   )r   rP   rQ   r   rR   r   r   r   r   rS   rV   rW   rX   rY   rœ   Úfuncs                   r*   r   r   œ  sÁ   € ô4 �>‰>×&Ñ&Ô(´ñ 5Ù-8ó5ô 2ô Ø^ó
ð 	
ð €Ü1Ø�N¨eô
‰
ˆˆ7ñ ”5—9‘9×)Ñ)Ô+ÜÐSÓTÐTá”u—y‘y×-Ñ-Ô/Ü"‰ä#ˆáØØØØØØ#Ø#ØØØØØ%Øör+   )NFFFF)Ú__doc__Útypingr   r8   r   Ú	optimizerr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   Ú__all__r   rŒ   r:   rd   r€   r¤   r   rA   r+   r*   Ú<module>r±      sÏ  ðá 8å ã Ý ÷÷ ÷ ÷ ð$ �GÐ
€ôHˆIô HðX!LðD	ð 
ˆð 	ð 
Ðð 	Ø	ˆð 	Ø	ˆð 	Ø	Ðð ðñE1ð „ðlDØ�‰LðDà�‰<ðDð �‰<ðDð �V‘ð	Dð
 �f‘ðDð ðDð ðDð ðDð ðDð ðDð ðDð ðDð ðDð 
óDðNo
Ø�‰Lðo
à�‰<ðo
ð �‰<ðo
ð �V‘ð	o
ð
 �f‘ðo
ð ðo
ð ðo
ð ðo
ð ðo
ð ðo
ð ðo
ð ðo
ð ðo
ð 
óo
ñl  Ð1EÔFð  ØØØ Øñ;Ø�‰Lð;à�‰<ð;ð �‰<ð;ð �V‘ð	;ð
 �f‘ð;ð �D‰[ð;ð ð;ð ð;ð ð;ð ð;ð ð;ð ð;ð  ð!;ð" ð#;ð$ 
ò%;ó Gñ;r+   