+
    QV-jÐ[  ã                  óÚ  a € 0 t $ R t^ RIHt ^ RIt^ RIt^ RIHt ^ RI	H
t
 ^ RIHtHt ^ RIt^ RIHt ^ RIHt ^RIHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHtHtHtH t H!t!H"t"H#t#H$t$ ]'       d   ^RI%H&t& ^RIH't' ]PP                  ! ])4      t*]
 ! R R4      4       t+R R lt,]]+.]-]].]/]3,          3,          3,          t0R R lt1R9R R llt2R R lt3R R lt4R R lt5R R lt6R R  lt7R! R" lt8R# R$ lt9R% R& lt:R' R( lt;R) R* lt<R+ R, lt=R- R. lt>R/ R0 lt?R1 R2 lt@R3 R4 ltAR5 R6 ltB]P†                  ]Pˆ                  ]PŠ                  ]PŒ                  ]PŽ                  ]P�                  ]P’                  ]P”                  ]P–                  ]P˜                  ]Pš                  ]Pœ                  ]Pž                  ]P                   ]P¢                  .tR]P¦                  ]P¨                  ]Pª                  ]P¬                  ]P®                  ]P°                  .tY]P´                  ]P¶                  .t\]Pº                  ]P¼                  .t_]PÀ                  ]PÂ                  ]PÄ                  .tc]PÈ                  ]3]PÊ                  ]4]PÌ                  ]4]PÎ                  ]5]PÐ                  ]6]PÒ                  ]8]PÔ                  ]9]PÖ                  ]:]PØ                  ];]PÚ                  ]?]PÜ                  ]B]PÞ                  ]>]Pà                  ]>/].Pã                  ]R]74      C].Pã                  ]Y]<4      C].Pã                  ]\]=4      C].Pã                  ]_]@4      C].Pã                  ]c]A4      CtrR7]sR8&   R# ):z,
Optimizer utilities for the Trainer class.
)ÚannotationsN)ÚCallable)Ú	dataclass)ÚTYPE_CHECKINGÚAny)Úversion)Únn)Ú	Adafactor)ÚLayerWiseDummyOptimizer)Úcheck_target_module_exists)ÚOptimizerNamesÚParallelMode)	Úis_apollo_torch_availableÚis_bitsandbytes_availableÚis_galore_torch_availableÚis_grokadamw_availableÚis_lomo_availableÚis_schedulefree_availableÚis_torch_optimi_availableÚis_torchao_availableÚ	strtobool)ÚPreTrainedModel)ÚTrainingArgumentsc                  óN   € ] tR t^6t$ RtR]R&   R]R&   R]R&   R]R&   R	]R
&   RtR# )ÚOptimizerContextz0Context object passed to all optimizer handlers.r   ÚargszPreTrainedModel | NoneÚmodelúdict[str, Any]Úoptimizer_kwargsÚadam_kwargsúdict[str, str]Ú
optim_args© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__r"   ó    Úo/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/trainer_optimizer.pyr   r   6   s#   ‡ á:à
ÓØ!Ó!Ø$Ó$ØÓØ×r*   r   c               ó    € V ^8„  d   QhRRRR/# )é   Úoptim_args_strz
str | NoneÚreturnr    r"   )Úformats   "r+   Ú__annotate__r1   A   s   € ÷ ñ  jð °^ñ r*   c                óœ   € V '       g   / # / pV P                  RR4      P                  R4       F  pVP                  R4      w  r4WAV&   K  	  V# )z8Parse optimizer arguments from a comma-separated string.Ú Ú Ú,Ú=)ÚreplaceÚsplit)r.   r!   ÚmappingÚkeyÚvalues   &    r+   Ú_parse_optim_argsr<   A   sP   € çØˆ	Ø€JØ!×)Ñ)¨#¨rÓ2×8Ñ8¸Ö=ˆØ—]‘] 3Ó'‰
ˆØ�3‹ñ >ð Ðr*   c               ó    € V ^8„  d   QhRRRR/# )r-   Úoptimizer_cls_or_factoryr   r/   Úboolr"   )r0   s   "r+   r1   r1   P   s   € ÷ ñ °3ð ¸4ñ r*   c                óŠ   € \        V \        4      '       d-   \        V \        P                  P
                  4      '       d   R# R# )a  
Check if the returned value from a handler is a factory rather than an Optimizer class.

Factory callables are used for complex optimizers like Muon or Dion that need to:
- Split parameters between multiple internal optimizers
- Handle complex sharding logic
- Access the full model structure for parameter grouping

Args:
    optimizer_cls_or_factory: The first element returned by an optimizer handler.

Returns:
    `bool`: True if it's not an Optimizer class (i.e., likely a factory), False if it's an Optimizer class.
FT)Ú
isinstanceÚtypeÚ
issubclassÚtorchÚoptimÚ	Optimizer)r>   s   &r+   Úis_optimizer_factoryrG   P   s2   € ô  Ð*¬D×1Ò1´jÐAYÔ[`×[fÑ[f×[pÑ[p×6qÒ6qÙÙr*   c               ó8   € V ^8„  d   QhRRRRRRRRR	RR
RRRRR/# )r-   r   r   r   r   Úoptimizer_nameÚstrÚoptimizer_mappingr   Úoptim_kwargsr   Úis_layerwise_supportedr?   r/   útuple[Any, dict[str, Any]]r"   )r0   s   "r+   r1   r1   e   sb   € ÷ V+ñ V+Ø
ðV+àðV+ð ðV+ð &ð	V+ð
 !ðV+ð %ðV+ð !ðV+ð  ñV+r*   c                óV  a€ VP                  4       P                  R4      pV'       d7   V P                  \        P                  8X  d   V'       d   \        RV R24      hW2,          pV P                  f   \        RV R24      h\        V P                  \        \        34      '       g   \        RV P                   24      hVf   \        RV R24      h\        V P                  \        4      ;'       d     V P                  P                  R	R
4      R8H  p	. p
VP                  4        F–  w  r¼\        V P                  VRR7      w  rÞ\        V\        P                   4      '       g.   V'       d$   V'       g   \"        P%                  V RV R24       Kl  V'       g   V	'       g   K~  V
P'                  VR,           4       K˜  	  \)        V
4      ^ 8X  d   \        RV RV P                   R24      hVP+                  4        UUu. uF  w  ppWú9   g   K  VNK  	  pppVP+                  4        UUu. uF  w  ppWú9  g   K  VNK  	  pppRV/RV/VC.pV'       d²   V P,                  ^8w  d   \        RV R24      h/ oV F  pV! RV./.3/ VB SV&   K  	  V F  pV! RV./VC.3/ VB SV&   K  	  V3R lpVP/                  4        F(  pVP0                  '       g   K  VP3                  V4       K*  	  \4        pVP7                  RS/4       VP7                  RV/4       W…3# u uppi u uppi )zª
Helper function to set up low-rank optimizers like GaLore and Apollo.

These optimizers apply low-rank projections to specific target modules (typically linear layers).
Ú	layerwisezLayer-wise z" does not support DDP at this timez1You need to define `optim_target_modules` to use z optimizerszX`optim_target_modules` must be a list of strings, a regex string, or 'all-linear'. Got: z'You need to pass a model to initialize z optimizer.Ú_Ú-z
all-linearT)Úreturn_is_regexz matched but ignored. z only supports linear layers.z.weightzNo target modules found for z (z).Úparamsz
Layerwise z( does not support gradient accumulation!c                ó„   <€ V P                   e1   SV ,          P                  4        SV ,          P                  4        R # R # )N)ÚgradÚstepÚ	zero_grad)ÚparamÚoptimizer_dicts   &€r+   Úoptimizer_hookÚ1_setup_low_rank_optimizer.<locals>.optimizer_hook®   s4   ø€ Ø�z‰zÒ%Ø˜uÕ%×*Ñ*Ô,Ø˜uÕ%×/Ñ/Ö1ñ &r*   rZ   )ÚlowerÚendswithÚparallel_moder   ÚDISTRIBUTEDÚNotImplementedErrorÚoptim_target_modulesÚ
ValueErrorrA   ÚlistrJ   Ú	TypeErrorr7   Únamed_modulesr   r   ÚLinearÚloggerÚwarningÚappendÚlenÚnamed_parametersÚgradient_accumulation_stepsÚ
parametersÚrequires_gradÚ"register_post_accumulate_grad_hookr
   Úupdate)r   r   rI   rK   rL   r   rM   Úis_layerwiseÚoptimizer_clsÚ
all_linearÚtarget_params_namesÚmodule_nameÚmoduleÚtarget_module_existsÚis_regexÚnÚpÚtarget_paramsÚnon_target_paramsÚparam_groupsrY   r[   rZ   s   &&&&&&&               @r+   Ú_setup_low_rank_optimizerr   e   s6  ø€ ð "×'Ñ'Ó)×2Ñ2°;Ó?€Lß˜×*Ñ*¬l×.FÑ.FÔF×KaÜ! K°Ð/?Ð?aÐ"bÓcÐcà%Õ5€Mà× Ñ Ò(ÜÐLÈ^ÐL\Ð\gÐhÓiÐiä�d×/Ñ/´$¼°×=Ò=ÜðØ×-Ñ-Ð.ð0ó
ð 	
ð
 ‚}ÜÐBÀ>ÐBRÐR]Ð^Ó_Ð_ô 	�4×,Ñ,¬cÓ2×rÐr°t×7PÑ7P×7XÑ7XÐY\Ð^aÓ7bÐfrÑ7rð ð ÐØ$×2Ñ2Ö4ÑˆÜ)CØ×%Ñ% {ÀDô*
Ñ&Ðô ˜&¤"§)¡)×,Ò,ß#¯HÜ—‘ + Ð.DÀ^ÐDTÐTqÐrÔsÙç#¯JÙà×"Ñ" ;°Õ#:Ö;ñ  5ô ÐÓ 1Ô$ÜÐ7¸Ð7GÀrÈ$×JcÑJcÐIdÐdfÐgÓhÐhà#(×#9Ñ#9Ô#;ÔXÑ#;™4˜1˜a¸qÑ?W—Q�QÑ#;€MÑXØ',×'=Ñ'=Ô'?Ô`Ñ'?™t˜q !À1ÑC_Ÿ˜Ñ'?ÐÑ`ð 
Ð$Ð%Ø	�=Ð1 LÐ1ð€L÷
 Ø×+Ñ+¨qÔ0Ü˜z¨.Ð)9Ð9aÐbÓcÐcàˆÛ&ˆEÙ$1°H¸u¸gÐ3FÐ2GÑ$\ÐK[Ñ$\ˆN˜5Ó!ñ 'ã"ˆEÙ$1°H¸u¸gÐ3VÈÐ3VÐ2WÑ$lÐ[kÑ$lˆN˜5Ó!ñ #õ	2ð
 ×%Ñ%Ö'ˆEØ×"×"Ò"Ø×8Ñ8¸ÖHñ (ô 0ˆØ×ÑÐ!1°>Ð BÔCà×Ñ˜X |Ð4Ô5ØÐ*Ð*ùó? YùÛ`s   ÈLÈLÈ-L%È=L%c               ó    € V ^8„  d   QhRRRR/# ©r-   Úctxr   r/   rN   r"   )r0   s   "r+   r1   r1   Ã   s   € ÷ +ñ +Ð(ð +Ð-Gñ +r*   c                ód   € V P                   P                  RRRR/4       \        V P                   3# )zGet Adafactor optimizer.Úscale_parameterFÚrelative_step)r   rq   r	   )r‚   s   &r+   Ú_get_adafactorr†   Ã   s1   € à×Ñ×ÑÐ!2°E¸?ÈEÐ RÔSÜ�c×*Ñ*Ð*Ð*r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   É   s   € ÷ 'ñ 'Ð*ð 'Ð/Iñ 'r*   c                óþ   € ^ RI Hp V P                  P                  V P                  4       V P
                  P                  \        P                  8X  d   V P                  P                  RR/4       WP                  3# )z/Get PyTorch AdamW optimizer (regular or fused).©ÚAdamWÚfusedT)	Útorch.optimrŠ   r   rq   r   r   rE   r   ÚADAMW_TORCH_FUSED©r‚   rŠ   s   & r+   Ú_get_adamw_torchr�   É   sZ   € å!à×Ñ×Ñ §¡Ô0Ø
‡x�x‡~�~œ×9Ñ9Ô9Ø×Ñ×#Ñ# W¨d OÔ4Ø×&Ñ&Ð&Ð&r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   Ó   s    € ÷ Tñ TÐ.ð TÐ3Mñ Tr*   c                ó¨   €  ^ RI Hp V P                  P                  V P                  4       WP                  3#   \
         d    \        R4      hi ; i)z'Get Torch XLA syncfree AdamW optimizer.r‰   z7Trainer failed to import syncfree AdamW from torch_xla.)Útorch_xla.amp.syncfreerŠ   r   rq   r   ÚImportErrorrc   rŽ   s   & r+   Ú_get_adamw_torch_xlar”   Ó   sO   € ðTÝ0à×Ñ×#Ñ# C§O¡OÔ4Ø×*Ñ*Ð*Ð*øÜô TÜÐRÓSÐSðTúó	   ‚7: ºAc               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   Þ   s    € ÷ Pñ PÐ$4ð PÐ9Sñ Pr*   c                ó¨   €  ^ RI Hp V P                  P                  V P                  4       WP                  3#   \
         d    \        R4      hi ; i)zGet NPU Fused AdamW optimizer.)ÚNpuFusedAdamWz3Trainer failed to import FusedAdamW from torch_npu.)Útorch_npu.optimr˜   r   rq   r   r“   rc   )r‚   r˜   s   & r+   Ú_get_adamw_torch_npu_fusedrš   Þ   sO   € ðPÝ1à×Ñ×#Ñ# C§O¡OÔ4Ø×2Ñ2Ð2Ð2øÜô PÜÐNÓOÐOðPúr•   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   é   s   € ÷ //ñ //Ð%5ð //Ð:Tñ //r*   c                óÚ  € \        4       '       g   \        R4      h^ RIHpHpHp V P                  P                  pRV9   pRV9   d   ^M^ pRpV P                  pRV9   d   TpEM¶RV9   d3   TpRV P                  P                  V P                  P                  3/pEM}R	V9   d   TpV P                  pEMfR
V9   Ed_   ^ RIHp	 T	pR\        V P                  P                  RV P                  P                  4      4      \        V P                  P                  RV P                  P                  4      4      \        V P                  P                  RR4      4      3R\        V P                  P                  RR4      4      R\        V P                  P                  RV P                  P                  4      4      /pRV P                  9   d    \!        V P                  R,          4      VR&   RV P                  9   d    \!        V P                  R,          4      VR&   RV/p
R	V9  d   WZR&   V P"                  P%                  V4       V P"                  P%                  V
4       WpP"                  3# )z;Get bitsandbytes optimizer (AdamW, Lion, RMSprop variants).ziYou need to install `bitsandbytes` in order to use bitsandbytes optimizers: `pip install -U bitsandbytes`)rŠ   ÚLionÚRMSpropÚpagedÚ8bitNÚadamÚlionÚbetasÚrmspropÚademamix)ÚAdEMAMixÚbeta1Úbeta2Úbeta3g§èH.ÿï?Úalphag      @ÚepsÚt_alphaÚt_beta3Ú
optim_bitsÚis_paged)r   r“   Úbitsandbytes.optimrŠ   r�   rž   r   rE   r   Ú
adam_beta1Ú
adam_beta2r!   r¦   ÚfloatÚgetÚadam_epsilonÚintr   rq   )r‚   rŠ   r�   rž   Ú
optim_namer¯   r®   rs   Úadditional_optim_kwargsr¦   Ú
bnb_kwargss   &          r+   Ú_get_bitsandbytes_optimizerrº   é   s  € ä$×&Ò&ÜØwó
ð 	
÷ 8Ñ7à—‘—‘€JØ˜*Ñ$€HØ 
Ô*‘°€JØ€MØ!Ÿo™oÐà�ÔØŠØ	�:Ô	ØˆØ#*¨S¯X©X×-@Ñ-@À#Ç(Á(×BUÑBUÐ,VÐ"WÒØ	�jÔ	 ØˆØ"%§.¡.ÒØ	�zÕ	!Ý/à ˆàÜ�c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°&Ó9Ó:ðð
 ”U˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ø”5˜Ÿ™×+Ñ+¨E°3·8±8×3HÑ3HÓIÓJð#
Ðð ˜Ÿ™Ô&Ü14°S·^±^ÀIÕ5NÓ1OÐ# IÑ.Ø˜Ÿ™Ô&Ü14°S·^±^ÀIÕ5NÓ1OÐ# IÑ.à 
Ð+€JØ˜
Ô"Ø!)�:Ñà×Ñ×ÑÐ 7Ô8Ø×Ñ×Ñ 
Ô+Ø×.Ñ.Ð.Ð.r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1     s    € ÷ Qñ QÐ!1ð QÐ6Pñ Qr*   c                ó$  €  ^ RI Hp V P                  P                  V P                  4       V P                  P                  R\        V P                  P                  RR4      4      R\        \        V P                  P                  RR4      4      R\        \        V P                  P                  RR4      4      R\        \        V P                  P                  RR4      4      /4       WP                  3#   \         d    \        R	4      hi ; i)
z!Get AnyPrecision AdamW optimizer.)ÚAnyPrecisionAdamWÚuse_kahan_summationÚFalseÚmomentum_dtypeÚfloat32Úvariance_dtypeÚcompensation_buffer_dtypeÚbfloat16z4Please install https://github.com/pytorch/torchdistx)Útorchdistx.optimizersr½   r   rq   r   r   r!   r´   ÚgetattrrD   r“   rc   )r‚   r½   s   & r+   Ú_get_adamw_anyprecisionrÇ     sæ   € ðQÝ;à×Ñ×#Ñ# C§O¡OÔ4Ø×Ñ×#Ñ#à%¤y°·±×1CÑ1CÐDYÐ[bÓ1cÓ'dØ ¤'¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aØ ¤'¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aØ+¬WÜ˜3Ÿ>™>×-Ñ-Ð.IÈ:ÓVó.ð	ô		
ð !×"6Ñ"6Ð6Ð6øÜô QÜÐOÓPÐPðQús   ‚C5C8 Ã8Dc               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   0  s   € ÷ 	#ñ 	#Ð"ð 	#Ð'Añ 	#r*   c                ót  € V P                   P                  4       pV P                  '       dr   R F3  pW P                  9   g   K  \        V P                  V,          4      W&   K5  	  RV P                  9   d(   V P                  R,          P	                  4       R9   VR&   \
        P                  P                  V3# )zGet SGD optimizer.Únesterov)ÚmomentumÚ	dampeningÚweight_decay©ÚtrueÚ1Úyes)r   Úcopyr!   r³   r]   rD   rE   ÚSGD©r‚   Úkwargsr:   s   &  r+   Ú_get_sgdrÖ   0  s‹   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~€~Û<ˆCØ—n‘nÖ$Ü# C§N¡N°3Õ$7Ó8�“ñ =ð ˜Ÿ™Ô'Ø!$§¡°
Õ!;×!AÑ!AÓ!CÐG[Ñ![ˆF�:ÑÜ�;‰;�?‰?˜FÐ"Ð"r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   <  s   € ÷ 'ñ 'Ð&ð 'Ð+Eñ 'r*   c                ó  € V P                   P                  4       pV P                  '       d:   R F3  pW P                  9   g   K  \        V P                  V,          4      W&   K5  	  \        P
                  P                  V3# )zGet Adagrad optimizer.)Úlr_decayrÍ   r«   )r   rÒ   r!   r³   rD   rE   ÚAdagradrÔ   s   &  r+   Ú_get_adagradrÛ   <  s_   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~€~Û6ˆCØ—n‘nÖ$Ü# C§N¡N°3Õ$7Ó8�“ñ 7ô �;‰;×Ñ Ð&Ð&r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   F  s   € ÷ 	'ñ 	'Ð&ð 	'Ð+Eñ 	'r*   c                ót  € V P                   P                  4       pV P                  '       dr   R F3  pW P                  9   g   K  \        V P                  V,          4      W&   K5  	  RV P                  9   d(   V P                  R,          P	                  4       R9   VR&   \
        P                  P                  V3# )zGet RMSprop optimizer.Úcentered)rË   rª   r«   rÍ   rÎ   )r   rÒ   r!   r³   r]   rD   rE   rž   rÔ   s   &  r+   Ú_get_rmsproprß   F  sŽ   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~€~Û?ˆCØ—n‘nÖ$Ü# C§N¡N°3Õ$7Ó8�“ñ @ð ˜Ÿ™Ô'Ø!$§¡°
Õ!;×!AÑ!AÓ!CÐG[Ñ![ˆF�:ÑÜ�;‰;×Ñ Ð&Ð&r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   R  s   € ÷ +ñ +Ð/ð +Ð4Nñ +r*   c                ó2  € \        4       '       g   \        R4      h^ RIHpHpHp \        P                  V\        P                  V\        P                  V\        P                  V\        P                  V\        P                  V/pR\        V P                  P                  R^€4      4      R\        V P                  P                  R^È4      4      R\!        V P                  P                  RR4      4      RV P                  P                  RR4      /p\#        V P$                  V P&                  V P$                  P(                  WEV P*                  4      w  rgV P$                  P(                  \        P                  8X  d   VP-                  R	R
RR
/4       Wg3# )zGet GaLore optimizer.z�You need to install `galore_torch` in order to use GaLore optimizers. Install it with `pip install git+https://github.com/jiaweizzhao/GaLore`)ÚGaLoreAdafactorÚGaLoreAdamWÚGaLoreAdamW8bitÚrankÚupdate_proj_gapÚscaleg      Ð?Ú	proj_typeÚstdr„   Fr…   )r   r“   Úgalore_torchrâ   rã   rä   r   ÚGALORE_ADAMWÚGALORE_ADAMW_8BITÚGALORE_ADAFACTORÚGALORE_ADAMW_LAYERWISEÚGALORE_ADAMW_8BIT_LAYERWISEÚGALORE_ADAFACTOR_LAYERWISEr¶   r!   Úpopr³   r   r   r   rE   r   rq   )r‚   râ   rã   rä   rK   Úgalore_optim_kwargsrs   r   s   &       r+   Ú_get_galore_optimizerró   R  sQ  € ä$×&Ò&ÜðVó
ð 	
÷ KÑJô 	×#Ñ# [Ü×(Ñ(¨/Ü×'Ñ'¨Ü×-Ñ-¨{Ü×2Ñ2°OÜ×1Ñ1°?ðÐð 	”�C—N‘N×&Ñ& v¨sÓ3Ó4Øœ3˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJØ”�s—~‘~×)Ñ)¨'°4Ó8Ó9Ø�S—^‘^×'Ñ'¨°UÓ;ð	Ðô '@Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->ÐUX×UiÑUió'Ñ#€Mð ‡x�x‡~�~œ×8Ñ8Ô8Ø×ÑÐ!2°E¸?ÈEÐ RÔSØÐ*Ð*r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   s  s   € ÷ ñ Ð/ð Ð4Nñ r*   c                óÊ  € \        4       '       g   \        R4      h^ RIHp \        P
                  V\        P                  V/pR\        V P                  P                  R^€4      4      RV P                  P                  RR4      RV P                  P                  RR4      R\        V P                  P                  R^È4      4      R	\        V P                  P                  R	R
4      4      RV P                  P                  RR4      /pVP                  V P                  4       \        V P                  V P                  V P                  P                   W#V P"                  4      # )zGet Apollo optimizer.zŒYou need to install `apollo_torch` in order to use APOLLO optimizers. Install it with `pip install git+https://github.com/zhuhanqing/APOLLO`)ÚAPOLLOAdamWrå   ÚprojÚrandomÚ
scale_typeÚchannelræ   rç   ç      ð?rè   ré   )r   r“   Úapollo_torchrö   r   ÚAPOLLO_ADAMWÚAPOLLO_ADAMW_LAYERWISEr¶   r!   rñ   r³   rq   r   r   r   r   rE   r   )r‚   rö   rK   Úapollo_optim_kwargss   &   r+   Ú_get_apollo_optimizerr   s  s%  € ä$×&Ò&ÜðUó
ð 	
õ )ô 	×#Ñ# [Ü×-Ñ-¨{ðÐð 	”�C—N‘N×&Ñ& v¨sÓ3Ó4Ø�—‘×"Ñ" 6¨8Ó4Ø�c—n‘n×(Ñ(¨°yÓAØœ3˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJØ”�s—~‘~×)Ñ)¨'°3Ó7Ó8Ø�S—^‘^×'Ñ'¨°UÓ;ðÐð ×Ñ˜sŸ™Ô/ä$Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->ÐUX×UiÑUióð r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   �  s   € ÷ /ñ /Ð-ð /Ð2Lñ /r*   c                ó   € \        4       '       g   \        R4      hV P                  f   \        R4      h^ RIHpHp RV P                  P                  9   d   TMTpV P                  P                  RV P                  /4       W0P                  3# )zGet LOMO optimizer.zjYou need to install `lomo_optim` in order to use LOMO optimizers. Install it with `pip install lomo-optim`zMYou need to pass a `model` in order to correctly initialize a LOMO optimizer.)ÚAdaLomoÚLomoÚadar   )r   r“   r   rc   Ú
lomo_optimr  r  r   rE   r   rq   )r‚   r  r  rs   s   &   r+   Ú_get_lomo_optimizerr  �  sz   € ä×ÒÜð7ó
ð 	
ð
 ‡y�yÒÜÐhÓiÐiç(à$¨¯©¯©Ô6‘G¸D€MØ×Ñ×Ñ ¨#¯)©)Ð 4Ô5Ø×.Ñ.Ð.Ð.r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   ¢  s   € ÷ +ñ +Ð(ð +Ð-Gñ +r*   c                ó  € \        4       '       g   \        R4      h^ RIHp V P                  P                  R\        V P                  P                  RR4      4      R\        V P                  P                  RR4      4      R\        V P                  P                  RR4      4      R	\        V P                  P                  R	R4      4      R
\        V P                  P                  R
R4      4      /4       WP                  3# )zGet GrokAdamW optimizer.z5Please install grokadamw with `pip install grokadamw`)Ú	GrokAdamWÚ
alpha_initg\�Âõ(\ï?Úlambç       @Úgammagš™™™™™¹?Úgrokking_signal_decay_rateÚgradient_clippingrû   )	r   rc   Ú	grokadamwr
  r   rq   r³   r!   r´   )r‚   r
  s   & r+   Ú_get_grokadamwr  ¢  sÎ   € ä!×#Ò#ÜÐPÓQÐQå#à×Ñ×Ñàœ% §¡× 2Ñ 2°<ÀÓ FÓGØ”E˜#Ÿ.™.×,Ñ,¨V°SÓ9Ó:Ø”U˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ø(¬%°·±×0BÑ0BÐC_ÐadÓ0eÓ*fØ¤ s§~¡~×'9Ñ'9Ð:MÈsÓ'SÓ!Tð	
ôð ×*Ñ*Ð*Ð*r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   µ  s   € ÷  /ñ  /Ð 0ð  /Ð5Oñ  /r*   c                ó°  € \        4       '       dM   \        P                  ! \        P                  P                  R4      4      \        P                  ! R4      8  d   \        R4      h\        P                  ! \        P                  P                  R4      4      \        P                  ! R4      8:  d   \        R4      h\        P                  ! \        P                  P                  R4      4      \        P                  ! R4      8¼  d
   ^ RIHpHp M^ RI	HpHp V P                  P                  \        P                  8X  d   TpMTpV P                  P                  R	V P                   P#                  R	R
4      R\%        V P                   P#                  RR4      4      /4       V P                  P                  V P&                  4       W0P                  3# )z%Get TorchAO 4-bit or 8-bit optimizer.Útorchaoz0.4.0z®You need to have `torchao>=0.4.0` in order to use torch 4-bit optimizers. Install it with `pip install torchao` or follow the instructions here: https://github.com/pytorch/aorD   z2.4z¹You need to have `torch>2.4` in order to use torch 4-bit optimizers. Install it with `pip install --upgrade torch` it is available on pipy. Otherwise, you need to install torch nightly.z0.11.0)Ú	AdamW4bitÚ	AdamW8bitÚ
block_sizeé   Úbf16_stochastic_roundr¿   )r   r   ÚparseÚ	importlibÚmetadatar“   Útorchao.optimr  r  Útorchao.prototype.low_bit_optimr   rE   r   ÚADAMW_TORCH_4BITr   rq   r!   r´   r   r   )r‚   r  r  rs   s   &   r+   Ú_get_torchao_optimizerr!  µ  sU  € ä×!Ò!¤W§]¢]´9×3EÑ3E×3MÑ3MÈiÓ3XÓ%YÔ\c×\iÒ\iÐjqÓ\rÔ%rÜð,ó
ð 	
ô
 ‡}‚}”Y×'Ñ'×/Ñ/°Ó8Ó9¼W¿]º]È5Ó=QÔQÜð<ó
ð 	
ô ‡}‚}”Y×'Ñ'×/Ñ/°	Ó:Ó;¼w¿}º}ÈXÓ?VÔVß6Ð6çHà
‡x�x‡~�~œ×8Ñ8Ô8Ø!‰à!ˆà×Ñ×Ñà˜#Ÿ.™.×,Ñ,¨\¸3Ó?Ø#¤Y¨s¯~©~×/AÑ/AÐBYÐ[bÓ/cÓ%dð	
ôð ×Ñ×Ñ §¡Ô0Ø×.Ñ.Ð.Ð.r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1   Ø  s   € ÷ )/ñ )/Ð&6ð )/Ð;Uñ )/r*   c                ó`  € \        4       '       g   \        R4      h^ RIHpHp / pRpV P
                  P                  \        P                  8X  d4   \        R4      '       g   \        R4      h^ RIH	p TpV P                  pRpMoV P
                  P                  \        P                  8X  d   TpV P                  pM7V P
                  P                  \        P                  8X  d   TpM\        R4      hV P
                  P                  VR	&   V'       d   V P
                  P                  VR
&   VP!                  R\#        V P$                  P'                  RR4      4      R\#        V P$                  P'                  RR4      4      /4       V P(                  P!                  V4       W`P(                  3# )zGet ScheduleFree optimizer.zwYou need to install `schedulefree` in order to use schedulefree optimizers. Install it with `pip install schedulefree.`)ÚAdamWScheduleFreeÚSGDScheduleFreeTz1.4.0z‚You need to install `schedulefree>=1.4.0` in order to use RAdamScheduleFree optimizer. Install it with `pip install schedulefree.`)ÚRAdamScheduleFreeFzInvalid schedulefree optimizerrÍ   Úwarmup_stepsÚweight_lr_powerr  Úrg        )r   r“   Úschedulefreer$  r%  r   rE   r   ÚSCHEDULE_FREE_RADAMr&  r   ÚSCHEDULE_FREE_ADAMWÚSCHEDULE_FREE_SGDrc   rÍ   r'  rq   r³   r!   r´   r   )r‚   r$  r%  r¸   Úrequire_warmupr&  rs   s   &      r+   Ú_get_schedule_free_optimizerr/  Ø  sc  € ä$×&Ò&Üð:ó
ð 	
÷ @à ÐØ€Nà
‡x�x‡~�~œ×;Ñ;Ô;Ü(¨×1Ò1Üð>óð õ 	3à)ˆØ"%§/¡/ÐØ‰Ø	�‰�‰œ>×=Ñ=Ô	=Ø)ˆØ"%§/¡/ÑØ	�‰�‰œ>×;Ñ;Ô	;Ø'‰äÐ9Ó:Ð:à.1¯h©h×.CÑ.CÐ˜NÑ+ßØ25·(±(×2GÑ2GÐ Ñ/Ø×"Ñ"àœu S§^¡^×%7Ñ%7Ð8IÈ3Ó%OÓPØ”�s—~‘~×)Ñ)¨#¨sÓ3Ó4ð	
ôð ×Ñ×ÑÐ 7Ô8Ø×.Ñ.Ð.Ð.r*   c               ó    € V ^8„  d   QhRRRR/# r�   r"   )r0   s   "r+   r1   r1     s   € ÷ -ñ -Ð+ð -Ð0Jñ -r*   c                ó&  € \        4       '       g   \        R4      h^ RIHp V P                  P                  RR4      pVe   \        V4      pV P                  P                  RR4      pVe   \        V4      pV P                  P                  V P                  R&   R\        V P                  P                  RR4      4      RVRV/pV P                  P                  V P                  4       V P                  P                  V4       WP                  3# )	z,Get StableAdamW optimizer from torch-optimi.zwYou need to install `torch-optimi` in order to use stable_adamw optimizers. Install it with `pip install torch-optimi`.)ÚStableAdamWÚmax_lrNÚ	kahan_sumrÍ   Údecouple_lrF)r   r“   Úoptimir2  r!   rñ   r³   r?   r   rÍ   r   r   rq   )r‚   r2  r3  r4  Ústable_adamw_kwargss   &    r+   Ú_get_stable_adamwr8    sï   € ä$×&Ò&Üð:ó
ð 	
õ #à�^‰^×Ñ ¨$Ó/€FØÒÜ�v“ˆà—‘×"Ñ" ;°Ó5€IØÒÜ˜“Oˆ	à&)§h¡h×&;Ñ&;€C‡O�O�NÑ#à”t˜CŸN™N×.Ñ.¨}¸eÓDÓEØ�&Ø�YðÐð ×Ñ×Ñ §¡Ô0Ø×Ñ×ÑÐ 3Ô4Ø×,Ñ,Ð,Ð,r*   zdict[str, OptimizerHandler]Ú_OPTIMIZER_HANDLERS)T)tÚ__conditional_annotations__r'   Ú
__future__r   Úimportlib.metadatar  ÚloggingÚcollections.abcr   Údataclassesr   Útypingr   r   rD   Ú	packagingr   r   Úoptimizationr	   Útrainer_pt_utilsr
   Útrainer_utilsr   Útraining_argsr   r   Úutilsr   r   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Ú	getLoggerr#   rh   r   r<   ÚtupleÚdictrJ   ÚOptimizerHandlerrG   r   r†   r�   r”   rš   rº   rÇ   rÖ   rÛ   rß   ró   r   r  r  r!  r/  r8  Ú	ADAMW_BNBÚ
ADAMW_8BITÚPAGED_ADAMWÚPAGED_ADAMW_8BITÚADEMAMIXÚADEMAMIX_8BITÚPAGED_ADEMAMIXÚPAGED_ADEMAMIX_8BITÚLIONÚ	LION_8BITÚ
PAGED_LIONÚPAGED_LION_8BITÚRMSPROP_BNBÚRMSPROP_8BITÚRMSPROP_32BITÚ_BITSANDBYTES_OPTIMIZERSrë   rì   rí   rî   rï   rð   Ú_GALORE_OPTIMIZERSrý   rþ   Ú_APOLLO_OPTIMIZERSr   ÚADAMW_TORCH_8BITÚ_TORCHAO_OPTIMIZERSr+  r,  r-  Ú_SCHEDULE_FREE_OPTIMIZERSÚ	ADAFACTORÚADAMW_TORCHr�   ÚADAMW_TORCH_XLAÚADAMW_TORCH_NPU_FUSEDÚADAMW_ANYPRECISIONrÓ   ÚADAGRADÚRMSPROPÚ	GROKADAMWÚSTABLE_ADAMWÚLOMOÚADALOMOÚfromkeysr9  r(   )r:  s   @r+   Ú<module>rm     sR  øðôõ #ã Û Ý $Ý !ß %ã Ý Ý å #Ý 5Ý 5ß 7÷
÷ 
õ 
÷ Ý/Ý0à	×	Ò	˜8Ó	$€ð ÷ð ó ðõð Ð-Ð.°°c¸4ÀÀSÀ½>Ð6IÕ0JÐJÕKÐ õ÷*V+õ|+õ'õTõPõ//õdQõ*	#õ'õ	'õ+õBõ:/õ$+õ& /õF)/õX-ðD ×ÑØ×ÑØ×ÑØ×#Ñ#Ø×ÑØ× Ñ Ø×!Ñ!Ø×&Ñ&Ø×ÑØ×ÑØ×ÑØ×"Ñ"Ø×ÑØ×ÑØ× Ñ ðÐ ð& ×ÑØ×$Ñ$Ø×#Ñ#Ø×)Ñ)Ø×.Ñ.Ø×-Ñ-ðÐ ð ×ÑØ×)Ñ)ðÐ ð ×#Ñ#Ø×#Ñ#ðÐ ð ×&Ñ&Ø×&Ñ&Ø×$Ñ$ðÐ ð ×Ñ˜nØ×ÑÐ 0Ø×$Ñ$Ð&6Ø×"Ñ"Ð$8Ø×(Ñ(Ð*DØ×%Ñ%Ð'>Ø×Ñ˜Ø×Ñ˜LØ×Ñ˜LØ×Ñ˜nØ×ÑÐ!2Ø×ÑÐ,Ø×ÑÐ/ð4ð ‡m�mÐ,Ð.IÓJð4ð ‡m�mÐ&Ð(=Ó>ð4ð  ‡m�mÐ&Ð(=Ó>ð!4ð" ‡m�mÐ'Ð)?Ó@ð#4ð$ ‡m�mÐ-Ð/KÓLð%4Ð Ð0õ r*   