+
    UV-já,  ã                   ó  € ^ RI t ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RI	H
t
 ^ RIHt ^RIHt R#t ! R R4      tRR	0tR
 tR R ltR tR tR R ltR R ltR$R ltR tR tR tR tR%R ltR tR tR t R t!R R lt"R R  lt#R! R" lt$R# )&é    N)ÚPath)ÚUnion©Útree_flatten)Ú	LoRaLayerc                   ó:   € ] tR t^tRtRtRtRtRtRt	Rt
RtR	tR
tR# )ÚColorsz[95mz[94mz[96mz[92mz[93mz[91mz[0mz[1mz[4m© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚHEADERÚOKBLUEÚOKCYANÚOKGREENÚWARNINGÚFAILÚENDCÚBOLDÚ	UNDERLINEÚ__static_attributes__r
   ó    Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/trainer/utils.pyr	   r	      s/   † Ø€FØ€FØ€FØ€GØ€GØ€DØ€DØ€DØ„Ir   r	   Úgemma3nÚ
qwen3_omnic                ó^   a€ \        V 4      P                  oV3R lpV\        V 4      n        R# )zD
Update all instances of type(layer) to use gradient checkpointing.
c                 óp   <a € VV 3R  lp\         P                  ! V4      ! S P                  4       .VO5/ VB # )c                 ó>   <€ SP                  V 4       S! S.VO5/ VB # ©N)Úupdate)ÚparamsÚargsÚkwargsÚfnÚmodels   &*,€€r   Úinner_fnÚ:grad_checkpoint.<locals>.checkpointed_fn.<locals>.inner_fn%   s$   ø€ Ø�L‰L˜Ô Ù�eÐ-˜dÒ- fÑ-Ð-r   )ÚmxÚ
checkpointÚtrainable_parameters)r&   r#   r$   r'   r%   s   f*, €r   Úcheckpointed_fnÚ(grad_checkpoint.<locals>.checkpointed_fn$   s1   ù€ ö	.ô �}Š}˜XÔ& u×'AÑ'AÓ'CÐUÀdÒUÈfÑUÐUr   N)ÚtypeÚ__call__)Úlayerr,   r%   s   & @r   Úgrad_checkpointr1      s(   ø€ ô 
ˆe‹×	Ñ	€BõVð +„DˆƒKÖr   c          
      óT   € V ^8„  d   QhR\         R\         R\         R\        R\        /# )é   ÚitersÚstepÚwarmup_stepsÚlearning_rateÚmin_learning_rate)ÚintÚfloat)Úformats   "r   Ú__annotate__r<   .   sA   € ÷ Rñ RÜðRä
ðRô ðRô ð	Rô
 ñRr   c                 óô   € W8  d   W1V,          ,          # W,
          W,
          ,          pR ^\         P                  ! \         P                  V,          4      ,           ,          pWCV,
          V,          ,           # )g      à?)ÚmathÚcosÚpi)r4   r5   r6   r7   r8   ÚprogressÚcosine_decays   &&&&&  r   Úget_learning_raterC   .   s]   € ð ÔØ |Õ 3Õ4Ð4àÕ#¨Õ(<Õ=€HØ˜!œdŸhšh¤t§w¡w°Õ'9Ó:Õ:Õ;€LØÐ0AÕ AÀ\ÕQÕQÐQr   c                 ó¨   € VP                  R 4      pT pV F8  pVP                  4       '       d   V\        V4      ,          pK-  \        W44      pK:  	  V# ©Ú.)ÚsplitÚisdigitr9   Úgetattr)r&   ÚnameÚpartsÚmoduleÚparts   &&   r   Úget_module_by_namerN   =   sH   € Ø�J‰J�s‹O€EØ€FÛˆØ�<‰<�>Š>ØœC ›IÕ&ŠFä˜VÓ*ŠFñ	 ð
 €Mr   c                 ó:  € VP                  R 4      pT pVRR  F8  pVP                  4       '       d   V\        V4      ,          pK-  \        WE4      pK:  	  VR,          P                  4       '       d   W$\        VR,          4      &   R# \	        WCR,          V4       R# )rF   Néÿÿÿÿ)rG   rH   r9   rI   Úsetattr)r&   rJ   Ú
new_modulerK   rL   rM   s   &&&   r   Úset_module_by_namerS   H   sz   € Ø�J‰J�s‹O€EØ€FØ�c�r“
ˆØ�<‰<�>Š>ØœC ›IÕ&ŠFä˜VÓ*ŠFñ	 ð
 ˆR…y×Ñ×ÒØ!+Œs�5˜•9‹~Óä�˜b�	 :Ö.r   c                ó<   € V ^8„  d   QhR\         R\        R\         /# )r3   ÚalphaÚrankÚreturn)r:   r9   )r;   s   "r   r<   r<   V   s!   € ÷ ñ ”uð ¤Cð ¬Eñ r   c                 ó   € W,          # r    r
   )rU   rV   s   &&r   Ú_lora_scalerY   V   s
   € Ø�<Ðr   c                ó<   € V ^8„  d   QhR\         R\         R\         /# )r3   Ú	model_keyÚlanguage_keyrW   )Ústr)r;   s   "r   r<   r<   Z   s&   € ÷ Hñ H¤ð H´Cð H¼Cñ Hr   c                 ó$   € V'       d   V  R V 2# T # rE   r
   )r[   r\   s   &&r   Ú_linear_layer_keyr_   Z   s   € ß,8ˆiˆ[˜˜,˜Ð(ÐG¸iÐGr   c                 óv  € \        V \        P                  \        P                  34      '       di   V'       d1   ^ RIHp VP                  V VR,          VR,          VR,          R7      # ^ RIHp VP                  V VR,          VR,          VR,          R7      # \        R\        V 4      P                   R24      h)	r   )Ú
DoRALinearrV   ÚscaleÚdropout)Úrrb   rc   )Ú
LoRALinearzCan't convert layer of type z to LoRA)Ú
isinstanceÚnnÚLinearÚQuantizedLinearÚmlx_lm.tuner.dorara   Ú	from_baseÚmlx_lm.tuner.lorare   Ú
ValueErrorr.   r   )r0   Úlora_parametersÚuse_dorara   re   s   &&&  r   Ú_to_lorarp   ^   s±   € Ü�%œ"Ÿ)™)¤R×%7Ñ%7Ð8×9Ò9ßÝ4à×'Ñ'ØØ! &Õ)Ø% gÕ.Ø'¨	Õ2ð	 (ó ð õ 	1à×#Ñ#ØØ˜fÕ%Ø! 'Õ*Ø# IÕ.ð	 $ó 
ð 	
ô Ð3´D¸³K×4HÑ4HÐ3IÈÐRÓ
SÐSr   c           	      ó~  € \        V4      p. pV P                  P                  4        F‘  w  rV\        V\        P
                  \        P                  34      '       g   K7  VP                  R 4      R,          V9   g   KV  \        V P                  V\        Wb4      4       VP                  \        RV4      4       K“  	  V# )rF   Úlanguage_modelrP   )Úsetrr   Únamed_modulesrf   rg   rh   ri   rG   rS   rp   Úappendr_   )r&   Úlinear_layersrn   ÚtargetsÚkeysrJ   rL   s   &&&    r   Ú_apply_language_lora_layersry   v   s—   € Ü�-Ó €GØ€DØ×,Ñ,×:Ñ:Ö<‰ˆÜ�fœrŸy™y¬"×*<Ñ*<Ð=×>Ô>Ø�z‰z˜#‹˜rÕ" gÖ-Ü"Ø×(Ñ(ØÜ˜VÓ5ôð
 —‘Ô-Ð.>ÀÓEÖFñ =ð €Kr   c                 ó.  € \        VR ,          4      pVP                  RR4      pVR8H  pVR8X  d   V # RV9  d)   ^ RIHp V! V VP                  R\        4      VVR7       V # VR,           F&  p\        W4      p\        V V\        WrVR7      4       K(  	  V # )	rn   Úfine_tune_typeÚloraÚdoraÚfullrx   )Úlinear_to_lora_layersÚ
num_layers)ro   )ÚdictÚgetÚmlx_lm.tuner.utilsr   ÚDEFAULT_LORA_NUM_LAYERSrN   rS   rp   )r&   Úconfigrn   r{   ro   r   rJ   rL   s   &&      r   Ú_apply_lora_layersr†   …   s§   € Ü˜6Ð"3Õ4Ó5€OØ—Z‘ZÐ 0°&Ó9€NØ Ñ'€Hà˜ÔØˆà�_Ô$Ý<áØØ�J‰J�|Ô%<Ó=ØØõ		
ð ˆà ×'Ð'ˆÜ# EÓ0ˆÜØØÜ�V°xÔ@ö	
ñ (ð €Lr   c                 ó8   € R RR\         RRV RVR\        W4      //# )r{   r|   r€   rn   rV   rc   rb   )r„   rY   )rV   rU   rc   s   &&&r   Ú_lora_configrˆ   ¢   s3   € à˜&ØÔ-ØØ�DØ�wØ”[ Ó-ð
ðð r   c           
      óœ   € \        V P                  4      p\        V VVR ,          VP                  RR4      VP                  RR4      RR7      # )rV   rU   çš™™™™™¹?rc   T)rV   rU   rc   Úlegacy)Úfind_all_linear_namesrr   Úget_peft_modelr‚   )r&   r…   Úlist_of_moduless   && r   Ú_apply_legacy_lora_layersr�   ®   sK   € Ü+¨E×,@Ñ,@ÓA€OÜØØØ�F�^Ø�j‰j˜ #Ó&Ø—
‘
˜9 cÓ*Øôð r   c                 óê  € V'       d   \        V 4       V'       Ed   V P                  P                  4        Fˆ  w  r‰\        V	\        P
                  4      '       g#   \        V	\        P                  4      '       g   KG  VP                  R 4      R,          V9   g   Kf  \        W’W44      p
\        V P                  WŠ4       KŠ  	  / V P                  n        W P                  P                  R&   W0P                  P                  R&   W@P                  P                  R&   M9\        W#V4      p\        WVR,          4      VR,          R&   W°P                  n        V'       d   \        V P                  4       V # )rF   rV   rU   rc   rn   rx   rP   )Úfreeze_modelrr   rt   rf   rg   rh   ri   rG   r   rS   r…   r|   rˆ   ry   Úprint_trainable_parameters)r&   rv   rV   rU   rc   ÚfreezeÚverboser‹   rJ   rL   Ú
lora_layerr…   s   &&&&&&&&    r   r�   r�   º   s  € ÷ Ü�UÔç€vØ!×0Ñ0×>Ñ>Ö@‰LˆDÜ˜&¤"§)¡)×,Ò,´
¸6Ä2×CUÑCU×0VÔ0VØ—:‘:˜c“? 2Õ&¨-Ö7Ü!*¨6¸Ó!H�JÜ& u×';Ñ';¸TÖNñ	 Að ˆ�‰ÔØ$(�‰×Ñ˜&Ñ!Ø%*�‰×Ñ˜'Ñ"Ø'.�‰×Ñ˜)Ò$ä˜d¨7Ó3ˆÜ,GØ &Ð):Õ";ó-
ˆÐ Õ! &Ñ)ð #�‰ÔçÜ" 5×#7Ñ#7Ô8à€Lr   c                 ó¬  € 0 RmpV P                  4        FP  w  r#VP                  R4      ^ ,          pW!9   g   K%  \        W4      '       g   K8   W ,          P                  4        KR  	  R#   \         d^     ^ RIHp Y ,          pT! TP                  4       R R7      pT F  w  rxTP                  RR7       K  	   Kª    \         d      K¹  i ; ii ; i)	rr   rF   r   c                 ó6   € \        V \        P                  4      # r    ©rf   rg   ÚModule©Úms   &r   Ú<lambda>Úfreeze_model.<locals>.<lambda>ù   s   € ¼jÈÌBÏIÉIÔ>Vr   ©Úis_leafF)ÚrecurseN>
   ÚalignerÚ	connectorÚaudio_towerÚembed_audioÚembed_visionÚmm_projectorÚvision_modelÚvision_towerrr   Úmulti_modal_projector)rt   rG   Úhasattrr“   Ú	ExceptionÚ	mlx.utilsr   Úleaf_modules)	r&   Útop_level_to_freezerJ   rL   r   ÚtopÚleavesÚ_r›   s	   &        r   r‘   r‘   ß   sÃ   € òÐð ×+Ñ+Ö-‰ˆØ�z‰z˜#‹˜qÕ!ˆØÖ&¬7°5×+?Ô+?ðØ˜Õ ×'Ñ'Ö)ó	 .øô
 ô ð
Ý6à &Õ*�CÙ)Ø×(Ñ(Ó*Ñ4Vô�Fó !'™˜ØŸ™¨˜Ö/ô !'øä ô Ûðúðús+   ÁA+Á+CÁ7AB?Â?CÃ
CÃCÃCc                 ó*  a€ \         P                  p\         P                  p\        4       p. ROpV P	                  4        F®  w  op\
        ;QJ d    V3R lV 4       F  '       g   K   RM	  RM! V3R lV 4       4      '       d   KH  \        WQ4      '       g   \        WR4      '       g   Kl  SP                  R4      pTP                  \        V4      ^8X  d
   V^ ,          MVR,          4       K°  	  RV9   d   VP                  R4       \        V4      # )r¦   c              3   ó,   <"  € T F	  qS9   x € K  	  R # 5ir    r
   )Ú.0Ú
mm_keywordrJ   s   & €r   Ú	<genexpr>Ú(find_all_linear_names.<locals>.<genexpr>  s   øé € ÐHÑ4G j˜TÖ!Ó4Gùs   ƒTFrF   Úlm_head)r¦   r¨   Úvision_resamplerr¡   rP   )rg   rh   ri   rs   rt   Úanyrf   rG   ÚaddÚlenÚremoveÚlist)r&   ÚclsÚquantized_clsÚlora_module_namesÚmultimodal_keywordsrL   ÚnamesrJ   s   &      @r   rŒ   rŒ     sÏ   ø€ Ü
�)‰)€CÜ×&Ñ&€MÜ›ÐòÐð ×+Ñ+Ö-‰ˆˆfß‹3ÔHÑ4GÓH�3�3Š3ÔHÑ4GÓH×HÒHÙÜ�f×"Ò"¤j°×&GÔ&GØ—J‘J˜s“OˆEØ×!Ñ!¬c°%«j¸A¬o %¨¦(À5ÈÅ9ÖMñ .ð Ð%Ô%Ø× Ñ  Ô+ÜÐ!Ó"Ð"r   c                 ó~   a€ R  o\        V P                  4       R R7      p\        V3R lV 4       4      R,          pV# )c                 ó  € \        V \        P                  \        P                  34      '       d/   V P                  P
                  ^ V P                  ,          ,          # \        R \        V P                  4       4       4       4      # )é    c              3   ó>   "  € T F  w  rVP                   x € K  	  R # 5ir    ©Úsize©r´   r±   Úvs   &  r   r¶   Ú4count_parameters.<locals>.nparams.<locals>.<genexpr>  ó   é € ÐCÑ&B™d˜a�1—6–6Ó&Bùó   ‚©
rf   rg   ri   ÚQuantizedEmbeddingÚweightrÉ   ÚbitsÚsumr   Ú
parametersrš   s   &r   ÚnparamsÚ!count_parameters.<locals>.nparams  óW   € Ü�aœ"×,Ñ,¬b×.CÑ.CÐD×EÒEØ—8‘8—=‘= B¨!¯&©&¥LÕ1Ð1ÜÑC¤l°1·<±<³>Ô&BÓCÓCÐCr   c                 ó6   € \        V \        P                  4      # r    r˜   rš   s   &r   rœ   Ú"count_parameters.<locals>.<lambda>  ó   € ´
¸1¼b¿i¹iÔ0Hr   rž   c              3   ó8   <"  € T F  w  rS! V4      x € K  	  R # 5ir    r
   ©r´   r±   r›   rÕ   s   &  €r   r¶   Ú#count_parameters.<locals>.<genexpr>   ó   øé € Ð6©¡ ‘'˜!—*�*«ùó   ƒé@B )r   r­   rÓ   )r&   r­   Útotal_prÕ   s   &  @r   Úcount_parametersrâ     s@   ø€ òDô
  Ø×ÑÓÑ&Hô€Lô Ô6©Ó6Ó6¸Õ>€Gà€Nr   c           	      ó&  a€ R  o\        V P                  4       R R7      p\        V3R lV 4       4      R,          p\        R \        V P                  4       4       4       4      R,          p\	        RV RV RV^d,          V,          R R	24       R
# )c                 ó  € \        V \        P                  \        P                  34      '       d/   V P                  P
                  ^ V P                  ,          ,          # \        R \        V P                  4       4       4       4      # )rÆ   c              3   ó>   "  € T F  w  rVP                   x € K  	  R # 5ir    rÈ   rÊ   s   &  r   r¶   Ú>print_trainable_parameters.<locals>.nparams.<locals>.<genexpr>)  rÍ   rÎ   rÏ   rš   s   &r   rÕ   Ú+print_trainable_parameters.<locals>.nparams&  r×   r   c                 ó6   € \        V \        P                  4      # r    r˜   rš   s   &r   rœ   Ú,print_trainable_parameters.<locals>.<lambda>,  rÚ   r   rž   c              3   ó8   <"  € T F  w  rS! V4      x € K  	  R # 5ir    r
   rÜ   s   &  €r   r¶   Ú-print_trainable_parameters.<locals>.<genexpr>.  rÞ   rß   c              3   ó>   "  € T F  w  rVP                   x € K  	  R # 5ir    rÈ   rÊ   s   &  r   r¶   rë   0  s   é € ÐJÑI‘t�qˆA�FŽFÓIùrÎ   z#trainable params: z M || all params: z M || trainable%: z.3fÚ%Nrà   )r   r­   rÓ   r+   Úprint)r&   r­   rá   Útrainable_prÕ   s   &   @r   r’   r’   %  s—   ø€ òDô
  Ø×ÑÓÑ&Hô€Lô Ô6©Ó6Ó6¸Õ>€GäÑJœ|¨E×,FÑ,FÓ,HÔIÓJÓJÈUÕRð ô 
Ø
˜k˜]Ð*<¸W¸IÐEWÐYdÐgjÕYjÐmtÕYtÐvyÐWzÐz{Ð|ör   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r3   r&   Úadapter_pathrW   )rg   r™   r]   )r;   s   "r   r<   r<   8  s)   € ÷ $ñ $œRŸY™Yð $´cð $¼b¿i¹iñ $r   c                ó@  € \        V RR4      '       d5   ^ RIHp V! V P                  P                  V4      V P                  n        V # \        V4      pVP                  4       '       g   \        RV 24      h\        VR,          R4      ;_uu_ 4       p\        P                  ! V4      pRV9  d   RV9  d   \        R	4      hR
R
R
4       RX9   d   \        W4      p M\        W4      p V P                  \        VR,          4      RR7       V #   + '       g   i     LS; i)zÞ
Apply LoRA layers to the model.

Args:
    model (nn.Module): The neural network model.
    adapter_path (str): Path to the adapter configuration file.

Returns:
    nn.Module: The updated model with LoRA layers applied.
Ú_is_text_modelF)Úload_adaptersz!The adapter path does not exist: úadapter_config.jsonrd   rV   rn   z3The adapter does not have lora params in the configNzadapters.safetensors)Ústrict)rI   Úmlx_lm.utilsrô   rr   Ú_modelr   ÚexistsÚFileNotFoundErrorÚopenÚjsonÚloadrm   r†   r�   Úload_weightsr]   )r&   rñ   rô   Úfr…   s   &&   r   Úapply_lora_layersr   8  sû   € ô ˆuÐ&¨×.Ò.Ý.á&3Ø× Ñ ×'Ñ'¨ó'
ˆ×ÑÔ#ð ˆä˜Ó%€Là×Ñ× Ò ÜÐ"CÀLÀ>Ð RÓSÐSä	ˆlÐ2Õ2°C×	8Ô	8¸AÜ—’˜1“ˆØ˜ÔÐ$5¸VÔ$CÜÐRÓSÐS÷ 
9ð
 ˜FÔ"Ü" 5Ó1‰ä)¨%Ó8ˆà	×Ñ”s˜<Ð*@Õ@ÓAÈ%ÐÔPà€L÷ 
9×	8ús   Â0DÄD	c                ó8   € V ^8„  d   QhR\         P                  /# )r3   r&   )rg   r™   )r;   s   "r   r<   r<   _  s   € ÷ 
ñ 
œBŸI™Iñ 
r   c                óü  aa€ \        V4      p\        4       pV P                  4        F«  w  opS'       d   SP                  R4      ^ ,          MRo\        ;QJ d     VV3R lV 4       F  '       g   K   RM	  RM! VV3R lV 4       4      '       g   Kl  \	        VR4      '       g   K€  VP                  4        TP                  S;'       g    S4       K­  	  V'       g   \        RRP                  V4      4       R	# R	# )
z±Unfreeze modules whose qualified names match any of the given patterns.

This scans model.named_modules() so nested components like
"vision_tower.layers.0" are handled as well.
rF   Ú c              3   óF   <"  € T F  qS8H  ;'       g    VS9   x € K  	  R # 5ir    r
   )r´   rJ   Ú	full_namer¯   s   & €€r   r¶   Ú#unfreeze_modules.<locals>.<genexpr>i  s#   øé € ÐHÁ¸˜‘×3Ð3 ¨Ñ!2Ô3Ãùs   ƒ!“!TFÚunfreezez@[warn] unfreeze_modules: no matching modules found for patterns:z, N)	rs   rt   rG   rº   rª   r  r»   rî   Újoin)r&   Úmodule_namesrw   ÚfoundÚsubr  r¯   s   &&   @@r   Úunfreeze_modulesr  _  s°   ù€ ô �,Ó€GÜ‹E€EØ×-Ñ-Ö/‰ˆ	�3ß)2ˆi�o‰o˜cÓ" 1Ö%¸ˆß‹3ÕHÁÓH�3�3Š3ÕHÁÓH×HÔHÜ�s˜J×'Ô'Ø—‘”Ø—	‘	˜#×*Ð* Ö+ñ 0÷ ÜØNØ�I‰I�lÓ#ö	
ñ r   c                óf   € V ^8„  d   QhR\         P                  R\        \        \        3,          /# )r3   r&   Úadapter_file)rg   r™   r   r]   r   )r;   s   "r   r<   r<   t  s+   € ÷ Añ AœŸ	™	ð A´´s¼D°yÕ1Añ Ar   c                ó  € \        V4      pVP                  P                  RRR7       \        V R4      '       dx   \        V P                  R4      '       d\   \        VP                  R,          R4      ;_uu_ 4       p\        P                  ! V P                  P                  V^R7       RRR4       \        V P                  4       4      p\        P                  ! \        V4      \        V4      4       R#   + '       g   i     LT; i)	z Save adapter weights and config.T)ÚparentsÚexist_okr…   r|   rõ   Úw)ÚindentN)r   ÚparentÚmkdirrª   r…   rû   rü   Údumpr|   r   r+   r)   Úsave_safetensorsr]   r�   )r&   r  Úpathrÿ   Úflattened_trees   &&   r   Úsave_adapterr  t  s²   € ä�Ó€DØ‡K�K×Ñ˜d¨TÐÔ2ô ˆu�h×Ò¤G¨E¯L©L¸&×$AÒ$AÜ�$—+‘+Ð 5Õ5°s×;Ô;¸qÜ�IŠI�e—l‘l×'Ñ'¨°1Õ5÷ <ô " %×"<Ñ"<Ó">Ó?€NÜ×Òœ˜LÓ)¬4°Ó+?Ö@÷ <×;ús   Á<.C6Ã6D	rP   )F)é
   rŠ   rŠ   TTF)%rü   r>   Úpathlibr   Útypingr   Úmlx.coreÚcorer)   Úmlx.nnrg   r¬   r   r|   r   r„   r	   Únot_supported_for_trainingr1   rC   rN   rS   rY   r_   rp   ry   r†   rˆ   r�   r�   r‘   rŒ   râ   r’   r   r  r  r
   r   r   Ú<module>r"     s    ðÛ Û Ý Ý å Ý Ý "å àÐ ÷	ñ 	ð (¨Ð6Ð ò+õ Ròò/õõHôTò0òò:	ò	ô"òJòD#ò,òõ&$õN
÷*Ar   