+
    TV-j‹  ã                   óÜ   € ^ RI t ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RI	H
t
 ^ RIHt ^ RIHtHt ^RIHtHt ^RIHt ^RIHtHt ^RIHtHtHt R	 R
 ltRR R lltR R ltR R lt R t!R# )é    N)ÚPath©ÚDict)Útree_flattenÚtree_unflatten)ÚQuantizedSwitchLinearÚSwitchLinear)Úget_total_parameters)ÚDoRAEmbeddingÚ
DoRALinear)ÚLoRAEmbeddingÚ
LoRALinearÚLoRASwitchLinearc                ó$   € V ^8„  d   QhR\         /# )é   Úschedule_configr   )Úformats   "Úc/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/tuner/utils.pyÚ__annotate__r      s   € ÷ !ñ !¤Dñ !ó    c                ób  € \        \        P                  V R,          4      pV R,          pV^ ,          pV! V!  pV P                  R^ 4      ;p'       d\   V P                  RR4      p\        P                  P	                  WcV4      p\        P                  P                  Wt.V^,           .4      # V# )z7
Build a learning rate schedule from the given config.
ÚnameÚ	argumentsÚwarmupÚwarmup_initg        )ÚgetattrÚoptÚ
schedulersÚgetÚlinear_scheduleÚjoin_schedules)r   Úschedule_fnr   Ú
initial_lrÚbound_schedule_fnÚwarmup_stepsr   Ú	warmup_fns   &       r   Úbuild_scheduler'      s«   € ô œ#Ÿ.™.¨/¸&Õ*AÓB€KØ Õ,€IØ˜1•€JÙ# YÑ/ÐØ&×*Ñ*¨8°QÓ7Ð7€|Ö7Ø%×)Ñ)¨-¸Ó=ˆÜ—N‘N×2Ñ2Ø \ó
ˆ	ô �~‰~×,Ñ,ØÐ*¨\¸AÕ-=Ð,>ó
ð 	
ð !Ð r   c                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# )r   ÚmodelÚ
num_layersÚconfigÚuse_dora)ÚnnÚModuleÚintr   Úbool)r   s   "r   r   r   &   s:   € ÷ H;ñ H;Ü�9‰9ðH;äðH;ô ðH;ô ñ	H;r   c           	     ód  aaa€ VV3R lpSP                  RR4      ;of5   \        4       oV3R lpV P                   F  pVP                  V4       K  	  V P                  \	        V^ 4      ) R  FZ  pVP                  4        UUu. uF  w  rxVS9   g   K  Wt! V4      3NK  	  p	ppV	'       g   K@  VP                  \        V	4      4       K\  	  V P                  4        UUu. uF  w  rxVS9   g   K  Wt! V4      3NK  	  p
ppV
'       d   V P                  \        V
4      4       R# R# u uppi u uppi )a   
Convert some of the models linear layers to lora layers.

Args:
    model (nn.Module): The neural network model.
    num_layers (int): The number of blocks to convert to lora layers
    starting from the last layer.
    config (dict): More configuration parameters for LoRA, including the
      rank, scale, and optional layer keys.
    use_dora (bool): If True, uses DoRA instead of LoRA.
      Default: ``False``
c                 óÎ  <€ S'       g<   \        V R 4      '       d*   V P                  SR,          SR,          SR,          R7      # \        V \        P                  \        P
                  34      '       d   S'       d   \        M\        pM³\        V \        \        34      '       d1   S'       d"   \        \        V 4      P                   R24      h\        pMg\        V \        P                  \        P                  34      '       d   S'       d   \         M\"        pM"\        R\        V 4      P                   R24      hVP%                  V SR,          SR,          SR,          R7      # )Úto_loraÚrankÚscaleÚdropout)Úrr5   r6   z doesn't support DoRA yet.zCan't convert layer of type z to LoRA)Úhasattrr3   Ú
isinstancer-   ÚLinearÚQuantizedLinearr   r   r	   r   Ú
ValueErrorÚtypeÚ__name__r   Ú	EmbeddingÚQuantizedEmbeddingr   r   Ú	from_base)ÚlayerÚ	LoRALayerr+   r,   s   & €€r   r3   Ú&linear_to_lora_layers.<locals>.to_lora9   s  ø€ ßœG E¨9×5Ò5Ø—=‘=Ø˜•.Ø˜W•oØ˜yÕ)ð !ó ð ô �eœbŸi™i¬×);Ñ);Ð<×=Ò=ß&.�
´J‰IÜ˜¤Ô.CÐD×EÒEßÜ ¤D¨£K×$8Ñ$8Ð#9Ð9SÐ!TÓUÐUÜ(‰IÜ˜¤§¡¬b×.CÑ.CÐD×EÒEß)1�´}‰IäØ.¬t°E«{×/CÑ/CÐ.DÀHÐMóð ð ×"Ñ"ØØ�V�nØ˜•/Ø˜9Õ%ð	 #ó 
ð 	
r   ÚkeysNc                 ó  <€ \         P                  \         P                  \        \        \         P
                  \         P                  3p\        VR 4      '       g   \        W4      '       d   SP                  V 4       R# R# )r3   N)
r-   r:   r;   r	   r   r?   r@   r8   r9   Úadd)ÚpÚmÚtypesrE   s   && €r   Úget_keys_for_loraÚ0linear_to_lora_layers.<locals>.get_keys_for_loraX   sY   ø€ ä—	‘	Ü×"Ñ"ÜÜ%Ü—‘Ü×%Ñ%ðˆEô �q˜)×$Ò$¬
°1×(<Ò(<Ø—‘˜–ñ )=r   )r   ÚsetÚlayersÚapply_to_modulesÚmaxÚnamed_modulesÚupdate_modulesr   )r)   r*   r+   r,   r3   rK   ÚlÚkrI   Úlora_layersÚlora_modulesrE   s   &&ff       @r   Úlinear_to_lora_layersrW   &   s  ú€ ö&
ð8 —
‘
˜6 4Ó(Ð(ˆÒ1Ü‹uˆõ
	ð —”ˆAØ×ÑÐ0Ö1ñ ð �\‰\œ3˜z¨1Ó-Ð-Ð/Ó0ˆØ34·?±?Ô3DÔRÑ3D©4¨1ÈÈTÉ	”˜˜7 1›:“Ñ3DˆÑRß‰;Ø×Ñœ^¨KÓ8Ö9ñ 1ð
 16×0CÑ0CÔ0EÔSÑ0E©¨ÈÈdÉ”O�Q˜ ›
“OÑ0E€LÑSßØ×Ñœ^¨LÓ9Ö:ñ ùó Sùó Ts   ÂD&ÂD&Ã D,Ã0D,c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r   r)   Úadapter_pathÚreturn)r-   r.   Ústr)r   s   "r   r   r   q   s)   € ÷ ñ œŸ™ð ´#ð ¼"¿)¹)ñ r   c           	     óà  € \        V4      pVP                  4       '       g   \        RV 24      h\        VR,          R4      ;_uu_ 4       p\        P
                  ! R/ \        P                  ! V4      B pRRR4       \        XRR4      pVR8w  d'   \        V VP                  VP                  VR8H  R	7       V P                  \        VR
,          4      RR7       V #   + '       g   i     Lo; i)zå
Load any fine-tuned adapters / layers.

Args:
    model (nn.Module): The neural network model.
    adapter_path (str): Path to the adapter configuration file.

Returns:
    nn.Module: The updated model with LoRA layers applied.
z!The adapter path does not exist: zadapter_config.jsonr7   NÚfine_tune_typeÚloraÚfullÚdora)r,   zadapters.safetensorsF)Ústrict© )r   ÚexistsÚFileNotFoundErrorÚopenrJ   ÚSimpleNamespaceÚjsonÚloadr   rW   r*   Úlora_parametersÚload_weightsr[   )r)   rY   Úfidr+   r]   s   &&   r   Úload_adaptersrl   q   sÏ   € ô ˜Ó%€LØ×Ñ× Ò ÜÐ"CÀLÀ>Ð RÓSÐSÜ	ˆlÐ2Õ2°C×	8Ô	8¸CÜ×&Ò&Ñ8¬¯ª°3«Ñ8ˆ÷ 
9ä˜VÐ%5°vÓ>€NØ˜ÔÜØØ×ÑØ×"Ñ"Ø$¨Ñ.õ		
ð 
×Ñ”s˜<Ð*@Õ@ÓAÈ%ÐÔPØ€L÷ 
9×	8ús   Á+CÃC-	c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r   r)   rZ   )r-   r.   )r   s   "r   r   r   �   s"   € ÷ ñ œbŸi™ið ¬B¯I©Iñ r   c                óø   € . pV P                  4        F9  w  r#\        V\        4      '       g   K  VP                  W#P                  34       K;  	  \        V4      ^ 8”  d   V P                  \        V4      4       V # )zš
Remove the LoRA layers from the model.

Args:
    model (nn.Module): The model with LoRA layers.

Returns:
    nn.Module: The model without LoRA layers.
)rQ   r9   r   ÚappendÚlinearÚlenrR   r   )r)   Úreset_layersr   Úmodules   &   r   Úremove_lora_layersrt   �   sg   € ð €LØ×+Ñ+Ö-‰ˆÜ�fœj×)Ô)Ø×Ñ §}¡}Ð 5Ö6ñ .ô ˆ<Ó˜1ÔØ×Ñœ^¨LÓ9Ô:Ø€Lr   c           	      óÖ   € \        V 4      R ,          p\        R \        V P                  4       4       4       4      R ,          p\	        RV^d,          V,          R RVR RVR R24       R# )g    €„.Ac              3   ó>   "  € T F  w  rVP                   x € K  	  R # 5i)N)Úsize)Ú.0Ú_Úvs   &  r   Ú	<genexpr>Ú-print_trainable_parameters.<locals>.<genexpr>£   s   é € ÐJÑI‘t�qˆA�FŽFÓIùs   ‚zTrainable parameters: z.3fz% (zM/zM)N)r
   Úsumr   Útrainable_parametersÚprint)r)   Útotal_pÚtrainable_ps   &  r   Úprint_trainable_parametersr‚       sr   € Ü" 5Ó)¨CÕ/€GäÑJœ|¨E×,FÑ,FÓ,HÔIÓJÓJÈSÕPð ô 
Ø
  +°Õ"3°gÕ"=¸sÐ Cð DØ˜Ð˜B˜w s˜m¨2ð	/ör   )F)"rg   rJ   Úpathlibr   Útypingr   Úmlx.coreÚcoreÚmxÚmlx.nnr-   Úmlx.optimizersÚ
optimizersr   Ú	mlx.utilsr   r   Úmodels.switch_layersr   r	   Úutilsr
   r`   r   r   r^   r   r   r   r'   rW   rl   rt   r‚   rb   r   r   Ú<module>rŽ      sG   ðã Û Ý Ý å Ý Ý ß 2ç FÝ (ß +ß =Ñ =õ!÷(H;õVõ8ô&r   