+
    TV-j—  ã                   óz   € ^ RI t ^ RIHt ^ RIHt  ! R R]P                  4      t ! R R]P                  4      tR# )é    Nc                   ó�   a a€ ] tR t^	t o]RV3R lR ll4       tRV3R lR lltRV3R lV 3R llltR tR t	R	 t
R
 tRtVtV ;t# )Ú
DoRALinearc                óF   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[/# )é   ÚlinearÚrÚdropoutÚscale)ÚnnÚLinearÚintÚfloat)ÚformatÚ__classdict__s   "€Úb/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/tuner/dora.pyÚ__annotate__ÚDoRALinear.__annotate__   s5   ø€ ÷ ñ Ù—	‘	ðáðñ ðñ ñ	ó    c                óì   € V P                   P                  w  rE\        V \        P                  4      '       d   V^ V P
                  ,          ,          p\        VVVVVR7      pVP                  V 4       V# )é    )Ú
input_dimsÚoutput_dimsr   r	   r
   )ÚweightÚshapeÚ
isinstancer   ÚQuantizedLinearÚbitsr   Ú
set_linear)r   r   r	   r
   r   r   Údora_lins   &&&&   r   Ú	from_baseÚDoRALinear.from_base
   sj   € ð #)§-¡-×"5Ñ"5ÑˆÜ�fœb×0Ñ0×1Ò1Ø˜" §¡Õ+Õ+ˆJÜØ!Ø#ØØØô
ˆð 	×Ñ˜FÔ#Øˆr   c                ó    <€ V ^8„  d   QhRS[ /# ©r   Ú
dequantize©Úbool)r   r   s   "€r   r   r       s   ø€ ÷ ñ ™tñ r   c                óÜ  € V P                   pR V9   pV P                  4       pVP                  w  rV\        P                  ! WeRR7      pV P
                  V P                  P                  ,          pV P                  P                  p	WHV	,          P                  VP                  4      ,           pV P                  \        P                  P                  V^R7      ,          p
V
R,          V,          Vn        V'       d   VP                   Vn        V P#                  4       '       dJ   V'       gB   \        P$                  P'                  VVP(                  VP*                  VP,                  R7      pV# )ÚbiasF©r(   ©Úaxis©Ú
group_sizer   Úmode©:NNNN)r   Ú_dequantized_weightr   r   r   r
   Úlora_bÚTÚlora_aÚastypeÚdtypeÚmÚmxÚlinalgÚnormr   r(   Ú_is_quantizedr   Úfrom_linearr-   r   r.   )Úselfr$   r   r(   r   r   r   Úfused_linearr1   r3   Ú
norm_scales   &&         r   ÚfuseÚDoRALinear.fuse    s  € Ø—‘ˆØ˜ÑˆØ×)Ñ)Ó+ˆà"(§,¡,ÑˆÜ—y’y ¸uÔEˆà—‘˜dŸk™kŸm™mÕ+ˆØ—‘—‘ˆØ F�?×2Ñ2°6·<±<Ó@Õ@ˆØ—V‘VœbŸi™iŸn™n¨V¸!˜nÓ<Õ<ˆ
Ø(¨Õ1°FÕ:ˆÔçØ &§¡ˆLÔà×Ñ×Ò¯
Ü×-Ñ-×9Ñ9ØØ!×,Ñ,Ø—[‘[Ø—[‘[ð	 :ó ˆLð Ðr   c                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[RS[/# )r   r   r   r   r	   r
   r(   )r   r   r&   )r   r   s   "€r   r   r   :   sG   ø€ ÷ 7ñ 7áð7ñ ð7ñ ð	7ñ
 ð7ñ ð7ñ ñ7r   c                ó|  <€ \         SV `  4        V P                  \        P                  ! WVR 7      4       \        P
                  ! VR7      V n        WPn        ^\        P                  ! V4      ,          p\        P                  P                  V) VW3R7      V n        \        P                  ! W23R7      V n        R# )r)   ©Úp©ÚlowÚhighr   ©r   N)ÚsuperÚ__init__r   r   r   ÚDropoutr	   r
   ÚmathÚsqrtr7   ÚrandomÚuniformr3   Úzerosr1   )r<   r   r   r   r	   r
   r(   Ú	__class__s   &&&&&&&€r   rJ   ÚDoRALinear.__init__:   s’   ø€ ô 	‰ÑÔð 	�‰œŸ	š	 *ÀÔEÔFÜ—z’z GÔ,ˆŒð Œ
ð ”D—I’I˜jÓ)Õ)ˆÜ—i‘i×'Ñ'Ø�ØØ�/ð (ó 
ˆŒô
 —h’h aÐ%5Ô6ˆŽr   c                ó´   € Wn         \        P                  P                  V P	                  4       P                  \        P                  4      ^R7      V n        R# )z1
Set the self.linear layer and recompute self.m.
r*   N)r   r7   r8   r9   r0   r4   Úfloat32r6   )r<   r   s   &&r   r   ÚDoRALinear.set_linearU   s:   € ð ŒÜ—‘—‘ × 8Ñ 8Ó :× AÑ AÄ"Ç*Á*Ó MÐTU�ÓVˆŽr   c           	     ó^  € V P                   P                  pV P                  4       '       d�   \        P                  ! VV P                   P
                  V P                   P                  V P                   P                  V P                   P                  V P                   P                  R7      pV# )zE
Return the weight of linear layer and dequantize it if is quantized
r,   )
r   r   r:   r7   r$   ÚscalesÚbiasesr-   r   r.   )r<   r   s   & r   r0   ÚDoRALinear._dequantized_weight\   s{   € ð —‘×#Ñ#ˆØ×Ñ×ÒÜ—]’]ØØ—‘×"Ñ"Ø—‘×"Ñ"ØŸ;™;×1Ñ1Ø—[‘[×%Ñ%Ø—[‘[×%Ñ%ôˆFð ˆr   c                óJ   € \        V P                  \        P                  4      # )N)r   r   r   r   )r<   s   &r   r:   ÚDoRALinear._is_quantizedl   s   € Ü˜$Ÿ+™+¤r×'9Ñ'9Ó:Ð:r   c                óÎ  € V P                  4       pWP                  ,          pV P                  V4      V P                  ,          V P                  ,          pW0P
                  V,          P                  VP                  4      ,           pW P
                  V P                  P                  ,          V P                  P                  ,          ,           p\        P                  ! \        P                  P                  V^R7      4      pV P                  V,          P                  VP                  4      V,          pRV P                  9   d   WPP                  P                  ,           pV# )é   r*   r(   )r0   r2   r	   r3   r1   r
   r4   r5   r7   Ústop_gradientr8   r9   r6   r   r(   )r<   ÚxÚwÚyÚzÚoutÚadaptedÚdenoms   &&      r   Ú__call__ÚDoRALinear.__call__o   sè   € à×$Ñ$Ó&ˆØ—‘�Gˆà�\‰\˜!‹_˜tŸ{™{Õ*¨d¯k©kÕ9ˆØ—:‘: •>×)Ñ)¨!¯'©'Ó2Õ2ˆð —z‘z D§K¡K§M¡MÕ1°T·[±[·]±]ÕBÕBˆÜ× Ò ¤§¡§¡°¸a Ó!@ÓAˆð �v‰v˜�~×%Ñ% a§g¡gÓ.°Õ4ˆà�T—[‘[Ô ØŸ™×(Ñ(Õ(ˆCØˆ
r   )r	   r   r3   r1   r6   r
   ©é   ç        ç      4@©F)ri   rj   rk   F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr    r?   rJ   r   r0   r:   rf   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rQ   r   s   @@r   r   r   	   sJ   ù‡ € Ø÷ñ ó ð÷*ò ÷47õ 7ò6Wòò ;÷ò r   r   c                   óŒ   a a€ ] tR t^ƒt oRV3R lR lltRV3R lR lltRV3R lV 3R llltV3R lR ltR	 tR
 t	Rt
VtV ;t# )ÚDoRAEmbeddingc                óF   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[/# )r   Ú	embeddingr   r	   r
   )r   Ú	Embeddingr   r   )r   r   s   "€r   r   ÚDoRAEmbedding.__annotate__„   s5   ø€ ÷ ñ Ù—<‘<ðáðñ ðñ ñ	r   c                óÎ   € V P                   P                  w  rE\        V \        P                  4      '       d   \        R 4      h\        VVVVVR7      pVP                  V 4       V# )z0DoRAEmbedding does not yet support quantization.)Únum_embeddingsÚdimsr   r	   r
   )r   r   r   r   r   Ú
ValueErrorrw   Úset_embedding)ry   r   r	   r
   r}   r~   Údora_embeddings   &&&&   r   r    ÚDoRAEmbedding.from_base„   sh   € ð  )×/Ñ/×5Ñ5Ñˆô �i¤×!3Ñ!3×4Ò4ÜÐOÓPÐPÜ&Ø)ØØØØô
ˆð 	×$Ñ$ YÔ/ØÐr   c                ó    <€ V ^8„  d   QhRS[ /# r#   r%   )r   r   s   "€r   r   r{   ™   s   ø€ ÷ ñ ™tñ r   c                ó®  € V P                   pVP                  pVP                  w  rE\        P                  ! WE4      pV P
                  V P                  ,          pV P                  pW7V,          P                  VP                  4      ,           pV P                  \        P                  P                  V^R7      ,          p	V	R,          V,          Vn        V# )r]   r*   r/   )ry   r   r   r   rz   r
   r3   r1   r4   r5   r6   r7   r8   r9   )
r<   r$   ry   r   r}   r~   Úfused_embeddingr3   r1   r>   s
   &&        r   r?   ÚDoRAEmbedding.fuse™   s�   € Ø—N‘Nˆ	Ø×!Ñ!ˆà%Ÿ|™|ÑˆÜŸ,š, ~Ó<ˆà—‘˜dŸk™kÕ)ˆØ—‘ˆØ F�?×2Ñ2°6·<±<Ó@Õ@ˆØ—V‘VœbŸi™iŸn™n¨V¸!˜nÓ<Õ<ˆ
Ø!+¨GÕ!4°vÕ!=ˆÔàÐr   c          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[/# )r   r}   r~   r   r	   r
   )r   r   )r   r   s   "€r   r   r{   ¨   s=   ø€ ÷ 0ñ 0áð0ñ ð0ñ ð	0ñ
 ð0ñ ñ0r   c                óx  <€ \         SV `  4        V P                  \        P                  ! W4      4       \        P
                  ! VR 7      V n        WPn        ^\        P                  ! V4      ,          p\        P                  P                  V) VW3R7      V n        \        P                  ! W23R7      V n        R# )rC   rE   rH   N)rI   rJ   r€   r   rz   rK   r	   r
   rL   rM   r7   rN   rO   r3   rP   r1   )r<   r}   r~   r   r	   r
   rQ   s   &&&&&&€r   rJ   ÚDoRAEmbedding.__init__¨   s’   ø€ ô 	‰ÑÔð 	×Ñœ2Ÿ<š<¨Ó=Ô>Ü—z’z GÔ,ˆŒð Œ
ð ”D—I’I˜nÓ-Õ-ˆÜ—i‘i×'Ñ'Ø�ØØ!Ð%ð (ó 
ˆŒô
 —h’h a YÔ/ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   ry   )r   ÚModule)r   r   s   "€r   r   r{   Â   s   ø€ ÷ :ñ :¡r§y¡yñ :r   c                ór   € Wn         \        P                  P                  VP                  ^R7      V n        R# )r]   r*   N)ry   r7   r8   r9   r   r6   )r<   ry   s   &&r   r€   ÚDoRAEmbedding.set_embeddingÂ   s%   € Ø"ŒÜ—‘—‘ 	× 0Ñ 0°q�Ó9ˆŽr   c                ó¼  € V P                  V4      pV P                  V P                  V,          ,          V P                  ,          pW P	                  V4      P                  VP                  4      ,           pW#,           p\        P                  ! \        P                  P                  VRR7      4      pV P                  V,          V,          R,          V,          pV# )r]   r*   éÿÿÿÿ).N)ry   r
   r3   r1   r	   r4   r5   r7   r^   r8   r9   r6   ©r<   r_   ra   rb   rc   rd   re   s   &&     r   rf   ÚDoRAEmbedding.__call__Æ   s›   € Ø�N‰N˜1ÓˆØ�J‰J˜Ÿ™ Q�Õ'¨$¯+©+Õ5ˆØ—,‘,˜q“/×(Ñ(¨¯©Ó1Õ1ˆð •%ˆÜ× Ò ¤§¡§¡°¸b Ó!AÓBˆð �v‰v�a�y˜5Õ  )Õ,¨sÕ2ˆàˆ
r   c                ó^  € V P                   P                  V4      pV P                  V4      V P                  P                  ,          V P
                  P                  ,          pW P                  V,          P                  VP                  4      ,           pV P                   P                  V P                  V P
                  ,          V P                  ,          ,           p\        P                  ! \        P                  P                  V^R7      4      pV P                  V,          V,          pV# )r]   r*   )ry   Ú	as_linearr	   r1   r2   r3   r
   r4   r5   r   r7   r^   r8   r9   r6   r�   s   &&     r   r“   ÚDoRAEmbedding.as_linearÔ   sÂ   € Ø�N‰N×$Ñ$ QÓ'ˆØ�\‰\˜!‹_˜tŸ{™{Ÿ}™}Õ,°·±·±Õ=ˆØ—:‘: •>×)Ñ)¨!¯'©'Ó2Õ2ˆð —.‘.×'Ñ'¨4¯:©:¸¿¹Õ+CÀtÇ{Á{Õ*RÕRˆÜ× Ò ¤§¡§¡°¸a Ó!@ÓAˆð �v‰v˜�~ Õ$ˆàˆ
r   )r	   ry   r3   r1   r6   r
   rh   rl   )rm   rn   ro   rp   r    r?   rJ   r€   rf   r“   rr   rs   rt   ru   s   @@r   rw   rw   ƒ   s<   ù‡ € ÷ò ÷*ò ÷0õ 0÷4:ð :ò÷ò r   rw   )	rL   Úmlx.coreÚcorer7   Úmlx.nnr   r‹   r   rw   © r   r   Ú<module>r™      s5   ðó å Ý ôw�—‘ô wôt]�B—I‘Iö ]r   