+
    TV-jï  ã                   ó¦  € ^ RI t ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RI	H
t
 ^RIHtHt ^RIHt ] ! R R4      4       t ! R R	]
P"                  4      t ! R
 R]
P"                  4      t ! R R]
P"                  4      t ! R R]
P"                  4      t ! R R]
P"                  4      t ! R R]
P"                  4      t ! R R]
P"                  4      tR# )é    N)Ú	dataclass)ÚTuple)Úcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchMLPc                   óX   a € ] tR t^t o RtRt^ t^ t^ t^t	^t
]R 4       tV 3R ltRtV tR# )Ú	ModelArgsi È  i 
  c                ó°   € T ! R/ VP                  4        UUu/ uF.  w  r#V\        P                  ! V 4      P                  9   g   K,  W#bK0  	  uppB # u uppi )N© )ÚitemsÚinspectÚ	signatureÚ
parameters)ÚclsÚparamsÚkÚvs   &&  Úg/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/phixtral.pyÚ	from_dictÚModelArgs.from_dict   s[   € áñ 
ð #ŸL™LœNôá*‘D�AØœ×)Ò)¨#Ó.×9Ñ9Ñ9ô �’Ù*òñ
ð 	
ùós   ˜)AÁAc                óz   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )	é   Ú
model_typeÚ	num_vocabÚ	model_dimÚ	num_headsÚ
num_layersÚ
rotary_dimÚnum_experts_per_tokÚnum_local_experts)ÚstrÚint)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚModelArgs.__annotate__   sj   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ñ ñ ñ Ñò ó    r   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r    Úclassmethodr   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r$   s   @r   r	   r	      sE   ø‡ € ð €IØ€IØ€IØ€JØ€JØ ÐØÐàñ
ó ð
÷ ƒ r'   r	   c                   óH   a a€ ] tR t^%t oV3R lV 3R lltRR ltRtVtV ;t# )ÚRoPEAttentionc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   Údimsr   r   )r"   )r#   r$   s   "€r   r%   ÚRoPEAttention.__annotate__&   s"   ø€ ÷ .ñ .™Sð .©Sð .¹cñ .r'   c                óæ   <€ \         SV `  4        W n        \        P                  ! VR R7      V n        \        P                  ! V^V,          4      V n        \        P                  ! W4      V n        R# )F)ÚtraditionalN)	ÚsuperÚ__init__r   ÚnnÚRoPEÚropeÚLinearÚWqkvÚout_proj)Úselfr3   r   r   Ú	__class__s   &&&&€r   r8   ÚRoPEAttention.__init__&   sK   ø€ Ü‰ÑÔà"Œä—G’G˜J°EÔ:ˆŒ	Ü—I’I˜d A¨¥HÓ-ˆŒ	ÜŸ	š	 $Ó-ˆŽr'   c           	     óâ  € V P                  V4      p\        P                  ! V^RR7      w  rVpV P                  pVP                  w  ršpVP                  WšVR4      P                  ^ ^^^4      pVP                  WšVR4      P                  ^ ^^^4      pVP                  WšVR4      P                  ^ ^^^4      pVeM   V P                  WSP                  R7      pV P                  WcP                  R7      pVP                  Wg4      w  rgM"V P                  V4      pV P                  V4      pVP                  \        P                  4      p\        P                  ! ^VP                  R,          ,          4      p\        VP                  \        P                  4      VVVVVR7      P                  VP                  4      pVP!                  ^^4      P                  WšR4      pV P#                  V4      # )é   ©Úaxis)Úoffset)ÚcacheÚscaleÚmaskéÿÿÿÿ)r=   ÚmxÚsplitr   ÚshapeÚreshapeÚ	transposer;   rF   Úupdate_and_fetchÚastypeÚfloat32ÚmathÚsqrtr   ÚdtypeÚmoveaxisr>   )r?   ÚxrI   rG   ÚqkvÚqueriesÚkeysÚvaluesr   ÚBÚLÚDrH   Úoutputs   &&&&          r   Ú__call__ÚRoPEAttention.__call__/   s¢  € Ø�i‰i˜‹lˆÜ "§¢¨¨a°bÔ 9Ñˆ�vð —N‘Nˆ	Ø—-‘-‰ˆˆað —/‘/ !¨	°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ�|‰|˜A )¨RÓ0×:Ñ:¸1¸aÀÀAÓFˆØ—‘  i°Ó4×>Ñ>¸qÀ!ÀQÈÓJˆð ÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDà—.‘.¤§¡Ó,ˆô —	’	˜!˜gŸm™m¨BÕ/Õ/Ó0ˆä-Ø�N‰Nœ2Ÿ:™:Ó&ØØØØØô
÷ ‰&�—‘Ó
ð 	ð —‘  AÓ&×.Ñ.¨q°RÓ8ˆà�}‰}˜VÓ$Ð$r'   )r=   r   r>   r;   ©NN©	r(   r)   r*   r+   r8   r`   r.   r/   Ú__classcell__©r@   r$   s   @@r   r1   r1   %   s   ù‡ € ÷.ó .÷%%ô %%r'   r1   c                   óP   a a€ ] tR t^Wt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMOEc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r   ÚargsÚdimÚ
hidden_dim)r	   r"   )r#   r$   s   "€r   r%   ÚMOE.__annotate__X   s'   ø€ ÷ 	Lñ 	L™Yð 	L©Sð 	L¹cñ 	Lr'   c                óH  <€ \         SV `  4        W n        W0n        VP                  V n        VP                  V n        \        V P                  V P                  V P
                  R R7      V n        \        P                  ! VP                  V P
                  RR7      V n        R# )T)ÚbiasFN)r7   r8   rj   rk   r    Únum_expertsr   r   Ú
switch_mlpr9   r<   r   Úgate)r?   ri   rj   rk   r@   s   &&&&€r   r8   ÚMOE.__init__X   sw   ø€ Ü‰ÑÔØŒØ$ŒØ×1Ñ1ˆÔØ#'×#;Ñ#;ˆÔ Ü#Ø�H‰H�d—o‘o t×'7Ñ'7¸dô
ˆŒô —I’I˜dŸn™n¨d×.>Ñ.>ÀUÔKˆŽ	r'   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rW   Úreturn©rK   Úarray)r#   r$   s   "€r   r%   rl   c   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r'   c                ó‚  € V P                  V4      pV P                  p\        P                  ! \        P                  ! V) V^,
          RR7      4      RRV13,          p\        P
                  ! W$RR7      p\        P                  ! VRRR7      pV P                  W4      pWeR,          ,          P                  R	R7      pV# )
é   )ÚkthrE   .NrD   T)rE   ÚpreciserJ   ).Néþÿÿÿ)	rq   r   rK   Ústop_gradientÚargpartitionÚtake_along_axisÚsoftmaxrp   Úsum)r?   rW   Úgatesr   ÚindsÚscoresÚys   &&     r   r`   ÚMOE.__call__c   sŸ   € Ø—	‘	˜!“ˆà×$Ñ$ˆÜ×Ò¤§¢°°¸AÀ½EÈÔ KÓLÈSÐRTÐSTÐRTÈWÕUˆÜ×#Ò# E°bÔ9ˆÜ—’˜F¨°TÔ:ˆà�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆàˆr'   )rj   rq   rk   ro   r   rp   rc   re   s   @@r   rg   rg   W   s    ù‡ € ÷	Ló 	L÷÷ ð r'   rg   c                   óD   a a€ ] tR t^qt oV3R lV 3R lltR tRtVtV ;t# )ÚParallelBlockc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfig©r	   )r#   r$   s   "€r   r%   ÚParallelBlock.__annotate__r   s   ø€ ÷ /ñ /™yñ /r'   c                óð   <€ \         SV `  4        VP                  pV^,          p\        W!P                  VP
                  4      V n        \        P                  ! V4      V n	        \        WV4      V n        R# )é   N)r7   r8   r   r1   r   r   Úmixerr9   Ú	LayerNormÚlnrg   Úmoe)r?   rŠ   r3   Úmlp_dimsr@   s   &&  €r   r8   ÚParallelBlock.__init__r   sZ   ø€ Ü‰ÑÔØ×ÑˆØ˜!•8ˆÜ" 4×)9Ñ)9¸6×;LÑ;LÓMˆŒ
Ü—,’,˜tÓ$ˆŒÜ�v XÓ.ˆŽr'   c                óˆ   € V P                  V4      pV P                  WBV4      pV P                  V4      pWV,           V,           # ©N©r‘   r�   r’   )r?   rW   rI   rG   ÚhÚattn_hÚff_hs   &&&&   r   r`   ÚParallelBlock.__call__z   s9   € Ø�G‰G�A‹JˆØ—‘˜A UÓ+ˆØ�x‰x˜‹{ˆØ�}˜qÕ Ð r'   r—   rc   re   s   @@r   r‡   r‡   q   s   ù‡ € ÷/ó /÷!ò !r'   r‡   c                   óD   a a€ ] tR t^�t oV3R lV 3R lltR tRtVtV ;t# )ÚTransformerDecoderc                ó    <€ V ^8„  d   QhRS[ /# r‰   r‹   )r#   r$   s   "€r   r%   ÚTransformerDecoder.__annotate__‚   s   ø€ ÷ Kñ K™yñ Kr'   c                ó´   <€ \         SV `  4        \        V4      V n        \	        VP
                  4       Uu. uF  p\        V4      NK  	  upV n        R # u upi r–   )r7   r8   ÚEmbdÚembdÚranger   r‡   r˜   )r?   rŠ   Úir@   s   && €r   r8   ÚTransformerDecoder.__init__‚   sD   ø€ Ü‰ÑÔÜ˜“LˆŒ	Ü16°v×7HÑ7HÔ1IÓJÑ1I¨A”- Ö'Ñ1IÑJˆŽùÒJs   ·Ac                óº   € V P                  V4      pVf   R .\        V P                  4      ,          p\        V P                  V4       F  w  rEV! WV4      pK  	  V# r–   )r¢   Úlenr˜   Úzip)r?   rW   rI   rG   ÚlayerÚcs   &&&&  r   r`   ÚTransformerDecoder.__call__‡   sP   € Ø�I‰I�a‹LˆØŠ=Ø�FœS §¡›[Õ(ˆEä˜DŸF™F EÖ*‰HˆEÙ�a˜qÓ!ŠAñ +àˆr'   )r¢   r˜   rc   re   s   @@r   r�   r�   �   s   ù‡ € ÷Kó K÷
ò r'   r�   c                   óD   a a€ ] tR t^‘t oV3R lV 3R lltR tRtVtV ;t# )r¡   c                ó    <€ V ^8„  d   QhRS[ /# r‰   r‹   )r#   r$   s   "€r   r%   ÚEmbd.__annotate__’   s   ø€ ÷ Dñ D™yñ Dr'   c                ó„   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        R # r–   )r7   r8   r9   Ú	Embeddingr   r   Úwte©r?   rŠ   r@   s   &&€r   r8   ÚEmbd.__init__’   s,   ø€ Ü‰ÑÔÜ—<’< × 0Ñ 0°&×2BÑ2BÓCˆŽr'   c                ó$   € V P                  V4      # r–   ©r±   )r?   rW   s   &&r   r`   ÚEmbd.__call__–   s   € Ø�x‰x˜‹{Ðr'   rµ   rc   re   s   @@r   r¡   r¡   ‘   s   ù‡ € ÷Dó D÷ò r'   r¡   c                   óD   a a€ ] tR t^št oV3R lV 3R lltR tRtVtV ;t# )Ú
OutputHeadc                ó$   <€ V ^8„  d   QhRS[ RR/# )r   rŠ   rt   Nr‹   )r#   r$   s   "€r   r%   ÚOutputHead.__annotate__›   s   ø€ ÷ Dñ D™yð D¨Tñ Dr'   c                óÎ   <€ \         SV `  4        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  4      V n        R # r–   )	r7   r8   r9   r�   r   r‘   r<   r   Úlinearr²   s   &&€r   r8   ÚOutputHead.__init__›   sB   ø€ Ü‰ÑÔÜ—,’,˜v×/Ñ/Ó0ˆŒÜ—i’i × 0Ñ 0°&×2BÑ2BÓCˆŽr'   c                óB   € V P                  V P                  V4      4      # r–   ©r¼   r‘   )r?   Úinputss   &&r   r`   ÚOutputHead.__call__    s   € Ø�{‰{˜4Ÿ7™7 6›?Ó+Ð+r'   r¿   rc   re   s   @@r   r¸   r¸   š   s   ù‡ € ÷Dó D÷
,ò ,r'   r¸   c                   ój   a a€ ] tR t^¤t oV3R lV 3R lltRV3R lR lltR t]R 4       tRt	Vt
V ;t# )	ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r‰   r‹   )r#   r$   s   "€r   r%   ÚModel.__annotate__¥   s   ø€ ÷ ñ ™yñ r'   c                ó’   <€ \         SV `  4        VP                  V n        \        V4      V n        \        V4      V n        Wn        R # r–   )r7   r8   r   r�   Útransformerr¸   Úlm_headri   r²   s   &&€r   r8   ÚModel.__init__¥   s9   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ-¨fÓ5ˆÔÜ! &Ó)ˆŒØŽ	r'   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   rW   rI   rt   ru   )r#   r$   s   "€r   r%   rÅ   ¬   s5   ø€ ÷ ñ á�8‰8ðñ �h‰hðñ
 
�‰ñr'   c                óf   € Vf   \        W4      pV P                  WV4      pV P                  V4      # r–   )r   rÇ   rÈ   )r?   rW   rI   rG   r„   s   &&&& r   r`   ÚModel.__call__¬   s3   € ð Š<Ü(¨Ó2ˆDà×Ñ˜Q eÓ,ˆØ�|‰|˜A‹Ðr'   c                óŽ  € R V9  d   V# \        V P                  P                  4       F“  pRV 2pR F…  pR F|  pV RV RV 2V9   g   K  \        V P                  P                  4       Uu. uF  pVP	                  V RV RV RV 24      NK!  	  pp\
        P                  ! V4      W RV RV 2&   K~  	  K‡  	  K•  	  V# u upi )z$transformer.h.0.moe.mlp.0.fc1.weightztransformer.h.z.moe.mlp.0.Ú.z	.moe.mlp.z.moe.switch_mlp.)Úfc1Úfc2)ÚweightÚscalesÚbiasesrn   )r£   ri   r   r    ÚpoprK   Ústack)r?   ÚweightsÚlÚprefixÚnr   ÚeÚto_joins   &&      r   ÚsanitizeÚModel.sanitize¹   sæ   € Ø1¸Ô@ØˆNÜ�t—y‘y×+Ñ+Ö,ˆAØ% a SÐ)ˆFÛ#�Û?�AØ ˜ ¨Q¨C¨q°°Ð4¸Ö?ô &+¨4¯9©9×+FÑ+FÔ%Gó#á%G ð $ŸK™K¨6¨(°)¸A¸3¸aÀ¸sÀ!ÀAÀ3Ð(GÖHÙ%Gð  ð #ô GIÇhÂhÈwÓFW˜ (Ð*:¸1¸#¸Q¸q¸cÐ BÓCó @ó $ñ -ð ˆùò#s   Á-%C
c                ó.   € V P                   P                  # r–   )rÇ   r˜   )r?   s   &r   ÚlayersÚModel.layersÈ   s   € à×Ñ×!Ñ!Ð!r'   )ri   rÈ   r   rÇ   rb   )r(   r)   r*   r+   r8   r`   rÜ   Úpropertyrß   r.   r/   rd   re   s   @@r   rÃ   rÃ   ¤   s7   ù‡ € ÷ó ÷ò òð ñ"ó ÷"ð "r'   rÃ   )r   rS   Údataclassesr   Útypingr   Úmlx.coreÚcorerK   Úmlx.nnr9   Úbaser   r   Úswitch_layersr   r	   ÚModuler1   rg   r‡   r�   r¡   r¸   rÃ   r   r'   r   Ú<module>rê      s©   ðó Û Ý !Ý å Ý ç EÝ $ð ÷
ð 
ó ð
ô*/%�B—I‘Iô /%ôdˆ"�)‰)ô ô4!�B—I‘Iô !ô ˜Ÿ™ô ô ˆ2�9‰9ô ô,�—‘ô ,ô&"ˆB�I‰Iö &"r'   