+
    TV-j  ã                   óh  € ^ RI t ^ RIHt ^ RIHtHtHtHt ^ RIH	t
 ^ RIHt ^RIHtHtHt ^RIHt ^RIHt ] ! R R]4      4       t ! R	 R
]P,                  4      t ! R R]P,                  4      t ! R R]P,                  4      t ! R R]P,                  4      t ! R R]P,                  4      tR# )é    N)Ú	dataclass)ÚDictÚListÚOptionalÚUnion)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚSuScaledRoPE)Ú	SwitchGLUc                   ód   a € ] tR t^t o RtRtRtRt^ t^ t	^t
RtRtRtRt^t^tRtV 3R	 ltR
tV tR# )Ú	ModelArgsÚphimoei@}  i   i   i   g�íµ ÷Æ°>Ng     ˆÃ@c                óþ   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[S[ S[S[S[S[,          3,          3,          ;R&   S[;R&   S[;R&   S[;R&   # )é   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚmax_position_embeddingsÚ original_max_position_embeddingsÚrms_norm_epsÚrope_scalingÚnum_local_expertsÚnum_experts_per_tokÚ
rope_theta)ÚstrÚintÚfloatr   r   r   )ÚformatÚ__classdict__s   "€Úe/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/phimoe.pyÚ__annotate__ÚModelArgs.__annotate__   sÑ   ø‡ ‚ áÑñ ñ Ññ ñ Ññ	 ñ
 Ñ!ñ ñ Ññ ñ Ñ!ñ ñ Ñ ñ ñ !Ñ)ñ ñ '*Ñ0ñ ñ Ññ ñ ‘s™E¡%©©e­Ð"4Õ5Ð5Õ6Ñ=ñ ñ Ññ ñ Ñ ñ ñ Ñò ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r$   s   @r%   r   r      sZ   ø‡ € à€JØ€JØ€KØ!ÐØÐØ!ÐØ ÐØ#)ÐØ,0Ð$Ø€LØ9=€LØÐØ ÐØ€J÷ ƒ r(   r   c                   óT   a a€ ] tR t^ t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r#   r$   s   "€r%   r&   ÚAttention.__annotate__!   s   ø€ ÷ 
ñ 
™Yñ 
r(   c                óÞ  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pVP                  V,          pVR	,          V n        \        P                  ! W#V,          RR7      V n
        \        P                  ! W$V,          RR7      V n        \        P                  ! W$V,          RR7      V n        \        P                  ! W5,          VRR7      V n        \        VVP                  VP                   VP"                  VP$                  R,          VP$                  R,          VP$                  R,          VP$                  R,          R7      V n        R# )
g      à?T©ÚbiasÚshort_factorÚlong_factorÚshort_mscaleÚlong_mscale)Úbaser   r   r;   r<   r=   r>   Ng      à¿)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr   r   r   r   r   Úrope)Úselfr5   ÚdimrB   rC   Úhead_dimÚ	__class__s   &&    €r%   rA   ÚAttention.__init__!   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à×#Ñ# wÕ.ˆØ˜t•^ˆŒ
ä—i’i ¨xÕ%7¸dÔCˆŒÜ—i’i °(Õ%:ÀÔFˆŒÜ—i’i °(Õ%:ÀÔFˆŒÜ—i’i Õ 2°C¸dÔCˆŒä ØØ—‘Ø$(×$@Ñ$@Ø-1×-RÑ-RØ×*Ñ*¨>Õ:Ø×)Ñ)¨-Õ8Ø×*Ñ*¨>Õ:Ø×)Ñ)¨-Õ8ô	
ˆŽ	r(   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# ©r   ÚxÚmaskÚreturn©ÚmxÚarrayr   )r#   r$   s   "€r%   r&   r7   ;   s:   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ
 
�‰ñ#r(   c           	     ó  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )é   )Úoffset)ÚcacherD   rT   éÿÿÿÿ)ÚshaperG   rH   rI   ÚreshaperB   Ú	transposerC   rK   r[   Úupdate_and_fetchr
   rD   rJ   )rL   rS   rT   r\   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r%   Ú__call__ÚAttention.__call__;   sS  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆð —/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r(   )rH   rB   rC   rJ   rG   rK   rD   rI   ©NN©	r*   r+   r,   r-   rA   ri   r/   r0   Ú__classcell__©rO   r$   s   @@r%   r2   r2       s   ù‡ € ÷
ó 
÷4#÷ #ò #r(   r2   c                   óP   a a€ ] tR t^Yt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚPhiMoESparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# r4   r6   )r#   r$   s   "€r%   r&   Ú!PhiMoESparseMoeBlock.__annotate__Z   s   ø€ ÷ Uñ U™Yñ Ur(   c                óp  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        \        P                  ! V P                  V P                  R R7      V n        \        V P                  V P
                  V P                  4      V n        R# )Fr9   N)r@   rA   r   Ú
hidden_dimr   Úffn_dimr   Únum_expertsr   Útop_krE   rF   Úgater   Ú
switch_mlp©rL   r5   rO   s   &&€r%   rA   ÚPhiMoESparseMoeBlock.__init__Z   s   ø€ Ü‰ÑÔØ×*Ñ*ˆŒØ×-Ñ-ˆŒØ×1Ñ1ˆÔØ×-Ñ-ˆŒ
ä—I’I˜dŸo™o¨t×/?Ñ/?ÀeÔLˆŒ	Ü# D§O¡O°T·\±\À4×CSÑCSÓTˆŽr(   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rS   rU   ©rW   rX   )r#   r$   s   "€r%   r&   rr   d   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r(   c                ó‚  € V P                  V4      pV P                  p\        P                  ! \        P                  ! V) V^,
          RR7      RRV13,          4      p\        P
                  ! W$RR7      p\        P                  ! VRRR7      pV P                  W4      pWeR,          ,          P                  R	R7      pV# )
rZ   )ÚkthÚaxis.N)r€   T)r€   Úpreciser]   ).Néþÿÿÿ)	rx   rw   rW   Ústop_gradientÚargpartitionÚtake_along_axisÚsoftmaxry   Úsum)rL   rS   ÚgatesÚkÚindsÚscoresÚys   &&     r%   ri   ÚPhiMoESparseMoeBlock.__call__d   s�   € Ø—	‘	˜!“ˆà�J‰JˆÜ×Ò¤§¢°°¸AÀ½EÈÔ KÈCÐQSÐRSÐQSÈGÕ TÓUˆÜ×#Ò# E°bÔ9ˆÜ—’˜F¨°TÔ:ˆà�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆàˆr(   )ru   rx   rt   rv   ry   rw   rl   rn   s   @@r%   rp   rp   Y   s    ù‡ € ÷Uó U÷÷ ð r(   rp   c                   óT   a a€ ] tR t^rt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚPhiMoEDecoderLayerc                ó    <€ V ^8„  d   QhRS[ /# r4   r6   )r#   r$   s   "€r%   r&   ÚPhiMoEDecoderLayer.__annotate__s   s   ø€ ÷ 	
ñ 	
™Yñ 	
r(   c                óJ  <€ \         SV `  4        VP                  V n        \        V4      V n        \        V4      V n        \        P                  ! VP                  VP                  R 7      V n
        \        P                  ! VP                  VP                  R 7      V n        R# ©)ÚepsN)r@   rA   r   r2   Ú	self_attnrp   Úblock_sparse_moerE   Ú	LayerNormr   Úinput_layernormÚpost_attention_layernormrz   s   &&€r%   rA   ÚPhiMoEDecoderLayer.__init__s   su   ø€ Ü‰ÑÔØ×+Ñ+ˆÔä" 4›ˆŒÜ 4°TÓ :ˆÔÜ!Ÿ|š|¨D×,<Ñ,<À$×BSÑBSÔTˆÔÜ(*¯ªØ×Ñ $×"3Ñ"3ô)
ˆÖ%r(   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# rR   rV   )r#   r$   s   "€r%   r&   r‘   ~   s:   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ
 
�‰ñr(   c                óº   € TpV P                  V4      pV P                  WRVR 7      pWE,           pTpV P                  V4      pV P                  V4      pWE,           pV# ))rT   r\   )r˜   r•   r™   r–   )rL   rS   rT   r\   ÚresidualÚhidden_statess   &&&&  r%   ri   ÚPhiMoEDecoderLayer.__call__~   sf   € ð ˆØ×,Ñ,¨QÓ/ˆØŸ™ }Àu˜ÓMˆØ Õ0ˆà ˆØ×5Ñ5°mÓDˆØ×-Ñ-¨mÓ<ˆØ Õ0ˆàÐr(   )r–   r   r˜   r™   r•   rk   rl   rn   s   @@r%   r�   r�   r   s   ù‡ € ÷	
ó 	
÷÷ ò r(   r�   c                   óT   a a€ ] tR t^‘t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚPhiMoEModelc                ó    <€ V ^8„  d   QhRS[ /# r4   r6   )r#   r$   s   "€r%   r&   ÚPhiMoEModel.__annotate__’   s   ø€ ÷ Jñ J™Yñ Jr(   c                ó„  <€ \         SV `  4        Wn        VP                  V n        \        P
                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        V4      NK  	  upV n        \        P                  ! VP                  VP                  R 7      V n        R# u upi r“   )r@   rA   r5   r   rE   Ú	Embeddingr   Úembed_tokensÚranger   r�   Úlayersr—   r   Únorm)rL   r5   Ú_rO   s   && €r%   rA   ÚPhiMoEModel.__init__’   sˆ   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒäŸLšL¨¯©¸$×:JÑ:JÓKˆÔÜ9>¸t×?UÑ?UÔ9VÓWÑ9V°AÔ)¨$Ö/Ñ9VÑWˆŒÜ—L’L ×!1Ñ!1°t×7HÑ7HÔIˆŽ	ùò Xs   Á.B=c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   ÚinputsrU   r}   )r#   r$   s   "€r%   r&   r£   ›   s'   ø€ ÷ ñ á—‘ðñ 
�‰ñ	r(   c                óü   € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2^ ,          4      p\	        V P                  V4       F  w  rVV! W4V4      pK  	  V P                  V4      # ©N)r¦   Úlenr¨   r	   Úzipr©   )rL   r­   r\   ÚhrT   ÚlayerÚcs   &&&    r%   ri   ÚPhiMoEModel.__call__›   so   € ð
 ×Ñ˜fÓ%ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEÜ$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr(   )r5   r¦   r¨   r©   r   r¯   rl   rn   s   @@r%   r¡   r¡   ‘   s    ù‡ € ÷Jó J÷÷ ò r(   r¡   c                   ój   a a€ ] tR t^¬t oV3R lV 3R lltRV3R lR lltR t]R 4       tRt	Vt
V ;t# )	ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r4   r6   )r#   r$   s   "€r%   r&   ÚModel.__annotate__­   s   ø€ ÷ Oñ O™Yñ Or(   c                óÖ   <€ \         SV `  4        VP                  V n        Wn        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# )Tr9   N)r@   rA   r   r5   r¡   ÚmodelrE   rF   r   r   Úlm_headrz   s   &&€r%   rA   ÚModel.__init__­   sH   ø€ Ü‰ÑÔØŸ/™/ˆŒØŒ	Ü  Ó&ˆŒ
Ü—y’y ×!1Ñ!1°4·?±?ÈÔNˆŽr(   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r­   r}   )r#   r$   s   "€r%   r&   r¹   ´   s   ø€ ÷ !ñ !á—‘ñ!r(   c                óF   € V P                  W4      pV P                  V4      # r¯   )r»   r¼   )rL   r­   r\   Úouts   &&& r%   ri   ÚModel.__call__´   s!   € ð
 �j‰j˜Ó'ˆØ�|‰|˜CÓ Ð r(   c                ó’  € R V9  d   V# \        V P                  P                  4       F•  pRV 2pR F‡  w  rER F|  pV RV RV 2V9   g   K  \        V P                  P                  4       Uu. uF  pVP	                  V RV RV RV 24      NK!  	  pp\
        P                  ! V4      W RV RV 2&   K~  	  K‰  	  K—  	  V# u upi )z3model.layers.0.block_sparse_moe.experts.0.w1.weightzmodel.layers.z.block_sparse_moe.experts.0.Ú.z.block_sparse_moe.experts.z.block_sparse_moe.switch_mlp.))Úw1Ú	gate_proj)Úw2Ú	down_proj)Úw3Úup_proj)ÚweightÚscalesÚbiases)r§   r5   r   r   ÚpoprW   Ústack)	rL   ÚweightsÚlÚprefixÚnÚmr‰   ÚeÚto_joins	   &&       r%   ÚsanitizeÚModel.sanitize¼   sô   € Ø@ÈÔOØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛU‘�Û7�AØ ˜Ð!=¸a¸SÀÀ!ÀÐEÈÖPô
 &+¨4¯9©9×+FÑ+FÔ%Gó	#ñ &H ð $ŸK™KØ#) (Ð*DÀQÀCÀqÈÈÈ1ÈQÈCÐ Pöñ &Hð	  ð #ô ŸHšH WÓ-ð   (Ð*GÈÀsÈ!ÈAÈ3Ð OÓPó 8ó Vñ 4ð ˆùò#s   Á/%C
c                ó.   € V P                   P                  # r¯   )r»   r¨   )rL   s   &r%   r¨   ÚModel.layersÐ   s   € à�z‰z× Ñ Ð r(   )r5   r¼   r»   r   r¯   )r*   r+   r,   r-   rA   ri   rÖ   Úpropertyr¨   r/   r0   rm   rn   s   @@r%   r·   r·   ¬   s9   ù‡ € ÷Oó O÷!ò !òð( ñ!ó ÷!ð !r(   r·   )ÚmathÚdataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerW   Úmlx.nnrE   r?   r   r	   r
   Ú
rope_utilsr   Úswitch_layersr   r   ÚModuler2   rp   r�   r¡   r·   r)   r(   r%   Ú<module>rä      s�   ðã Ý !ß .Ó .å Ý ç TÑ TÝ $Ý $ð ô �ó  ó ð ô"6#�—	‘	ô 6#ôr˜2Ÿ9™9ô ô2˜Ÿ™ô ô>�"—)‘)ô ô6&!ˆB�I‰Iö &!r(   