+
    TV-jž2  ã                   óÜ  € ^ RI t ^ RIHt ^ RIHtHtHtHtHt ^ RI	H
t ^ RIHt ^RIHt ^RIHtHtHt ^RIHtHt ^RIHt ^RIHt ] ! R	 R
]4      4       t ! R R]P8                  4      t ! R R]P8                  4      t ! R R]P8                  4      t ! R R]P8                  4      t  ! R R]P8                  4      t! ! R R]P8                  4      t" ! R R]P8                  4      t#R# )é    N)Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnion)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)Úinitialize_rope)Ú	SwitchGLUc                   óŒ   a € ] tR t^t o RtRtRtRt^ t^ t	^t
^@tRtRtRtRtR	t^€t^t^t^tR
tRtRt^t^tRtR
tV 3R ltRtV tR# )Ú	ModelArgsi  i   i   i   i   gñhãˆµøä>i'  NFTgÏ÷Sã¥›@Úsigmoidc                óž  <€ V ^8„  d   Qh/ S[ ;R&   S[S[ ,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )é   Ú
model_typeÚlayer_typesÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚmoe_intermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚhead_dimÚmax_position_embeddingsÚrms_norm_epsÚ
rope_thetaÚrope_scalingÚtie_word_embeddingsÚnum_expertsÚnum_experts_per_tokÚnum_shared_expertsÚnum_dense_layersÚ
route_normÚroute_scaleÚ
score_funcÚn_groupÚ
topk_groupÚsliding_windowÚmup_enabled)Ústrr   ÚintÚfloatr   r   r   Úbool)ÚformatÚ__classdict__s   "€Úd/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/afmoe.pyÚ__annotate__ÚModelArgs.__annotate__   sp  ø‡ ‚ á�Oñ ñ ‘c•Ññ ñ Ññ	 ñ
 Ññ ñ Ñ!ñ ñ Ñ%ñ ñ Ññ ñ Ñ!ñ ñ Ñ ñ ñ Ññ ñ !Ñ)ñ ñ Ññ ñ Ññ ñ ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?ñ ñ  Ñ%ñ! ñ$ Ññ% ñ& Ñ ñ' ñ( Ññ) ñ* Ññ+ ñ, Ññ- ñ. Ññ/ ñ0 Ññ1 ñ2 Ññ3 ñ4 Ññ5 ñ6 Ññ7 ñ8 Ñò9 ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r5   s   @r6   r   r      s�   ø‡ € ð €JØ€KØ!ÐØ!%ÐØÐØ!ÐØ ÐØ€HØ#)ÐØ€LØ€JØ;?€LØ %Ðà€KØ ÐØÐØÐØ€JØ€KØ€JØ€GØ€JØ€NØ€K÷9 ƒ r9   r   c                   óX   a a€ ] tR t^0t oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   ÚargsÚis_local_attention)r   r3   )r4   r5   s   "€r6   r7   ÚAttention.__annotate__1   s   ø€ ÷ (ñ (™Yð (¹Dñ (r9   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        W n        V P                  R,          V n	        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  ,          V P                  RR7      V n        \        P                   ! V P                  VP"                  R7      V n        \        P                   ! V P                  VP"                  R7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        V'       d?   \+        V P                  VP,                  RVP.                  VP0                  4      V n        R# RV n        R# )g      à?F©Úbias©ÚepsNg      à¿)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr   rF   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRMSNormr!   Úq_normÚk_normÚ	gate_projr   r"   r#   r    Úrope)ÚselfrE   rF   Ú	__class__s   &&&€r6   rN   ÚAttention.__init__1   s¦  ø€ Ü‰ÑÔà×+Ñ+ˆÔØ×/Ñ/ˆŒØ×2Ñ2ˆŒØŸ™ˆŒØ"4Ôà—]‘] DÕ(ˆŒ
ä—i’iØ×Ñ˜dŸl™l¨T¯]©]Õ:Àô
ˆŒô —i’iØ×Ñ˜dŸo™o°·±Õ=ÀEô
ˆŒô —i’iØ×Ñ˜dŸo™o°·±Õ=ÀEô
ˆŒô —i’iØ�L‰L˜4Ÿ=™=Õ(¨$×*:Ñ*:Àô
ˆŒô —j’j §¡°D×4EÑ4EÔFˆŒÜ—j’j §¡°D×4EÑ4EÔFˆŒäŸšØ×Ñ˜dŸl™l¨T¯]©]Õ:Àô
ˆŒ÷ Ü'Ø—‘Ø—‘ØØ×!Ñ!Ø×,Ñ,óˆDŽIð ˆDŽIr9   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r4   r5   s   "€r6   r7   rG   [   sH   ø€ ÷ +#ñ +#á�8‰8ð+#ñ ‘r—x‘xÕ ð+#ñ ™�}ð	+#ñ
 
�‰ñ+#r9   c           	     óB  € VP                   w  rEpV P                  V4      pV P                  V4      pV P                  V4      p	VP	                  WEV P
                  V P                  4      P                  ^ ^^^4      pVP	                  WEV P                  V P                  4      P                  ^ ^^^4      pV	P	                  WEV P                  V P                  4      P                  ^ ^^^4      p	V P                  V4      pV P                  V4      pV P                  '       dn   V P                  e`   Ve:   V P                  WsP                  R7      pV P                  WƒP                  R7      pM"V P                  V4      pV P                  V4      pVe   VP                  W‰4      w  r‰\        WxW“V P                   VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
\"        P$                  ! V P'                  V4      4      pW«,          p
V P)                  V
4      # )r   )Úoffset)rd   rQ   rc   éÿÿÿÿ)ÚshaperT   rU   rV   ÚreshaperO   r   Ú	transposerP   rY   rZ   rF   r\   rj   Úupdate_and_fetchr   rQ   rg   r   r[   rW   )r]   rb   rc   rd   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputÚgates   &&&&        r6   Ú__call__ÚAttention.__call__[   sÀ  € ð —'‘'‰ˆˆaà—+‘+˜a“.ˆØ�{‰{˜1‹~ˆØ—‘˜Q“ˆà—/‘/ !¨¯©°d·m±mÓD×NÑNØˆq�!�Qó
ˆð �|‰|˜A $§/¡/°4·=±=ÓA×KÑKÈAÈqÐRSÐUVÓWˆØ—‘  d§o¡o°t·}±}ÓE×OÑOØˆq�!�Qó
ˆð —+‘+˜gÓ&ˆØ�{‰{˜4Ó ˆà×"×"Ð" t§y¡yÒ'<ØÒ ØŸ)™) G·L±L˜)ÓA�Ø—y‘y ¯l©l�yÓ;‘àŸ)™) GÓ,�Ø—y‘y “�àÒØ ×1Ñ1°$Ó?‰LˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆä�zŠz˜$Ÿ.™.¨Ó+Ó,ˆØ•ˆà�{‰{˜6Ó"Ð"r9   )r[   r   r   rF   rZ   rU   rO   rP   rW   rY   rT   r\   rQ   rV   ©F©NN©	r;   r<   r=   r>   rN   rx   r@   rA   Ú__classcell__©r^   r5   s   @@r6   rC   rC   0   s   ù‡ € ÷(õ (÷T+#÷ +#ò +#r9   rC   c                   óT   a a€ ] tR t^‰t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMLPc                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# )r   rE   r   )r   r   r1   )r4   r5   s   "€r6   r7   ÚMLP.__annotate__Š   s   ø€ ÷ >ñ >™Yð >¹8ÁC½=ñ >r9   c                ó  <€ \         SV `  4        VP                  pVe   TMVP                  p\        P
                  ! W4RR7      V n        \        P
                  ! WCRR7      V n        \        P
                  ! W4RR7      V n        R # )NFrI   )	rM   rN   r   r   rR   rS   r[   Ú	down_projÚup_proj)r]   rE   r   ÚdimÚ
hidden_dimr^   s   &&&  €r6   rN   ÚMLP.__init__Š   sl   ø€ Ü‰ÑÔà×Ñˆð !Ò,ñ à×'Ñ'ð 	ô Ÿš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr9   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   re   ©rg   rh   )r4   r5   s   "€r6   r7   r‚   ˜   s   ø€ ÷ Jñ J™RŸX™Xñ Jr9   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # ©N)r„   r	   r[   r…   ©r]   rb   s   &&r6   rx   ÚMLP.__call__˜   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr9   )r„   r[   r…   rŒ   r|   r~   s   @@r6   r€   r€   ‰   s!   ù‡ € ÷>õ >÷J÷ Jð Jr9   r€   c                   óT   a a€ ] tR t^œt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )Ú	MoERouterz;Router module that wraps the gate for proper weight naming.c                ó    <€ V ^8„  d   QhRS[ /# ©r   rE   ©r   )r4   r5   s   "€r6   r7   ÚMoERouter.__annotate__Ÿ   s   ø€ ÷ Nñ N™Yñ Nr9   c                óˆ   <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        R# ©FrI   N)rM   rN   rR   rS   r   r%   rw   ©r]   rE   r^   s   &&€r6   rN   ÚMoERouter.__init__Ÿ   s.   ø€ Ü‰ÑÔÜ—I’I˜d×.Ñ.°×0@Ñ0@ÀuÔMˆŽ	r9   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   rb   re   rŠ   )r4   r5   s   "€r6   r7   r”   £   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r9   c                ó$   € V P                  V4      # rŒ   ©rw   r�   s   &&r6   rx   ÚMoERouter.__call__£   s   € Ø�y‰y˜‹|Ðr9   rœ   )
r;   r<   r=   r>   Ú__doc__rN   rx   r@   rA   r}   r~   s   @@r6   r�   r�   œ   s#   ù‡ € ÙE÷Nó N÷÷ ð r9   r�   c                   óP   a a€ ] tR t^§t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚAfmoeMoEc                ó    <€ V ^8„  d   QhRS[ /# r’   r“   )r4   r5   s   "€r6   r7   ÚAfmoeMoE.__annotate__¨   s   ø€ ÷ Xñ X™Yñ Xr9   c                ól  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        \        V4      V n        \        P                  ! VP                  34      V n        \        VP                   VP"                  VP                  4      V n        VP&                  ^ 8”  d1   VP"                  VP&                  ,          p\)        WR7      V n        R# R# )r   )r   N)rM   rN   rE   r%   r&   r)   r*   r+   r,   r-   r�   Úrouterrg   ÚzerosÚexpert_biasr   r   r   Úexpertsr'   r€   Úshared_experts)r]   rE   Úshared_intermediate_sizer^   s   && €r6   rN   ÚAfmoeMoE.__init__¨   sò   ø€ Ü‰ÑÔØŒ	Ø×+Ñ+ˆÔØ#'×#;Ñ#;ˆÔ ØŸ/™/ˆŒØ×+Ñ+ˆÔØŸ/™/ˆŒØ—|‘|ˆŒØŸ/™/ˆŒä “oˆŒäŸ8š8 T×%5Ñ%5Ð$7Ó8ˆÔä Ø×ÑØ×&Ñ&Ø×Ñó
ˆŒð ×"Ñ" QÔ&à×*Ñ*¨T×-DÑ-DÕDð %ô #& dÔ"WˆDÖñ	 'r9   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rš   rŠ   )r4   r5   s   "€r6   r7   r¢   Ã   s#   ø€ ÷ ,ñ ,™"Ÿ(™(ð ,¡r§x¡xñ ,r9   c                óz  € V P                  V4      pV P                  R 8X  d5   \        P                  ! VP	                  \        P
                  4      4      pM5\        P                  ! VP	                  \        P
                  4      R
R7      pW0P                  ,           pV P                  ^8”  dò   \        P                  ! VR
V P                  R
3R7      p\        P                  ! V^R
R7      P                  R
RR7      pV P                  V P                  ,
          p\        P                  ! WV^,
          RR7      RRV1R3,          p\        P                  ! V\        P                  ! V4      \        P                   ! R	4      RR7      p\        P"                  ! VRR
4      pV P$                  p\        P                  ! V) V^,
          R
R7      RRV13,          p\        P&                  ! W8R
R7      p	V P(                  '       d-   V P$                  ^8”  d   V	P                  R
RR7      p
Wš,          p	W�P*                  ,          p	V P-                  W4      pW¹R,          ,          P                  RR7      P	                  VP.                  4      pV P0                  P2                  ^ 8”  d   W°P5                  V4      ,           pV# )r   )Úaxis)r­   rl   T)r­   Úkeepdims)Úkthr­   .N:NNNg        rk   éþÿÿÿ).N)r¤   r+   rg   r   ÚastypeÚfloat32Úsoftmaxr¦   r,   Ú	unflattenÚtopkÚsumr-   ÚargpartitionÚput_along_axisÚstop_gradientrh   Úflattenr&   Útake_along_axisr)   r*   r§   ÚdtyperE   r'   r¨   )r]   rb   ÚgatesÚscoresÚselection_scoresÚgroup_scoresÚkÚ	group_idxÚindsÚselected_scoresÚdenominatorÚys   &&          r6   rx   ÚAfmoeMoE.__call__Ã   s"  € Ø—‘˜A“ˆà�?‰?˜iÔ'Ü—Z’Z §¡¬R¯Z©ZÓ 8Ó9‰Fä—Z’Z §¡¬R¯Z©ZÓ 8¸rÔBˆFð "×$4Ñ$4Õ4Ðð �<‰<˜!ÔÜ!Ÿ|š|Ø  r°$·,±,ÀÐ1Cô Ðô Ÿ7š7Ð#3°Q¸RÔ@×DÑDØ $ð Eó ˆLð —‘˜tŸ™Õ.ˆAÜŸš¨¸a½%ÀbÔIÈ#ÈrÐPQÈrÐSTÈ*ÕUˆIÜ!×0Ò0Ø ¤"×"2Ò"2°9Ó"=¼r¿xºxÈ»}ÐSUô Ðô  "ŸzšzÐ*:¸BÀÓCÐð ×$Ñ$ˆÜ�ŠÐ 0Ð0°a¸!µeÀ"ÔEÀcÈ2ÈAÈ2ÀgÕNˆä×,Ò,¨VÀÔCˆà�?�?ˆ?˜t×7Ñ7¸!Ô;Ø)×-Ñ-°2ÀÐ-ÓEˆKØ-Õ;ˆOà)×,<Ñ,<Õ<ˆà�L‰L˜Ó!ˆØ Õ+Õ+×0Ñ0°bÐ0Ó9×@Ñ@ÀÇÁÓIˆà�9‰9×'Ñ'¨!Ô+Ø×'Ñ'¨Ó*Õ*ˆAàˆr9   )rE   r¦   r§   r,   r%   r&   r)   r*   r¤   r+   r¨   r-   r|   r~   s   @@r6   r    r    §   s    ù‡ € ÷Xó X÷6,÷ ,ð ,r9   r    c                   óX   a a€ ] tR t^òt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚDecoderLayerc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r   rE   Ú	layer_idxÚuse_sliding)r   r1   r3   )r4   r5   s   "€r6   r7   ÚDecoderLayer.__annotate__ó   s'   ø€ ÷ Vñ V™Yð V±3ð VÁTñ Vr9   c                ój  <€ \         SV `  4        VP                  V n        W0n        W n        \        WR 7      V n        W!P                  8  d   \        V4      V n	        M\        V4      V n	        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        R# ))rF   rK   N)rM   rN   r   rÌ   rË   rC   Ú	self_attnr(   r€   Úmlpr    rR   rX   r!   Úinput_layernormÚpost_attention_layernormÚpre_mlp_layernormÚpost_mlp_layernorm)r]   rE   rË   rÌ   r^   s   &&&&€r6   rN   ÚDecoderLayer.__init__ó   sÖ   ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ&ÔØ"Œä" 4ÔHˆŒà×,Ñ,Ô,Ü˜4“yˆD�Hä “~ˆDŒHä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ô "$§¢¨D×,<Ñ,<À$×BSÑBSÔ!TˆÔÜ"$§*¢*¨T×-=Ñ-=À4×CTÑCTÔ"UˆÖr9   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ra   rf   )r4   r5   s   "€r6   r7   rÍ     sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr9   c                óè   € V P                  V P                  V4      W#4      pV P                  V4      pW,           pV P                  V P	                  V4      4      pV P                  V4      pWT,           # rŒ   )rÏ   rÑ   rÒ   rÐ   rÓ   rÔ   )r]   rb   rc   rd   ÚrÚhs   &&&&  r6   rx   ÚDecoderLayer.__call__  sf   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ×)Ñ)¨!Ó,ˆØ�Eˆà�H‰H�T×+Ñ+¨AÓ.Ó/ˆØ×#Ñ# AÓ&ˆØ�uˆr9   )	r   rÑ   rË   rÐ   rÒ   rÔ   rÓ   rÏ   rÌ   rz   r{   r|   r~   s   @@r6   rÉ   rÉ   ò   s    ù‡ € ÷Võ V÷(÷ ò r9   rÉ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú
AfmoeModeli  c                ó    <€ V ^8„  d   QhRS[ /# r’   r“   )r4   r5   s   "€r6   r7   ÚAfmoeModel.__annotate__  s   ø€ ÷ ñ ™Yñ r9   c           
     ó  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                  ! VP                  VP                  4      V n        \        V P
                  4       UUu. uF  w  r#\        WVR 8H  R7      NK  	  uppV n        \        P                  ! VP                  VP                   R7      V n        V P
                  P%                  R4      V n        RV n        \        V P                  4       F   w  r$VP*                  '       g   K  W n         R# 	  R# u uppi )Úsliding_attention)rE   rË   rÌ   rK   Úfull_attentionN)rM   rN   rE   r   r   r   r.   r/   r   rR   Ú	EmbeddingÚembed_tokensÚ	enumeraterÉ   ÚlayersrX   r!   ÚnormÚindexÚfa_idxÚswa_idxrÌ   )r]   rE   ÚidxÚ
layer_typeÚlayerr^   s   &&   €r6   rN   ÚAfmoeModel.__init__  s1  ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ×+Ñ+ˆÔØ"×1Ñ1ˆÔØ×+Ñ+ˆÔØ×+Ñ+ˆÔäŸLšL¨¯©¸$×:JÑ:JÓKˆÔô
 $-¨T×-=Ñ-=Ô#>ô	
ñ $?‘�ô Ø°jÐDWÑ6W÷ñ $?ò	
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	à×&Ñ&×,Ñ,Ð-=Ó>ˆŒØˆŒÜ# D§K¡KÖ0‰JˆCØ× × Ò Ø"”Úó 1ùó
s   ÃE;c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   ÚinputsrŠ   )r4   r5   s   "€r6   r7   rÞ   1  s   ø€ ÷ ñ á—‘ñr9   c                ó   € V P                  V4      pV P                  '       d(   V\        P                  ! V P                  4      ,          pVf   R .\        V P                  4      ,          p\        W2V P                  ,          4      pR pV P                  e)   \        W2V P                  ,          V P                  R7      p\        V P                  V4       F%  w  rgVP                  '       d   TMTpV! W8VR7      pK'  	  V P                  V4      # )N)Úwindow_size)rd   )rã   r/   ÚmathÚsqrtr   Úlenrå   r   rè   ré   r.   ÚziprÌ   ræ   )	r]   rð   rd   rÙ   Úfa_maskÚswa_maskrì   Úcrc   s	   &&&      r6   rx   ÚAfmoeModel.__call__1  sÚ   € ð
 ×Ñ˜fÓ%ˆà××ÐØ”D—I’I˜d×.Ñ.Ó/Õ/ˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEä'¨°·±Õ+=Ó>ˆØˆØ�<‰<Ò#Ü,Ø˜Ÿ™Õ&°D×4GÑ4GôˆHô ˜DŸK™K¨Ö/‰HˆEØ$×0×0Ð0‘8°gˆDÙ�a QÔ'ŠAñ 0ð �y‰y˜‹|Ðr9   )rE   rã   rè   r   r   rå   r/   ræ   r   r.   ré   r   rŒ   r|   r~   s   @@r6   rÜ   rÜ     s   ù‡ € ÷ó ÷4÷ ò r9   rÜ   c                   ó�   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR t]R 4       tR t	]R	 4       t
]R
 4       tRtVtV ;t# )ÚModeliL  c                ó    <€ V ^8„  d   QhRS[ /# r’   r“   )r4   r5   s   "€r6   r7   ÚModel.__annotate__M  s   ø€ ÷ Tñ T™Yñ Tr9   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# r–   )rM   rN   rE   r   rÜ   Úmodelr$   rR   rS   r   r   Úlm_headr—   s   &&€r6   rN   ÚModel.__init__M  sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ Ó%ˆŒ
à×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r9   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rï   rŠ   )r4   r5   s   "€r6   r7   rþ   V  s   ø€ ÷ 
ñ 
á—‘ñ
r9   c                óÐ   € V P                  W4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# rŒ   )r   rE   r$   rã   Ú	as_linearr  )r]   rð   rd   Úouts   &&& r6   rx   ÚModel.__call__V  sX   € ð
 �j‰j˜Ó'ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r9   c                ót  € VP                  4        UUu/ uF  w  r#R V9  g   K  W#bK  	  pppV P                  P                  '       d   VP                  RR4       \	        V P                  P
                  4       F¯  pW@P                  P                  8  d   K  RV 2pR F…  pR F|  pV RV RV 2V9   g   K  \	        V P                  P                  4       Uu. uF  pVP                  V RV RV RV 24      NK!  	  pp\        P                  ! V4      W RV RV 2&   K~  	  K‡  	  K±  	  V# u uppi u upi )	zrotary_emb.inv_freqzlm_head.weightNzmodel.layers.z.mlp.experts.0.Ú.z.mlp.experts.)r…   r„   r[   )ÚweightÚscalesÚbiases)
ÚitemsrE   r$   ÚpopÚranger   r(   r%   rg   Ústack)	r]   ÚweightsrÁ   ÚvÚlÚprefixÚnÚeÚto_joins	   &&       r6   ÚsanitizeÚModel.sanitizeb  s:  € à$+§M¡M¤OÔV¡O™D˜AÐ7LÐTUÑ7U”4�1’4¡OˆÑVà�9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/ô �t—y‘y×2Ñ2Ö3ˆAØ—9‘9×-Ñ-Ô-ÙØ$ Q CÐ(ˆFÛ:�Û7�AØ ˜ °°°1°Q°CÐ8¸GÖCô &+¨4¯9©9×+@Ñ+@Ô%Aó#á%A ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÖLÙ%Að  ð #ô DFÇ8Â8ÈGÓCT˜ (¨-¸°s¸!¸A¸3Ð ?Ó@ó 8ó ;ñ	 4ð ˆùó' Wùò#s   ”D/¤D/Ã%D5
c                ó.   € V P                   P                  # rŒ   )r   rå   )r]   s   &r6   rå   ÚModel.layersy  s   € à�z‰z× Ñ Ð r9   c                ó¸   € V P                    Uu. uF?  pVP                  '       d!   \        V P                  P                  R 7      M	\        4       NKA  	  up# u upi ))Úmax_size)rå   rÌ   r   r   r.   r   )r]   rì   s   & r6   Ú
make_cacheÚModel.make_cache}  sX   € ð Ÿšó
ñ %�ð ×$×$Ð$ô  ¨¯©×)BÑ)BÕCä“Yòñ %ñ
ð 	
ùò 
s
   �A¦.Ac                ó   € R  pV# )c                 ó   € R V 9  # )r¦   r:   )rÁ   s   &r6   Ú	predicateÚ'Model.cast_predicate.<locals>.predicate‰  s   € Ø ¨Ñ)Ð)r9   r:   ©r]   r"  s   & r6   Úcast_predicateÚModel.cast_predicate‡  s   € ò	*ð Ðr9   c                ó   € R  pV# )c                 ó    € R V 9   d   R^@R^/# R# )zrouter.gateÚ
group_sizeÚbitsTr:   )ÚpathÚ_s   &&r6   r"  Ú(Model.quant_predicate.<locals>.predicate�  s   € Ø Ô$Ø$ b¨&°!Ð4Ð4Ùr9   r:   r$  s   & r6   Úquant_predicateÚModel.quant_predicateŽ  s   € ò	ð
 Ðr9   )rE   r  r   r   rŒ   )r;   r<   r=   r>   rN   rx   r  Úpropertyrå   r  r%  r.  r@   rA   r}   r~   s   @@r6   rü   rü   L  sf   ù‡ € ÷Tó T÷
ò 
òð. ñ!ó ð!ò
ð ñó ðð ñó ÷ð r9   rü   )$ró   Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorerg   Úmlx.nnrR   Úactivationsr	   Úbaser
   r   r   rd   r   r   Ú
rope_utilsr   Úswitch_layersr   r   ÚModulerC   r€   r�   r    rÉ   rÜ   rü   r:   r9   r6   Ú<module>r;     sÂ   ðó Ý !ß 3Õ 3å Ý å ß TÑ Tß +Ý 'Ý $ð ô�ó ó ðô<V#�—	‘	ô V#ôrJˆ"�)‰)ô Jô&�—	‘	ô ôHˆr�y‰yô HôV!�2—9‘9ô !ôH3�—‘ô 3ôlIˆB�I‰Iö Ir9   