+
    TV-jf*  ã                   óì  € ^ RI Ht ^ RIHtHtHt ^ RIHt ^ RI	H
t
 ^RIHt ^RIHtHtHt ^RIHtHt ^RIHt ^RIHt ] ! R	 R
]4      4       t] ! R R]4      4       t ! R R]
P4                  4      t ! R R]
P4                  4      t ! R R]
P4                  4      t ! R R]
P4                  4      t ! R R]
P4                  4      t ! R R]
P4                  4      t  ! R R]
P4                  4      t!R# )é    )Ú	dataclass)ÚAnyÚOptionalÚUnionN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚChunkedKVCacheÚKVCache)Úinitialize_rope)Ú	SwitchGLUc                   ó8   a € ] tR t^t o ^tRtRtV 3R ltRtV t	R# )ÚTextArgsi    gš™™™™™¹?c                ó"  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )é   Úattention_biasÚattention_chunk_sizeÚhead_dimÚhidden_sizeÚinterleave_moe_layer_stepÚintermediate_sizeÚintermediate_size_mlpÚmax_position_embeddingsÚ
model_typeÚnum_attention_headsÚnum_experts_per_tokÚnum_hidden_layersÚnum_key_value_headsÚnum_local_expertsÚrms_norm_epsÚrope_scalingÚ
rope_thetaÚuse_qk_normÚ
vocab_sizeÚattn_temperature_tuningÚfloor_scaleÚ
attn_scale)ÚboolÚintÚstrÚfloatr   )ÚformatÚ__classdict__s   "€Úe/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/llama4.pyÚ__annotate__ÚTextArgs.__annotate__   s  ø‡ ‚ áÑñ ñ Ññ ñ �Mñ	 ñ
 Ññ ñ  #Ñ"ñ ñ Ññ ñ Ññ ñ !Ñ ñ ñ �Oñ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ  Ññ! ñ" Ññ# ñ$ Ññ% ñ& Ññ' ñ( �Oñ) ñ* !Ñ$ñ+ ñ, Ññ- ñ. Ñò/ ó    © N)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r&   r'   r(   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r.   s   @r/   r   r      s   ø‡ € ð* $%ÐØ€KØ€J÷/ ƒ r2   r   c                   ó2   a € ] tR t^*t o R tV 3R ltRtV tR# )Ú	ModelArgsc                óN   € \         P                  V P                  4      V n        R # ©N)r   Ú	from_dictÚtext_config©Úselfs   &r/   Ú__post_init__ÚModelArgs.__post_init__/   s   € Ü#×-Ñ-¨d×.>Ñ.>Ó?ˆÖr2   c                óH   <€ V ^8„  d   Qh/ S[ S[S[3,          ;R&   S[;R&   # )r   rA   r   )r   r   Údictr+   )r-   r.   s   "€r/   r0   ÚModelArgs.__annotate__*   s%   ø‡ ‚ á‘x¡�~Õ&Ñ&ñ ñ �Oò r2   )rA   N)r4   r5   r6   r7   rD   r8   r9   r:   r;   s   @r/   r=   r=   *   s   ø‡ € ò
@÷ ƒ r2   r=   c                   óT   a a€ ] tR t^3t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   ÚargsÚ	layer_idx©r   r*   )r-   r.   s   "€r/   r0   ÚAttention.__annotate__4   s   ø€ ÷ "ñ "™Xð "±#ñ "r2   c                óà  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        p\        V^,           ^,          ^ 8g  4      V n        VP                  V n	        VP                  V n
        VP                  V n        VP                  ;'       g    VP                  V,          ;V n        pVR,          V n        \        VR4      '       d   VP                  pMRp\         P"                  ! W4V,          VR7      V n        \         P"                  ! W5V,          VR7      V n        \         P"                  ! W5V,          VR7      V n        \         P"                  ! WF,          W7R7      V n        VP,                  ;'       d    V P                  V n        V P                  '       d6   \/        VVP0                  RVP2                  VP4                  R7      V n        R# R# )é   r   F©ÚbiasT)ÚtraditionalÚscaling_configr   Ng      à¿)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr*   Úuse_roper&   r'   r(   r   ÚscaleÚhasattrr   ÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr$   r   r#   r"   r   Úrope)	rC   rM   rN   ÚdimrY   rZ   r   r   Ú	__class__s	   &&&     €r/   rX   ÚAttention.__init__4   sw  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*ä˜Y¨�]¨aÕ/°1Ñ4Ó5ˆŒØ'+×'CÑ'CˆÔ$Ø×+Ñ+ˆÔØŸ/™/ˆŒà#'§=¡=×#OÐ#O°D×4DÑ4DÈÕ4OÐOˆŒ˜à˜t•^ˆŒ
Ü�4Ð)×*Ò*Ø!×0Ñ0‰Nà"ˆNä—i’i ¨xÕ%7¸nÔMˆŒÜ—i’i °(Õ%:ÀÔPˆŒÜ—i’i °(Õ%:ÀÔPˆŒÜ—i’i Õ 2°CÔMˆŒà×+Ñ+×=Ð=°·±ˆÔà�=�=ˆ=Ü'ØØ—‘Ø Ø#×0Ñ0Ø(,×(DÑ(DôˆDŽIñ r2   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r-   r.   s   "€r/   r0   rP   X   sH   ø€ ÷ .#ñ .#á�8‰8ð.#ñ ‘r—x‘xÕ ð.#ñ ™�}ð	.#ñ
 
�‰ñ.#r2   c           	     óH  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	Ve   VP                  p
M^ p
V P                  '       d%   V P                  WzR7      pV P                  WŠR7      pV P                  '       dE   \        P                  P                  VRRR7      p\        P                  P                  VRRR7      pV P                  '       dÀ   V P                  '       g®   \        P                   ! \        P"                  ! \        P$                  ! V
^,           W¥,           ^,           4      V P&                  ,          4      R,           4      V P(                  ,          R,           pVR,          pW{,          P+                  VP,                  4      pVe   VP/                  W‰4      w  r‰\1        WxW“V P2                  VR7      pVP                  ^ ^^^4      P	                  WER4      pV P5                  V4      # )	rR   N)Úoffsetg�íµ ÷Æ°>)ÚweightÚepsg      ð?)rl   r\   rk   éÿÿÿÿ©:NNNN)Úshaper`   ra   rb   ÚreshaperY   Ú	transposerZ   rr   r[   rd   r$   ro   ÚfastÚrms_normr&   ÚlogÚfloorÚaranger'   r(   ÚastypeÚdtypeÚupdate_and_fetchr
   r\   rc   )rC   rj   rk   rl   ÚBÚLÚDÚqueriesÚkeysÚvaluesrr   Úattn_scalesÚoutputs   &&&&         r/   Ú__call__ÚAttention.__call__X   s  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà—/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÒØ—\‘\‰FàˆFà�=�=ˆ=Ø—i‘i �iÓ7ˆGØ—9‘9˜T�9Ó1ˆDà××ÐÜ—g‘g×&Ñ& w°tÀÐ&ÓFˆGÜ—7‘7×#Ñ# D°¸4Ð#Ó@ˆDà×'×'Ð'°··°ä—’Ü—H’HœRŸYšY v°¥z°6µ:Àµ>ÓBÀT×EUÑEUÕUÓVØõóð —/‘/õ	"ð
 õð ð & gÕ.ˆKØÕ,×4Ñ4°W·]±]ÓCˆGàÒØ ×1Ñ1°$Ó?‰LˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r2   )r(   r&   r'   r   ra   rY   rZ   rc   r`   rd   r\   r$   r[   rb   ©NN©	r4   r5   r6   r7   rX   rŠ   r9   r:   Ú__classcell__©rf   r.   s   @@r/   rJ   rJ   3   s   ù‡ € ÷"ó "÷H.#÷ .#ò .#r2   rJ   c                   óT   a a€ ] tR t^‰t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMLPc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   rM   r   )r=   r*   )r-   r.   s   "€r/   r0   ÚMLP.__annotate__Š   s   ø€ ÷ >ñ >™Yð >¹3ñ >r2   c                ó  <€ \         SV `  4        VP                  pT;'       g    VP                  p\        P
                  ! W4R R7      V n        \        P
                  ! WCR R7      V n        \        P
                  ! W4R R7      V n        R# ©FrS   N)	rW   rX   r   r   r^   r_   Ú	gate_projÚ	down_projÚup_proj)rC   rM   r   re   Ú
hidden_dimrf   s   &&&  €r/   rX   ÚMLP.__init__Š   sb   ø€ Ü‰ÑÔà×ÑˆØ&×@Ð@¨$×*@Ñ*@ˆ
äŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr2   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   rm   ©ro   rp   )r-   r.   s   "€r/   r0   r“   ”   s   ø€ ÷ Jñ J™RŸX™Xñ Jr2   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # r?   )r—   r   r–   r˜   )rC   rj   s   &&r/   rŠ   ÚMLP.__call__”   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr2   )r—   r–   r˜   r?   r�   r�   s   @@r/   r‘   r‘   ‰   s!   ù‡ € ÷>õ >÷J÷ Jð Jr2   r‘   c                   óD   a a€ ] tR t^˜t oV 3R ltV3R lR ltRtVtV ;t# )ÚMoEc                ó|  <€ \         SV `  4        VP                  V n        V P                  ^8X  g   Q R4       hVP                  V n        \        VP                  VP                  V P
                  4      V n	        \        P                  ! VP                  VP                  RR7      V n        \        V4      V n        R# )rR   z!Only 1 expert per token supportedFrS   N)rW   rX   r   Útop_kr    Únum_expertsr   r   r   Úexpertsr^   r_   Úrouterr‘   Úshared_expert©rC   rM   rf   s   &&€r/   rX   ÚMoE.__init__™   s’   ø€ Ü‰ÑÔØ×-Ñ-ˆŒ
Ø�z‰z˜QŒÐCÐ CÓCˆØ×1Ñ1ˆÔÜ Ø×Ñ˜d×4Ñ4°d×6FÑ6Fó
ˆŒô —i’i × 0Ñ 0°$×2HÑ2HÈuÔUˆŒÜ  ›YˆÖr2   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rœ   r�   )r-   r.   s   "€r/   r0   ÚMoE.__annotate__¤   s   ø€ ÷ +ñ +™RŸX™Xñ +r2   c                óØ  € V P                  V4      pV P                  p\        P                  ! V) V^,
          RR7      RRV13,          p\        P                  ! W$RR7      p\        P
                  ! VP                  \        P                  4      4      P                  VP                  4      pV P                  W,          V4      P                  ^4      pW`P                  V4      ,           # )rR   )ÚkthÚaxis.N©r®   ru   )r¦   r£   ro   ÚargpartitionÚtake_along_axisÚsigmoidr   Úfloat32r€   r¥   Úsqueezer§   )rC   rj   ÚlogitsÚkÚindicesÚscoresÚouts   &&     r/   rŠ   ÚMoE.__call__¤   sª   € Ø—‘˜Q“ˆØ�J‰JˆÜ—/’/ 6 '¨q°1­u¸2Ô>¸sÀBÀQÀB¸wÕGˆÜ×#Ò# F¸"Ô=ˆÜ—’˜FŸM™M¬"¯*©*Ó5Ó6×=Ñ=¸a¿g¹gÓFˆà�l‰l˜1�: wÓ/×7Ñ7¸Ó:ˆØ×'Ñ'¨Ó*Õ*Ð*r2   )r¥   r¤   r¦   r§   r£   r�   r�   s   @@r/   r¡   r¡   ˜   s   ù‡ € õ	'÷+÷ +ð +r2   r¡   c                   óT   a a€ ] tR t^¯t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó&   <€ V ^8„  d   QhRS[ RS[/# rL   rO   )r-   r.   s   "€r/   r0   ÚTransformerBlock.__annotate__°   s   ø€ ÷ ñ ™Xð ±#ñ r2   c                ó(  <€ \         SV `  4        VP                  V n        VP                  V n        \	        W4      V n        W!P                  ,          VP                  ^,
          8H  V n        V P                  '       d   \        V4      V n	        M\        WP                  4      V n	        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        Wn        R# )rR   ©rt   N)rW   rX   r   r   rJ   Ú	self_attnr   Úis_moe_layerr¡   Úfeed_forwardr‘   r   r^   ÚRMSNormr!   Úinput_layernormÚpost_attention_layernormrM   )rC   rM   rN   rf   s   &&&€r/   rX   ÚTransformerBlock.__init__°   sÏ   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø×+Ñ+ˆÔÜ" 4Ó3ˆŒØ&×)GÑ)GÕGØ×*Ñ*¨QÕ.ñ
ˆÔð ××ÐÜ # D£	ˆDÕä # D×*DÑ*DÓ EˆDÔä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð Ž	r2   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ri   rn   )r-   r.   s   "€r/   r0   r¾   Ã   sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r2   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# r?   )rÁ   rÅ   rÃ   rÆ   )rC   rj   rk   rl   ÚrÚhr¹   s   &&&&   r/   rŠ   ÚTransformerBlock.__call__Ã   sM   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ×Ñ˜d×;Ñ;¸AÓ>Ó?ˆØ�eˆØˆ
r2   )rM   rÃ   r   rÅ   rÂ   r   rÆ   rÁ   rŒ   r�   r�   s   @@r/   r¼   r¼   ¯   s   ù‡ € ÷ó ÷&
÷ 
ò 
r2   r¼   c                   óT   a a€ ] tR t^Ðt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú
LlamaModelc                ó    <€ V ^8„  d   QhRS[ /# ©r   rM   ©r   )r-   r.   s   "€r/   r0   ÚLlamaModel.__annotate__Ñ   s   ø€ ÷ 	>ñ 	>™Xñ 	>r2   c                óî  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        V P                  ^ 8”  g   Q h\
        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        W4      NK  	  upV n        \
        P                  ! VP                  VP                  R7      V n        VP                  V n        R# u upi )r   rÀ   N)rW   rX   rM   r%   r   r^   Ú	Embeddingr   Úembed_tokensÚranger¼   ÚlayersrÄ   r!   Únormr   )rC   rM   Úirf   s   && €r/   rX   ÚLlamaModel.__init__Ñ   s¶   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ô"Ð"Ð"ÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔÜ:?À×@VÑ@VÔ:WÓXÑ:W°QÔ'¨Ö0Ñ:WÑXˆŒÜ—J’J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	Ø$(×$=Ñ$=ˆÖ!ùò Ys   ÂC2c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   Úinputsr�   )r-   r.   s   "€r/   r0   rÒ   Ü   s   ø€ ÷ #ñ #á—‘ñ#r2   c                ó\  € V P                  V4      pVec   \        V4       F,  w  rEV^,           ^,          ^ 8w  g   K  VP                  4        K.  	  V^ ,          P                  pV^ ,          P                  pM^ p^ pWsP
                  ^,          ,           p\        P                  ! Wh4      p	\        P                  ! Wx4      R,          p
\        P                  ! W�P                  ,          W P                  ,          ,
          4      pWš8*  pV^ 8H  V,          pVf   R .\        V P                  4      ,          p\        W2^,          4      p\        \        V P                  V4      4       F2  w  pw  rõV^,           ^,          ^ 8g  pV'       d   TMTpV! VVVR7      pK4  	  V P                  V4      # )N)rl   rv   )rÕ   Ú	enumerateÚmaybe_trim_frontÚstart_positionrr   rw   ro   r~   Úabsr   Úlenr×   r	   ÚziprØ   )rC   rÝ   rl   rË   ÚidxÚcÚstartrr   ÚendÚlindsÚrindsÚ	block_posÚ	token_posÚ
chunk_maskÚglobal_maskÚlayerÚuse_chunked_attentionrk   s   &&&               r/   rŠ   ÚLlamaModel.__call__Ü   sb  € ð
 ×Ñ˜fÓ%ˆàÒÜ# EÖ*‘�Ø˜!•G˜q•= AÖ%Ø×&Ñ&Ö(ñ +ð ˜!•H×+Ñ+ˆEØ˜1•X—_‘_‰FàˆEØˆFØ—w‘w˜q•zÕ!ˆÜ—	’	˜%Ó%ˆÜ—	’	˜&Ó& wÕ/ˆÜ—F’FØ×/Ñ/Õ/°E×=VÑ=VÕ4VÕWó
ˆ	ð ‘Nˆ	Ø 1‘n¨	Õ1ˆ
àŠ=Ø�FœS §¡Ó-Õ-ˆEä+¨A°Q­xÓ8ˆä(¬¨T¯[©[¸%Ó)@ÖA‰OˆC‘�%Ø%(¨1¥W°¥M°QÑ$6Ð!ß!6‘:¸KˆDÙ�a˜ QÔ'ŠAñ  Bð
 �y‰y˜‹|Ðr2   )rM   r   rÕ   r×   rØ   r   r%   r?   r�   r�   s   @@r/   rÎ   rÎ   Ð   s   ù‡ € ÷	>ó 	>÷#÷ #ò #r2   rÎ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLanguageModeli  c                ó    <€ V ^8„  d   QhRS[ /# rÐ   rÑ   )r-   r.   s   "€r/   r0   ÚLanguageModel.__annotate__  s   ø€ ÷ 
ñ 
™Xñ 
r2   c                ó  <€ \         SV `  4        Wn        VP                  V n        \	        V P                  4      V n        \        P                  ! V P                  P                  V P                  P                  R R7      V n
        R# r•   )rW   rX   rM   r   rÎ   Úmodelr^   r_   r   r%   Úlm_headr¨   s   &&€r/   rX   ÚLanguageModel.__init__  sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ §	¡	Ó*ˆŒ
Ü—y’yØ�I‰I×!Ñ! 4§9¡9×#7Ñ#7¸eô
ˆŽr2   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rÜ   r�   )r-   r.   s   "€r/   r0   rõ     s   ø€ ÷ !ñ !á—‘ñ!r2   c                óF   € V P                  W4      pV P                  V4      # r?   )r÷   rø   )rC   rÝ   rl   r¹   s   &&& r/   rŠ   ÚLanguageModel.__call__  s!   € ð
 �j‰j˜Ó'ˆØ�|‰|˜CÓ Ð r2   )rM   rø   r÷   r   r?   r�   r�   s   @@r/   ró   ró     s   ù‡ € ÷
ó 
÷!÷ !ò !r2   ró   c                   óp   a a€ ] tR tRt oV3R lV 3R lltR
V3R lR lltR t]R 4       tR t	R	t
VtV ;t# )ÚModeli  c                ó    <€ V ^8„  d   QhRS[ /# rÐ   )r=   )r-   r.   s   "€r/   r0   ÚModel.__annotate__  s   ø€ ÷ >ñ >™Yñ >r2   c                ó†   <€ \         SV `  4        Wn        VP                  V n        \	        VP
                  4      V n        R # r?   )rW   rX   rM   r   ró   rA   Úlanguage_modelr¨   s   &&€r/   rX   ÚModel.__init__  s1   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ+¨D×,<Ñ,<Ó=ˆÖr2   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rÜ   r�   )r-   r.   s   "€r/   r0   r     s   ø€ ÷ 2ñ 2á—‘ñ2r2   c                ó$   € V P                  W4      # r?   )r  )rC   rÝ   rl   s   &&&r/   rŠ   ÚModel.__call__  s   € ð
 ×"Ñ" 6Ó1Ð1r2   c                óR  € R  pVP                  4        UUu/ uF  w  r4V! V4      '       d   K  W4bK  	  ppp\        V P                  P                  P                  4       F½  pRV R2pV R2V9   dn   VP                  V R24      pV R2pV R2p\        P                  ! V^R	R7      w  rš\        P                  ! V	^^4      W&   \        P                  ! V
^^4      W&   V R2V9   g   KŒ  VP                  V R24      p\        P                  ! V^^4      W R2&   K¿  	  V# u uppi )
c                 ó&   € R V 9   ;'       g    RV 9   # )Úvision_modelÚmulti_modal_projectorr3   )r¶   s   &r/   Ú	to_removeÚ!Model.sanitize.<locals>.to_remove$  s   € Ø! QÑ&×FÐFÐ*AÀQÑ*FÐFr2   zlanguage_model.model.layers.z.feed_forward.expertsz.gate_up_projz.gate_proj.weightz.up_proj.weightr¯   z
.down_projz.down_proj.weightru   )	ÚitemsrÖ   rM   rA   r   Úpopro   ÚsplitÚswapaxes)rC   Úweightsr  r¶   ÚvÚlÚprefixÚgate_kÚup_kr–   r˜   r—   s   &&          r/   ÚsanitizeÚModel.sanitize#  s*  € ò	Gð %,§M¡M¤OÔH¡O™D˜A¹9ÀQ¿<”4�1’4¡OˆÑHô �t—y‘y×,Ñ,×>Ñ>Ö?ˆAØ3°A°3Ð6KÐLˆFØ�˜Ð'¨7Ô2Ø—K‘K 6 (¨-Ð 8Ó9�Ø"˜8Ð#4Ð5�Ø ˜ Ð1�Ü%'§X¢X¨a°¸Ô%<Ñ"�	Ü"$§+¢+¨i¸¸AÓ">�‘Ü "§¢¨G°Q¸Ó :�‘Ø�˜Ð$¨Ö/Ø#ŸK™K¨6¨(°*Ð(=Ó>�	Ü8:¿ºÀIÈqÐRSÓ8T�˜(Ð"3Ð4Ó5ñ @ð ˆùó Is
   —D#®D#c                óB   € V P                   P                  P                  # r?   )r  r÷   r×   rB   s   &r/   r×   ÚModel.layers9  s   € à×"Ñ"×(Ñ(×/Ñ/Ð/r2   c                ó(  € V P                   P                  P                  p. p\        \	        V P
                  4      4       FM  pV^,           ^,          ^ 8w  d   VP                  \        V4      4       K4  VP                  \        4       4       KO  	  V# )rR   )	rM   rA   r   rÖ   rã   r×   Úappendr   r   )rC   Ú
chunk_sizeÚcachesrÙ   s   &   r/   Ú
make_cacheÚModel.make_cache=  sj   € Ø—Y‘Y×*Ñ*×?Ñ?ˆ
ØˆÜ”s˜4Ÿ;™;Ó'Ö(ˆAØ�A•˜�{˜aÔØ—‘œn¨ZÓ8Ö9à—‘œg›iÖ(ñ	 )ð
 ˆr2   )rM   r  r   r?   )r4   r5   r6   r7   rX   rŠ   r  Úpropertyr×   r  r9   r:   rŽ   r�   s   @@r/   rþ   rþ     s<   ù‡ € ÷>ó >÷2ò 2òð, ñ0ó ð0÷ò r2   rþ   )"Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorero   Úmlx.nnr^   Úactivationsr   Úbaser   r	   r
   rl   r   r   Ú
rope_utilsr   Úswitch_layersr   r   r=   ÚModulerJ   r‘   r¡   r¼   rÎ   ró   rþ   r3   r2   r/   Ú<module>r,     sØ   ðõ "ß 'Ñ 'å Ý å ß TÑ Tß *Ý 'Ý $ð ôˆ}ó ó ðð2 ô@�ó @ó ð@ôS#�—	‘	ô S#ôlJˆ"�)‰)ô Jô+ˆ"�)‰)ô +ô.�r—y‘yô ôB/�—‘ô /ôd!�B—I‘Iô !ô&0ˆB�I‰Iö 0r2   