+
    TV-jQ2  ã                   óô  € ^ RI t ^ RIHt ^ RIHtHtHtHt ^ RIH	t
 ^ RIHt ^RIHt ^RIHtHtHtHt ^RIHtHt ^RIHt ] ! R R	]4      4       t ! R
 R]P4                  4      t ! R R]P4                  4      t]
P:                  R 4       t ! R R]P4                  4      t ! R R]P4                  4      t  ! R R]P4                  4      t! ! R R]P4                  4      t" ! R R]P4                  4      t#R# )é    N)Ú	dataclass)ÚAnyÚListÚOptionalÚUnion)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Ú	SwitchGLUc                   óF   a € ] tR t^t o RtRtRtRtRtR t	V 3R lt
RtV tR# )Ú	ModelArgsÚautoFTNc                óD  € V P                   R 8X  d-   \        P                  ! V P                  ^,          4      V n         V P                  fP   \        V P                  4       Uu. uF(  pWP                  ,          V P                  8X  d   RMRNK*  	  upV n        R# R# u upi )r   NÚ	attentionÚmamba)	Úmamba_dt_rankÚmathÚceilÚhidden_sizeÚlayers_block_typeÚrangeÚnum_hidden_layersÚattn_layer_periodÚattn_layer_offset)ÚselfÚis   & Úd/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/jamba.pyÚ__post_init__ÚModelArgs.__post_init__/   s”   € Ø×Ñ Ô'Ü!%§¢¨4×+;Ñ+;¸bÕ+@Ó!AˆDÔØ×!Ñ!Ò)ô ˜t×5Ñ5Ô6ó&ñ 7�Að ×1Ñ1Õ1°T×5KÑ5KÔKñ  à ò!ñ 7ñ&ˆDÖ"ñ *ùò&s   Á#.Bc                ód  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[ S[3,          ;R&   S[;R&   S[;R&   S[S[S[ ,          ,          ;R&   S[;R&   # )é   Ú
model_typer   Úintermediate_sizer   Únum_attention_headsÚnum_key_value_headsr   r   Úexpert_layer_offsetÚexpert_layer_periodÚmamba_d_convÚmamba_d_stateÚmamba_expandÚnum_expertsÚnum_experts_per_tokÚrms_norm_epsÚmax_position_embeddingsÚ
vocab_sizer   Úmamba_proj_biasÚmamba_conv_biasr   Útie_word_embeddings)ÚstrÚintÚfloatr   Úboolr   r   )ÚformatÚ__classdict__s   "€r!   Ú__annotate__ÚModelArgs.__annotate__   s=  ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ  Ññ! ñ" Ññ# ñ$ !Ñ ñ% ñ& �Oñ' ñ( ™™c˜•?Ñ+ñ) ñ* Ñ!ñ+ ñ, Ñ ñ- ñ.  ¡¡S¥	Õ*Ñ1ñ/ ñ0 Ñ$ò1 ó    )r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r4   r5   r   r6   r"   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r<   s   @r!   r   r      s.   ø‡ € ð( &,€MØ!€OØ €OØ-1ÐØ $Ðò÷5 ƒ r?   r   c                   óP   a a€ ] tR t^=t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚJambaMLPc                ó    <€ V ^8„  d   QhRS[ /# ©r%   Úargs©r   )r;   r<   s   "€r!   r=   ÚJambaMLP.__annotate__>   s   ø€ ÷ Yñ Y™Yñ Yr?   c                óP  <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP
                  VP                  R R7      V n        R# ©F©ÚbiasN)	ÚsuperÚ__init__ÚnnÚLinearr   r'   Ú	gate_projÚup_projÚ	down_proj©r   rK   Ú	__class__s   &&€r!   rS   ÚJambaMLP.__init__>   sq   ø€ Ü‰ÑÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŽr?   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r%   ÚxÚreturn©ÚmxÚarray)r;   r<   s   "€r!   r=   rM   D   s'   ø€ ÷ Jñ J™"Ÿ(™(ð J¡r§x¡xñ Jr?   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # ©N)rX   r   rV   rW   )r   r^   s   &&r!   Ú__call__ÚJambaMLP.__call__D   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr?   )rX   rV   rW   ©	r@   rA   rB   rC   rS   re   rE   rF   Ú__classcell__©rZ   r<   s   @@r!   rH   rH   =   s#   ù‡ € ÷Yó Y÷J÷ Jð Jr?   rH   c                   óT   a a€ ] tR t^Ht oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚJambaAttentionc                ó    <€ V ^8„  d   QhRS[ /# rJ   rL   )r;   r<   s   "€r!   r=   ÚJambaAttention.__annotate__I   s   ø€ ÷ 
ñ 
™Yñ 
r?   c                óô  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  VP                  ,          V n        V P
                  R,          V n        \        P                  ! VP                  VP                  V P
                  ,          RR7      V n	        \        P                  ! VP                  VP                  V P
                  ,          RR7      V n
        \        P                  ! VP                  VP                  V P
                  ,          RR7      V n        \        P                  ! VP                  V P
                  ,          VP                  RR7      V n        R# )g      à?FrP   Ng      à¿)rR   rS   r(   r)   r   Úhead_dimÚscalerT   rU   Úq_projÚk_projÚv_projÚo_projrY   s   &&€r!   rS   ÚJambaAttention.__init__I   s  ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ Ø×(Ñ(¨D×,DÑ,DÕDˆŒØ—]‘] DÕ(ˆŒ
ä—i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ×$Ñ$ t§}¡}Õ4°d×6FÑ6FÈUô
ˆŽr?   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r%   r^   ÚmaskÚcacher_   ©ra   rb   r   r   )r;   r<   s   "€r!   r=   rm   ]   sH   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™�}ð	#ñ
 
�‰ñ#r?   c           	     óh  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	Ve   VP                  W‰4      w  r‰\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )é   )ry   rp   rx   éÿÿÿÿ)Úshaperq   rr   rs   Úreshaper(   Ú	transposer)   Úupdate_and_fetchr   rp   rt   )r   r^   rx   ry   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r!   re   ÚJambaAttention.__call__]   s  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà—/‘/ !¨×(@Ñ(@À"ÓE×OÑOØˆq�!�Qó
ˆð �|‰|˜A $×":Ñ":¸BÓ?×IÑIÈ!ÈQÐPQÐSTÓUˆØ—‘  d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÒØ ×1Ñ1°$Ó?‰LˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r?   )ro   rr   r(   r)   rt   rq   rp   rs   ©NNrg   ri   s   @@r!   rk   rk   H   s   ù‡ € ÷
ó 
÷(#÷ #ò #r?   rk   c                 ó    € W,          V,           # rd   © )ÚaÚbÚcs   &&&r!   Úfmar�   z   s   € à�5�1�9Ðr?   c                   óT   a a€ ] tR t^t oV3R lV 3R lltRR ltR tR tRtVt	V ;t
# )ÚJambaMambaMixerc                ó    <€ V ^8„  d   QhRS[ /# rJ   rL   )r;   r<   s   "€r!   r=   ÚJambaMambaMixer.__annotate__€   s   ø€ ÷ +Rñ +R™Yñ +Rr?   c           	     ó@  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  VP                  ,          V n        VP                  V n
        VP                  V n        VP                  V n        \        P                   ! V P                  V P                  ^,          V P                  R7      V n        \        P$                  ! V P                  V P                  V P                  V P                  V P                  ^ R7      V n        \        P                   ! V P                  V P                  V P                  ^,          ,           RR7      V n        \        P                   ! V P                  V P                  RR7      V n        \,        P.                  ! \,        P0                  ! RV P                  R,           4      P3                  ^V P                  .4      V P                  ^ R7      p\,        P4                  ! V4      V n        \,        P8                  ! V P                  .4      V n        \        P                   ! V P                  V P                  V P                  R7      V n        \        P>                  ! V P                  VP@                  R7      V n!        \        P>                  ! V P                  VP@                  R7      V n"        \        P>                  ! V P                  VP@                  R7      V n#        R# )	r%   rP   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsrQ   ÚpaddingFTg      ð?)ÚrepeatsÚaxis©ÚepsN)$rR   rS   r   r-   Ússm_state_sizer,   Úconv_kernel_sizer.   r'   r   Útime_step_rankr5   Úuse_conv_biasr4   Úuse_biasrT   rU   Úin_projÚConv1dÚconv1dÚx_projÚdt_projra   ÚrepeatÚaranger   ÚlogÚA_logÚonesr„   Úout_projÚRMSNormr1   Údt_layernormÚb_layernormÚc_layernorm)r   rK   ÚArZ   s   && €r!   rS   ÚJambaMambaMixer.__init__€   s;  ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ"×0Ñ0ˆÔØ $× 1Ñ 1ˆÔØ!%×!2Ñ!2°T×5EÑ5EÕ!EˆÔØ"×0Ñ0ˆÔØ!×1Ñ1ˆÔØ×,Ñ,ˆŒä—y’yØ×Ñ˜d×4Ñ4°qÕ8¸t¿}¹}ô
ˆŒô —i’iØ×.Ñ.Ø×/Ñ/Ø×-Ñ-Ø×)Ñ)Ø×#Ñ#Øô
ˆŒô —i’iØ×"Ñ"Ø×Ñ $×"5Ñ"5¸Õ"9Õ9Øô
ˆŒô
 —y’y ×!4Ñ!4°d×6LÑ6LÐSWÔXˆŒä�IŠIÜ�IŠI�c˜4×.Ñ.°Õ4Ó5×=Ñ=¸qÀ$×BUÑBUÐ>VÓWØ×*Ñ*Øô
ˆô
 —V’V˜A“YˆŒ
Ü—’˜$×0Ñ0Ð1Ó2ˆŒäŸ	š	Ø×"Ñ" D×$4Ñ$4¸4¿=¹=ô
ˆŒô ŸJšJ t×':Ñ':À×@QÑ@QÔRˆÔÜŸ:š: d×&9Ñ&9¸t×?PÑ?PÔQˆÔÜŸ:š: d×&9Ñ&9¸t×?PÑ?PÔQˆÖr?   c                ó€  € VP                   ^,          pV P                  pV P                  V4      p\        P                  ! VV P
                  V P
                  V P                  ,           .RR7      w  rxp	V P                  V4      V P                  V4      V P                  V	4      r˜p\        P                  ! V P                  V4      4      p\        P                  ! Wq,          R4      \        P                  ! VR4      ,          p
\        P                  ! \        P                  ! VR4      V,          4      p\        V4       F5  pVe$   \!        W;RV3,          V
RV3,          4      V
RV3&   V
RV3,          pK7  	  V
\        P                  ! V	R4      ,          P#                  R4      pWÕV,          ,           pWÚR,          3# )r|   ©rœ   ºNNNr}   éþÿÿÿ)r·   r}   )r~   r„   r§   ra   Úsplitr¡   rŸ   r°   r±   r²   rT   Úsoftplusr¨   Úexpand_dimsÚexpr   r�   Úsqueeze)r   r^   r³   ÚstateÚTr„   ÚdeltaBCÚdeltar‚   ÚCÚ	new_stateÚdtAÚtÚys   &&&&          r!   Ússm_stepÚJambaMambaMixer.ssm_step­   sd  € Ø�G‰G�A�JˆØ�F‰FˆØ—+‘+˜a“.ˆÜ—h’hØØ× Ñ  $×"5Ñ"5¸×8KÑ8KÕ"KÐLØô
‰ˆ�!ð
 ×'Ñ'¨Ó.°×0@Ñ0@ÀÓ0CÀT×EUÑEUÐVWÓEX�!ˆÜ—’˜DŸL™L¨Ó/Ó0ˆÜ—N’N 5¥9¨bÓ1´B·N²NÀ1ÀbÓ4IÕIˆ	Ü�fŠf”R—^’^ E¨2Ó.°Õ2Ó3ˆô �q–ˆAØÒ Ü"% e°°A°­Y¸	À!ÀQÀ$½Ó"H�	˜!˜Q˜$‘Ø˜a ˜d•OŠEñ ð œŸš¨¨2Ó.Õ.×7Ñ7¸Ó;ˆØ�A•�IˆØ˜EÕ"Ð"Ð"r?   c                ó
  € V P                  V4      pVP                  ^RR7      w  rV P                  pVe   \        P                  ! W!.^R7      pM#\        P
                  ! VRV^,
          ^ 3R.4      pV P                  V4      pVRV^,
          ) R1R3,          p\        P                  ! V4      p\        P                  ! V P                  4      ) p	V P                  WV4      w  r£V P                  \        WZ4      4      pWRV33# )r%   )Úindices_or_sectionsrœ   Nr¶   r·   r}   )r   r   )r¤   r¹   r    ra   ÚconcatenateÚpadr¦   rT   Úsilur¼   r¬   rÇ   r®   r   )r   r^   Ú
conv_stateÚ	ssm_stateÚxzÚzÚKÚx_fullÚconv_outr³   rÆ   s   &&&&       r!   Ú_process_sequenceÚ!JambaMambaMixer._process_sequenceÄ   sá   € Ø�\‰\˜!‹_ˆØ�x‰x¨A°BˆxÓ7‰ˆØ×!Ñ!ˆØÒ!Ü—^’^ Z O¸!Ô<‰Fä—V’V˜A ¨¨Q­°¨
°FÐ;Ó<ˆFØ—;‘;˜vÓ&ˆØ˜A  Q¥˜x™z¨1Ð,Õ-ˆ
Ü�GŠG�HÓˆÜ�VŠV�D—J‘JÓÐˆØ—}‘} Q¨9Ó5‰ˆØ�M‰Mœ& ›,Ó'ˆØ˜yÐ)Ð)Ð)r?   c                ó~   € Vf   R R rCMV^ ,          V^,          rCV P                  WV4      w  pw  r4Ve	   W2^ &   WB^&   V# rd   )rÕ   )r   r^   ry   rÎ   rÏ   rˆ   s   &&&   r!   re   ÚJambaMambaMixer.__call__Ô   sU   € ØŠ=Ø$(¨$™	à$)¨!¥H¨e°A­h˜	à*.×*@Ñ*@Ø˜9ó+
Ñ'ˆÑ'�ð ÒØ!�!‰HØ �!‰Hàˆr?   )r¬   r„   r±   r²   r¦   r    r°   r¨   r   r¤   r'   r®   rŸ   r¡   r£   r¢   r§   rd   )r@   rA   rB   rC   rS   rÇ   rÕ   re   rE   rF   rh   ri   s   @@r!   r’   r’      s&   ù‡ € ÷+Ró +RôZ#ò.*÷ ò r?   r’   c                   óP   a a€ ] tR t^åt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚJambaSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# rJ   rL   )r;   r<   s   "€r!   r=   Ú JambaSparseMoeBlock.__annotate__æ   s   ø€ ÷ 
ñ 
™Yñ 
r?   c                ó
  <€ \         SV `  4        VP                  V n        \        P                  ! VP
                  VP                  R R7      V n        \        VP
                  VP                  VP                  4      V n
        R# rO   )rR   rS   r0   rT   rU   r   r/   Úrouterr   r'   Ú
switch_mlprY   s   &&€r!   rS   ÚJambaSparseMoeBlock.__init__æ   sa   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ ä—i’i × 0Ñ 0°$×2BÑ2BÈÔOˆŒÜ#Ø×Ñ˜d×4Ñ4°d×6FÑ6Fó
ˆŽr?   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r]   r`   )r;   r<   s   "€r!   r=   rÜ   ï   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r?   c                ó‚  € V P                  V4      pV P                  p\        P                  ! \        P                  ! V) V^,
          RR7      RRV13,          4      p\        P
                  ! W$RR7      p\        P                  ! VRRR7      pV P                  W4      pWeR,          ,          P                  R	R7      pV# )
r|   )Úkthrœ   .Nr¶   T)rœ   Úpreciser}   ).Nr¸   )	rÞ   r0   ra   Ústop_gradientÚargpartitionÚtake_along_axisÚsoftmaxrß   Úsum)r   r^   ÚgatesÚkÚindsÚscoresrÆ   s   &&     r!   re   ÚJambaSparseMoeBlock.__call__ï   sŸ   € Ø—‘˜A“ˆØ×$Ñ$ˆÜ×Ò¤§¢°°¸AÀ½EÈÔ KÈCÐQSÐRSÐQSÈGÕ TÓUˆÜ×#Ò# E°bÔ9ˆÜ—’˜F¨°TÔ:ˆØ�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆØˆr?   )r0   rÞ   rß   rg   ri   s   @@r!   rÚ   rÚ   å   s   ù‡ € ÷
ó 
÷÷ ð r?   rÚ   c                   óT   a a€ ] tR t^út oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚJambaDecoderLayerc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r%   rK   Ú
layer_typeÚ	layer_idx)r   r7   r8   )r;   r<   s   "€r!   r=   ÚJambaDecoderLayer.__annotate__û   s'   ø€ ÷ Tñ T™Yð T±Cð TÁCñ Tr?   c                ó  <€ \         SV `  4        VR 8H  V n        V P                  '       d   \        V4      V n        M\        V4      V n        VP                  ^8”  d0   W1P                  ,           VP                  ,          ^ 8X  d   \        pM\        pV! V4      V n        \        P                  ! VP                  VP                   R7      V n        \        P                  ! VP                  VP                   R7      V n        R# )r   r�   N)rR   rS   Úis_attnrk   Ú	self_attnr’   r   r/   r*   r+   rÚ   rH   Úfeed_forwardrT   r¯   r   r1   Úinput_layernormÚpre_ff_layernorm)r   rK   rò   ró   Úffn_layer_classrZ   s   &&&& €r!   rS   ÚJambaDecoderLayer.__init__û   s»   ø€ Ü‰ÑÔØ! [Ñ0ˆŒØ�<�<ˆ<Ü+¨DÓ1ˆD�Nä(¨Ó.ˆDŒJà×Ñ˜qÔ Ø×5Ñ5Õ5¸×9QÑ9QÕQÐUVÔVä1‰Oä&ˆOÙ+¨DÓ1ˆÔÜ!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ "§
¢
¨4×+;Ñ+;À×ARÑARÔ SˆÖr?   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rw   rz   )r;   r<   s   "€r!   r=   rô     sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr?   c                ó  € V P                   '       d#   V P                  V P                  V4      W#4      pM!V P                  V P                  V4      V4      pW,           pWPP	                  V P                  V4      4      ,           pV# rd   )rö   r÷   rù   r   rø   rú   )r   r^   rx   ry   ÚhÚrÚouts   &&&&   r!   re   ÚJambaDecoderLayer.__call__  sm   € ð �<�<ˆ<Ø—‘˜t×3Ñ3°AÓ6¸ÓD‰Aà—
‘
˜4×/Ñ/°Ó2°EÓ:ˆAØ�EˆØ×#Ñ# D×$9Ñ$9¸!Ó$<Ó=Õ=ˆØˆ
r?   )rø   rù   rö   r   rú   r÷   rŠ   rg   ri   s   @@r!   rð   rð   ú   s    ù‡ € ÷Tó T÷$÷ ò r?   rð   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú
JambaModeli  c                ó    <€ V ^8„  d   QhRS[ /# rJ   rL   )r;   r<   s   "€r!   r=   ÚJambaModel.__annotate__  s   ø€ ÷ 
=ñ 
=™Yñ 
=r?   c           	     óâ  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       UUu. uF  w  r#\        WV4      NK  	  uppV n
        \        P                  ! VP
                  VP                  R 7      V n        VP                  P                  R4      V n        VP                  P                  R4      V n        R# u uppi )r�   r   r   N)rR   rS   rT   Ú	Embeddingr3   r   Úembed_tokensÚ	enumerater   rð   Úlayersr¯   r1   Úfinal_layernormÚindexÚattn_idxÚssm_idx)r   rK   ÚidxrÅ   rZ   s   &&  €r!   rS   ÚJambaModel.__init__  s¶   ø€ Ü‰ÑÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô $ D×$:Ñ$:Ô;ô
á;‘�ô ˜d sÖ+Ù;ò
ˆŒô  "Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔØ×.Ñ.×4Ñ4°[ÓAˆŒØ×-Ñ-×3Ñ3°GÓ<ˆŽùó
s   ÁC+c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# ©r%   Úinputsry   r_   rz   )r;   r<   s   "€r!   r=   r  )  s5   ø€ ÷ 'ñ 'á—‘ð'ñ ™�}ð'ñ 
�‰ñ	'r?   c                óv  € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2V P                  ,          4      p\        W2V P                  ,          4      p\        V P                  V4       F%  w  rgVP                  '       d   TMTpV! W8VR7      pK'  	  V P                  V4      # )N)rx   ry   )
r	  Úlenr  r
   r  r   r  Úziprö   r  )	r   r  ry   rÿ   Ú	attn_maskÚssm_maskÚlayerr�   rx   s	   &&&      r!   re   ÚJambaModel.__call__)  s™   € ð
 ×Ñ˜fÓ%ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä)¨!°4·=±=Õ-AÓBˆ	Ü" 1¨D¯L©LÕ&9Ó:ˆä˜DŸK™K¨Ö/‰HˆEØ %§§ ‘9°8ˆDÙ�a¨!Ô,ŠAñ 0ð ×#Ñ# AÓ&Ð&r?   )r  r	  r  r  r  rd   rg   ri   s   @@r!   r  r    s   ù‡ € ÷
=ó 
=÷'÷ 'ò 'r?   r  c                   ó€   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR tR t]R 4       t	]R	 4       t
R
tVtV ;t# )ÚModeli=  c                ó    <€ V ^8„  d   QhRS[ /# rJ   rL   )r;   r<   s   "€r!   r=   ÚModel.__annotate__>  s   ø€ ÷ Tñ T™Yñ Tr?   c                óþ   <€ \         SV `  4        VP                  V n        Wn        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# rO   )rR   rS   r&   rK   r  Úmodelr6   rT   rU   r   r3   Úlm_headrY   s   &&€r!   rS   ÚModel.__init__>  sY   ø€ Ü‰ÑÔØŸ/™/ˆŒØŒ	Ü Ó%ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r?   c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# r  rz   )r;   r<   s   "€r!   r=   r  F  s5   ø€ ÷ 
ñ 
á—‘ð
ñ ™�}ð
ñ 
�‰ñ	
r?   c                óÐ   € V P                  W4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# rd   )r!  rK   r6   r	  Ú	as_linearr"  )r   r  ry   r  s   &&& r!   re   ÚModel.__call__F  sX   € ð
 �j‰j˜Ó'ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r?   c                óÔ   € . pV P                   P                   FK  pVP                  '       d   VP                  \	        4       4       K0  VP                  \        ^R7      4       KM  	  V# )r%   )Úsize)r!  r  rö   Úappendr   r   )r   Úcachesr  s   &  r!   Ú
make_cacheÚModel.make_cacheR  sJ   € ØˆØ—Z‘Z×&Ô&ˆEØ�}�}ˆ}Ø—‘œg›iÖ(à—‘œk¨qÔ1Ö2ñ	 'ð
 ˆr?   c                óP  a	€ \        VP                  4       4       F<  w  r#R V9   g   K  VP                  R,          ^8w  g   K(  VP                  ^^4      W&   K>  	  V P                  P
                  '       d   VP                  RR4       \        V P                  P                  4       Fõ  pRV R2o	\        ;QJ d-    V	3R lVP                  4        4       F  '       g   K   RM"	  RM! V	3R lVP                  4        4       4      '       g   Kg  R Fˆ  pR F  p\        \        V4      4       Uu. uF3  pS	 RV R	V R	V 2V9   g   K  VP                  S	 RV R	V R	V 24      NK5  	  ppV'       g   K^  \        P                  ! V4      VS	 R
V R	V 2&   K�  	  KŠ  	  K÷  	  V# u upi )zconv1d.weightzlm_head.weightNzmodel.layers.z.feed_forwardc              3   óJ   <"  € T F  qP                  S R 24      x € K  	  R# 5i)ú	.experts.N)Ú
startswith)Ú.0ÚkeyÚbases   & €r!   Ú	<genexpr>Ú!Model.sanitize.<locals>.<genexpr>e  s"   øé € ÐTÁ^¸c—~‘~¨¨¨iÐ&8×9Ð9Ã^ùs   ƒ #TFr0  Ú.z.switch_mlp.r}   )rV   rX   rW   )ÚweightrQ   ÚscalesÚbiases)ÚlistÚitemsr~   ÚmoveaxisrK   r6   Úpopr   r   Úanyr†   r  ra   Ústack)
r   Úweightsrë   ÚvÚlÚprojÚnameÚeÚexpert_tensorsr4  s
   &&       @r!   ÚsanitizeÚModel.sanitize[  sx  ø€ Ü˜Ÿ™›Ö)‰DˆAØ !Ö#¨¯©°­°qÖ(8ØŸZ™Z¨¨1Ó-�“
ñ *ð �9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/ä�t—y‘y×2Ñ2Ö3ˆAØ" 1 # ]Ð3ˆDß“3ÔTÀWÇ\Á\Ä^ÓT—3—3’3ÔTÀWÇ\Á\Ä^ÓT×TÒTÙã=�ÛB�Dô "'¤s¨7£|Ô!4ó&á!4˜AØ"˜V 9¨Q¨C¨q°°°a¸°vÐ>À'ÑIô I˜Ÿ™ t f¨I°a°S¸¸$¸¸qÀÀÐ$GÖHÙ!4ð #ð &÷
 &‘~ÜFHÇhÂhØ*óG˜ 4 &¨°T°F¸!¸D¸6Ð BÓCó Có >ñ 4ð" ˆùò&s   Ä/F#
Å F#
c                ó.   € V P                   P                  # rd   )r!  r  )r   s   &r!   r  ÚModel.layersv  s   € à�z‰z× Ñ Ð r?   c                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )rÞ   Ú
group_sizeÚbitsT)Úendswith)ÚpathÚ_s   &&r!   Ú	predicateÚ(Model.quant_predicate.<locals>.predicate|  s#   € Ø�}‰}˜X×&Ò&Ø$ b¨&°!Ð4Ð4Ùr?   rŒ   )r   rS  s   & r!   Úquant_predicateÚModel.quant_predicatez  s   € ò	ð
 Ðr?   )rK   r"  r!  r&   rd   )r@   rA   rB   rC   rS   re   r,  rH  Úpropertyr  rU  rE   rF   rh   ri   s   @@r!   r  r  =  sR   ù‡ € ÷Tó T÷
ò 
òòð6 ñ!ó ð!ð ñó ÷ð r?   r  )$r   Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorera   Úmlx.nnrT   Úactivationsr   r4  r	   r
   r   r   ry   r   r   Úswitch_layersr   r   ÚModulerH   rk   Úcompiler�   r’   rÚ   rð   r  r  rŒ   r?   r!   Ú<module>ra     sÜ   ðó Ý !ß -Ó -å Ý å ÷ó ÷ (Ý $ð ô$�ó $ó ð$ôNJˆr�y‰yô Jô/#�R—Y‘Yô /#ðd ‡�ñó ðôc�b—i‘iô côL˜"Ÿ)™)ô ô*˜Ÿ	™	ô ôD'�—‘ô 'ôBDˆB�I‰Iö Dr?   