+
    UV-jì8  ã                   óÈ  € ^ RI HtHtHtHt ^ RIHt ^ RIH	t	 ^ RI
Ht ^ RIHt ^ RIHt ^RIHtHtHt ^RIHtHt ^RIHt R	 R
 ltR R lt ! R R]	P6                  4      t ! R R]	P6                  4      t ! R R]	P6                  4      t ! R R]	P6                  4      t ! R R]	P6                  4      t  ! R R]	P6                  4      t! ! R R]	P6                  4      t"R# )é    )ÚAnyÚDictÚOptionalÚUnionN)Úswiglu)Úinitialize_rope)Ú	SwitchGLU)ÚLanguageModelOutputÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache©ÚModelConfigc                ó€   € V ^8„  d   QhR\         R\        \        \        \        \        3,          3,          R\        /# ©é   ÚargsÚrope_configÚreturn)r   r   Ústrr   Úfloat)Úformats   "Úo/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/laguna/language.pyÚ__annotate__r      s:   € ÷ Añ A”[ð A¬t´C¼¼uÄc¸zÕ9JÐ4JÕ/Kð AÔPUñ Aó    c                 óL   € \        VP                  R V P                  4      4      # )Ú
rope_theta)r   Úgetr   )r   r   s   &&r   Ú
_rope_baser       s   € Ü�—‘ ¨t¯©Ó?Ó@Ð@r   c                ó€   € V ^8„  d   QhR\         R\        \        \        \        \        3,          3,          R\
        /# r   )r   r   r   r   r   Úint)r   s   "r   r   r      s5   € ÷ (ñ (”[ð (¬t´C¼¼uÄc¸zÕ9JÐ4JÕ/Kð (ÔPSñ (r   c                 óp   € \        VP                  R R4      4      p\        V P                  V,          4      # )Úpartial_rotary_factorç      ð?)r   r   r"   Úhead_dim)r   r   Úpartials   && r   Ú
_rope_dimsr(      s,   € Ü�K—O‘OÐ$;¸SÓAÓB€GÜˆt�}‰}˜wÕ&Ó'Ð'r   c                   óP   a a€ ] tR t^t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPc                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   ÚdimÚ
hidden_dim)r"   )r   Ú__classdict__s   "€r   r   ÚMLP.__annotate__   s   ø€ ÷ >ñ >™Cð >©Sñ >r   c                óÒ   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W!R R7      V n        \        P                  ! WR R7      V n        R# ©F©ÚbiasN)ÚsuperÚ__init__ÚnnÚLinearÚ	gate_projÚ	down_projÚup_proj)Úselfr,   r-   Ú	__class__s   &&&€r   r5   ÚMLP.__init__   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r   ©ÚmxÚarray)r   r.   s   "€r   r   r/   "   s   ø€ ÷ Jñ J™RŸX™Xñ Jr   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # ©N)r9   r   r8   r:   )r;   Úxs   &&r   Ú__call__ÚMLP.__call__"   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr   )r9   r8   r:   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r5   rE   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r<   r.   s   @@r   r*   r*      s!   ù‡ € ÷>ó >÷J÷ Jð Jr   r*   c                   óP   a a€ ] tR t^&t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚLagunaTopKRouterc                ó    <€ V ^8„  d   QhRS[ /# ©r   r   r   )r   r.   s   "€r   r   ÚLagunaTopKRouter.__annotate__'   s   ø€ ÷ Eñ E™[ñ Er   c                ó\  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! VP                  VP                  R R7      V n        \        P                  ! VP                  34      V n        R# r1   )r4   r5   Únum_experts_per_tokÚtop_kÚnorm_topk_probÚmoe_router_use_sigmoidÚuse_sigmoidÚmoe_router_logit_softcappingÚrouter_logit_softcappingr6   r7   Úhidden_sizeÚnum_expertsÚprojr@   ÚzerosÚe_score_correction_bias©r;   r   r<   s   &&€r   r5   ÚLagunaTopKRouter.__init__'   s   ø€ Ü‰ÑÔØ×-Ñ-ˆŒ
Ø"×1Ñ1ˆÔØ×6Ñ6ˆÔØ(,×(IÑ(IˆÔ%Ü—I’I˜d×.Ñ.°×0@Ñ0@ÀuÔMˆŒ	Ü')§x¢x°×1AÑ1AÐ0CÓ'DˆÖ$r   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          /# ©r   rD   r   )r@   rA   Útuple)r   r.   s   "€r   r   rT   0   s1   ø€ ÷ +ñ +™"Ÿ(™(ð +¡u©R¯X©X±r·x±xÐ-?Õ'@ñ +r   c                ó  € VP                   pV P                  V4      P                  \        P                  4      pV P
                  R 8”  d0   V P
                  p\        P                  ! W4,          4      V,          pV P                  '       d   \        P                  ! V4      M\        P                  ! VRR7      pWPP                  P                  VP                   4      ,           pV P                  p\        P                  ! \        P                  ! V) V^,
          RR7      RRV13,          4      p\        P                  ! WXRR7      p	V P                  '       d!   V	\        P                   ! V	RRR7      ,          p	W‰P                  V4      3# )g        ©Úaxis)Úkthri   .NT)ri   Úkeepdimséÿÿÿÿ)Údtyper_   Úastyper@   Úfloat32r\   ÚtanhrZ   ÚsigmoidÚsoftmaxra   rW   Ústop_gradientÚargpartitionÚtake_along_axisrX   Úsum)
r;   rD   rm   ÚlogitsÚcÚscoresÚcorrected_scoresÚkÚindsÚweightss
   &&        r   rE   ÚLagunaTopKRouter.__call__0   s!  € Ø—‘ˆØ—‘˜1“×$Ñ$¤R§Z¡ZÓ0ˆØ×(Ñ(¨3Ô.Ø×-Ñ-ˆAÜ—W’W˜V�ZÓ(¨1Õ,ˆFà'+×'7×'7Ð'7”—’˜FÔ#¼R¿ZºZÈÐUWÔ=XˆØ!×$@Ñ$@×$GÑ$GÈÏÉÓ$UÕUÐà�J‰JˆÜ×ÒÜ�OŠOÐ-Ð-°1°qµ5¸rÔBÀ3ÈÈÈÀ7ÕKó
ˆô ×$Ò$ V¸Ô;ˆØ××ÐØ¤§¢ w°RÀ$Ô GÕGˆGØ—^‘^ EÓ*Ð*Ð*r   )ra   rX   r_   r\   rW   rZ   rG   rO   s   @@r   rQ   rQ   &   s    ù‡ € ÷Eó E÷+÷ +ð +r   rQ   c                   óP   a a€ ] tR t^Dt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚLagunaSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# rS   r   )r   r.   s   "€r   r   Ú!LagunaSparseMoeBlock.__annotate__E   s   ø€ ÷ Yñ Y™[ñ Yr   c                óJ  <€ \         SV `  4        VP                  '       d   \        R 4      hVP                  V n        \        V4      V n        \        VP                  VP                  VP                  4      V n        \        VP                  VP                  4      V n        R# )z7moe_apply_router_weight_on_input=True is not supported.N)r4   r5   Ú moe_apply_router_weight_on_inputÚNotImplementedErrorÚmoe_routed_scaling_factorÚrouted_scaling_factorrQ   Úgater	   r]   Úmoe_intermediate_sizer^   Ú
switch_mlpr*   Úshared_expert_intermediate_sizeÚshared_expertrb   s   &&€r   r5   ÚLagunaSparseMoeBlock.__init__E   sˆ   ø€ Ü‰ÑÔØ×0×0Ð0Ü%ØIóð ð &*×%CÑ%CˆÔ"Ü$ TÓ*ˆŒ	Ü#Ø×Ñ˜d×8Ñ8¸$×:JÑ:Jó
ˆŒô ! ×!1Ñ!1°4×3WÑ3WÓXˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# re   r?   )r   r.   s   "€r   r   r‚   R   s#   ø€ ÷ )ñ )™"Ÿ(™(ð )¡r§x¡xñ )r   c                ó  € V P                  V4      w  r#V P                  W4      p\        P                  ! WCR,          ,          RR7      pV P                  R8w  d   W@P                  ,          pW@P                  V4      ,           # ).rh   r%   ©.Néþÿÿÿ)rˆ   rŠ   r@   rv   r‡   rŒ   )r;   rD   r|   ry   Úys   &&   r   rE   ÚLagunaSparseMoeBlock.__call__R   sg   € Ø—y‘y “|‰ˆØ�O‰O˜AÓ$ˆÜ�FŠF�1˜iÕ(Õ(¨rÔ2ˆØ×%Ñ%¨Ô,Ø×.Ñ.Õ.ˆAØ×%Ñ% aÓ(Õ(Ð(r   )rˆ   r‡   rŒ   rŠ   rG   rO   s   @@r   r€   r€   D   s    ù‡ € ÷Yó Y÷)÷ )ð )r   r€   c                   óT   a a€ ] tR t^[t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   r   Ú	layer_idx©r   r"   )r   r.   s   "€r   r   ÚAttention.__annotate__\   s   ø€ ÷ 1
ñ 1
™[ð 1
±Sñ 1
r   c                óÚ  <€ \         SV `  4        VP                  V,          V n        VP                  V n        VP                  V n        V P                  R,          V n        VP                  R8H  V n	        \        VP                  4      V n        VP                  V,          R8H  V n        V P                  '       d   VP                  MRV n        VP                  p\        P                   ! W0P                  V P                  ,          VP"                  R7      V n        \        P                   ! W0P
                  V P                  ,          VP"                  R7      V n        \        P                   ! W0P
                  V P                  ,          VP"                  R7      V n        \        P                   ! V P                  V P                  ,          W1P*                  R7      V n        V P                  '       dY   V P                  '       d   V P                  MV P                  V P                  ,          p\        P                   ! W4RR7      V n        V P                  '       d:   VP0                  '       d(   \2        P4                  ! V P                  34      V n        MRV n        \        P8                  ! V P                  VP:                  R7      V n        \        P8                  ! V P                  VP:                  R7      V n        V P                  '       d   VP@                  e   VP@                  MVPB                  p\E        \G        W4      \I        W4      RVVPJ                  R7      V n&        R# )	g      à?zper-headÚsliding_attentionNr2   F©Úeps)ÚbaseÚtraditionalÚscaling_configÚmax_position_embeddingsg      à¿)'r4   r5   Únum_attention_heads_per_layerÚn_headsÚnum_key_value_headsÚ
n_kv_headsr&   ÚscaleÚgatingÚgate_per_headÚboolÚlayer_typesÚ
is_slidingÚsliding_windowr]   r6   r7   Úqkv_biasÚq_projÚk_projÚv_projÚattention_biasÚo_projÚg_projÚswa_attention_sink_enabledr@   r`   ÚsinkÚRMSNormÚrms_norm_epsÚq_normÚk_normÚswa_rope_parametersÚrope_parametersr   r(   r    r¢   Úrope)r;   r   r˜   r,   Úgate_dimr   r<   s   &&&   €r   r5   ÚAttention.__init__\   sI  ø€ Ü‰ÑÔà×9Ñ9¸)ÕDˆŒØ×2Ñ2ˆŒØŸ™ˆŒØ—]‘] DÕ(ˆŒ
Ø!Ÿ[™[¨JÑ6ˆÔÜ˜4Ÿ;™;Ó'ˆŒØ×*Ñ*¨9Õ5Ð9LÑLˆŒØ59·_·_°_˜d×1Ò1È$ˆÔà×ÑˆÜ—i’i §\¡\°D·M±MÕ%AÈÏÉÔVˆŒÜ—i’iØ—‘ 4§=¡=Õ0°t·}±}ô
ˆŒô —i’iØ—‘ 4§=¡=Õ0°t·}±}ô
ˆŒô —i’iØ�L‰L˜4Ÿ=™=Õ(¨#×4GÑ4Gô
ˆŒð �;�;ˆ;à $× 2× 2Ð 2�—’¸¿¹ÀtÇ}Á}Õ8Tð ô Ÿ)š) C¸Ô>ˆDŒKà�?�?ˆ?˜t×>×>Ð>ÜŸš $§,¡, Ó1ˆD�IàˆDŒIä—j’j §¡°D×4EÑ4EÔFˆŒÜ—j’j §¡°D×4EÑ4EÔFˆŒð ��ˆ 4×#;Ñ#;Ò#Gð ×$Ò$à×%Ñ%ð 	ô
 $Ü�tÓ)Ü˜DÓ.ØØ&Ø$(×$@Ñ$@ô
ˆŽ	r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   rD   ÚmaskÚcacher   ©r@   rA   r   r   )r   r.   s   "€r   r   rš   �   sH   ø€ ÷ 0#ñ 0#á�8‰8ð0#ñ ‘r—x‘xÕ ð0#ñ ™�}ð	0#ñ
 
�‰ñ0#r   c           
     ó`  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pV P	                  VP                  WEV P                  V P                  4      4      P                  ^ ^^^4      pV P                  VP                  WEV P                  V P                  4      4      P                  ^ ^^^4      pV	P                  WEV P                  V P                  4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        VVV	VV P                  VV P                   R7      p
V
P                  ^ ^^^4      P                  WER4      p
V P"                  '       dÇ   \$        P&                  ! V P)                  V4      P+                  \,        P.                  4      4      P+                  V
P0                  4      pV P2                  '       dR   V
P                   pV
P                  WEV P                  V P                  4      VR,          ,          P                  V4      p
MW«,          p
V P5                  V
4      # )r   )Úoffset)rÃ   r§   rÂ   Úsinksrl   r�   )Úshaper¯   r°   r±   r¹   Úreshaper¤   r&   Ú	transposerº   r¦   r½   rÆ   Úupdate_and_fetchr   r§   r¶   r¨   r6   Úsoftplusr´   rn   r@   ro   rm   r©   r³   )r;   rD   rÂ   rÃ   ÚBÚLÚ_ÚqueriesÚkeysÚvaluesÚoutputrˆ   rÈ   s   &&&&         r   rE   ÚAttention.__call__�   s  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆØ—+‘+Ø�O‰O˜A $§,¡,°·±Ó>ó
ç
‰)�A�q˜!˜QÓ
ð 	ð �{‰{Ø�L‰L˜˜tŸ™°·±Ó>ó
ç
‰)�A�q˜!˜QÓ
ð 	ð —‘  d§o¡o°t·}±}ÓE×OÑOØˆq�!�Qó
ˆð ÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-ØØØØØ—*‘*ØØ—)‘)ô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆà�;�;ˆ;Ü—;’;˜tŸ{™{¨1›~×4Ñ4´R·Z±ZÓ@ÓA×HÑHÈÏÉÓVˆDØ×!×!Ð!ØŸ™�à—N‘N 1¨¯©°t·}±}ÓEÈÈYÍÕWß‘'˜%“.ñ ð  ��à�{‰{˜6Ó"Ð"r   )r´   r©   r¨   r&   r¬   rº   r°   r¤   r¦   r³   r¹   r¯   r½   r§   r¶   r­   r±   ©NNrG   rO   s   @@r   r•   r•   [   s   ù‡ € ÷1
ó 1
÷f0#÷ 0#ò 0#r   r•   c                   óT   a a€ ] tR t^Ât oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# r—   r™   )r   r.   s   "€r   r   ÚDecoderLayer.__annotate__Ã   s   ø€ ÷ :ñ :™[ð :±Sñ :r   c                ó$  <€ \         SV `  4        \        W4      V n        W!P                  9  dB   VP
                  ^ 8”  d1   V^,           VP                  ,          ^ 8X  d   \        V4      V n        M%\        VP                  VP                  4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        VP"                  V,          V n        R# )r   r�   N)r4   r5   r•   Ú	self_attnÚmlp_only_layersr^   Údecoder_sparse_stepr€   Úmlpr*   r]   Úintermediate_sizer6   r·   r¸   Úinput_layernormÚpost_attention_layernormr«   Úattention_type©r;   r   r˜   r<   s   &&&€r   r5   ÚDecoderLayer.__init__Ã   sÇ   ø€ Ü‰ÑÔÜ" 4Ó3ˆŒØ×1Ñ1Ô1Ø×Ñ˜qÔ  i°!¥m°t×7OÑ7OÕ%OÐSTÔ%Tä+¨DÓ1ˆD�Hä˜4×+Ñ+¨T×-CÑ-CÓDˆDŒHä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð #×.Ñ.¨yÕ9ˆÖr   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rÁ   rÄ   )r   r.   s   "€r   r   rÙ   Ó   sH   ø€ ÷ 	ñ 	á�8‰8ð	ñ ‘r—x‘xÕ ð	ñ ™�}ð		ñ
 
�‰ñ	r   c                ó¤   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           # rC   )rÛ   rà   rÞ   rá   )r;   rD   rÂ   rÃ   ÚrÚhs   &&&&  r   rE   ÚDecoderLayer.__call__Ó   sF   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�uˆr   )râ   rà   rÞ   rá   rÛ   rÕ   rG   rO   s   @@r   r×   r×   Â   s   ù‡ € ÷:ó :÷ 	÷ 	ò 	r   r×   c                   óT   a a€ ] tR t^ßt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLagunaModelc                ó    <€ V ^8„  d   QhRS[ /# rS   r   )r   r.   s   "€r   r   ÚLagunaModel.__annotate__à   s   ø€ ÷ 
ñ 
™[ñ 
r   c                óZ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        W4      NK  	  upV n        \
        P                  ! VP                  VP                  R 7      V n        VP                  P!                  R4      V n        RVP                  9   d#   VP                  P!                  R4      V n        R# RV n        R# u upi )r�   Úfull_attentionrœ   N)r4   r5   r   Ú
vocab_sizeÚnum_hidden_layersr6   Ú	Embeddingr]   Úembed_tokensÚranger×   Úlayersr·   r¸   Únormr«   ÚindexÚfa_idxÚswa_idxrã   s   && €r   r5   ÚLagunaModel.__init__à   sí   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔä;@À×AWÑAWÔ;Xó
Ù;X¨iŒL˜Ö)Ñ;Xñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	Ø×&Ñ&×,Ñ,Ð-=Ó>ˆŒð # d×&6Ñ&6Ô6ð ×Ñ×"Ñ"Ð#6Ó7ð 	Žð ð 	Žùò
s   Á?D(c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# )r   ÚinputsÚinputs_embedsr   )r@   rA   r   )r   r.   s   "€r   r   rí   ñ   s:   ø€ ÷ ñ á—‘ðñ  ¡§¡Õ)ð	ñ
 
�‰ñr   c                óÈ  € Ve   TpMV P                  V4      pVf   R .\        V P                  4      ,          p\        WBV P                  ,          4      pV P
                  e3   \        WBV P
                  ,          V P                  P                  R7      p\        V P                  V4       F#  w  rxVP                  R8X  d   XMTp	V! WIV4      pK%  	  V P                  V4      # )N)Úwindow_sizerœ   )ró   Úlenrõ   r   rø   rù   r   r­   Úziprâ   rö   )
r;   rü   rÃ   rý   rè   Ú	full_maskÚsliding_maskÚlayerrx   rÂ   s
   &&&&      r   rE   ÚLagunaModel.__call__ñ   sÍ   € ð Ò$Ø‰Aà×!Ñ! &Ó)ˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEä)¨!°4·;±;Õ-?Ó@ˆ	Ø�<‰<Ò#Ü0Ø˜Ÿ™Õ&°D·I±I×4LÑ4LôˆLô ˜DŸK™K¨Ö/‰HˆEð ×'Ñ'Ð+>Ô>ñ àð ñ
 �a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr   )r   ró   rø   rõ   rö   rñ   rù   rð   rÕ   rG   rO   s   @@r   rë   rë   ß   s   ù‡ € ÷
ó 
÷"÷ ò r   rë   c                   óÂ   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR tR tR tR	 t	]
R
 4       t]
R 4       t]
R 4       t]
R 4       t]
R 4       tR tRtVtV ;t# )ÚLanguageModeli  c                ó    <€ V ^8„  d   QhRS[ /# rS   r   )r   r.   s   "€r   r   ÚLanguageModel.__annotate__  s   ø€ ÷ Tñ T™[ñ Tr   c                ó
  <€ \         SV `  4        Wn        Wn        VP                  V n        \        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# r1   )r4   r5   r   ÚconfigÚ
model_typerë   ÚmodelÚtie_word_embeddingsr6   r7   r]   rð   Úlm_headrb   s   &&€r   r5   ÚLanguageModel.__init__  s^   ø€ Ü‰ÑÔØŒ	ØŒØŸ/™/ˆŒÜ  Ó&ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r   c                óž   <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[/# )r   rü   Úinput_embeddingsrý   r   )r   r@   rA   r
   )r   r.   s   "€r   r   r	    sN   ø€ ÷ /ñ /á™Ÿ™Õ"ð/ñ #¡2§8¡8Õ,ð	/ñ
  ¡§¡Õ)ð/ñ 
ñ/r   c                ó  € Vf   VP                  R4      pVf   TpV P                  WV4      pV P                  P                  '       d'   V P                  P                  P                  V4      pMV P                  V4      p\        VR7      # )NÚ	input_ids)rw   )r   r  r   r  ró   Ú	as_linearr  r
   )r;   rü   rÃ   r  rý   ÚkwargsÚouts   &&&&&, r   rE   ÚLanguageModel.__call__  su   € ð Š>Ø—Z‘Z Ó,ˆFØÒ Ø,ˆMØ�j‰j˜¨Ó6ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8‰Cà—,‘,˜sÓ#ˆCÜ"¨#Ô.Ð.r   c                óŽ  € V P                   P                  '       d   VP                  R R4       V P                  V4      pV P	                  V4      pV P                  V4      pVP                  4        UUu/ uFB  w  r#RV9  g   K  VP                  R4      '       d   K'  VP                  R4      '       d   K@  W#bKD  	  upp# u uppi )zlm_head.weightNzrotary_emb.inv_freqz.self_attn.k_scalez.self_attn.v_scale)r   r  ÚpopÚ_unpack_compressed_tensorsÚ_remap_router_weightsÚ_stack_expertsÚitemsÚendswith)r;   r}   r{   Úvs   &&  r   ÚsanitizeÚLanguageModel.sanitize,  s´   € Ø�9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/à×1Ñ1°'Ó:ˆØ×,Ñ,¨WÓ5ˆØ×%Ñ% gÓ.ˆð  Ÿ™œô
á'‘�Ø$¨AÑ-ô ð —J‘JÐ3×4ô ð —J‘JÐ3×4ô	 ˆAŠDÙ'ò
ð 	
ùó 
s   Á5CÂCÂCÂ7Cc                óÐ  € \         ;QJ d    R  V 4       F  '       g   K   RM	  RM! R  V 4       4      '       g   V# / pVP                  4        EF  w  r4VP                  R4      '       d�   VR\        R4      )  pV R2V9   du   V R2V9   di   W R2,          pW R2,          P	                  \
        P                  4      W% R2&   WbV R	2&   RV,          P                  VP                  4      W% R
2&   K¨  Kª  K¬  VP                  R4      '       g   VP                  R4      '       d5   VP                  R^4      ^ ,          R,           pV R2V9   d   EK  WBV&   EK  WBV&   EK  	  V# )c              3   óB   "  € T F  qP                  R 4      x € K  	  R# 5i)ú.weight_shapeN©r  )Ú.0r{   s   & r   Ú	<genexpr>Ú;LanguageModel._unpack_compressed_tensors.<locals>.<genexpr><  s   é € Ð@¹°1—:‘:˜o×.Ð.»ùs   ‚TFr%  NÚweight_shapeÚweight_packedÚweight_scaleÚweightÚscalesÚbiasesz.weight_packedz.weight_scaleÚ.iøÿÿÿ)
Úanyr  r  r   Úviewr@   Úuint32rn   rm   Úrsplit)r;   r}   Únew_weightsr{   r   rŸ   r.  s   &&     r   r  Ú(LanguageModel._unpack_compressed_tensors;  sW  € ß‹sÑ@¹Ó@�s�sŠsÑ@¹Ó@×@Ò@ØˆNàˆØ—M‘M—O‰DˆAØ�z‰z˜/×*Ò*ØÐ/œC Ó/Ð/Ð0�à�f˜MÐ*¨gÔ5Ø˜& Ð-°Ô8à$ v¨\Ð%:Õ;�FØ3:¸VÀ=Ð;QÕ3R×3WÑ3WÜŸ	™	ó4�K &¨ Ñ0ð 4: 4 &¨ Ñ0Ø46¸µK×3GÑ3GÈÏÉÓ3U�K &¨ Ó0ñ 9ñ 6ð —‘Ð,×-Ò-°·±¸O×1LÒ1LØ—x‘x  QÓ'¨Õ*¨SÕ0�Ø�V˜<Ð(¨GÔ3ÚØ!"˜A”à!"˜A”ñ' $ð( Ðr   c                óæ   € \        V P                  P                  4       FM  pR V R2pV R2pWA9   d   VP                  V4      W R2&   V R2pWQ9   g   K7  VP                  V4      W R2&   KO  	  V# )úmodel.layers.ú.mlpz.gate.weightz.gate.proj.weightz .experts.e_score_correction_biasz.gate.e_score_correction_bias)rô   r   rñ   r  )r;   r}   r˜   ÚprefixÚgate_weightÚlegacy_biass   &&    r   r  Ú#LanguageModel._remap_router_weightsV  s‹   € Ü˜tŸy™y×:Ñ:Ö;ˆIØ$ Y K¨tÐ4ˆFØ#˜H LÐ1ˆKØÔ%Ø8?¿¹ÀKÓ8P�˜(Ð"3Ð4Ñ5à#˜HÐ$DÐEˆKØÖ%ØDKÇKÁKØóE�˜(Ð"?Ð@ÓAñ <ð ˆr   c                ó~  € \        V P                  P                  4       F”  pR V R2pR F…  pR F|  pV RV RV 2pWa9  d   K  \        P                  ! \        V P                  P
                  4       Uu. uF  pVP                  V RV RV RV 24      NK!  	  up4      W RV RV 2&   K~  	  K‡  	  K–  	  V# u upi )r8  r9  z.experts.0.r0  z	.experts.z.switch_mlp.)r8   r:   r9   )r-  r.  r/  )rô   r   rñ   r@   Ústackr^   r  )r;   r}   r˜   r:  r_   ÚsuffixÚ	first_keyÚes   &&      r   r  ÚLanguageModel._stack_expertsd  s×   € Ü˜tŸy™y×:Ñ:Ö;ˆIØ$ Y K¨tÐ4ˆFÛ=�Û<�FØ#) (¨+°d°V¸1¸V¸HÐ E�IØ Ô/Ù ÜFHÇhÂhô &+¨4¯9©9×+@Ñ+@Ô%Aóá%A ð $ŸK™K¨6¨(°)¸A¸3¸aÀ¸vÀQÀvÀhÐ(OÖPÙ%AñóG�G˜h l°4°&¸¸&¸ÐBÓCó	 =ó >ñ <ð ˆùòs   Á6%B:c                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )zmlp.gate.projÚ
group_sizeÚbitsTr&  )ÚpathrÏ   s   &&r   Ú	predicateÚ0LanguageModel.quant_predicate.<locals>.predicatev  s#   € Ø�}‰}˜_×-Ò-Ø$ b¨&°!Ð4Ð4Ùr   © ©r;   rI  s   & r   Úquant_predicateÚLanguageModel.quant_predicatet  s   € ò	ð
 Ðr   c                ó   € R  pV# )c                 ó   € R V 9  # )ra   rK  )r{   s   &r   rI  Ú/LanguageModel.cast_predicate.<locals>.predicate  s   € Ø,°AÑ5Ð5r   rK  rL  s   & r   Úcast_predicateÚLanguageModel.cast_predicate}  s   € ò	6ð Ðr   c                ó.   € V P                   P                  # rC   )r  rõ   ©r;   s   &r   rõ   ÚLanguageModel.layers„  s   € à�z‰z× Ñ Ð r   c                ó.   € V P                   P                  # rC   )r   r&   rU  s   &r   r&   ÚLanguageModel.head_dimˆ  s   € à�y‰y×!Ñ!Ð!r   c                ó.   € V P                   P                  # rC   )r   r¥   rU  s   &r   r¦   ÚLanguageModel.n_kv_headsŒ  s   € à�y‰y×,Ñ,Ð,r   c                óæ   € V P                    Uu. uFV  pVP                  R 8X  d9   V P                  P                  e!   \	        V P                  P                  R7      M	\        4       NKX  	  up# u upi )rœ   )Úmax_size)rõ   râ   r   r­   r   r   )r;   r  s   & r   Ú
make_cacheÚLanguageModel.make_cache�  sm   € ð Ÿšó
ñ %�ð	 ×'Ñ'Ð+>Ô>Ø—I‘I×,Ñ,Ò8ô  ¨¯©×)AÑ)AÕBô “Yòñ
 %ñ
ð 	
ùò 
s   �AA.)r   r  r  r  r  )NNNN)rH   rI   rJ   rK   r5   rE   r!  r  r  r  ÚpropertyrM  rR  rõ   r&   r¦   r]  rL   rM   rN   rO   s   @@r   r  r    s�   ù‡ € ÷Tó T÷/ò /ò&
òò6òð  ñó ðð ñó ðð ñ!ó ð!ð ñ"ó ð"ð ñ-ó ð-÷	
ò 	
r   r  )#Útypingr   r   r   r   Úmlx.coreÚcorer@   Úmlx.nnr6   Úmlx_lm.models.activationsr   Úmlx_lm.models.rope_utilsr   Úmlx_lm.models.switch_layersr	   rŸ   r
   r   r   rÃ   r   r   r  r   r    r(   ÚModuler*   rQ   r€   r•   r×   rë   r  rK  r   r   Ú<module>rh     s¯   ðß -Ó -å Ý Ý ,Ý 4Ý 1÷ñ ÷
 -Ý õAõ(ô
Jˆ"�)‰)ô Jô+�r—y‘yô +ô<)˜2Ÿ9™9ô )ô.d#�—	‘	ô d#ôN�2—9‘9ô ô:-�"—)‘)ô -ô`J
�B—I‘Iö J
r   