+
    TV-j‹D  ã                   óø  € ^ RI t ^ RIHt ^ RIHtHtHtHt ^ RIH	t
 ^ RIHt ^ RIHtHtHt ^RIHt ^RIHtHtHt ^RIHtHt ^RIHt ^R	IHt ^R
IHt ] ! R R]4      4       t  ! R R]PB                  4      t" ! R R]PB                  4      t# ! R R]PB                  4      t$ ! R R]PB                  4      t% ! R R]PB                  4      t& ! R R]PB                  4      t' ! R R]PB                  4      t(R# )é    N)Ú	dataclass)ÚAnyÚDictÚOptionalÚTuple)Úshard_inplaceÚshard_linearÚsum_gradients)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	CacheListÚKVCache)ÚMultiLinear)Úinitialize_rope)Ú	SwitchGLUc                   ó8   a € ] tR t^t o RtRtRtV 3R ltRtV t	R# )Ú	ModelArgsFNc                ón  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   # )é   Ú
model_typeÚattention_methodÚzero_expert_typeÚhidden_sizeÚffn_hidden_sizeÚmoe_topkÚexpert_ffn_hidden_sizeÚn_routed_expertsÚzero_expert_numÚ
num_layersÚ
vocab_sizeÚmax_position_embeddingsÚnum_attention_headsÚkv_lora_rankÚq_lora_rankÚqk_rope_head_dimÚqk_nope_head_dimÚ
v_head_dimÚrouted_scaling_factorÚrms_norm_epsÚ
rope_thetaÚmla_scale_q_loraÚmla_scale_kv_loraÚattention_biasÚnorm_topk_probÚrouter_biasÚrope_scaling)ÚstrÚintÚfloatÚboolr   r   )ÚformatÚ__classdict__s   "€Úl/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/longcat_flash.pyÚ__annotate__ÚModelArgs.__annotate__   sa  ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ �Mñ ñ  Ññ ñ Ññ ñ Ññ ñ �Oñ ñ �Oñ ñ !Ñ ñ ñ Ññ ñ Ññ ñ  Ññ! ñ" Ññ# ñ$ Ññ% ñ& �Oñ' ñ( !Ñ ñ) ñ* Ññ+ ñ, Ññ- ñ. Ññ/ ñ0 Ññ1 ñ2 Ññ3 ñ4 Ñ ñ5 ñ6 Ññ7 ñ8 ™4•.Ñ'ò9 ó    © )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r0   r1   r2   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r8   s   @r9   r   r      s   ø‡ € ð4 !€NØ€KØ#'€L÷9 ƒ r<   r   c                   óT   a a€ ] tR t^0t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLongcatFlashMLAc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r7   r8   s   "€r9   r:   ÚLongcatFlashMLA.__annotate__1   s   ø€ ÷ C
ñ C
™Yñ C
r<   c                ó   <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  VP                  ,           V n        V P                  R
,          V n	        V P                  fE   \        P                  ! VP                  V P                  V P                  ,          RR7      V n        M¤\        P                  ! VP                  V P                  VP                  R7      V n        \        P                   ! V P                  4      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! VP                  V P
                  V P                  ,           VP                  R7      V n        \        P                   ! V P
                  4      V n        \+        V P                  V P
                  V P                  4      V n        \+        V P
                  V P                  V P                  4      V n        \        P                  ! V P                  VP                  ,          VP                  VP                  R7      V n        VP2                  '       d*   VP                  V P                  ,          R ,          V n        VP4                  '       d*   VP                  V P
                  ,          R ,          V n        VP6                  e‰   VP6                  P9                  R^ 4      pV'       de   VP6                  R,          pV^8”  dK   RV,          \:        P<                  ! V4      ,          R,           pV P                  V,          V,          V n	        \?        V P                  VP@                  RVP6                  VPB                  R	7      V n"        R# )g      à?NF©ÚbiasÚmscale_all_dimÚfactorgš™™™™™¹?g      ð?T)ÚdimsÚbaseÚtraditionalÚscaling_configr#   g      à¿)#ÚsuperÚ__init__r$   r'   r(   r%   r&   r)   Úqk_head_dimÚscaleÚnnÚLinearr   Úq_projr/   Úq_a_projÚRMSNormÚq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormr   Úembed_qÚunembed_outÚo_projr-   r.   r2   ÚgetÚmathÚlogr   r,   r#   Úrope)ÚselfrI   rO   Úscaling_factorÚsÚ	__class__s   &&   €r9   rV   ÚLongcatFlashMLA.__init__1   s  ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø $× 5Ñ 5ˆÔØ $× 5Ñ 5ˆÔØ ×-Ñ-ˆÔØ×+Ñ+ˆÔØŸ/™/ˆŒà×0Ñ0°4×3HÑ3HÕHˆÔØ×%Ñ% tÕ+ˆŒ
à×ÑÒ#ÜŸ)š)Ø× Ñ Ø×(Ñ(¨4×+;Ñ+;Õ;ØôˆD�Kô ŸIšIØ× Ñ  $×"2Ñ"2¸×9LÑ9LôˆDŒMô "$§¢¨D×,<Ñ,<Ó!=ˆDÔÜŸIšIØ× Ñ Ø×(Ñ(¨4×+;Ñ+;Õ;ØôˆDŒMô #%§)¢)Ø×ÑØ×Ñ × 5Ñ 5Õ5Ø×$Ñ$ô#
ˆÔô
 !Ÿjšj¨×):Ñ):Ó;ˆÔÜ"Ø×!Ñ! 4×#4Ñ#4°d×6NÑ6Nó
ˆŒô 'Ø×Ñ˜tŸ™°×0HÑ0Hó
ˆÔô —i’iØ×$Ñ$ t§¡Õ6Ø×ÑØ×$Ñ$ô
ˆŒð × × Ð Ø%)×%5Ñ%5¸×8HÑ8HÕ%HÈSÕ$PˆDÔ!Ø×!×!Ð!Ø&*×&6Ñ&6¸×9JÑ9JÕ&JÈsÕ%RˆDÔ"à×ÑÒ(Ø!×.Ñ.×2Ñ2Ð3CÀQÓGˆNßØ!%×!2Ñ!2°8Õ!<�Ø! AÔ%Ø˜nÕ,¬t¯xªx¸Ó/GÕGÈ#ÕM�AØ!%§¡¨a¥°!Õ!3�D”Jä#Ø×&Ñ&Ø—‘ØØ×,Ñ,Ø$(×$@Ñ$@ô
ˆŽ	r<   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r7   r8   s   "€r9   r:   rK   v   sH   ø€ ÷ =#ñ =#á�8‰8ð=#ñ ‘r—x‘xÕ ð=#ñ ™�}ð	=#ñ
 
�‰ñ=#r<   c           	     ó  € VP                   w  rEpV P                  f   V P                  V4      pM/V P                  V P	                  V P                  V4      4      4      pVP                  WEV P                  V P                  4      P                  ^ ^^^4      pV P                  e   WpP                  ,          p\        P                  ! WpP                  .RR7      w  r‰V P                  V4      p
\        P                  ! W P                  .RR7      w  r«VP                  WE^V P                   4      P                  ^ ^^^4      pV P#                  V
4      pV P$                  e   WÀP$                  ,          pVe   VP&                  M^ pV P)                  W�4      p	V P)                  W½4      p\        P*                  ! V^R7      pVe   VP-                  WË4      w  rËW�P.                  ,          VP1                  RR4      ,          pVe`   \        P2                  ! VV\        P4                  ! \        P6                  ! VP8                  4      P:                  VP8                  4      4      pV^8X  d   V P=                  V4      pT;ppM$V P=                  VRR7      pV P?                  V4      p\A        W�VW0P.                  VR7      pV^8X  d   V P?                  V4      pVP                  ^ ^^^4      P                  WER4      pV PC                  V4      # )N©ÚaxisF)Ú	transpose)rr   rX   rq   éÿÿÿÿéþÿÿÿ)"Úshaper&   r[   r_   r^   r\   Úreshaper$   rW   rz   r-   ru   Úsplitr(   r`   r%   r'   ra   r.   Úoffsetrh   Úexpand_dimsÚupdate_and_fetchrX   ÚswapaxesÚwhererv   ÚfinfoÚdtypeÚminrb   rc   r   rd   )ri   rp   rq   rr   ÚBÚLÚ_ÚqÚq_nopeÚq_peÚcompressed_kvÚk_peÚ	kv_latentr€   Ú	pe_scoresÚkÚvÚoutputs   &&&&              r9   Ú__call__ÚLongcatFlashMLA.__call__v   s‰  € ð —'‘'‰ˆˆaà×ÑÒ#Ø—‘˜A“‰Aà—‘˜d×0Ñ0°·±¸qÓ1AÓBÓCˆAà�I‰I�a˜D×4Ñ4°d×6FÑ6FÓG×QÑQØˆq�!�Qó
ˆð × Ñ Ò,Ø×)Ñ)Õ)ˆAä—x’x ×$9Ñ$9Ð#:ÀÔD‰ˆà×/Ñ/°Ó2ˆÜ Ÿhšh }×7HÑ7HÐ6IÐPRÔSÑˆØ�|‰|˜A ! T×%:Ñ%:Ó;×EÑEÀaÈÈAÈqÓQˆØ×'Ñ'¨Ó6ˆ	à×!Ñ!Ò-Ø!×$:Ñ$:Õ:ˆIà!&Ò!2�—’¸ˆØ�y‰y˜Ó&ˆØ�y‰y˜Ó&ˆä—N’N 9°1Ô5ˆ	àÒØ#×4Ñ4°YÓE‰OˆIàŸJ™JÕ&¨$¯-©-¸¸BÓ*?Õ?ˆ	ØÒÜŸšØØÜ—’œŸš )§/¡/Ó2×6Ñ6¸	¿¹ÓHóˆIð �Œ6Ø—\‘\ &Ó)ˆFØÐˆA‘à—‘˜Y°%�Ó8ˆAØ× Ñ  Ó+ˆAä-Ø�q ¯Z©Z¸iô
ˆð �Œ6Ø×%Ñ% fÓ-ˆFà×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r<   )rb   ra   r`   r%   r.   r-   r$   rd   r^   r\   r_   r&   r[   rW   r(   r'   rh   rX   rc   r)   ©NN©	r>   r?   r@   rA   rV   r•   rC   rD   Ú__classcell__©rl   r8   s   @@r9   rF   rF   0   s!   ù‡ € ÷C
ó C
÷J=#÷ =#ò =#r<   rF   c                   óT   a a€ ] tR t^¶t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚLongcatFlashMLPc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   rI   Ú	is_expert)r   r6   )r7   r8   s   "€r9   r:   ÚLongcatFlashMLP.__annotate__·   s   ø€ ÷ Nñ N™Yð N±4ñ Nr<   c                óR  <€ \         SV `  4        V'       d   VP                  MVP                  p\        P
                  ! VP                  VR R7      V n        \        P
                  ! VP                  VR R7      V n        \        P
                  ! W1P                  R R7      V n	        R# ©FrM   N)
rU   rV   r   r   rY   rZ   r   Ú	gate_projÚup_projÚ	down_proj)ri   rI   rž   r   rl   s   &&& €r9   rV   ÚLongcatFlashMLP.__init__·   so   ø€ Ü‰ÑÔß5>�d×1Ò1ÀD×DXÑDXˆäŸš 4×#3Ñ#3°[ÀuÔMˆŒÜ—y’y ×!1Ñ!1°;ÀUÔKˆŒÜŸš ;×0@Ñ0@ÀuÔMˆŽr<   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rp   rs   )ru   rv   )r7   r8   s   "€r9   r:   rŸ   ¿   s'   ø€ ÷ Jñ J™"Ÿ(™(ð J¡r§x¡xñ Jr<   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # ©N)r¤   r   r¢   r£   )ri   rp   s   &&r9   r•   ÚLongcatFlashMLP.__call__¿   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr<   )r¤   r¢   r£   )Fr˜   rš   s   @@r9   rœ   rœ   ¶   s#   ù‡ € ÷Nõ N÷J÷ Jð Jr<   rœ   c                   óP   a a€ ] tR t^Ãt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚLongcatFlashTopkRouterc                ó    <€ V ^8„  d   QhRS[ /# rH   rJ   )r7   r8   s   "€r9   r:   Ú#LongcatFlashTopkRouter.__annotate__Ä   s   ø€ ÷ Jñ J™Yñ Jr<   c                óÀ  <€ \         SV `  4        Wn        VP                  V n        VP
                  VP                  ,           V n        VP                  V n        VP                  V n        VP                  V n	        \        P                  ! VP                  V P
                  V P                  R 7      V n        \        P                  ! V P
                  34      V n        R# )rM   N)rU   rV   Úconfigr   Útop_kr   r    r*   r0   r1   rY   rZ   r   Ú
classifierru   ÚzerosÚe_score_correction_bias©ri   rI   rl   s   &&€r9   rV   ÚLongcatFlashTopkRouter.__init__Ä   s¦   ø€ Ü‰ÑÔØŒØ—]‘]ˆŒ
Ø $× 5Ñ 5¸×8LÑ8LÕ LˆÔØ%)×%?Ñ%?ˆÔ"Ø"×1Ñ1ˆÔØ×+Ñ+ˆÔäŸ)š)Ø×Ñ˜d×3Ñ3¸$×:JÑ:Jô
ˆŒô (*§x¢x°×1FÑ1FÐ0HÓ'IˆÖ$r<   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          /# )r   Úhidden_statesrs   )ru   rv   r   )r7   r8   s   "€r9   r:   r­   Ò   s1   ø€ ÷ 8ñ 8¡b§h¡hð 8±5¹¿¹Á2Ç8Á8Ð9KÕ3Lñ 8r<   c                óî  € VP                   pV P                  V4      p\        P                  ! VRR7      pW@P                  ,           p\        P
                  ! WPP                  ) RR7      RV P                  ) R13,          p\        P                  ! WFRR7      pV P                  '       d)   \        P                  ! VRRR7      R,           pWx,          pWpP                  ,          pWgP                  V4      3# )	é   rx   )Úkthry   .NT©ry   Úkeepdimsg#B’¡œÇ;r{   )r†   r±   ru   Úsoftmaxr³   Úargpartitionr°   Útake_along_axisr0   Úsumr*   Úastype)	ri   r·   r†   Úrouter_logitsÚscoresÚcorrected_scoresÚtopk_indicesÚtopk_weightsÚdenominators	   &&       r9   r•   ÚLongcatFlashTopkRouter.__call__Ò   sÇ   € à×#Ñ#ˆØŸ™¨Ó6ˆÜ—’˜M°Ô3ˆà!×$@Ñ$@Õ@ÐÜ—’Ð'7¿j¹j¸[ÈrÔRØ�$—*‘*�‘Ðõ
ˆô ×)Ò)¨&ÀRÔHˆà××ÐÜŸ&š& °BÀÔFÈÕNˆKØ'Õ5ˆLà#×&@Ñ&@Õ@ˆà×0Ñ0°Ó7Ð7Ð7r<   )r±   r¯   r³   r   r0   r*   r1   r°   r˜   rš   s   @@r9   r«   r«   Ã   s    ù‡ € ÷Jó J÷8÷ 8ð 8r<   r«   c                   óD   a a€ ] tR t^çt oV3R lV 3R lltR tRtVtV ;t# )ÚLongcatFlashMoEc                ó    <€ V ^8„  d   QhRS[ /# rH   rJ   )r7   r8   s   "€r9   r:   ÚLongcatFlashMoE.__annotate__è   s   ø€ ÷ #ñ #™Yñ #r<   c                óF  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        \        VP                  VP                  VP
                  4      V n        \        V4      V n        R V n        R # r¨   )rU   rV   r¯   r   Únum_experts_per_tokr   r    r   r   r   r   Ú
switch_mlpr«   ÚrouterÚsharding_groupr´   s   &&€r9   rV   ÚLongcatFlashMoE.__init__è   s†   ø€ Ü‰ÑÔØŒØ#'§=¡=ˆÔ Ø $× 5Ñ 5ˆÔØ#×3Ñ3ˆÔØ $× 5Ñ 5ˆÔä#Ø×ÑØ×'Ñ'Ø×!Ñ!ó
ˆŒô -¨TÓ2ˆŒØ"ˆÖr<   c                óz  € V P                   e   \        V P                   4      ! V4      pV P                  V4      w  r#W P                  8¬  p\        P
                  ! V^ V4      p\        P
                  ! VRV4      pV P                  W4      pWeR,          ,          p\        P                  ! VRR7      pV P                   e+   \        P                  P                  W€P                   R7      pV P                  R8X  g   Q h\        P                  ! \        P
                  ! WCR4      R	RR7      p	W�V	,          ,           pV# )
Ng        rx   ©ÚgroupÚidentityTr»   ).Nr|   r{   )rÑ   r
   rÐ   r   ru   r„   rÏ   rÀ   ÚdistributedÚall_sumr   )
ri   r·   rÅ   rÆ   rq   Úregular_weightsÚregular_outputsÚweighted_outputsÚfinal_outputÚidentity_weights_sums
   &&        r9   r•   ÚLongcatFlashMoE.__call__ù   s  € Ø×ÑÒ*Ü)¨$×*=Ñ*=Ô>¸}ÓMˆMà%)§[¡[°Ó%?Ñ"ˆð ×4Ñ4Ñ4ˆÜ—x’x  a¨Ó6ˆÜŸ(š( 4¨¨lÓ;ˆàŸ/™/¨-ÓFˆà*¸YÕ-GÕGÐÜ—v’vÐ.°RÔ8ˆà×ÑÒ*ÜŸ>™>×1Ñ1Ø×$7Ñ$7ð 2ó ˆLð
 ×$Ñ$¨
Ô2Ð2Ð2Ü!ŸvšvÜ�HŠH�T¨Ó-°BÀô 
Ðð $Ð6JÕ&JÕJˆàÐr<   )r¯   r   rÎ   rÐ   rÑ   rÏ   r    r   r˜   rš   s   @@r9   rÊ   rÊ   ç   s   ù‡ € ÷#ó #÷"ò r<   rÊ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLongcatFlashDecoderLayeri  c                ó    <€ V ^8„  d   QhRS[ /# rH   rJ   )r7   r8   s   "€r9   r:   Ú%LongcatFlashDecoderLayer.__annotate__  s   ø€ ÷ 
ñ 
™Yñ 
r<   c                ó\  <€ \         SV `  4        VP                  V n        \        V4      V n        \        ^4       Uu. uF  p\        V4      NK  	  upV n        \        ^4       Uu. uF  p\        VR4      NK  	  upV n	        \        ^4       Uu. uF/  p\        P                  ! VP                  VP                  R7      NK1  	  upV n        \        ^4       Uu. uF/  p\        P                  ! VP                  VP                  R7      NK1  	  upV n        R# u upi u upi u upi u upi )r   F)ÚepsN)rU   rV   r   rÊ   ÚmlpÚrangerF   Ú	self_attnrœ   ÚmlpsrY   r]   r+   Úinput_layernormÚpost_attention_layernorm)ri   rI   rŠ   rl   s   && €r9   rV   Ú!LongcatFlashDecoderLayer.__init__  sí   ø€ Ü‰ÑÔØ×+Ñ+ˆÔÜ" 4Ó(ˆŒä9>¸q¼ÓB¹°Aœ/¨$Ö/¹ÑBˆŒÜ;@À¼8ÓD¹8°a”_ T¨5Ö1¹8ÑDˆŒ	äINÈqÌó 
ÙIQÀAŒB�JŠJ�t×'Ñ'¨T×->Ñ->×?Éñ 
ˆÔô JOÈqÌó)
ÙIQÀAŒB�JŠJ�t×'Ñ'¨T×->Ñ->×?Éñ)
ˆÖ%ùò CùÚDùò 
ùò)
s   ¾DÁ'DÂ5D$Ã5D)c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ro   rt   )r7   r8   s   "€r9   r:   râ   '  sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr<   c                ó�  € TpR pVf   Rp\        ^4       F¬  pTpV P                  V,          ! V4      pV P                  V,          ! WBW6,          R7      pWt,           pTpV P                  V,          ! V4      pV^ 8X  d   V P	                  V4      pV P
                  V,          ! V4      pWt,           pV^8X  g   K¤  WE,           pK®  	  V# )N)rq   rr   r—   )ræ   ré   rç   rê   rå   rè   )ri   rp   rq   rr   r·   Úshortcut_mlp_outputÚiÚresiduals   &&&&    r9   r•   Ú!LongcatFlashDecoderLayer.__call__'  sÁ   € ð ˆØ"ÐàŠ=Ø ˆEä�q–ˆAØ$ˆHà ×0Ñ0°Ö3°MÓBˆMØ ŸN™N¨1Ö-¨mÈeÍhÔWˆMØ$Õ4ˆMà$ˆHØ ×9Ñ9¸!Ö<¸]ÓKˆMà�AŒvØ&*§h¡h¨}Ó&=Ð#à ŸI™I ažL¨Ó7ˆMØ$Õ4ˆMà�AŽvØ -Õ C’ñ# ð& Ðr<   )r   ré   rå   rè   rê   rç   r—   r˜   rš   s   @@r9   rà   rà     s   ù‡ € ÷
ó 
÷÷ ò r<   rà   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLongcatFlashModeliI  c                ó    <€ V ^8„  d   QhRS[ /# rH   rJ   )r7   r8   s   "€r9   r:   ÚLongcatFlashModel.__annotate__J  s   ø€ ÷ Dñ D™Yñ Dr<   c                óv  <€ \         SV `  4        VP                  V n        \        P                  ! VP
                  VP                  4      V n        \        VP                  4       Uu. uF  p\        V4      NK  	  upV n
        \        P                  ! VP                  VP                  4      V n        R # u upi r¨   )rU   rV   r!   rY   Ú	Embeddingr"   r   Úembed_tokensræ   rà   Úlayersr]   r+   Únorm)ri   rI   Úidxrl   s   && €r9   rV   ÚLongcatFlashModel.__init__J  s�   ø€ Ü‰ÑÔØŸ/™/ˆŒÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔÜAFÀtÇÁÔAWÓXÑAW¸#Ô/°Ö5ÑAWÑXˆŒÜ—J’J˜t×/Ñ/°×1BÑ1BÓCˆŽ	ùò Ys   Á(B6c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# )r   rp   rr   rs   rt   )r7   r8   s   "€r9   r:   rõ   Q  s5   ø€ ÷ ñ á�8‰8ðñ ™�}ðñ 
�‰ñ	r<   c                óþ   € V P                  V4      pVf   R.V P                  ,          p\        W2^ ,          ^ ,          RR7      p\        V P                  V4       F  w  rVV! W4VR7      pK  	  V P                  V4      # )NT)Úreturn_array)rr   r—   )rø   r!   r   Úziprù   rú   )ri   rp   rr   Úhrq   ÚlayerÚcs   &&&    r9   r•   ÚLongcatFlashModel.__call__Q  sp   € ð
 ×Ñ˜aÓ ˆàŠ=Ø!�N T§_¡_Õ4ˆEä$ Q¨a­°­À$ÔGˆä˜DŸK™K¨Ö/‰HˆEÙ�a QÔ'ŠAñ 0ð �y‰y˜‹|Ðr<   )rø   rù   rú   r!   r¨   r˜   rš   s   @@r9   ró   ró   I  s    ù‡ € ÷Dó D÷÷ ò r<   ró   c                   ó¦   a a€ ] tR tRt oV3R lV 3R lltRV3R lR llt]R 4       t]R 4       t]R 4       t	R	 t
R
 tRV3R lR lltRtVtV ;t# )ÚModelic  c                ó    <€ V ^8„  d   QhRS[ /# rH   rJ   )r7   r8   s   "€r9   r:   ÚModel.__annotate__d  s   ø€ ÷ Pñ P™Yñ Pr<   c                óÖ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# r¡   )rU   rV   rI   r   ró   ÚmodelrY   rZ   r   r"   Úlm_headr´   s   &&€r9   rV   ÚModel.__init__d  sH   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ& tÓ,ˆŒ
Ü—y’y ×!1Ñ!1°4·?±?ÈÔOˆŽr<   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r   Úinputsrr   rt   )r7   r8   s   "€r9   r:   r  k  s'   ø€ ÷ !ñ !á—‘ð!ñ ™�}ñ!r<   c                óF   € V P                  W4      pV P                  V4      # r¨   )r
  r  )ri   r  rr   Úouts   &&& r9   r•   ÚModel.__call__k  s!   € ð
 �j‰j˜Ó'ˆØ�|‰|˜CÓ Ð r<   c                ó.   € V P                   P                  # r¨   )r
  rù   )ri   s   &r9   rù   ÚModel.layerss  s   € à�z‰z× Ñ Ð r<   c                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )r±   Ú
group_sizeÚbitsT)Úendswith)ÚpathrŠ   s   &&r9   Ú	predicateÚ(Model.quant_predicate.<locals>.predicatey  s#   € Ø�}‰}˜\×*Ò*Ø$ b¨&°!Ð4Ð4Ùr<   r=   ©ri   r  s   & r9   Úquant_predicateÚModel.quant_predicatew  s   € ò	ð
 Ðr<   c                ó   € R  pV# )c                 ó   € R V 9  # )r³   r=   )r’   s   &r9   r  Ú'Model.cast_predicate.<locals>.predicate‚  s   € Ø,°AÑ5Ð5r<   r=   r  s   & r9   Úcast_predicateÚModel.cast_predicate€  s   € ò	6ð Ðr<   c                ó€  € \        V P                  P                  4       F•  pR V 2pR F‡  w  rER F|  pV RV RV 2V9   g   K  \        V P                  P                  4       Uu. uF  pVP	                  V RV RV RV 24      NK!  	  pp\
        P                  ! V4      W RV RV 2&   K~  	  K‰  	  K—  	  \        V P                  P                  4       EF#  p\        ^4       EF  p	R V RV	 2pV R2p
W¡9   g   K  V P                  P                  pV P                  P                  V P                  P                  ,           pV R2V9   pVP	                  V
4      pV'       d•   V P                  P                  pVP	                  V R24      pVP	                  V R24      pVP                  R,          ^ ,          V,          pVVP                  R,          ,          p\
        P                  ! VVVVVR	7      pVP                  W¼R4      p\
        P                  ! VR
RV P                  P                  1R
3,          P                  RR4      4      p\
        P                  ! VR
V P                  P                  R1R
3,          4      pV'       dW   \
        P                   ! VXXR	7      w  ppp\
        P                   ! VVVR	7      w  pppVW R2&   VW R2&   VW R2&   VW R2&   VW R2&   VW R2&   EK  	  EK&  	  / pVP#                  4        F#  w  rnVP%                  R4      '       d   K  VVV&   K%  	  V# u upi )zmodel.layers.z.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.z.self_attn.z.kv_b_proj.weightz.kv_b_proj.scalesz.kv_b_proj.biases)r  r  :NNNNz.embed_q.scalesz.embed_q.biasesz.unembed_out.scalesz.unembed_out.biasesz.embed_q.weightz.unembed_out.weightz	model.mtp))Úw1r¢   )Úw2r¤   )Úw3r£   )ÚweightÚscalesÚbiasesr{   r|   )ræ   rI   r!   r   Úpopru   Ústackr$   r(   r)   r%   r}   Ú
dequantizer~   Ú
contiguousrƒ   ÚquantizeÚitemsÚ
startswith)ri   ÚweightsÚlÚprefixÚnÚmr’   ÚeÚto_joinrï   Úkv_b_keyÚ	num_headsÚhead_dimÚ	quantizedr“   rQ   r*  r+  r  r  ÚwkÚwvÚwk_sÚwk_bÚwv_sÚwv_bÚnew_weightss   &&                         r9   ÚsanitizeÚModel.sanitize‡  s?  € Ü�t—y‘y×+Ñ+Ö,ˆAØ$ Q CÐ(ˆFÛU‘�Û7�AØ ˜ °°°1°Q°CÐ8¸GÖCô &+¨4¯9©9×+EÑ+EÔ%Fó#á%F ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÖLÙ%Fð  ð #ô GIÇhÂhÈwÓFW˜ (Ð*:¸1¸#¸Q¸q¸cÐ BÓCó 8ó Vñ -ô �t—y‘y×+Ñ+×,ˆAÜ˜1—X�Ø(¨¨¨;°q°cÐ:�Ø$˜XÐ%6Ð7�ØÖ&Ø $§	¡	× =Ñ =�IØ#Ÿy™y×9Ñ9¸D¿I¹I×<PÑ<PÕP�HØ#) (Ð*;Ð <ÀÑ G�IØŸ™ HÓ-�Aç Ø#Ÿy™y×5Ñ5˜Ø!(§¡°¨xÐ7HÐ-IÓ!J˜Ø!(§¡°¨xÐ7HÐ-IÓ!J˜Ø !§¡¨¥¨bÕ 0°TÕ9˜Ø%)¨V¯\©\¸"Õ-=Õ%=˜
ÜŸMšMØ˜v v°DÀZô˜ð Ÿ	™	 )°rÓ:�AÜŸšØ˜!Ð9˜tŸy™y×9Ñ9Ð9¸1Ð<Õ=×FÑFÀrÈ2ÓNó�Bô Ÿš q¨¨D¯I©I×,FÑ,FÑ,HÈ!Ð)KÕ'LÓM�Bç Ü)+¯ªØ T°jô*™˜˜D $ô *,¯ªØ T°jô*™˜˜D $ð ?C˜ (¨/Ð :Ñ;Ø>B˜ (¨/Ð :Ñ;ØBF˜ (Ð*=Ð >Ñ?ØBF˜ (Ð*=Ð >Ñ?à:<�G˜h oÐ6Ñ7Ø>@�G˜hÐ&9Ð:Ô;ôM ñ -ðR ˆØ—M‘M–O‰DˆAØ�|‰|˜K×(Ò(ÙØˆK˜‹Nñ $ð Ðùòi#s   Á&%L;
c                óŒ   € V P                   P                   Uu. uF  p\        \        4       \        4       4      NK!  	  up# u upi r¨   )r
  rù   r   r   )ri   rŠ   s   & r9   Ú
make_cacheÚModel.make_cacheÃ  s1   € Ø9=¿¹×9JÒ9JÓKÑ9J°A”	œ'›)¤W£YÖ/Ñ9JÑKÐKùÒKs   ™%Ac                óX   <€ V ^8„  d   QhRS[ S[P                  P                  ,          /# )r   rÕ   )r   ru   r×   ÚGroup)r7   r8   s   "€r9   r:   r  Æ  s&   ø€ ÷ 'Yñ 'Y™8¡B§N¡N×$8Ñ$8Õ9ñ 'Yr<   c                ó¬  a	a
€ T;'       g    \         P                  P                  4       pVP                  4       pVP	                  4       pV P
                  P                   EFî  pVP                   FÜ  pVP                  f   \        VP                  RVR7      Vn
        M\        VP                  RVR7      Vn        \        VP                  RVR7      Vn        V;P                  V,          un        VP                  pW6,          o
S
V,           o	V	V
3R lpVP                  P                  V4       VP                   P                  V4       KÞ  	  VP"                   FZ  p\        VP$                  RVR7      Vn        \        VP&                  RVR7      Vn        \        VP(                  RVR7      Vn        K\  	  WP*                  n        \/        VP*                  P0                  P$                  RVR7       \/        VP*                  P0                  P&                  RVR7       \/        VP*                  P0                  P(                  RVR7       EKñ  	  R # )Nzall-to-shardedrÔ   zsharded-to-allc                 ó   <€ V SS # r¨   r=   )ÚwÚehÚshs   &€€r9   Úshard_headsÚ Model.shard.<locals>.shard_headsÛ  s   ø€ Ø˜R ˜8�Or<   )ru   r×   ÚinitÚsizeÚrankr
  rù   rç   r&   r	   r[   r_   rd   r$   rb   Úapplyrc   rè   r¢   r£   r¤   rå   rÑ   r   rÏ   )ri   rÕ   ÚNrU  r  Úattnr;  rQ  rå   rO  rP  s   &&       @@r9   ÚshardÚModel.shardÆ  sÇ  ù€ Ø×.Ð.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ�z‰z‹|ˆà—Z‘Z×&Õ&ˆEØŸœ�Ø×#Ñ#Ò+Ü".ØŸ™Ð%5¸Uô#�D•Kô %1ØŸ™Ð'7¸uô%�D”Mô +¨4¯;©;Ð8HÐPUÔV�”Ø×(Ò(¨QÕ.Õ(Ø ×4Ñ4�	ØÕ%�Ø˜)•^�ö$ð —‘×"Ñ" ;Ô/Ø× Ñ ×&Ñ& {Ö3ñ' (ð* —z”z�Ü ,Ø—M‘MÐ#3¸5ô!�”ô +¨3¯;©;Ð8HÐPUÔV�”Ü ,Ø—M‘MÐ#3¸5ô!�–ñ "ð (-�I‰IÔ$Ü˜%Ÿ)™)×.Ñ.×8Ñ8Ð:JÐRWÕXÜ˜%Ÿ)™)×.Ñ.×6Ñ6Ð8HÐPUÕVÜ˜%Ÿ)™)×.Ñ.×8Ñ8Ð:JÐRW×XÐXóE 'r<   )rI   r  r
  r   r¨   )r>   r?   r@   rA   rV   r•   Úpropertyrù   r  r"  rE  rH  rY  rC   rD   r™   rš   s   @@r9   r  r  c  su   ù‡ € ÷Pó P÷!ò !ð ñ!ó ð!ð ñó ðð ñó ðò:òxL÷'Y÷ 'Yò 'Yr<   r  ))rf   Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcoreru   Úmlx.nnrY   Úmlx.nn.layers.distributedr   r	   r
   Úactivationsr   rR   r   r   r   rr   r   r   Úmlar   Ú
rope_utilsr   Úswitch_layersr   r   ÚModulerF   rœ   r«   rÊ   rà   ró   r  r=   r<   r9   Ú<module>rg     sÉ   ðÛ Ý !ß -Ó -å Ý ß PÑ På ß TÑ Tß %Ý Ý 'Ý $ð ô(�ó (ó ð(ô<C#�b—i‘iô C#ôL
J�b—i‘iô 
Jô!8˜RŸY™Yô !8ôH.�b—i‘iô .ôb.˜rŸy™yô .ôb˜Ÿ	™	ô ô4JYˆB�I‰Iö JYr<   