Ë
    îÿæi‹D  ã                   óö  — d dl Z d dlmZ d dlmZmZmZmZ d dlm	Z
 d dlmZ d dlmZmZmZ ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ e G d„ de«      «       Z  G d„ dejB                  «      Z" G d„ dejB                  «      Z# G d„ dejB                  «      Z$ G d„ dejB                  «      Z% G d„ dejB                  «      Z& G d„ dejB                  «      Z' G d„ dejB                  «      Z(y)é    N)Ú	dataclass)ÚAnyÚDictÚOptionalÚTuple)Úshard_inplaceÚshard_linearÚsum_gradientsé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	CacheListÚKVCache)ÚMultiLinear)Úinitialize_rope)Ú	SwitchGLUc                   ó.  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed<   dZ	eed<   dZ
ee   ed<   y)Ú	ModelArgsÚ
model_typeÚattention_methodÚzero_expert_typeÚhidden_sizeÚffn_hidden_sizeÚmoe_topkÚexpert_ffn_hidden_sizeÚn_routed_expertsÚzero_expert_numÚ
num_layersÚ
vocab_sizeÚmax_position_embeddingsÚnum_attention_headsÚkv_lora_rankÚq_lora_rankÚqk_rope_head_dimÚqk_nope_head_dimÚ
v_head_dimÚrouted_scaling_factorÚrms_norm_epsÚ
rope_thetaÚmla_scale_q_loraÚmla_scale_kv_loraÚattention_biasFÚnorm_topk_probÚrouter_biasNÚrope_scaling)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatÚboolr/   r0   r1   r   r   © ó    úp/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/longcat_flash.pyr   r      s¯   … àƒOØÓØÓØÓØÓØƒMØÓØÓØÓØƒOØƒOØ Ó ØÓØÓØÓØÓØÓØƒOØ Ó ØÓØÓØÓØÓØÓØ €N�DÓ Ø€K�ÓØ#'€L�(˜4‘.Ô'r;   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚLongcatFlashMLAÚargsc                 ó^  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  |j                  z   | _        | j                  dz  | _	        | j                  €?t        j                  |j                  | j                  | j                  z  d¬«      | _        n�t        j                  |j                  | j                  |j                  ¬«      | _        t        j                   | j                  «      | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  |j                  | j
                  | j                  z   |j                  ¬«      | _        t        j                   | j
                  «      | _        t+        | j                  | j
                  | j                  «      | _        t+        | j
                  | j                  | j                  «      | _        t        j                  | j                  |j                  z  |j                  |j                  ¬«      | _        |j2                  r!|j                  | j                  z  dz  | _        |j4                  r!|j                  | j
                  z  dz  | _        |j6                  �g|j6                  j9                  dd«      }|rI|j6                  d   }|dkD  r5d	|z  t;        j<                  |«      z  d
z   }| j                  |z  |z  | _	        t?        | j                  |j@                  d|j6                  |jB                  ¬«      | _"        y )Ng      à¿F©Úbiasg      à?Úmscale_all_dimr   Úfactorr   gš™™™™™¹?g      ð?T)ÚdimsÚbaseÚtraditionalÚscaling_configr"   )#ÚsuperÚ__init__r#   r&   r'   r$   r%   r(   Úqk_head_dimÚscaleÚnnÚLinearr   Úq_projr.   Úq_a_projÚRMSNormÚq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormr   Úembed_qÚunembed_outÚo_projr,   r-   r1   ÚgetÚmathÚlogr   r+   r"   Úrope)Úselfr?   rC   Úscaling_factorÚsÚ	__class__s        €r<   rJ   zLongcatFlashMLA.__init__1   sÿ  ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø $× 5Ñ 5ˆÔØ $× 5Ñ 5ˆÔØ ×-Ñ-ˆÔØ×+Ñ+ˆÔØŸ/™/ˆŒà×0Ñ0°4×3HÑ3HÑHˆÔØ×%Ñ% tÑ+ˆŒ
à×ÑÐ#ÜŸ)™)Ø× Ñ Ø×(Ñ(¨4×+;Ñ+;Ñ;ØôˆD�Kô ŸI™IØ× Ñ  $×"2Ñ"2¸×9LÑ9LôˆDŒMô "$§¡¨D×,<Ñ,<Ó!=ˆDÔÜŸI™IØ× Ñ Ø×(Ñ(¨4×+;Ñ+;Ñ;ØôˆDŒMô #%§)¡)Ø×ÑØ×Ñ × 5Ñ 5Ñ5Ø×$Ñ$ô#
ˆÔô
 !Ÿj™j¨×):Ñ):Ó;ˆÔÜ"Ø×!Ñ! 4×#4Ñ#4°d×6NÑ6Nó
ˆŒô 'Ø×Ñ˜tŸ™°×0HÑ0Hó
ˆÔô —i‘iØ×$Ñ$ t§¡Ñ6Ø×ÑØ×$Ñ$ô
ˆŒð × Ò Ø%)×%5Ñ%5¸×8HÑ8HÑ%HÈSÑ$PˆDÔ!Ø×!Ò!Ø&*×&6Ñ&6¸×9JÑ9JÑ&JÈsÑ%RˆDÔ"à×ÑÐ(Ø!×.Ñ.×2Ñ2Ð3CÀQÓGˆNÙØ!%×!2Ñ!2°8Ñ!<�Ø! AÒ%Ø˜nÑ,¬t¯x©x¸Ó/GÑGÈ#ÑM�AØ!%§¡¨a¡°!Ñ!3�D”Jä#Ø×&Ñ&Ø—‘ØØ×,Ñ,Ø$(×$@Ñ$@ô
ˆ�	r;   ÚxÚmaskÚcacheÚreturnc           	      óè  — |j                   \  }}}| j                  €| j                  |«      }n/| j                  | j	                  | j                  |«      «      «      }|j                  ||| j                  | j                  «      j                  dddd«      }| j                  �|| j                  z  }t        j                  || j                  gd¬«      \  }}	| j                  |«      }
t        j                  |
| j                  gd¬«      \  }
}|j                  ||d| j                   «      j                  dddd«      }| j#                  |
«      }| j$                  �|| j$                  z  }|�|j&                  nd}| j)                  |	|«      }	| j)                  ||«      }t        j*                  |d¬«      }|�|j-                  ||«      \  }}|	| j.                  z  |j1                  dd«      z  }|�\t        j2                  ||t        j4                  t        j6                  |j8                  «      j:                  |j8                  «      «      }|dk(  r| j=                  |«      }|x}}n$| j=                  |d¬	«      }| j?                  |«      }tA        ||||| j.                  |¬
«      }|dk(  r| j?                  |«      }|j                  dddd«      j                  ||d«      }| jC                  |«      S )Nr   é   r   é   éÿÿÿÿ©ÚaxiséþÿÿÿF)Ú	transpose)rc   rL   rb   )"Úshaper%   rO   rS   rR   rP   Úreshaper#   rK   rl   r,   ÚmxÚsplitr'   rT   r$   r&   rU   r-   Úoffsetr\   Úexpand_dimsÚupdate_and_fetchrL   ÚswapaxesÚwhereÚarrayÚfinfoÚdtypeÚminrV   rW   r   rX   )r]   ra   rb   rc   ÚBÚLÚ_ÚqÚq_nopeÚq_peÚcompressed_kvÚk_peÚ	kv_latentrq   Ú	pe_scoresÚkÚvÚoutputs                     r<   Ú__call__zLongcatFlashMLA.__call__v   s«  € ð —'‘'‰ˆˆ1ˆaà×ÑÐ#Ø—‘˜A“‰Aà—‘˜d×0Ñ0°·±¸qÓ1AÓBÓCˆAà�I‰I�a˜˜D×4Ñ4°d×6FÑ6FÓG×QÑQØˆq�!�Qó
ˆð × Ñ Ð,Ø�D×)Ñ)Ñ)ˆAä—x‘x  D×$9Ñ$9Ð#:ÀÔD‰ˆ�à×/Ñ/°Ó2ˆÜ Ÿh™h }°t×7HÑ7HÐ6IÐPRÔSÑˆ�tØ�|‰|˜A˜q ! T×%:Ñ%:Ó;×EÑEÀaÈÈAÈqÓQˆØ×'Ñ'¨Ó6ˆ	à×!Ñ!Ð-Ø! D×$:Ñ$:Ñ:ˆIà!&Ð!2�—’¸ˆØ�y‰y˜˜vÓ&ˆØ�y‰y˜˜vÓ&ˆä—N‘N 9°1Ô5ˆ	àÐØ#×4Ñ4°YÀÓE‰OˆI�tà˜DŸJ™JÑ&¨$¯-©-¸¸BÓ*?Ñ?ˆ	ØÐÜŸ™ØØÜ—‘œŸ™ )§/¡/Ó2×6Ñ6¸	¿¹ÓHóˆIð �Š6Ø—\‘\ &Ó)ˆFØÐˆA‘à—‘˜Y°%�Ó8ˆAØ× Ñ  Ó+ˆAä-Ø�A�q ¨T¯Z©Z¸iô
ˆð �Š6Ø×%Ñ% fÓ-ˆFà×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r;   ©NN©r2   r3   r4   r   rJ   ro   rv   r   r   r‡   Ú__classcell__©r`   s   @r<   r>   r>   0   sZ   ø„ ðC
˜Yõ C
ðP $(Ø#ñ	=#à�8‰8ð=#ð �r—x‘xÑ ð=#ð ˜‰}ð	=#ð
 
�‰÷=#r;   r>   c                   ób   ‡ — e Zd Zddedefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚLongcatFlashMLPr?   Ú	is_expertc                 ó@  •— t         ‰| �  «        |r|j                  n|j                  }t	        j
                  |j                  |d¬«      | _        t	        j
                  |j                  |d¬«      | _        t	        j
                  ||j                  d¬«      | _	        y ©NFrA   )
rI   rJ   r   r   rM   rN   r   Ú	gate_projÚup_projÚ	down_proj)r]   r?   rŽ   r   r`   s       €r<   rJ   zLongcatFlashMLP.__init__·   sq   ø€ Ü‰ÑÔÙ5>�d×1Ò1ÀD×DXÑDXˆäŸ™ 4×#3Ñ#3°[ÀuÔMˆŒÜ—y‘y ×!1Ñ!1°;ÀUÔKˆŒÜŸ™ ;°×0@Ñ0@ÀuÔMˆ�r;   ra   rd   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)r“   r   r‘   r’   )r]   ra   s     r<   r‡   zLongcatFlashMLP.__call__¿   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr;   )F)
r2   r3   r4   r   r9   rJ   ro   rv   r‡   rŠ   r‹   s   @r<   r�   r�   ¶   s7   ø„ ñN˜Yð N°4õ NðJ˜"Ÿ(™(ð J r§x¡x÷ Jr;   r�   c                   óz   ‡ — e Zd Zdefˆ fd„Zdej                  deej                  ej                  f   fd„Zˆ xZ	S )ÚLongcatFlashTopkRouterr?   c                 ó´  •— t         ‰| �  «        || _        |j                  | _        |j
                  |j                  z   | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  |j                  | j
                  | j                  ¬«      | _        t        j                  | j
                  f«      | _        y )NrA   )rI   rJ   Úconfigr   Útop_kr   r   r)   r/   r0   rM   rN   r   Ú
classifierro   ÚzerosÚe_score_correction_bias©r]   r?   r`   s     €r<   rJ   zLongcatFlashTopkRouter.__init__Ä   s¨   ø€ Ü‰ÑÔØˆŒØ—]‘]ˆŒ
Ø $× 5Ñ 5¸×8LÑ8LÑ LˆÔØ%)×%?Ñ%?ˆÔ"Ø"×1Ñ1ˆÔØ×+Ñ+ˆÔäŸ)™)Ø×Ñ˜d×3Ñ3¸$×:JÑ:Jô
ˆŒô (*§x¡x°×1FÑ1FÐ0HÓ'IˆÕ$r;   Úhidden_statesrd   c                 ó¾  — |j                   }| j                  |«      }t        j                  |d¬«      }|| j                  z   }t        j
                  || j                   d¬«      d| j                   d …f   }t        j                  ||d¬«      }| j                  r t        j                  |dd¬«      dz   }||z  }|| j                  z  }||j                  |«      fS )Nrh   ri   )Úkthrj   .T©rj   Úkeepdimsg#B’¡œÇ;)rx   r›   ro   Úsoftmaxr�   Úargpartitionrš   Útake_along_axisr/   Úsumr)   Úastype)	r]   rŸ   rx   Úrouter_logitsÚscoresÚcorrected_scoresÚtopk_indicesÚtopk_weightsÚdenominators	            r<   r‡   zLongcatFlashTopkRouter.__call__Ò   sÐ   € à×#Ñ#ˆØŸ™¨Ó6ˆÜ—‘˜M°Ô3ˆà! D×$@Ñ$@Ñ@ÐÜ—‘Ð'7¸d¿j¹j¸[ÈrÔRØ�$—*‘*�‘Ðñ
ˆô ×)Ñ)¨&°,ÀRÔHˆà×ÒÜŸ&™& °BÀÔFÈÑNˆKØ'¨+Ñ5ˆLà# d×&@Ñ&@Ñ@ˆà˜\×0Ñ0°Ó7Ð7Ð7r;   )
r2   r3   r4   r   rJ   ro   rv   r   r‡   rŠ   r‹   s   @r<   r—   r—   Ã   s:   ø„ ðJ˜Yõ Jð8 b§h¡hð 8°5¸¿¹À2Ç8Á8Ð9KÑ3L÷ 8r;   r—   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongcatFlashMoEr?   c                 óF  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        t        |j                  |j                  |j
                  «      | _        t        |«      | _        d | _        y r•   )rI   rJ   r™   r   Únum_experts_per_tokr   r   r   r   r   r   Ú
switch_mlpr—   ÚrouterÚsharding_grouprž   s     €r<   rJ   zLongcatFlashMoE.__init__è   sˆ   ø€ Ü‰ÑÔØˆŒØ#'§=¡=ˆÔ Ø $× 5Ñ 5ˆÔØ#×3Ñ3ˆÔØ $× 5Ñ 5ˆÔä#Ø×ÑØ×'Ñ'Ø×!Ñ!ó
ˆŒô -¨TÓ2ˆŒØ"ˆÕr;   c                 óR  — | j                   � t        | j                   «      |«      }| j                  |«      \  }}|| j                  k\  }t	        j
                  |d|«      }t	        j
                  |d|«      }| j                  ||«      }||d   z  }t	        j                  |d¬«      }| j                   �+t        j                  j                  || j                   ¬«      }| j                  dk(  sJ ‚t	        j                  t	        j
                  ||d«      dd	¬
«      }	|||	z  z   }|S )Nr   g        ).Nrk   ri   ©ÚgroupÚidentityrh   Tr¢   )rµ   r
   r´   r   ro   ru   r³   r§   ÚdistributedÚall_sumr   )
r]   rŸ   r¬   r­   rb   Úregular_weightsÚregular_outputsÚweighted_outputsÚfinal_outputÚidentity_weights_sums
             r<   r‡   zLongcatFlashMoE.__call__ù   s   € Ø×ÑÐ*Ø>œM¨$×*=Ñ*=Ó>¸}ÓMˆMà%)§[¡[°Ó%?Ñ"ˆ�lð ˜t×4Ñ4Ñ4ˆÜ—x‘x  a¨Ó6ˆÜŸ(™( 4¨¨lÓ;ˆàŸ/™/¨-¸ÓFˆà*¨_¸YÑ-GÑGÐÜ—v‘vÐ.°RÔ8ˆà×ÑÐ*ÜŸ>™>×1Ñ1Ø D×$7Ñ$7ð 2ó ˆLð
 ×$Ñ$¨
Ò2Ð2Ð2Ü!Ÿv™vÜ�H‰H�T˜<¨Ó-°BÀô 
Ðð $ mÐ6JÑ&JÑJˆàÐr;   )r2   r3   r4   r   rJ   r‡   rŠ   r‹   s   @r<   r°   r°   ç   s   ø„ ð#˜Yõ #ö"r;   r°   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚLongcatFlashDecoderLayerr?   c                 óF  •— t         ‰| �  «        |j                  | _        t        |«      | _        t        d«      D �cg c]  }t        |«      ‘Œ c}| _        t        d«      D �cg c]  }t        |d«      ‘Œ c}| _	        t        d«      D �cg c]-  }t        j                  |j                  |j                  ¬«      ‘Œ/ c}| _        t        d«      D �cg c]-  }t        j                  |j                  |j                  ¬«      ‘Œ/ c}| _        y c c}w c c}w c c}w c c}w )Nrf   F)Úeps)rI   rJ   r   r°   ÚmlpÚranger>   Ú	self_attnr�   ÚmlpsrM   rQ   r*   Úinput_layernormÚpost_attention_layernorm)r]   r?   r|   r`   s      €r<   rJ   z!LongcatFlashDecoderLayer.__init__  sí   ø€ Ü‰ÑÔØ×+Ñ+ˆÔÜ" 4Ó(ˆŒä9>¸q¼ÓB¹°Aœ/¨$Õ/¸ÑBˆŒÜ;@À¼8ÓD¹8°a”_ T¨5Õ1¸8ÑDˆŒ	äINÈqÌó 
ÙIQÀAŒB�J‰J�t×'Ñ'¨T×->Ñ->Ö?Èñ 
ˆÔô JOÈqÌó)
ÙIQÀAŒB�J‰J�t×'Ñ'¨T×->Ñ->Ö?Èñ)
ˆÕ%ùò CùÚDùò 
ùò)
s   ¾DÁ%DÂ2DÃ2Dra   rb   rc   rd   c                 óH  — |}d }|€d}t        d«      D ]‹  }|} | j                  |   |«      } | j                  |   ||||   ¬«      }||z   }|} | j                  |   |«      }|dk(  r| j	                  |«      } | j
                  |   |«      }||z   }|dk(  sŒ‡||z   }Œ� |S )Nrˆ   rf   )rb   rc   r   r   )rÆ   rÉ   rÇ   rÊ   rÅ   rÈ   )r]   ra   rb   rc   rŸ   Úshortcut_mlp_outputÚiÚresiduals           r<   r‡   z!LongcatFlashDecoderLayer.__call__'  sÕ   € ð ˆØ"Ðàˆ=Ø ˆEä�q–ˆAØ$ˆHà3˜D×0Ñ0°Ñ3°MÓBˆMØ-˜DŸN™N¨1Ñ-¨mÀ$ÈeÐTUÉhÔWˆMØ$ }Ñ4ˆMà$ˆHØ<˜D×9Ñ9¸!Ñ<¸]ÓKˆMà�AŠvØ&*§h¡h¨}Ó&=Ð#à(˜DŸI™I a™L¨Ó7ˆMØ$ }Ñ4ˆMà�A‹vØ -Ð0CÑ C‘ð# ð& Ðr;   rˆ   r‰   r‹   s   @r<   rÂ   rÂ     sW   ø„ ð
˜Yõ 
ð" $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r;   rÂ   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	ˆ xZ
S )ÚLongcatFlashModelr?   c                 ól  •— t         ‰| �  «        |j                  | _        t        j                  |j
                  |j                  «      | _        t        |j                  «      D �cg c]  }t        |«      ‘Œ c}| _
        t        j                  |j                  |j                  «      | _        y c c}w r•   )rI   rJ   r    rM   Ú	Embeddingr!   r   Úembed_tokensrÆ   rÂ   ÚlayersrQ   r*   Únorm)r]   r?   Úidxr`   s      €r<   rJ   zLongcatFlashModel.__init__J  s�   ø€ Ü‰ÑÔØŸ/™/ˆŒÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔÜAFÀtÇÁÔAWÓXÑAW¸#Ô/°Õ5ÐAWÑXˆŒÜ—J‘J˜t×/Ñ/°×1BÑ1BÓCˆ�	ùò Ys   Á'B1ra   rc   rd   c                 óä   — | j                  |«      }|€dg| j                  z  }t        ||d   d   d¬«      }t        | j                  |«      D ]  \  }} ||||¬«      }Œ | j                  |«      S )Nrˆ   r   T)Úreturn_array)rc   )rÓ   r    r   ÚziprÔ   rÕ   )r]   ra   rc   Úhrb   ÚlayerÚcs          r<   r‡   zLongcatFlashModel.__call__Q  sv   € ð
 ×Ñ˜aÓ ˆàˆ=Ø!�N T§_¡_Ñ4ˆEä$ Q¨¨a©°©À$ÔGˆä˜DŸK™K¨Ö/‰HˆE�1Ù�a˜ QÔ'‰Að 0ð �y‰y˜‹|Ðr;   r•   r‰   r‹   s   @r<   rÐ   rÐ   I  sC   ø„ ðD˜Yõ Dð  $ñà�8‰8ðð ˜‰}ðð 
�‰÷	r;   rÐ   c                   óÊ   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   fd„Z	e
d„ «       Ze
d„ «       Ze
d„ «       Zd	„ Zd
„ Zddeej                   j"                     fd„Zˆ xZS )ÚModelr?   c                 óÔ   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        y r�   )rI   rJ   r?   r   rÐ   ÚmodelrM   rN   r   r!   Úlm_headrž   s     €r<   rJ   zModel.__init__d  sJ   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ& tÓ,ˆŒ
Ü—y‘y ×!1Ñ!1°4·?±?ÈÔOˆ�r;   Úinputsrc   c                 óH   — | j                  ||«      }| j                  |«      S r•   )rà   rá   )r]   râ   rc   Úouts       r<   r‡   zModel.__call__k  s#   € ð
 �j‰j˜ Ó'ˆØ�|‰|˜CÓ Ð r;   c                 ó.   — | j                   j                  S r•   )rà   rÔ   )r]   s    r<   rÔ   zModel.layerss  s   € à�z‰z× Ñ Ð r;   c                 ó   — d„ }|S )Nc                 ó0   — | j                  d«      rdddœS y)Nr›   é@   é   )Ú
group_sizeÚbitsT)Úendswith)Úpathr|   s     r<   Ú	predicatez(Model.quant_predicate.<locals>.predicatey  s   € Ø�}‰}˜\Ô*Ø&(°!Ñ4Ð4Ør;   r:   ©r]   rî   s     r<   Úquant_predicatezModel.quant_predicatew  s   € ò	ð
 Ðr;   c                 ó   — d„ }|S )Nc                 ó
   — d| vS )Nr�   r:   )r„   s    r<   rî   z'Model.cast_predicate.<locals>.predicate‚  s   € Ø,°AÐ5Ð5r;   r:   rï   s     r<   Úcast_predicatezModel.cast_predicate€  s   € ò	6ð Ðr;   c                 ó  — t        | j                  j                  «      D ]‹  }d|› �}dD ]  \  }}dD ]u  }|› d|› d|› �|v sŒt        | j                  j                  «      D �cg c]  }|j	                  |› d|› d|› d|› �«      ‘Œ  }}t        j                  |«      ||› d|› d|› �<   Œw Œ� Œ� t        | j                  j                  «      D �]ú  }t        d«      D �]è  }	d|› d	|	› �}|› d
�}
|
|v sŒ| j                  j                  }| j                  j                  | j                  j                  z   }|› d�|v }|j	                  |
«      }|r| j                  j                  }|j	                  |› d�«      }|j	                  |› d�«      }|j                  d   dz  |z  }||j                  d   z  }t        j                  |||||¬«      }|j                  ||d«      }t        j                  |d d …d | j                  j                  …d d …f   j                  dd«      «      }t        j                  |d d …| j                  j                  d …d d …f   «      }|rXt        j                   |¬«      \  }}}t        j                   |||¬«      \  }}}|||› d�<   |||› d�<   |||› d�<   |||› d�<   |||› d�<   |||› d�<   �Œë �Œý i }|j#                  «       D ]  \  }}|j%                  d«      rŒ|||<   Œ |S c c}w )Nzmodel.layers.))Úw1r‘   )Úw2r“   )Úw3r’   )ÚweightÚscalesÚbiasesz.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.rf   z.self_attn.z.kv_b_proj.weightz.kv_b_proj.scalesz.kv_b_proj.biasesrh   é    )rë   rê   rk   z.embed_q.scalesz.embed_q.biasesz.unembed_out.scalesz.unembed_out.biasesz.embed_q.weightz.unembed_out.weightz	model.mtp)rÆ   r?   r    r   Úpopro   Ústackr#   r'   r(   r$   rm   Ú
dequantizern   Ú
contiguousrt   ÚquantizeÚitemsÚ
startswith)r]   ÚweightsÚlÚprefixÚnÚmr„   ÚeÚto_joinrÍ   Úkv_b_keyÚ	num_headsÚhead_dimÚ	quantizedr…   rE   rù   rú   rë   rê   ÚwkÚwvÚwk_sÚwk_bÚwv_sÚwv_bÚnew_weightss                              r<   ÚsanitizezModel.sanitize‡  sR  € Ü�t—y‘y×+Ñ+Ö,ˆAØ$ Q CÐ(ˆFÛU‘��1Û7�AØ ˜ °°°1°Q°CÐ8¸GÒCô &+¨4¯9©9×+EÑ+EÔ%Fó#á%F ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÕLØ%Fð  ð #ô GIÇhÁhÈwÓFW˜ 6 (Ð*:¸1¸#¸Q¸q¸cÐ BÒCñ 8ñ Vð -ô �t—y‘y×+Ñ+×,ˆAÜ˜1—X�Ø(¨¨¨;°q°cÐ:�Ø$˜XÐ%6Ð7�Ø˜wÒ&Ø $§	¡	× =Ñ =�IØ#Ÿy™y×9Ñ9¸D¿I¹I×<PÑ<PÑP�HØ#) (Ð*;Ð <ÀÐ G�IØŸ™ HÓ-�Aá Ø#Ÿy™y×5Ñ5˜Ø!(§¡°¨xÐ7HÐ-IÓ!J˜Ø!(§¡°¨xÐ7HÐ-IÓ!J˜Ø !§¡¨¡¨bÑ 0°TÑ9˜Ø%)¨V¯\©\¸"Ñ-=Ñ%=˜
ÜŸM™MØ˜v v°DÀZô˜ð Ÿ	™	 )¨X°rÓ:�AÜŸ™Øš!Ð9˜tŸy™y×9Ñ9Ð9º1Ð<Ñ=×FÑFÀrÈ2ÓNó�Bô Ÿ™ qª¨D¯I©I×,FÑ,FÑ,HÊ!Ð)KÑ'LÓM�Bá Ü)+¯©Ø T°jô*™˜˜D $ô *,¯©Ø T°jô*™˜˜D $ð ?C˜ 6 (¨/Ð :Ñ;Ø>B˜ 6 (¨/Ð :Ñ;ØBF˜ 6 (Ð*=Ð >Ñ?ØBF˜ 6 (Ð*=Ð >Ñ?à:<�G˜v˜h oÐ6Ñ7Ø>@�G˜v˜hÐ&9Ð:Ó;òM ð -ðR ˆØ—M‘M–O‰DˆAˆqØ�|‰|˜KÔ(ØØˆK˜ŠNð $ð Ðùòi#s   Á##K>
c                 óˆ   — | j                   j                  D �cg c]  }t        t        «       t        «       «      ‘Œ  c}S c c}w r•   )rà   rÔ   r   r   )r]   r|   s     r<   Ú
make_cachezModel.make_cacheÃ  s1   € Ø9=¿¹×9JÒ9JÓKÑ9J°A”	œ'›)¤W£YÕ/Ð9JÑKÐKùÒKs   ™#?r¸   c                 óx  ‡	‡
— |xs t         j                  j                  «       }|j                  «       }|j	                  «       }| j
                  j                  D �]Ü  }|j                  D ]Í  }|j                  €t        |j                  d|¬«      |_
        nt        |j                  d|¬«      |_        t        |j                  d|¬«      |_        |xj                  |z  c_        |j                  }||z  Š
‰
|z   Š	ˆ	ˆ
fd„}|j                  j                  |«       |j                   j                  |«       ŒÏ |j"                  D ]Y  }t        |j$                  d|¬«      |_        t        |j&                  d|¬«      |_        t        |j(                  d|¬«      |_        Œ[ ||j*                  _        t/        |j*                  j0                  j$                  d|¬«       t/        |j*                  j0                  j&                  d|¬«       t/        |j*                  j0                  j(                  d|¬«       �Œß y )Nzall-to-shardedr·   zsharded-to-allc                 ó   •— | ‰‰ S r•   r:   )ÚwÚehÚshs    €€r<   Úshard_headsz Model.shard.<locals>.shard_headsÛ  s   ø€ Ø˜R ˜8�Or;   )ro   rº   ÚinitÚsizeÚrankrà   rÔ   rÇ   r%   r	   rO   rS   rX   r#   rV   ÚapplyrW   rÈ   r‘   r’   r“   rÅ   rµ   r   r³   )r]   r¸   ÚNr!  rÛ   Úattnr  r  rÅ   r  r  s            @@r<   ÚshardzModel.shardÆ  sÅ  ù€ ØÒ.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ�z‰z‹|ˆà—Z‘Z×&Õ&ˆEØŸœ�Ø×#Ñ#Ð+Ü".ØŸ™Ð%5¸Uô#�D•Kô %1ØŸ™Ð'7¸uô%�D”Mô +¨4¯;©;Ð8HÐPUÔV�”Ø×(Ò(¨QÑ.Õ(Ø ×4Ñ4�	Ø˜IÑ%�Ø˜)‘^�õ$ð —‘×"Ñ" ;Ô/Ø× Ñ ×&Ñ& {Õ3ð' (ð* —z”z�Ü ,Ø—M‘MÐ#3¸5ô!�”ô +¨3¯;©;Ð8HÐPUÔV�”Ü ,Ø—M‘MÐ#3¸5ô!�•ð "ð (-ˆE�I‰IÔ$Ü˜%Ÿ)™)×.Ñ.×8Ñ8Ð:JÐRWÕXÜ˜%Ÿ)™)×.Ñ.×6Ñ6Ð8HÐPUÕVÜ˜%Ÿ)™)×.Ñ.×8Ñ8Ð:JÐRW×XñE 'r;   r•   )r2   r3   r4   r   rJ   ro   rv   r   r   r‡   ÚpropertyrÔ   rð   ró   r  r  rº   ÚGroupr%  rŠ   r‹   s   @r<   rÞ   rÞ   c  sš   ø„ ðP˜Yõ Pð  $ñ!à—‘ð!ð ˜‰}ó!ð ñ!ó ð!ð ñó ðð ñó ðò:òxLñ'Y˜8 B§N¡N×$8Ñ$8Ñ9÷ 'Yr;   rÞ   ))rZ   Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorero   Úmlx.nnrM   Úmlx.nn.layers.distributedr   r	   r
   Úactivationsr   rF   r   r   r   rc   r   r   Úmlar   Ú
rope_utilsr   Úswitch_layersr   r   ÚModuler>   r�   r—   r°   rÂ   rÐ   rÞ   r:   r;   r<   Ú<module>r3     sÉ   ðÛ Ý !ß -Ó -å Ý ß PÑ På ß TÑ Tß %Ý Ý 'Ý $ð ô(�ó (ó ð(ô<C#�b—i‘iô C#ôL
J�b—i‘iô 
Jô!8˜RŸY™Yô !8ôH.�b—i‘iô .ôb.˜rŸy™yô .ôb˜Ÿ	™	ô ô4JYˆB�I‰Iõ JYr;   