Ë
    îÿæi¼  ã                   óˆ  — d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZmZmZ ddlmZ ddlmZ e G d„ d	e«      «       Z G d
„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionNé   )ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Úinitialize_rope)Ú	SwitchGLUc                   óî   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZe	e
eeeef   f      ed<   dZeed<   y)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚrms_norm_epsÚ
vocab_sizeÚlogits_scalingÚattention_multiplierÚembedding_multiplierÚresidual_multiplierÚmax_position_embeddingsÚnum_key_value_headsÚattention_biasÚ
rope_thetaÚnum_local_expertsÚnum_experts_per_tokNÚrope_scalingTÚtie_word_embeddings)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatÚboolr!   r   r   r   r"   © ó    úm/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/granitemoe.pyr   r      s“   … àƒOØÓØÓØÓØÓØÓØƒOØÓØÓØÓØÓØ Ó ØÓØÓØÓØÓØÓØ;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø $Ð˜Ô$r,   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚGraniteMoeAttentionÚargsc                 ód  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|j                  | _	        |j                  }t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  ||z  ||¬«      | _        t#        | j                  |j$                  d|j&                  |j(                  «      | _        y )N©ÚbiasF)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚhead_dimr   Úscaler   ÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr   r   r!   r   Úrope)Úselfr0   Údimr6   r7   r8   r   Ú	__class__s          €r-   r5   zGraniteMoeAttention.__init__&   sÿ   ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'×#3Ñ#3°wÑ#>Ð>ˆŒ˜à×.Ñ.ˆŒ
Ø×,Ñ,ˆÜ—i‘i  W¨xÑ%7¸nÔMˆŒÜ—i‘i  Z°(Ñ%:ÀÔPˆŒÜ—i‘i  Z°(Ñ%:ÀÔPˆŒÜ—i‘i ¨(Ñ 2°C¸nÔMˆŒä#Ø�M‰MØ�O‰OØØ×ÑØ×(Ñ(ó
ˆ�	r,   ÚxÚmaskÚcacheÚreturnc                 ó2  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r   é   )Úoffset)rF   r9   rE   )Úshaper<   r=   r>   Úreshaper6   Ú	transposer7   r@   rL   Úupdate_and_fetchr   r9   r?   )rA   rD   rE   rF   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r-   Ú__call__zGraniteMoeAttention.__call__>   sg  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà—/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r,   ©NN©r#   r$   r%   r   r5   ÚmxÚarrayr   r   rX   Ú__classcell__©rC   s   @r-   r/   r/   %   sW   ø„ ð
˜Yõ 
ð6 $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r,   r/   c                   óL   ‡ — e Zd Zdededefˆ fd„Zdej                  fd„Zˆ xZS )ÚGraniteMoeTopKGatingÚ
input_sizeÚnum_expertsÚtop_kc                 ó†   •— t         ‰| �  «        || _        || _        || _        t        j                  ||d¬«      | _        y ©NFr2   )r4   r5   rb   ra   rc   r:   r;   Úlayer)rA   ra   rb   rc   rC   s       €r-   r5   zGraniteMoeTopKGating.__init__]   s8   ø€ Ü‰ÑÔØ&ˆÔØ$ˆŒØˆŒ
Ü—Y‘Y˜z¨;¸UÔCˆ�
r,   Úhidden_statesc                 ó.  — | j                  |«      }t        j                  || j                   d¬«      d| j                   d …f   }t        j                  ||d¬«      }t        j
                  |j                  t        j                  «      d¬«      }||fS )NrI   )ÚkthÚaxis.©rj   )rf   r[   Úargpartitionrc   Útake_along_axisÚsoftmaxÚastypeÚfloat32)rA   rg   ÚlogitsÚ	top_k_idxÚtop_k_logitsÚtop_k_gatess         r-   rX   zGraniteMoeTopKGating.__call__d   s}   € Ø—‘˜MÓ*ˆÜ—O‘O F°·±°À"ÔEØ�$—*‘*�‘Ðñ
ˆ	ô ×)Ñ)¨&°)À"ÔEˆÜ—j‘j ×!4Ñ!4´R·Z±ZÓ!@ÀrÔJˆØ˜+Ð%Ð%r,   )	r#   r$   r%   r(   r5   r[   r\   rX   r]   r^   s   @r-   r`   r`   \   s1   ø„ ðD 3ð D°Sð DÀõ Dð& b§h¡h÷ &r,   r`   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚGraniteMoeMoEr0   c                 ó(  •— t         ‰| �  «        |j                  | _        |j                  | _        t        | j                  | j                  |j                  «      | _        t        | j                  |j                  |j                  ¬«      | _
        y )N)ra   rb   rc   )r4   r5   r   ra   r   r   r   Ú
switch_mlpr`   r    Úrouter©rA   r0   rC   s     €r-   r5   zGraniteMoeMoE.__init__o   ss   ø€ Ü‰ÑÔà×*Ñ*ˆŒØ×1Ñ1ˆÔÜ#Ø�O‰O˜T×-Ñ-¨t×/EÑ/Eó
ˆŒô +Ø—‘Ø×.Ñ.Ø×*Ñ*ô
ˆ�r,   rD   rG   c                 ó°   — | j                  |«      \  }}| j                  ||«      }||d   z  j                  d¬«      j                  |j                  «      S )N).Néþÿÿÿrk   )ry   rx   Úsumro   Údtype)rA   rD   Ú	token_idsÚgatesÚys        r-   rX   zGraniteMoeMoE.__call__}   sQ   € ØŸ;™; q›>Ñˆ	�5Ø�O‰O˜A˜yÓ)ˆØ�E˜)Ñ$Ñ$×)Ñ)¨rÐ)Ó2×9Ñ9¸!¿'¹'ÓBÐBr,   ©	r#   r$   r%   r   r5   r[   r\   rX   r]   r^   s   @r-   rv   rv   n   s-   ø„ ð
˜Yõ 
ðC˜"Ÿ(™(ð C r§x¡x÷ Cr,   rv   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚGraniteMoeDecoderLayerr0   c                 óD  •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  ¬«      | _        |j                  | _        y )N©Úeps)r4   r5   r/   Ú	self_attnrv   Úblock_sparse_moer:   ÚRMSNormr   r   Úinput_layernormÚpost_attention_layernormr   rz   s     €r-   r5   zGraniteMoeDecoderLayer.__init__„   sx   ø€ Ü‰ÑÔÜ,¨TÓ2ˆŒÜ -¨dÓ 3ˆÔÜ!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð $(×#;Ñ#;ˆÕ r,   rD   rE   rF   rG   c                 óÒ   — | j                  | j                  |«      ||«      }||| j                  z  z   }| j                  | j	                  |«      «      }||| j                  z  z   }|S ©N)rˆ   r‹   r   r‰   rŒ   )rA   rD   rE   rF   ÚrÚhÚouts          r-   rX   zGraniteMoeDecoderLayer.__call__Ž   sk   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ��D×,Ñ,Ñ,Ñ,ˆØ×!Ñ! $×"?Ñ"?ÀÓ"BÓCˆØ�!�d×.Ñ.Ñ.Ñ.ˆØˆ
r,   rY   rZ   r^   s   @r-   r„   r„   ƒ   sW   ø„ ð<˜Yõ <ð $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r,   r„   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚGraniteMoEModelr0   c                 ó~  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  «      | _        t        |j                  «      D �cg c]  }t        |¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        |j                  | _        y c c}w )N)r0   r†   )r4   r5   r0   r:   Ú	Embeddingr   r   Úembed_tokensÚranger   r„   ÚlayersrŠ   r   Únormr   )rA   r0   Ú_rC   s      €r-   r5   zGraniteMoEModel.__init__œ   s’   ø€ Ü‰ÑÔØˆŒ	ÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔä7<¸T×=SÑ=SÔ7Tó
Ù7T°!Ô"¨Ö-Ð7Tñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	Ø$(×$=Ñ$=ˆÕ!ùò	
s   ÁB:Úinputsc                 ó  — | j                  |«      | j                  z  }|€d gt        | j                  «      z  }t	        ||d   «      }t        | j                  |«      D ]  \  }} ||||¬«      }Œ | j                  |«      S )Nr   )rF   )r–   r   Úlenr˜   r
   Úzipr™   )rA   r›   rF   r�   rE   rf   Úcs          r-   rX   zGraniteMoEModel.__call__¦   s€   € ð
 ×Ñ˜fÓ%¨×(AÑ(AÑAˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆä˜DŸK™K¨Ö/‰HˆE�1Ù�a˜ QÔ'‰Að 0ð �y‰y˜‹|Ðr,   rŽ   r‚   r^   s   @r-   r“   r“   ›   s%   ø„ ð>˜Yõ >ð ñà—‘÷r,   r“   c                   ón   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
e	d„ «       Zˆ xZS )	ÚModelr0   c                 ó  •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s1t        j                  |j                  |j                  d¬«      | _        |j                  | _        y re   )r4   r5   r0   r   r“   Úmodelr"   r:   r;   r   r   Úlm_headr   rz   s     €r-   r5   zModel.__init__¹   sa   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ$ TÓ*ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆDŒLØ"×1Ñ1ˆÕr,   r›   c                 óÞ   — | j                  ||«      }| j                  j                  r&| j                   j                  j	                  |«      }n| j                  |«      }|| j                  z  S rŽ   )r£   r0   r"   r–   Ú	as_linearr¤   r   )rA   r›   rF   r‘   s       r-   rX   zModel.__call__Â   sZ   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8‰Cà—,‘,˜sÓ#ˆCØ�T×(Ñ(Ñ(Ð(r,   c                 ó¾  — d|vr|S t        | j                  j                  «      D ]Œ  }d|› d�}|› d�}|j                  |«      }t	        j
                  |dd¬«      \  }}|||j                  dd	«      <   |||j                  dd
«      <   |› d�}|j                  |«      ||j                  dd«      <   ŒŽ | j                  j                  r|j                  dd «       |S )Nz3model.layers.0.block_sparse_moe.input_linear.weightzmodel.layers.z.block_sparse_moez.input_linear.weightrJ   r   rk   Úinput_linearzswitch_mlp.gate_projzswitch_mlp.up_projz.output_linear.weightÚoutput_linearzswitch_mlp.down_projzlm_head.weight)r—   r0   r   Úpopr[   ÚsplitÚreplacer"   )rA   ÚweightsÚlÚprefixÚkeyÚvalueÚ	gate_projÚup_projs           r-   ÚsanitizezModel.sanitizeÎ   sè   € Ø@ÈÑOØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ'8Ð9ˆFØ�HÐ0Ð1ˆCØ—K‘K Ó$ˆEÜ!#§¡¨%°¸Ô!;ÑˆI�wØKTˆG�C—K‘K Ð0FÓGÑHØIPˆG�C—K‘K Ð0DÓEÑFØ�HÐ1Ð2ˆCØLSÏKÉKØóMˆG�C—K‘K Ð1GÓHÒIð 4ð �9‰9×(Ò(Ø�K‰KÐ(¨$Ô/Øˆr,   c                 ó   — d„ }|S )Nc                 ó0   — | j                  d«      rdddœS y)Nzblock_sparse_moe.router.layeré@   é   )Ú
group_sizeÚbitsT)Úendswith)Úpathrš   s     r-   Ú	predicatez(Model.quant_predicate.<locals>.predicateâ   s   € Ø�}‰}Ð<Ô=Ø&(°!Ñ4Ð4Ør,   r+   )rA   r½   s     r-   Úquant_predicatezModel.quant_predicateà   s   € ò	ð
 Ðr,   c                 ó.   — | j                   j                  S rŽ   )r£   r˜   )rA   s    r-   r˜   zModel.layersé   s   € à�z‰z× Ñ Ð r,   rŽ   )r#   r$   r%   r   r5   r[   r\   rX   r´   Úpropertyr¾   r˜   r]   r^   s   @r-   r¡   r¡   ¸   sR   ø„ ð2˜Yõ 2ð ñ
)à—‘ó
)òð$ ñó ðð ñ!ó ô!r,   r¡   )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorer[   Úmlx.nnr:   Úbaser	   r
   r   Ú
rope_utilsr   Úswitch_layersr   r   ÚModuler/   r`   rv   r„   r“   r¡   r+   r,   r-   Ú<module>rÊ      s    ðõ "ß -Ó -å Ý ç TÑ TÝ 'Ý $ð ô%�ó %ó ð%ô,4#˜"Ÿ)™)ô 4#ôn&˜2Ÿ9™9ô &ô$C�B—I‘Iô Cô*˜RŸY™Yô ô0�b—i‘iô ô:3!ˆB�I‰Iõ 3!r,   