Ë
    îÿæiÀJ  ã                   ó\  — d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ e G d„ de«      «       Z G d„ dej:                  «      Z G d„ dej:                  «      Z G d„ dej:                  «      Z  G d„ dej:                  «      Z! G d„ dej:                  «      Z" G d„ dej:                  «      Z# G d„ dej:                  «      Z$ G d„ dej:                  «      Z% G d„ dej:                  «      Z& G d„ d ej:                  «      Z'y)!é    )Ú	dataclass)ÚAnyÚListÚOptionalÚTupleNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Úinitialize_rope)Ú
ssm_update)Ú	SwitchGLUc                   óÔ  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   ee   ed<   eed<   eed<   dZ	e
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZeed<   dZeed<   d Zeed!<   d"Zeeef   ed#<   ed$efd%„«       Zy)&Ú	ModelArgsÚ
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚmax_position_embeddingsÚnum_attention_headsÚnum_key_value_headsÚattention_biasÚembedding_multiplierÚattention_multiplierÚlogits_scalingÚresidual_multiplierÚlayer_typesÚrms_norm_epsÚ
rope_thetaNÚnum_local_expertsÚnum_experts_per_tokÚshared_intermediate_sizeÚmamba_n_headsÚmamba_d_headÚmamba_proj_biasÚmamba_d_stateÚmamba_d_convÚmamba_n_groupsÚmamba_conv_biasFÚmlp_biasÚropeÚposition_embedding_typeTÚtie_word_embeddings)gü©ñÒMbP?g      Y@Útime_step_limitÚreturnc                 ó,   — t        | j                  «      S ©N)Úboolr%   ©Úselfs    ús/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/granitemoehybrid.pyÚuse_moezModelArgs.use_moeB   s   € ä�D×*Ñ*Ó+Ð+ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__Úintr7   Úfloatr   r%   r   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r1   r2   r3   r   Úpropertyr;   © r<   r:   r   r      sK  … ð ƒOØƒOØÓØÓØÓØ Ó ØÓØÓØÓØÓØÓØÓØÓØ�c‘ÓØÓØÓð (,Ð�x ‘}Ó+Ø)-Ð˜ #™Ó-Ø.2Ð˜h s™mÓ2ð $(€M�8˜C‘=Ó'Ø"&€L�(˜3‘-Ó&Ø&*€O�X˜d‘^Ó*Ø#'€M�8˜C‘=Ó'Ø"&€L�(˜3‘-Ó&Ø$(€N�H˜S‘MÓ(Ø&*€O�X˜d‘^Ó*ð €HˆdÓð $*Ð˜SÓ)Ø $Ð˜Ó$Ø+9€O�U˜5 %˜<Ñ(Ó9ð ð,˜ò ,ó ñ,r<   r   c                   ó|   ‡ — e Zd Zddedefˆ fd„Zd	dej                  dej                  dej                  fd„Zˆ xZ	S )
ÚGraniteMoeHybridRMSNormGatedr   Úepsc                 ód   •— t         ‰| �  «        || _        t        j                  |«      | _        y r6   )ÚsuperÚ__init__rH   ÚmxÚonesÚweight)r9   r   rH   Ú	__class__s      €r:   rK   z%GraniteMoeHybridRMSNormGated.__init__H   s%   ø€ Ü‰ÑÔØˆŒÜ—g‘g˜kÓ*ˆ�r<   Úhidden_statesÚgater4   c                 óˆ   — |�t        ||«      }t        j                  j                  || j                  | j
                  «      S r6   )r	   rL   ÚfastÚrms_normrN   rH   )r9   rP   rQ   s      r:   Ú__call__z%GraniteMoeHybridRMSNormGated.__call__M   s5   € ØÐÜ" 4¨Ó7ˆMÜ�w‰w×Ñ ¨t¯{©{¸D¿H¹HÓEÐEr<   )g�íµ ÷Æ°>r6   )
r=   r>   r?   rB   rC   rK   rL   ÚarrayrU   Ú__classcell__©rO   s   @r:   rG   rG   G   s@   ø„ ñ+ Cð +¨eõ +ñ
F b§h¡hð F°b·h±hð FÈ"Ï(É(÷ Fr<   rG   c                   ó�  ‡ — e Zd Zdefˆ fd„Zdej                  dee   deej                     dej                  fd„Z	dej                  d	ej                  d
ej                  dej                  dee   deej                     dej                  fd„Z
	 ddej                  deej                     dee   dej                  fd„Zˆ xZS )ÚGraniteMoeHybridMamba2MixerÚargsc                 ó  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _
        |j                  | _        |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  d| j                  z  | j                  z  z   | _        t#        j$                  | j                   | j                   |j                  d| j                   |j&                  ¬«      | _        | j                  | j                   z   | j                  z   }t#        j*                  | j                  ||j,                  ¬«      | _        t1        j2                  | j                  «      | _        t1        j6                  t1        j8                  d| j                  dz   t0        j:                  ¬«      «      | _        t1        j2                  | j                  «      | _        tA        | j                  |jB                  ¬«      | _"        t#        j*                  | j                  | j                  |j,                  ¬«      | _#        y )Né   r   )Úin_channelsÚout_channelsÚkernel_sizeÚpaddingÚgroupsÚbias©rc   r   ©Údtype©rH   )$rJ   rK   r(   Ú	num_headsr   r+   Ússm_state_sizer,   Úconv_kernel_sizer)   r   r-   Ún_groupsÚhead_dimr3   Úheads_per_groupÚconv_dimÚnnÚConv1dr.   Úconv1dÚLinearr*   Úin_projrL   rM   Údt_biasÚlogÚarangeÚfloat32ÚA_logÚDrG   r#   ÚnormÚout_proj)r9   r[   Úprojection_sizerO   s      €r:   rK   z$GraniteMoeHybridMamba2Mixer.__init__T   sÙ  ø€ Ü‰ÑÔØ×+Ñ+ˆŒØ×+Ñ+ˆÔØ"×0Ñ0ˆÔØ $× 1Ñ 1ˆÔØ!%×!3Ñ!3°d×6GÑ6GÑ!GˆÔØ×+Ñ+ˆŒØ×)Ñ)ˆŒØ#×3Ñ3ˆÔØ#Ÿ~™~°·±Ñ>ˆÔà×.Ñ.°°T·]±]Ñ1BÀT×EXÑEXÑ1XÑXˆŒä—i‘iØŸ™ØŸ™Ø×)Ñ)ØØ—=‘=Ø×%Ñ%ô
ˆŒð ×0Ñ0°4·=±=Ñ@À4Ç>Á>ÑQˆÜ—y‘yØ×Ñ˜o°D×4HÑ4Hô
ˆŒô —w‘w˜tŸ~™~Ó.ˆŒÜ—V‘VœBŸI™I a¨¯©¸!Ñ);Ä2Ç:Á:ÔNÓOˆŒ
Ü—‘˜Ÿ™Ó(ˆŒä0Ø×"Ñ"¨×(9Ñ(9ô
ˆŒ	ô Ÿ	™	Ø×"Ñ" D×$4Ñ$4¸4×;OÑ;Oô
ˆ�r<   Ú
conv_inputÚcacheÚmaskr4   c                 óò  — |�t        j                  |d   |d«      }|��|d   €It        j                  |j                  d   | j                  dz
  | j
                  f|j                  ¬«      }n|d   }t        j                  ||gd¬«      }| j                  dz
  }|j                  �q|j                  d   }t        j                  |j                  d||z
  «      }|d d …d f   t        j                  |«      z   d   }	t        j                  ||	d¬«      |d<   n;|d d …| d …d d …f   |d<   n(t        j                  |d| j                  dz
  dfdg«      }| j                  |«      }
t        j                  |
«      S )N©.Nr   r   re   ©Úaxis)r   r   )rL   ÚwhereÚzerosÚshaperj   rn   rf   ÚconcatenateÚlengthsÚcliprv   Útake_along_axisÚpadrq   ro   Úsilu)r9   r}   r~   r   Ú
conv_stateÚpadded_inputÚn_keepÚtÚendsÚ	positionsÚconv_outputs              r:   Ú_convz!GraniteMoeHybridMamba2Mixer._conv{   sh  € ð ÐÜŸ™ $ y¡/°:¸qÓAˆJàÑØ�Q‰xÐÜŸX™XØ×%Ñ% aÑ(¨$×*?Ñ*?À!Ñ*CÀTÇ]Á]ÐSØ$×*Ñ*ô‘
ð
 # 1™X�
ÜŸ>™>¨:°zÐ*BÈÔKˆLØ×*Ñ*¨QÑ.ˆFØ�}‰}Ð(Ø ×&Ñ& qÑ)�Ü—w‘w˜uŸ}™}¨a°°V±Ó<�Ø!¢! T '™]¬R¯Y©Y°vÓ->Ñ>À	ÑJ�	Ü×-Ñ-¨l¸IÈAÔN��a’à'ª¨F¨7©8²Q¨Ñ7��a’äŸ6™6Ø˜V d×&;Ñ&;¸aÑ&?ÀÐ%CÀVÐLóˆLð —k‘k ,Ó/ˆÜ�w‰w�{Ó#Ð#r<   rP   ÚBÚCÚdtc                 ó<  — |j                   \  }}}	|j                  ||| j                  | j                  «      }|j                  ||| j                  | j
                  «      }|j                  ||| j                  | j
                  «      }|r|d   }
|j                  }nd\  }
}t        || j                  ||| j                  j                  |j                  «      || j                  |
| j                  |«
      \  }}
|r|
|d<   |j                  ||| j                  «      S )Nr   ©NN)r†   Úreshaperh   rl   rk   ri   rˆ   r   rx   ry   Úastyperf   rt   r3   r   )r9   rP   r•   r–   r—   r~   r   Ú
batch_sizeÚseq_lenÚ_Ústaterˆ   Úys                r:   Ú_ssmz GraniteMoeHybridMamba2Mixer._ssm�   s  € ð "/×!4Ñ!4Ñˆ
�G˜Qà%×-Ñ-Ø˜ §¡°·±ó
ˆð �I‰I�j '¨4¯=©=¸$×:MÑ:MÓNˆØ�I‰I�j '¨4¯=©=¸$×:MÑ:MÓNˆÙØ˜!‘HˆEØ—m‘m‰Gà'‰NˆE�7äØØ�J‰JØØØ�F‰F�M‰M˜-×-Ñ-Ó.ØØ�L‰LØØ× Ñ Øó
‰ˆˆ5ñ ØˆE�!‰Hà�y‰y˜ W¨d×.DÑ.DÓEÐEr<   c                 ó  — | j                  |«      }t        j                  || j                  | j                  | j                  z   gd¬«      \  }}}| j                  |||«      }t        j                  || j                  | j                  | j                  | j                  z  z   gd¬«      \  }	}
}| j                  |	|
||||«      }|r|j                  |j                  d   «       | j                  ||«      }| j                  |«      S )Néÿÿÿÿr‚   r   )rs   rL   Úsplitr   rn   r”   rk   ri   r¡   Úadvancer†   rz   r{   )r9   rP   r   r~   Ú	projectedrQ   r}   r—   r“   Úhidden_states_ssmr•   r–   r    s                r:   rU   z$GraniteMoeHybridMamba2Mixer.__call__Ä   sø   € ð —L‘L Ó/ˆ	ä!Ÿx™xØØ×#Ñ# T×%;Ñ%;¸d¿m¹mÑ%KÐLØô 
Ñˆˆj˜"ð
 —j‘j ¨U°DÓ9ˆÜ"$§(¡(Øà×&Ñ&Ø×&Ñ&¨¯©¸×9LÑ9LÑ)LÑLðð ô#
ÑÐ˜1˜að �I‰IÐ'¨¨A¨r°5¸$Ó?ˆÙØ�M‰M˜!Ÿ'™' !™*Ô%Ø�I‰I�a˜ÓˆØ�}‰}˜QÓÐr<   r6   )r=   r>   r?   r   rK   rL   rV   r   r   r”   r¡   rU   rW   rX   s   @r:   rZ   rZ   S   s  ø„ ð%
˜Yõ %
ðN $à—H‘Hð $ð ˜Ñ$ð $ð �r—x‘xÑ ð	 $ð
 
�‰ó $ðD%Fà—x‘xð%Fð �8‰8ð%Fð �8‰8ð	%Fð
 �H‰Hð%Fð ˜Ñ$ð%Fð �r—x‘xÑ ð%Fð 
�‰ó%FðV (,ñ	 à—x‘xð ð �r—x‘xÑ ð ð ˜Ñ$ð	 ð
 
�‰÷ r<   rZ   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚGraniteMoeHybridAttentionr[   c                 ó‚  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|j                  | _	        |j                  }t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  ||z  ||¬«      | _        |j"                  dk7  }|r3t%        | j                  |j&                  dd |j(                  «      | _        y d | _        y )Nrd   ÚnopeF)rJ   rK   r   r   Ún_headsr   Ú
n_kv_headsrl   r   Úscaler   ro   rr   Úq_projÚk_projÚv_projÚo_projr1   r   r$   r   r0   )	r9   r[   Údimr¬   r­   rl   r   Úuse_roperO   s	           €r:   rK   z"GraniteMoeHybridAttention.__init__ã   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'×#3Ñ#3°wÑ#>Ð>ˆŒ˜à×.Ñ.ˆŒ
Ø×,Ñ,ˆÜ—i‘i  W¨xÑ%7¸nÔMˆŒÜ—i‘i  Z°(Ñ%:ÀÔPˆŒÜ—i‘i  Z°(Ñ%:ÀÔPˆŒÜ—i‘i ¨(Ñ 2°C¸nÔMˆŒð ×/Ñ/°6Ñ9ˆÙÜ'Ø—‘Ø—‘ØØØ×,Ñ,óˆD�Ið ˆD�Ir<   Úxr   r~   r4   c                 óN  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	| j                  �_|�;| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }n"| j                  |«      }| j                  |«      }|�|j                  ||	«      \  }}	t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )Nr£   r   r]   r   é   )Úoffset)r~   r®   r   )r†   r¯   r°   r±   rš   r¬   Ú	transposer­   r0   r¸   Úupdate_and_fetchr   r®   r²   )r9   rµ   r   r~   r•   ÚLry   ÚqueriesÚkeysÚvaluesÚoutputs              r:   rU   z"GraniteMoeHybridAttention.__call__  sy  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà—/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆð �9‰9Ð ØÐ ØŸ)™) G°E·L±L˜)ÓA�Ø—y‘y ¨e¯l©l�yÓ;‘àŸ)™) GÓ,�Ø—y‘y “�àÐØ ×1Ñ1°$¸Ó?‰LˆD�&ä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r<   r™   )r=   r>   r?   r   rK   rL   rV   r   r   rU   rW   rX   s   @r:   r©   r©   â   sY   ø„ ð˜Yõ ðB $(Ø#'ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜Ñ ð	#ð
 
�‰÷#r<   r©   c                   óL   ‡ — e Zd Zdededefˆ fd„Zdej                  fd„Zˆ xZS )ÚGraniteMoeHybridTopKGatingÚ
input_sizeÚnum_expertsÚtop_kc                 ó†   •— t         ‰| �  «        || _        || _        || _        t        j                  ||d¬«      | _        y ©NFrd   )rJ   rK   rÃ   rÂ   rÄ   ro   rr   Úlayer)r9   rÂ   rÃ   rÄ   rO   s       €r:   rK   z#GraniteMoeHybridTopKGating.__init__#  s8   ø€ Ü‰ÑÔØ&ˆÔØ$ˆŒØˆŒ
Ü—Y‘Y˜z¨;¸UÔCˆ�
r<   rP   c                 óö   — | j                  |«      }t        j                  || j                   d¬«      d| j                   d …f   }t        j                  ||d¬«      }t        j
                  |dd¬«      }||fS )Nr£   )Úkthrƒ   .r‚   T)Úpreciserƒ   )rÇ   rL   ÚargpartitionrÄ   rŠ   Úsoftmax)r9   rP   ÚlogitsÚ	top_k_idxÚtop_k_logitsÚtop_k_gatess         r:   rU   z#GraniteMoeHybridTopKGating.__call__*  sp   € Ø—‘˜MÓ*ˆÜ—O‘O F°·±°À"ÔEØ�$—*‘*�‘Ðñ
ˆ	ô ×)Ñ)¨&°)À"ÔEˆÜ—j‘j °tÀ"ÔEˆØ˜+Ð%Ð%r<   )	r=   r>   r?   rB   rK   rL   rV   rU   rW   rX   s   @r:   rÁ   rÁ   "  s1   ø„ ðD 3ð D°Sð DÀõ Dð& b§h¡h÷ &r<   rÁ   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚGraniteMoeHybridMoEr[   c                 ó(  •— t         ‰| �  «        |j                  | _        |j                  | _        t        | j                  | j                  |j                  «      | _        t        | j                  |j                  |j                  ¬«      | _
        y )N)rÂ   rÃ   rÄ   )rJ   rK   r   rÂ   r   r   r%   Ú
switch_mlprÁ   r&   Úrouter©r9   r[   rO   s     €r:   rK   zGraniteMoeHybridMoE.__init__5  ss   ø€ Ü‰ÑÔà×*Ñ*ˆŒØ×1Ñ1ˆÔÜ#Ø�O‰O˜T×-Ñ-¨t×/EÑ/Eó
ˆŒô 1Ø—‘Ø×.Ñ.Ø×*Ñ*ô
ˆ�r<   rµ   r4   c                 ó~   — | j                  |«      \  }}| j                  ||«      }||d   z  j                  d¬«      S )Nr�   éþÿÿÿr‚   )rÕ   rÔ   Úsum)r9   rµ   Ú	token_idsÚgatesr    s        r:   rU   zGraniteMoeHybridMoE.__call__C  sB   € ØŸ;™; q›>Ñˆ	�5Ø�O‰O˜A˜yÓ)ˆØ�E˜)Ñ$Ñ$×)Ñ)¨rÐ)Ó2Ð2r<   ©	r=   r>   r?   r   rK   rL   rV   rU   rW   rX   s   @r:   rÒ   rÒ   4  s*   ø„ ð
˜Yõ 
ð3˜"Ÿ(™(ð 3 r§x¡x÷ 3r<   rÒ   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚGraniteMoeHybridSharedMLPr[   c                 óì   •— t         ‰| �  «        t        j                  |j                  |j
                  dz  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        y )Nr]   Frd   )rJ   rK   ro   rr   r   r'   Úinput_linearÚoutput_linearrÖ   s     €r:   rK   z"GraniteMoeHybridSharedMLP.__init__J  s]   ø€ Ü‰ÑÔÜŸI™IØ×Ñ˜d×;Ñ;¸aÑ?Àeô
ˆÔô  ŸY™YØ×)Ñ)¨4×+;Ñ+;À%ô
ˆÕr<   rµ   r4   c                 óŒ   — t        j                  | j                  |«      dd¬«      \  }}| j                  t	        ||«      «      S )Nr]   r£   r‚   )rL   r¤   rà   rá   r	   )r9   rµ   rQ   Úups       r:   rU   z"GraniteMoeHybridSharedMLP.__call__S  s;   € Ü—8‘8˜D×-Ñ-¨aÓ0°!¸"Ô=‰ˆˆbØ×!Ñ!¤&¨¨rÓ"2Ó3Ð3r<   rÜ   rX   s   @r:   rÞ   rÞ   I  s*   ø„ ð
˜Yõ 
ð4˜"Ÿ(™(ð 4 r§x¡x÷ 4r<   rÞ   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚGraniteMoeHybridMLPr[   c                 ó  •— t         ‰| �  «        |j                  }|j                  }|j                  }t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _	        y )Nrd   )
rJ   rK   r   r   r/   ro   rr   Ú	gate_projÚ	down_projÚup_proj)r9   r[   r³   Ú
hidden_dimr/   rO   s        €r:   rK   zGraniteMoeHybridMLP.__init__Y  si   ø€ Ü‰ÑÔØ×ÑˆØ×+Ñ+ˆ
Ø—=‘=ˆäŸ™ 3¨
¸ÔBˆŒÜŸ™ :¨s¸ÔBˆŒÜ—y‘y  j°xÔ@ˆ�r<   r4   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S r6   )rè   r	   rç   ré   )r9   rµ   s     r:   rU   zGraniteMoeHybridMLP.__call__c  s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr<   rÜ   rX   s   @r:   rå   rå   X  s#   ø„ ðA˜Yõ AðJ˜RŸX™X÷ Jr<   rå   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚGraniteMoeHybridLayerr[   Ú
layer_typec                 ó"  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  |j                  |j                  ¬«      | _	        |dk(  rt        |«      | _        n$|dk(  rt        |«      | _        nt        d|› �«      ‚| j                  r!t        |«      | _        t#        |«      | _        nt'        |«      | _        t        j                  |j                  |j                  ¬«      | _        y )Nrg   ÚmambaÚ	attentionzUnknown layer type: )rJ   rK   rî   r!   r;   ro   ÚRMSNormr   r#   Úinput_layernormrZ   rð   r©   Ú	self_attnÚ
ValueErrorrÞ   Ú
shared_mlprÒ   Úblock_sparse_moerå   ÚmlpÚpost_attention_layernorm©r9   r[   rî   rO   s      €r:   rK   zGraniteMoeHybridLayer.__init__h  sÚ   ø€ Ü‰ÑÔØ$ˆŒØ#'×#;Ñ#;ˆÔ Ø—|‘|ˆŒä!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔà˜Ò Ü4°TÓ:ˆD�JØ˜;Ò&Ü6°tÓ<ˆD�NäÐ3°J°<Ð@ÓAÐAð �<Š<Ü7¸Ó=ˆDŒOÜ$7¸Ó$=ˆDÕ!ô +¨4Ó0ˆDŒHä(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÕ%r<   rµ   r   r~   r4   c                 ó”  — |}| j                  |«      }| j                  dk(  r| j                  |||¬«      }n| j                  |||¬«      }||| j                  z  z   }|}| j                  |«      }| j                  r(| j                  |«      }| j                  |«      }||z   }	n| j                  |«      }	||	| j                  z  z   }|S )Nrð   ©r   r~   )
ró   rî   rð   rô   r!   rù   r;   r÷   rö   rø   )
r9   rµ   r   r~   ÚresidualrP   ÚnormedÚmoe_outÚ
shared_outÚmlp_outs
             r:   rU   zGraniteMoeHybridLayer.__call__ƒ  sÑ   € ð ˆØ×,Ñ,¨QÓ/ˆà�?‰?˜gÒ%Ø ŸJ™J }¸4Àu˜JÓM‰Mà ŸN™N¨=¸tÈ5˜NÓQˆMà  =°4×3KÑ3KÑ#KÑKˆð !ˆØ×.Ñ.¨}Ó=ˆà�<Š<Ø×+Ñ+¨FÓ3ˆGØŸ™¨Ó0ˆJØ 
Ñ*‰Gà—h‘h˜vÓ&ˆGà  7¨T×-EÑ-EÑ#EÑEˆàÐr<   r™   )r=   r>   r?   r   r@   rK   rL   rV   r   r   rU   rW   rX   s   @r:   rí   rí   g  s^   ø„ ð
˜Yð 
°Cõ 
ð< $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r<   rí   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	ˆ xZ
S )ÚGraniteMoeHybridModelr[   c                 ó8  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  «      | _        |j                  D �cg c]  }t        ||«      ‘Œ c}| _
        t        j                  |j                  |j                  ¬«      | _        |j                  | _        d|j                  v r|j                  j                  d«      nd | _        d|j                  v r!|j                  j                  d«      | _        y d | _        y c c}w )Nrg   rñ   rð   )rJ   rK   r[   ro   Ú	Embeddingr   r   Úembed_tokensr"   rí   Úlayersrò   r#   rz   r   ÚindexÚfa_idxÚssm_idxrú   s      €r:   rK   zGraniteMoeHybridModel.__init__¥  só   ø€ Ü‰ÑÔØˆŒ	ÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔàFJ×FVÒFVó
ÙFV¸
Ô! $¨
Õ3ÐFVñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	Ø$(×$=Ñ$=ˆÔ!ð
 ˜d×.Ñ.Ñ.ð ×Ñ×"Ñ" ;Ô/àð 	Œð 07¸$×:JÑ:JÑ/JˆD×Ñ×"Ñ" 7Ó+ð 	�ØPTð 	�ùò
s   ÁDÚinputsr~   r4   c                 óª  — | j                  |«      | j                  z  }|€d gt        | j                  «      z  }d }d }| j                  �t        ||| j                     «      }| j                  �t        ||| j                     «      }t        | j                  |«      D ]#  \  }}|j                  dk(  r|n|} ||||¬«      }Œ% | j                  |«      S )Nrñ   rü   )r  r   Úlenr  r	  r   r
  r   Úziprî   rz   )	r9   r  r~   rP   Ú	attn_maskÚ
mamba_maskrÇ   Úcr   s	            r:   rU   zGraniteMoeHybridModel.__call__¹  sÏ   € ð
 ×)Ñ)¨&Ó1°D×4MÑ4MÑMˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEð ˆ	Øˆ
à�;‰;Ð"Ü-¨m¸UÀ4Ç;Á;Ñ=OÓPˆIØ�<‰<Ð#Ü(¨¸¸d¿l¹lÑ8KÓLˆJä˜DŸK™K¨Ö/‰HˆE�1Ø %× 0Ñ 0°KÒ ?‘9ÀZˆDÙ! -°dÀ!ÔD‰Mð 0ð �y‰y˜Ó'Ð'r<   r6   )r=   r>   r?   r   rK   rL   rV   r   r   rU   rW   rX   s   @r:   r  r  ¤  sA   ø„ ð
˜Yõ 
ð.  $ñ(à—‘ð(ð ˜‰}ð(ð 
�‰÷	(r<   r  c                   ó–   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	e
d„ «       Zd„ Zd	„ Ze
d
„ «       Zˆ xZS )ÚModelr[   c                 ó  •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s1t        j                  |j                  |j                  d¬«      | _        |j                  | _        y rÆ   )rJ   rK   r[   r   r  Úmodelr2   ro   rr   r   r   Úlm_headr    rÖ   s     €r:   rK   zModel.__init__Ô  sa   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ*¨4Ó0ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆDŒLØ"×1Ñ1ˆÕr<   r  r~   r4   c                 óà   — | j                  ||¬«      }| j                  j                  r&| j                   j                  j	                  |«      }n| j                  |«      }|| j                  z  S )N)r~   )r  r[   r2   r  Ú	as_linearr  r    )r9   r  r~   Úouts       r:   rU   zModel.__call__Ý  s\   € ð
 �j‰j˜ uˆjÓ-ˆà�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8‰Cà—,‘,˜sÓ#ˆCà�T×(Ñ(Ñ(Ð(r<   c                 ó.   — | j                   j                  S r6   )r  r  r8   s    r:   r  zModel.layersë  s   € à�z‰z× Ñ Ð r<   c                 óÔ   — g }| j                   D ]V  }|j                  dk(  r|j                  t        d¬«      «       Œ.|j                  dk(  sŒ>|j                  t	        «       «       ŒX |S )Nrð   r]   )Úsizerñ   )r  rî   Úappendr   r   )r9   ÚcachesrÇ   s      r:   Ú
make_cachezModel.make_cacheï  sW   € ØˆØ—[”[ˆEØ×Ñ 7Ò*Ø—‘œk¨qÔ1Õ2Ø×!Ñ! [Ó0Ø—‘œg›iÕ(ð	 !ð
 ˆr<   c                 ó<  — |j                  «       D ]2  \  }}d|v sŒ|j                  d   dk7  sŒ|j                  dd«      ||<   Œ4 | j                  j                  r d|v rœt        | j                  j                  «      D ]x  }d|› d�}|j                  |› d�«      }|j                  \  }}}|d d …d |dz  …d d …f   }	|d d …|dz  d …d d …f   }
|	||› d	�<   |
||› d
�<   |j                  |› d�«      ||› d�<   Œz |S | j                  j                  sŠd|v r†t        | j                  j                  «      D ]d  }d|› d�}|j                  |› d�«      }t        j                  |dd¬«      \  }	}
|	|d|› d�<   |
|d|› d�<   |j                  |› d�«      |d|› d�<   Œf |S )Nzconv1d.weightr£   r   r]   z3model.layers.0.block_sparse_moe.input_linear.weightzmodel.layers.z.block_sparse_moez.input_linear.weightz.switch_mlp.gate_proj.weightz.switch_mlp.up_proj.weightz.output_linear.weightz.switch_mlp.down_proj.weightz-model.layers.0.shared_mlp.input_linear.weightz.shared_mlpr   r‚   z.mlp.gate_proj.weightz.mlp.up_proj.weightz.mlp.down_proj.weight)
Úitemsr†   Úmoveaxisr[   r;   Úranger   ÚpoprL   r¤   )r9   ÚweightsÚkÚvÚlÚprefixÚinput_weightrž   Úexpert_hiddenrç   ré   s              r:   ÚsanitizezModel.sanitizeø  s÷  € à—M‘M–O‰DˆAˆqØ !Ò#¨¯©°©°qÓ(8ØŸZ™Z¨¨1Ó-�˜’
ð $ð �I‰I×ÒØEÈÑPä˜4Ÿ9™9×6Ñ6Ö7�Ø(¨¨Ð+<Ð=�à&Ÿ{™{¨f¨XÐ5IÐ+JÓK�Ø&2×&8Ñ&8Ñ#��= !ð )ªÐ,@¨m¸qÑ.@Ð,@Â!Ð)CÑD�	Ø&¢q¨-¸1Ñ*<Ñ*>ÂÐ'AÑB�ØCL�˜6˜(Ð">Ð?Ñ@ØAH�˜6˜(Ð"<Ð=Ñ>àCJÇ;Á;Ø�hÐ3Ð4óD�˜6˜(Ð">Ð?Ò@ð 8ðF ˆð# —	‘	×!Ò!Ø?À7ÑJä˜4Ÿ9™9×6Ñ6Ö7�Ø(¨¨¨;Ð7�ð  'Ÿ{™{¨f¨XÐ5IÐ+JÓK�ä%'§X¡X¨l¸AÀAÔ%FÑ"�	˜7ØDM�˜-¨ sÐ*?Ð@ÑAØBI�˜-¨ sÐ*=Ð>Ñ?àDKÇKÁKØ�hÐ3Ð4óE�˜-¨ sÐ*?Ð@ÒAð 8ð ˆr<   c                 ó   ‡ — ˆ fd„}|S )Nc                 ó^   •— ‰j                   j                  r| j                  d«      rdddœS y)Nzrouter.layeré@   é   )Ú
group_sizeÚbitsT)r[   r;   Úendswith)Úpathrž   r9   s     €r:   Ú	predicatez(Model.quant_predicate.<locals>.predicate*  s)   ø€ Ø�y‰y× Ò  T§]¡]°>Ô%BØ&(°!Ñ4Ð4Ør<   rE   )r9   r5  s   ` r:   Úquant_predicatezModel.quant_predicate(  s   ø€ ô	ð
 Ðr<   r6   )r=   r>   r?   r   rK   rL   rV   r   r   rU   rD   r  r  r,  r6  rW   rX   s   @r:   r  r  Ó  st   ø„ ð2˜Yõ 2ð  $ñ)à—‘ð)ð ˜‰}ð)ð 
�‰ó	)ð ñ!ó ð!òò.ð` ñó ôr<   r  )(Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerL   Úmlx.nnro   Úactivationsr	   Úbaser
   r   r   r   r~   r   r   Ú
rope_utilsr   Ússmr   Úswitch_layersr   r   ÚModulerG   rZ   r©   rÁ   rÒ   rÞ   rå   rí   r  r  rE   r<   r:   Ú<module>rB     sù   ðõ "ß -Ó -å Ý å ÷ó ÷ (Ý 'Ý Ý $ð ô-,�ó -,ó ð-,ô`	F 2§9¡9ô 	FôL  "§)¡)ô L ô^=# §	¡	ô =#ô@& §¡ô &ô$3˜"Ÿ)™)ô 3ô*4 §	¡	ô 4ôJ˜"Ÿ)™)ô Jô:˜BŸI™Iô :ôz,(˜BŸI™Iô ,(ô^\ˆB�I‰Iõ \r<   