+
    TV-jÀJ  ã                   ó^  € ^ RI Ht ^ RIHtHtHtHt ^ RIHt	 ^ RI
Ht ^RIHt ^RIHtHtHtHt ^RIHtHt ^RIHt ^RIHt ^R	IHt ] ! R
 R]4      4       t ! R R]P:                  4      t ! R R]P:                  4      t ! R R]P:                  4      t  ! R R]P:                  4      t! ! R R]P:                  4      t" ! R R]P:                  4      t# ! R R]P:                  4      t$ ! R R]P:                  4      t% ! R R]P:                  4      t& ! R R]P:                  4      t'R# ) é    )Ú	dataclass)ÚAnyÚListÚOptionalÚTupleN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Úinitialize_rope)Ú
ssm_update)Ú	SwitchGLUc                   ó€   a € ] tR t^t o RtRtRtRtRtRt	Rt
RtRtRtRtRtRtR	t]V 3R lR l4       tV 3R ltRtV tR# )
Ú	ModelArgsNFÚropeTc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úreturn)Úbool)ÚformatÚ__classdict__s   "€Úo/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/granitemoehybrid.pyÚ__annotate__ÚModelArgs.__annotate__C   s   ø€ ÷ ,ñ ,™ñ ,ó    c                ó,   € \        V P                  4      # ©N)r   Únum_local_experts©Úselfs   &r   Úuse_moeÚModelArgs.use_moeB   s   € ä�D×*Ñ*Ó+Ð+r   c                óH  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[S[ ,          ;R&   S[;R&   S[;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[;R&   S[ ;R&   S[;R&   S[S[S[3,          ;R&   # )r   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚmax_position_embeddingsÚnum_attention_headsÚnum_key_value_headsÚattention_biasÚembedding_multiplierÚattention_multiplierÚlogits_scalingÚresidual_multiplierÚlayer_typesÚrms_norm_epsÚ
rope_thetar"   Únum_experts_per_tokÚshared_intermediate_sizeÚmamba_n_headsÚmamba_d_headÚmamba_proj_biasÚmamba_d_stateÚmamba_d_convÚmamba_n_groupsÚmamba_conv_biasÚmlp_biasÚposition_embedding_typeÚtie_word_embeddingsÚtime_step_limit)ÚstrÚintr   Úfloatr   r   r   )r   r   s   "€r   r   r      sÊ  ø‡ ‚ ñ �Oñ ñ �Oñ	 ñ
 Ññ ñ Ññ ñ Ññ ñ !Ñ ñ ñ Ññ ñ Ññ ñ Ññ ñ  Ññ ñ  Ññ ñ Ññ ñ Ññ ñ  ‘c•Ññ! ñ" Ññ# ñ$ Ññ% ñ,  ¡•}Ñ+ñ- ñ. "¡#�Ñ-ñ/ ñ0 '¡s�mÑ2ñ1 ñ6 ™C•=Ñ'ñ7 ñ8 ™3•-Ñ&ñ9 ñ: ™d•^Ñ*ñ; ñ< ™C•=Ñ'ñ= ñ> ™3•-Ñ&ñ? ñ@ ™S•MÑ(ñA ñB ™d•^Ñ*ñC ñH ÑñI ñN !Ñ)ñO ñP Ñ$ñQ ñR ™5¡%˜<Õ(Ñ9òS r   © )gü©ñÒMbP?g      Y@)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r"   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   Úpropertyr%   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r   s   @r   r   r      st   ø‡ € ð, (,ÐØ)-ÐØ.2Ðð $(€MØ"&€LØ&*€OØ#'€MØ"&€LØ$(€NØ&*€Oð €Hð $*ÐØ $ÐØ+9€Oð ÷,ó ð,÷[ ƒ r   r   c                   óX   a a€ ] tR t^Gt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚGraniteMoeHybridRMSNormGatedc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r*   Úeps)rF   rG   )r   r   s   "€r   r   Ú)GraniteMoeHybridRMSNormGated.__annotate__H   s   ø€ ÷ +ñ +¡Cð +©eñ +r   c                óf   <€ \         SV `  4        W n        \        P                  ! V4      V n        R # r!   )ÚsuperÚ__init__rT   ÚmxÚonesÚweight)r$   r*   rT   Ú	__class__s   &&&€r   rX   Ú%GraniteMoeHybridRMSNormGated.__init__H   s#   ø€ Ü‰ÑÔØŒÜ—g’g˜kÓ*ˆŽr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   Úhidden_statesÚgater   ©rY   Úarray)r   r   s   "€r   r   rU   M   s3   ø€ ÷ Fñ F¡b§h¡hð F±b·h±hð FÉ"Ï(É(ñ Fr   c                óˆ   € Ve   \        W!4      p\        P                  P                  WP                  V P
                  4      # r!   )r   rY   ÚfastÚrms_normr[   rT   )r$   r_   r`   s   &&&r   Ú__call__Ú%GraniteMoeHybridRMSNormGated.__call__M   s1   € ØÒÜ" 4Ó7ˆMÜ�w‰w×Ñ ¯{©{¸D¿H¹HÓEÐEr   )rT   r[   )g�íµ ÷Æ°>r!   ©	rI   rJ   rK   rL   rX   rf   rO   rP   Ú__classcell__©r\   r   s   @@r   rR   rR   G   s!   ù‡ € ÷+õ +÷
F÷ Fò Fr   rR   c                   óx   a a€ ] tR t^St oV3R lV 3R lltV3R lR ltV3R lR ltR
V3R lR lltR	tVt	V ;t
# )ÚGraniteMoeHybridMamba2Mixerc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r   r   s   "€r   r   Ú(GraniteMoeHybridMamba2Mixer.__annotate__T   s   ø€ ÷ %
ñ %
™Yñ %
r   c           	     ól  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  VP                  ,          V n
        VP                  V n        VP                  V n        VP                  V n        V P                  V P                  ,          V n        V P                  ^V P                  ,          V P                  ,          ,           V n        \"        P$                  ! V P                   V P                   VP                  ^ V P                   VP&                  R7      V n        V P                  V P                   ,           V P                  ,           p\"        P*                  ! V P                  W!P,                  R7      V n        \0        P2                  ! V P                  4      V n        \0        P6                  ! \0        P8                  ! ^V P                  ^,           \0        P:                  R7      4      V n        \0        P2                  ! V P                  4      V n        \A        V P                  VPB                  R7      V n"        \"        P*                  ! V P                  V P                  VP,                  R7      V n#        R# )r   )Úin_channelsÚout_channelsÚkernel_sizeÚpaddingÚgroupsÚbias©rx   ©Údtype©rT   N)$rW   rX   r:   Ú	num_headsr*   r=   Ússm_state_sizer>   Úconv_kernel_sizer;   r+   r?   Ún_groupsÚhead_dimrD   Úheads_per_groupÚconv_dimÚnnÚConv1dr@   Úconv1dÚLinearr<   Úin_projrY   rZ   Údt_biasÚlogÚarangeÚfloat32ÚA_logÚDrR   r6   ÚnormÚout_proj)r$   ro   Úprojection_sizer\   s   && €r   rX   Ú$GraniteMoeHybridMamba2Mixer.__init__T   s×  ø€ Ü‰ÑÔØ×+Ñ+ˆŒØ×+Ñ+ˆÔØ"×0Ñ0ˆÔØ $× 1Ñ 1ˆÔØ!%×!3Ñ!3°d×6GÑ6GÕ!GˆÔØ×+Ñ+ˆŒØ×)Ñ)ˆŒØ#×3Ñ3ˆÔØ#Ÿ~™~°·±Õ>ˆÔà×.Ñ.°°T·]±]Õ1BÀT×EXÑEXÕ1XÕXˆŒä—i’iØŸ™ØŸ™Ø×)Ñ)ØØ—=‘=Ø×%Ñ%ô
ˆŒð ×0Ñ0°4·=±=Õ@À4Ç>Á>ÕQˆÜ—y’yØ×Ñ˜o×4HÑ4Hô
ˆŒô —w’w˜tŸ~™~Ó.ˆŒÜ—V’VœBŸIšI a¨¯©¸!Õ);Ä2Ç:Á:ÔNÓOˆŒ
Ü—’˜Ÿ™Ó(ˆŒä0Ø×"Ñ"¨×(9Ñ(9ô
ˆŒ	ô Ÿ	š	Ø×"Ñ" D×$4Ñ$4¸4×;OÑ;Oô
ˆŽr   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[ P                  ,          RS[ P                  /# )r   Ú
conv_inputÚcacheÚmaskr   ©rY   rb   r   r   )r   r   s   "€r   r   rq   {   sI   ø€ ÷  $ñ  $á—H‘Hð $ñ ™Õ$ð $ñ ‘r—x‘xÕ ð	 $ñ
 
�‰ñ $r   c                ód  € Ve    \         P                  ! VR,          V^ 4      pVEe6   V^ ,          fS   \         P                  ! VP                  ^ ,          V P                  ^,
          V P
                  3VP                  R7      pM	V^ ,          p\         P                  ! WA.^R7      pV P                  ^,
          pVP                  eƒ   VP                  ^,          p\         P                  ! VP                  ^ Wv,
          4      pVR,          \         P                  ! V4      ,           R,          p	\         P                  ! WY^R7      V^ &   M@VRV) R 1R3,          V^ &   M-\         P                  ! VRV P                  ^,
          ^ 3R.4      pV P                  V4      p
\        P                  ! V
4      # )Nrz   ©ÚaxisºNNN©.N)r›   N)r   r   )rY   ÚwhereÚzerosÚshaper   rƒ   r{   ÚconcatenateÚlengthsÚclipr‹   Útake_along_axisÚpadr†   r„   Úsilu)r$   r”   r•   r–   Ú
conv_stateÚpadded_inputÚn_keepÚtÚendsÚ	positionsÚconv_outputs   &&&&       r   Ú_convÚ!GraniteMoeHybridMamba2Mixer._conv{   s^  € ð ÒÜŸš $ y¥/°:¸qÓAˆJàÓØ�Q�xÒÜŸXšXØ×%Ñ% aÕ(¨$×*?Ñ*?À!Õ*CÀTÇ]Á]ÐSØ$×*Ñ*ô‘
ð
 # 1�X�
ÜŸ>š>¨:Ð*BÈÔKˆLØ×*Ñ*¨QÕ.ˆFØ�}‰}Ò(Ø ×&Ñ& qÕ)�Ü—w’w˜uŸ}™}¨a°µÓ<�Ø! '�]¬R¯YªY°vÓ->Õ>À	ÕJ�	Ü×-Ò-¨lÈAÔN��a’à'¨¨F¨7©8°Q¨Õ7��a’äŸ6š6Ø˜V d×&;Ñ&;¸aÕ&?ÀÐ%CÀVÐLóˆLð —k‘k ,Ó/ˆÜ�wŠw�{Ó#Ð#r   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[S[,          RS[S[ P                  ,          RS[ P                  /# )r   r_   ÚBÚCÚdtr•   r–   r   r—   )r   r   s   "€r   r   rq   �   s|   ø€ ÷ %Fñ %Fá—x‘xð%Fñ �8‰8ð%Fñ �8‰8ð	%Fñ
 �H‰Hð%Fñ ™Õ$ð%Fñ ‘r—x‘xÕ ð%Fñ 
�‰ñ%Fr   c                óJ  € VP                   w  rxp	VP                  WxV P                  V P                  4      pVP                  WxV P                  V P
                  4      pVP                  WxV P                  V P
                  4      pV'       d   V^,          p
VP                  pMRRrº\        VV P                  VVV P                  P                  VP                  4      VV P                  V
V P                  V4
      w  rÊV'       d   W¥^&   VP                  WxV P                  4      # )é   N)rŸ   Úreshaper}   r�   r€   r~   r¡   r   r�   rŽ   Úastyper{   r‰   rD   r+   )r$   r_   r°   r±   r²   r•   r–   Ú
batch_sizeÚseq_lenÚ_Ústater¡   Úys   &&&&&&&      r   Ú_ssmÚ GraniteMoeHybridMamba2Mixer._ssm�   s÷   € ð "/×!4Ñ!4Ñˆ
˜Qà%×-Ñ-Ø §¡°·±ó
ˆð �I‰I�j¨4¯=©=¸$×:MÑ:MÓNˆØ�I‰I�j¨4¯=©=¸$×:MÑ:MÓNˆßØ˜!•HˆEØ—m‘m‰Gà! 4�7äØØ�J‰JØØØ�F‰F�M‰M˜-×-Ñ-Ó.ØØ�L‰LØØ× Ñ Øó
‰ˆ÷ Ø�!‰Hà�y‰y˜¨d×.DÑ.DÓEÐEr   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r   r_   r–   r•   r   r—   )r   r   s   "€r   r   rq   Ä   sI   ø€ ÷  ñ  á—x‘xð ñ ‘r—x‘xÕ ð ñ ™Õ$ð	 ñ
 
�‰ñ r   c                ó4  € V P                  V4      p\        P                  ! VV P                  V P                  V P                  ,           .RR7      w  rVpV P                  WcV4      p\        P                  ! VV P                  V P                  V P                  V P                  ,          ,           .RR7      w  ršpV P                  WšW·W24      pV'       d#   VP                  VP                  ^,          4       V P                  WÅ4      pV P                  V4      # )r´   r™   éÿÿÿÿ)rˆ   rY   Úsplitr+   rƒ   r­   r€   r~   r¼   ÚadvancerŸ   r�   r�   )r$   r_   r–   r•   Ú	projectedr`   r”   r²   r¬   Úhidden_states_ssmr°   r±   r»   s   &&&&         r   rf   Ú$GraniteMoeHybridMamba2Mixer.__call__Ä   sê   € ð —L‘L Ó/ˆ	ä!ŸxšxØØ×#Ñ# T×%;Ñ%;¸d¿m¹mÕ%KÐLØô 
Ñˆ˜"ð
 —j‘j °DÓ9ˆÜ"$§(¢(Øà×&Ñ&Ø×&Ñ&¨¯©¸×9LÑ9LÕ)LÕLðð ô#
ÑÐ˜að �I‰IÐ'¨A°5Ó?ˆßØ�M‰M˜!Ÿ'™' !�*Ô%Ø�I‰I�aÓˆØ�}‰}˜QÓÐr   )r�   rŽ   r†   rƒ   r   r‰   r�   r‚   r*   rˆ   r+   r€   r�   r}   r�   r~   rD   r!   )rI   rJ   rK   rL   rX   r­   r¼   rf   rO   rP   ri   rj   s   @@r   rl   rl   S   s7   ù‡ € ÷%
ó %
÷N $ð  $÷D%Fð %F÷N ÷  ò  r   rl   c                   óT   a a€ ] tR t^ât oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGraniteMoeHybridAttentionc                ó    <€ V ^8„  d   QhRS[ /# rn   rp   )r   r   s   "€r   r   Ú&GraniteMoeHybridAttention.__annotate__ã   s   ø€ ÷ ñ ™Yñ r   c                ó¸  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pVP                  V,          ;V n        pVP                  V n	        VP                  p\        P                  ! W#V,          VR 7      V n        \        P                  ! W$V,          VR 7      V n        \        P                  ! W$V,          VR 7      V n        \        P                  ! W5,          W&R 7      V n        VP"                  R8g  pV'       d5   \%        V P                  VP&                  RRVP(                  4      V n        R# RV n        R# )ry   ÚnopeFN)rW   rX   r*   r.   Ún_headsr/   Ú
n_kv_headsr�   r2   Úscaler0   r„   r‡   Úq_projÚk_projÚv_projÚo_projrB   r   r7   r-   r   )	r$   ro   ÚdimrÌ   rÍ   r�   r0   Úuse_roper\   s	   &&      €r   rX   Ú"GraniteMoeHybridAttention.__init__ã   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'×#3Ñ#3°wÕ#>Ð>ˆŒ˜à×.Ñ.ˆŒ
Ø×,Ñ,ˆÜ—i’i ¨xÕ%7¸nÔMˆŒÜ—i’i °(Õ%:ÀÔPˆŒÜ—i’i °(Õ%:ÀÔPˆŒÜ—i’i Õ 2°CÔMˆŒð ×/Ñ/°6Ñ9ˆßÜ'Ø—‘Ø—‘ØØØ×,Ñ,óˆDŽIð ˆDŽIr   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   Úxr–   r•   r   )rY   rb   r   r   )r   r   s   "€r   r   rÉ     sI   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™Õ ð	#ñ
 
�‰ñ#r   c           	     óB  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	V P                  e`   Ve:   V P                  WsP                  R7      pV P                  WƒP                  R7      pM"V P                  V4      pV P                  V4      pVe   VP                  W‰4      w  r‰\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )r´   )Úoffset)r•   rÎ   r–   rÀ   )rŸ   rÏ   rÐ   rÑ   rµ   rÌ   Ú	transposerÍ   r   rÚ   Úupdate_and_fetchr   rÎ   rÒ   )r$   rØ   r–   r•   r°   ÚLrŽ   ÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r   rf   Ú"GraniteMoeHybridAttention.__call__  sa  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà—/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆð �9‰9Ò ØÒ ØŸ)™) G·L±L˜)ÓA�Ø—y‘y ¯l©l�yÓ;‘àŸ)™) GÓ,�Ø—y‘y “�àÒØ ×1Ñ1°$Ó?‰LˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r   )	r�   rÐ   rÌ   rÍ   rÒ   rÏ   r   rÎ   rÑ   ©NNrh   rj   s   @@r   rÇ   rÇ   â   s   ù‡ € ÷ó ÷<#÷ #ò #r   rÇ   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGraniteMoeHybridTopKGatingi"  c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   Ú
input_sizeÚnum_expertsÚtop_k)rF   )r   r   s   "€r   r   Ú'GraniteMoeHybridTopKGating.__annotate__#  s'   ø€ ÷ Dñ D¡3ð D±Sð DÁñ Dr   c                ó‚   <€ \         SV `  4        W n        Wn        W0n        \
        P                  ! WR R7      V n        R# ©Fry   N)rW   rX   rè   rç   ré   r„   r‡   Úlayer)r$   rç   rè   ré   r\   s   &&&&€r   rX   Ú#GraniteMoeHybridTopKGating.__init__#  s0   ø€ Ü‰ÑÔØ&ÔØ$ŒØŒ
Ü—Y’Y˜z¸UÔCˆŽ
r   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r_   ra   )r   r   s   "€r   r   rê   *  s   ø€ ÷ &ñ &¡b§h¡hñ &r   c                óþ   € V P                  V4      p\        P                  ! W P                  ) RR7      RV P                  ) R13,          p\        P                  ! W#RR7      p\        P
                  ! VRRR7      pW53# )r´   )Úkthrš   .Nr™   T)Úpreciserš   rÀ   )rí   rY   Úargpartitionré   r£   Úsoftmax)r$   r_   ÚlogitsÚ	top_k_idxÚtop_k_logitsÚtop_k_gatess   &&    r   rf   Ú#GraniteMoeHybridTopKGating.__call__*  sj   € Ø—‘˜MÓ*ˆÜ—O’O F·±°À"ÔEØ�$—*‘*�‘Ðõ
ˆ	ô ×)Ò)¨&À"ÔEˆÜ—j’j °tÀ"ÔEˆØÐ%Ð%r   )rç   rí   rè   ré   rh   rj   s   @@r   rå   rå   "  s    ù‡ € ÷Dó D÷&÷ &ð &r   rå   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGraniteMoeHybridMoEi4  c                ó    <€ V ^8„  d   QhRS[ /# rn   rp   )r   r   s   "€r   r   Ú GraniteMoeHybridMoE.__annotate__5  s   ø€ ÷ 
ñ 
™Yñ 
r   c                ó*  <€ \         SV `  4        VP                  V n        VP                  V n        \        V P                  V P                  VP                  4      V n        \        V P                  VP                  VP                  R 7      V n
        R# ))rç   rè   ré   N)rW   rX   r*   rç   r+   r   r"   Ú
switch_mlprå   r8   Úrouter©r$   ro   r\   s   &&€r   rX   ÚGraniteMoeHybridMoE.__init__5  ss   ø€ Ü‰ÑÔà×*Ñ*ˆŒØ×1Ñ1ˆÔÜ#Ø�O‰O˜T×-Ñ-¨t×/EÑ/Eó
ˆŒô 1Ø—‘Ø×.Ñ.Ø×*Ñ*ô
ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   rØ   r   ra   )r   r   s   "€r   r   rý   C  s#   ø€ ÷ 3ñ 3™"Ÿ(™(ð 3¡r§x¡xñ 3r   c                óˆ   € V P                  V4      w  r#V P                  W4      pWCR,          ,          P                  RR7      # ).r™   rœ   éþÿÿÿ)r   rÿ   Úsum)r$   rØ   Ú	token_idsÚgatesr»   s   &&   r   rf   ÚGraniteMoeHybridMoE.__call__C  s<   € ØŸ;™; q›>Ñˆ	Ø�O‰O˜AÓ)ˆØ˜)Õ$Õ$×)Ñ)¨rÐ)Ó2Ð2r   )r*   rç   r   rÿ   rh   rj   s   @@r   rû   rû   4  s   ù‡ € ÷
ó 
÷3÷ 3ð 3r   rû   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGraniteMoeHybridSharedMLPiI  c                ó    <€ V ^8„  d   QhRS[ /# rn   rp   )r   r   s   "€r   r   Ú&GraniteMoeHybridSharedMLP.__annotate__J  s   ø€ ÷ 
ñ 
™Yñ 
r   c                óú   <€ \         SV `  4        \        P                  ! VP                  VP
                  ^,          RR7      V n        \        P                  ! VP
                  VP                  RR7      V n        R# )r   Fry   N)rW   rX   r„   r‡   r*   r9   Úinput_linearÚoutput_linearr  s   &&€r   rX   Ú"GraniteMoeHybridSharedMLP.__init__J  s]   ø€ Ü‰ÑÔÜŸIšIØ×Ñ˜d×;Ñ;¸aÕ?Àeô
ˆÔô  ŸYšYØ×)Ñ)¨4×+;Ñ+;À%ô
ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  ra   )r   r   s   "€r   r   r  S  s#   ø€ ÷ 4ñ 4™"Ÿ(™(ð 4¡r§x¡xñ 4r   c                óŠ   € \         P                  ! V P                  V4      ^RR7      w  r#V P                  \	        W#4      4      # )r   r™   rÀ   )rY   rÁ   r  r  r   )r$   rØ   r`   Úups   &&  r   rf   Ú"GraniteMoeHybridSharedMLP.__call__S  s7   € Ü—8’8˜D×-Ñ-¨aÓ0°!¸"Ô=‰ˆØ×!Ñ!¤&¨Ó"2Ó3Ð3r   )r  r  rh   rj   s   @@r   r  r  I  s   ù‡ € ÷
ó 
÷4÷ 4ð 4r   r  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚGraniteMoeHybridMLPiX  c                ó    <€ V ^8„  d   QhRS[ /# rn   rp   )r   r   s   "€r   r   Ú GraniteMoeHybridMLP.__annotate__Y  s   ø€ ÷ Añ A™Yñ Ar   c                ó  <€ \         SV `  4        VP                  pVP                  pVP                  p\
        P                  ! W#VR 7      V n        \
        P                  ! W2VR 7      V n        \
        P                  ! W#VR 7      V n	        R# )ry   N)
rW   rX   r*   r+   rA   r„   r‡   Ú	gate_projÚ	down_projÚup_proj)r$   ro   rÓ   Ú
hidden_dimrA   r\   s   &&   €r   rX   ÚGraniteMoeHybridMLP.__init__Y  sc   ø€ Ü‰ÑÔØ×ÑˆØ×+Ñ+ˆ
Ø—=‘=ˆäŸš 3¸ÔBˆŒÜŸš :¸ÔBˆŒÜ—y’y °xÔ@ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r   ra   )r   r   s   "€r   r   r  c  s   ø€ ÷ Jñ J™RŸX™Xñ Jr   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # r!   )r  r   r  r  )r$   rØ   s   &&r   rf   ÚGraniteMoeHybridMLP.__call__c  s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr   )r  r  r  rh   rj   s   @@r   r  r  X  s#   ù‡ € ÷Aó A÷J÷ Jð Jr   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGraniteMoeHybridLayerig  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   ro   Ú
layer_type)r   rE   )r   r   s   "€r   r   Ú"GraniteMoeHybridLayer.__annotate__h  s   ø€ ÷ 
ñ 
™Yð 
±Cñ 
r   c                ó:  <€ \         SV `  4        W n        VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  R 7      V n	        VR8X  d   \        V4      V n        M&VR8X  d   \        V4      V n        M\        RV 24      hV P                  '       d"   \        V4      V n        \#        V4      V n        M\'        V4      V n        \
        P                  ! VP                  VP                  R 7      V n        R# )r|   ÚmambaÚ	attentionzUnknown layer type: N)rW   rX   r'  r4   r%   r„   ÚRMSNormr*   r6   Úinput_layernormrl   r*  rÇ   Ú	self_attnÚ
ValueErrorr  Ú
shared_mlprû   Úblock_sparse_moer  ÚmlpÚpost_attention_layernorm©r$   ro   r'  r\   s   &&&€r   rX   ÚGraniteMoeHybridLayer.__init__h  sÚ   ø€ Ü‰ÑÔØ$ŒØ#'×#;Ñ#;ˆÔ Ø—|‘|ˆŒä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔà˜Ô Ü4°TÓ:ˆD�JØ˜;Ô&Ü6°tÓ<ˆD�NäÐ3°J°<Ð@ÓAÐAð �<�<ˆ<Ü7¸Ó=ˆDŒOÜ$7¸Ó$=ˆDÕ!ô +¨4Ó0ˆDŒHä(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÖ%r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r×   ©rY   rb   r   r   )r   r   s   "€r   r   r(  ƒ  sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr   c                óÂ  € TpV P                  V4      pV P                  R 8X  d   V P                  WRVR7      pMV P                  WRVR7      pWEV P                  ,          ,           pTpV P                  V4      pV P                  '       d,   V P                  V4      pV P                  V4      pWx,           p	MV P                  V4      p	WIV P                  ,          ,           pV# )r*  ©r–   r•   )
r-  r'  r*  r.  r4   r3  r%   r1  r0  r2  )
r$   rØ   r–   r•   Úresidualr_   ÚnormedÚmoe_outÚ
shared_outÚmlp_outs
   &&&&      r   rf   ÚGraniteMoeHybridLayer.__call__ƒ  sÉ   € ð ˆØ×,Ñ,¨QÓ/ˆà�?‰?˜gÔ%Ø ŸJ™J }Àu˜JÓM‰Mà ŸN™N¨=È5˜NÓQˆMà °4×3KÑ3KÕ#KÕKˆð !ˆØ×.Ñ.¨}Ó=ˆà�<�<ˆ<Ø×+Ñ+¨FÓ3ˆGØŸ™¨Ó0ˆJØÕ*‰Gà—h‘h˜vÓ&ˆGà ¨T×-EÑ-EÕ#EÕEˆàÐr   )
r1  r-  r'  r*  r2  r3  r4   r.  r0  r%   rã   rh   rj   s   @@r   r%  r%  g  s   ù‡ € ÷
ó 
÷6÷ ò r   r%  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGraniteMoeHybridModeli¤  c                ó    <€ V ^8„  d   QhRS[ /# rn   rp   )r   r   s   "€r   r   Ú"GraniteMoeHybridModel.__annotate__¥  s   ø€ ÷ 
ñ 
™Yñ 
r   c                óL  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP                  4      V n        VP                   Uu. uF  p\        W4      NK  	  upV n
        \        P                  ! VP                  VP                  R 7      V n        VP                  V n        RVP                  9   d   VP                  P                  R4      MRV n        RVP                  9   d#   VP                  P                  R4      V n        R# RV n        R# u upi )r|   r+  Nr*  )rW   rX   ro   r„   Ú	Embeddingr)   r*   Úembed_tokensr5   r%  Úlayersr,  r6   r�   r1   ÚindexÚfa_idxÚssm_idxr4  s   && €r   rX   ÚGraniteMoeHybridModel.__init__¥  sï   ø€ Ü‰ÑÔØŒ	ÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔàFJ×FVÒFVó
ÙFV¸
Ô! $Ö3ÑFVñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	Ø$(×$=Ñ$=ˆÔ!ð
 ˜d×.Ñ.Ô.ð ×Ñ×"Ñ" ;Ô/àð 	Œð 07¸$×:JÑ:JÔ/JˆD×Ñ×"Ñ" 7Ó+ð 	ŽØPTð 	Žùò
s   ÁD!c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# ©r   Úinputsr•   r   r7  )r   r   s   "€r   r   rC  ¹  s5   ø€ ÷ (ñ (á—‘ð(ñ ™�}ð(ñ 
�‰ñ	(r   c                óÖ  € V P                  V4      V P                  ,          pVf   R .\        V P                  4      ,          pR pR pV P                  e   \        W2V P                  ,          4      pV P                  e   \        W2V P                  ,          4      p\        V P                  V4       F$  w  rgVP                  R8X  d   TMTpV! W8VR7      pK&  	  V P                  V4      # )Nr+  r9  )rF  r1   ÚlenrG  rI  r
   rJ  r   Úzipr'  r�   )	r$   rN  r•   r_   Ú	attn_maskÚ
mamba_maskrí   Úcr–   s	   &&&      r   rf   ÚGraniteMoeHybridModel.__call__¹  sÇ   € ð
 ×)Ñ)¨&Ó1°D×4MÑ4MÕMˆàŠ=Ø�FœS §¡Ó-Õ-ˆEð ˆ	Øˆ
à�;‰;Ò"Ü-¨mÀ4Ç;Á;Õ=OÓPˆIØ�<‰<Ò#Ü(¨¸d¿l¹lÕ8KÓLˆJä˜DŸK™K¨Ö/‰HˆEØ %× 0Ñ 0°KÔ ?‘9ÀZˆDÙ! -À!ÔDŠMñ 0ð �y‰y˜Ó'Ð'r   )ro   rF  r1   rI  rG  r�   rJ  r!   rh   rj   s   @@r   rA  rA  ¤  s   ù‡ € ÷
ó 
÷((÷ (ò (r   rA  c                   ó€   a a€ ] tR tRt oV3R lV 3R lltRV3R lR llt]R 4       tR tR t	]R	 4       t
R
tVtV ;t# )ÚModeliÓ  c                ó    <€ V ^8„  d   QhRS[ /# rn   rp   )r   r   s   "€r   r   ÚModel.__annotate__Ô  s   ø€ ÷ 2ñ 2™Yñ 2r   c                ó  <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g3   \        P                  ! VP                  VP                  R R7      V n        VP                  V n        R# rì   )rW   rX   ro   r(   rA  ÚmodelrC   r„   r‡   r*   r)   Úlm_headr3   r  s   &&€r   rX   ÚModel.__init__Ô  sb   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ*¨4Ó0ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŒLØ"×1Ñ1ˆÖr   c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# rM  r7  )r   r   s   "€r   r   rY  Ý  s5   ø€ ÷ )ñ )á—‘ð)ñ ™�}ð)ñ 
�‰ñ	)r   c                óð   € V P                  WR 7      pV P                  P                  '       d'   V P                   P                  P	                  V4      pMV P                  V4      pW0P                  ,          # ))r•   )r[  ro   rC   rF  Ú	as_linearr\  r3   )r$   rN  r•   Úouts   &&& r   rf   ÚModel.__call__Ý  s[   € ð
 �j‰j˜ˆjÓ-ˆà�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8‰Cà—,‘,˜sÓ#ˆCà×(Ñ(Õ(Ð(r   c                ó.   € V P                   P                  # r!   )r[  rG  r#   s   &r   rG  ÚModel.layersë  s   € à�z‰z× Ñ Ð r   c                óä   € . pV P                    F]  pVP                  R 8X  d   VP                  \        ^R7      4       K1  VP                  R8X  g   KD  VP                  \	        4       4       K_  	  V# )r*  )Úsizer+  )rG  r'  Úappendr   r   )r$   Úcachesrí   s   &  r   Ú
make_cacheÚModel.make_cacheï  sW   € ØˆØ—[”[ˆEØ×Ñ 7Ô*Ø—‘œk¨qÔ1Ö2Ø×!Ñ! [Ö0Ø—‘œg›iÖ(ñ	 !ð
 ˆr   c                ó‚  € VP                  4        F<  w  r#R V9   g   K  VP                  R,          ^8w  g   K(  VP                  ^^4      W&   K>  	  V P                  P                  '       dª   RV9   d£   \        V P                  P                  4       F}  pRV R2pVP                  V R24      pVP                  w  rxpVRRV^,          1R3,          p	VRV^,          R1R3,          p
W‘V R2&   W¡V R2&   VP                  V R	24      W R
2&   K  	  V# V P                  P                  '       gŽ   RV9   d‡   \        V P                  P                  4       Fc  pRV R2pVP                  V R24      p\        P                  ! V^^ R7      w  ršW‘RV R2&   W¡RV R2&   VP                  V R	24      VRV R2&   Ke  	  V# )zconv1d.weightz3model.layers.0.block_sparse_moe.input_linear.weightzmodel.layers.z.block_sparse_moez.input_linear.weightr›   Nz.switch_mlp.gate_proj.weightz.switch_mlp.up_proj.weightz.output_linear.weightz.switch_mlp.down_proj.weightz-model.layers.0.shared_mlp.input_linear.weightz.shared_mlpr™   z.mlp.gate_proj.weightz.mlp.up_proj.weightz.mlp.down_proj.weightrÀ   )
ÚitemsrŸ   Úmoveaxisro   r%   Úranger,   ÚpoprY   rÁ   )r$   ÚweightsÚkÚvÚlÚprefixÚinput_weightr¹   Úexpert_hiddenr  r  s   &&         r   ÚsanitizeÚModel.sanitizeø  së  € à—M‘M–O‰DˆAØ !Ö#¨¯©°­°qÖ(8ØŸZ™Z¨¨1Ó-�“
ñ $ð �I‰I××ÐØEÈÔPä˜4Ÿ9™9×6Ñ6Ö7�Ø(¨¨Ð+<Ð=�à&Ÿ{™{¨f¨XÐ5IÐ+JÓK�Ø&2×&8Ñ&8Ñ#� !ð )¨Ð,@¨m¸qÕ.@Ð,@À!Ð)CÕD�	Ø& q¨-¸1Õ*<Ñ*>ÀÐ'AÕB�ØCL˜6˜(Ð">Ð?Ñ@ØAH˜6˜(Ð"<Ð=Ñ>àCJÇ;Á;Ø�hÐ3Ð4óD�˜(Ð">Ð?Ó@ñ 8ðF ˆð# —	‘	×!×!Ð!Ø?À7ÔJä˜4Ÿ9™9×6Ñ6Ö7�Ø(¨¨¨;Ð7�ð  'Ÿ{™{¨f¨XÐ5IÐ+JÓK�ä%'§X¢X¨l¸AÀAÔ%FÑ"�	ØDM˜-¨ sÐ*?Ð@ÑAØBI˜-¨ sÐ*=Ð>Ñ?àDKÇKÁKØ�hÐ3Ð4óE�˜-¨ sÐ*?Ð@ÓAñ 8ð ˆr   c                ó   a € V 3R  lpV# )c                 óz   <€ SP                   P                  '       d   V P                  R 4      '       d   R^@R^/# R# )zrouter.layerÚ
group_sizeÚbitsT)ro   r%   Úendswith)Úpathr¹   r$   s   &&€r   Ú	predicateÚ(Model.quant_predicate.<locals>.predicate*  s3   ø€ Ø�y‰y× × Ð  T§]¡]°>×%BÒ%BØ$ b¨&°!Ð4Ð4Ùr   rH   )r$   r  s   f r   Úquant_predicateÚModel.quant_predicate(  s   ø€ õ	ð
 Ðr   )ro   r\  r3   r[  r(   r!   )rI   rJ   rK   rL   rX   rf   rM   rG  ri  rw  r�  rO   rP   ri   rj   s   @@r   rW  rW  Ó  sQ   ù‡ € ÷2ó 2÷)ò )ð ñ!ó ð!òò.ð` ñó ÷ð r   rW  )(Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerY   Úmlx.nnr„   Úactivationsr   Úbaser	   r
   r   r   r•   r   r   Ú
rope_utilsr   Ússmr   Úswitch_layersr   r   ÚModulerR   rl   rÇ   rå   rû   r  r  r%  rA  rW  rH   r   r   Ú<module>rŽ     sù   ðõ "ß -Ó -å Ý å ÷ó ÷ (Ý 'Ý Ý $ð ô-,�ó -,ó ð-,ô`	F 2§9¡9ô 	FôL  "§)¡)ô L ô^=# §	¡	ô =#ô@& §¡ô &ô$3˜"Ÿ)™)ô 3ô*4 §	¡	ô 4ôJ˜"Ÿ)™)ô Jô:˜BŸI™Iô :ôz,(˜BŸI™Iô ,(ô^\ˆB�I‰Iö \r   