+
    UV-j|3  ã                   ó`  € ^ RI Ht ^ RIHtHtHt ^ RIHt ^ RI	H
t
 ^ RIHt ^ RIHtHt ^RIHtHtHt ^RIHtHt ^RIHtHt ]! ]P4                  R	R
7      R 4       t ! R R]
P8                  4      t ! R R]
P8                  4      t ! R R]
P8                  4      t]P4                  R 4       t  ! R R]
P8                  4      t! ! R R]
P8                  4      t" ! R R]
P8                  4      t# ! R R]
P8                  4      t$ ! R R]
P8                  4      t% ! R R]
P8                  4      t&R# )é    )Úpartial)ÚAnyÚDictÚOptionalN)Úinitialize_rope)ÚSwiGLUÚ	SwitchGLU)ÚLanguageModelOutputÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)ÚModelConfigÚ
TextConfigT)Ú	shapelessc                 óž   € \         P                  ! \        P                  ! V4      R VR7      p\         P                  ! W) VR7      p W,          # )N)Úa_minÚa_max)ÚmxÚclipÚnnÚsilu)ÚxÚgateÚlimits   &&&Úp/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/step3p7/language.pyÚ_clamped_swiglur      s5   € ä�7Š7”2—7’7˜4“=¨°EÔ:€DÜ
�Š�˜ uÔ-€AØ�8€Oó    c                   óD   a a€ ] tR t^t oV3R lV 3R lltR tRtVtV ;t# )ÚClampedSwiGLUc                ó    <€ V ^8„  d   QhRS[ /# )é   r   )Úfloat)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚClampedSwiGLU.__annotate__   s   ø€ ÷ ñ ™eñ r   c                ó0   <€ \         SV `  4        Wn        R # ©N)ÚsuperÚ__init__r   )Úselfr   Ú	__class__s   &&€r   r+   ÚClampedSwiGLU.__init__   s   ø€ Ü‰ÑÔØŽ
r   c                ó.   € \        WV P                  4      # r)   )r   r   )r,   r   r   s   &&&r   Ú__call__ÚClampedSwiGLU.__call__   s   € Ü˜q¨¯
©
Ó3Ð3r   )r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r+   r0   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r-   r%   s   @@r   r    r       s   ù‡ € ÷ó ÷4ò 4r   r    c                   óD   a a€ ] tR t^"t oV3R lV 3R lltR tRtVtV ;t# )ÚStepRMSNormc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r"   Úhidden_sizeÚeps)Úintr#   )r$   r%   s   "€r   r&   ÚStepRMSNorm.__annotate__#   s   ø€ ÷ ñ ¡Cð ©eñ r   c                óh   <€ \         SV `  4        \        P                  ! V34      V n        W n        R # r)   )r*   r+   r   ÚonesÚweightr?   )r,   r>   r?   r-   s   &&&€r   r+   ÚStepRMSNorm.__init__#   s%   ø€ Ü‰ÑÔÜ—g’g˜{˜nÓ-ˆŒØŽr   c                óœ   € \         P                  P                  WP                  V P                  4      P                  VP                  4      # r)   )r   ÚfastÚrms_normrD   r?   ÚastypeÚdtype©r,   r   s   &&r   r0   ÚStepRMSNorm.__call__(   s0   € Ü�w‰w×Ñ §;¡;°·±Ó9×@Ñ@ÀÇÁÓIÐIr   )r?   rD   r2   r:   s   @@r   r<   r<   "   s   ù‡ € ÷ó ÷
Jò Jr   r<   c                   óH   a a€ ] tR t^,t oRV3R lV 3R llltR tRtVtV ;t# )ÚMLPc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r"   ÚconfigÚintermediate_size©r   r@   )r$   r%   s   "€r   r&   ÚMLP.__annotate__-   s   ø€ ÷ Qñ Q™zð Q¹cñ Qr   c                óN  <€ \         SV `  4        \        P                  ! VP                  VR R7      V n        \        P                  ! VP                  VR R7      V n        \        P                  ! W!P                  R R7      V n        V'       d   V^ 8”  d	   W0n        R# RV n        R# ©F©ÚbiasN)	r*   r+   r   ÚLinearr>   Ú	gate_projÚup_projÚ	down_projr   )r,   rP   rQ   Úswiglu_limitr-   s   &&&&€r   r+   ÚMLP.__init__-   sq   ø€ Ü‰ÑÔÜŸš 6×#5Ñ#5Ð7HÈuÔUˆŒÜ—y’y ×!3Ñ!3Ð5FÈUÔSˆŒÜŸšÐ#4×6HÑ6HÈuÔUˆŒß%1°lÀQÔ6F�\Ž
ÈDˆŽ
r   c                ó:  € V P                   eE   V P                  \        V P                  V4      V P	                  V4      V P                   4      4      # V P                  \
        P                  ! V P	                  V4      4      V P                  V4      ,          4      # r)   )r   r[   r   rZ   rY   r   r   rK   s   &&r   r0   ÚMLP.__call__4   sn   € Ø�:‰:Ò!Ø—>‘>Ü §¡¨Q£°·±ÀÓ1BÀDÇJÁJÓOóð ð �~‰~œbŸgšg d§n¡n°QÓ&7Ó8¸4¿<¹<È»?ÕJÓKÐKr   )r[   rY   r   rZ   )r   r2   r:   s   @@r   rN   rN   ,   s   ù‡ € ÷Qõ Q÷Lò Lr   rN   c                 ór  € \         P                  ! V P                  \         P                  4      4      pWQ,           p\         P                  ! V) V^,
          RR7      RRV13,          p\         P
                  ! WWRR7      pV'       d(   V\         P                  ! VRRR7      R,           ,          pWxV,          3# )	é   )ÚkthÚaxis.N©rc   T)rc   Úkeepdimsg#B’¡œÇ;éÿÿÿÿ)r   ÚsigmoidrI   Úfloat32ÚargpartitionÚtake_along_axisÚsum)	ÚgatesÚrouter_biasÚtop_kÚrouted_scaling_factorÚnorm_topk_probÚscoresÚcorrected_scoresÚtopk_indicesÚtopk_weightss	   &&&&&    r   Ú_moe_gate_selectru   <   s—   € ä�ZŠZ˜Ÿ™¤R§Z¡ZÓ0Ó1€FØÕ+ÐÜ—?’?Ð$4Ð#4¸%À!½)È"ÔMØˆVˆeˆVˆõ€Lô ×%Ò% fÀÔD€LßØ#Ü�FŠF�< b°4Ô8¸5Õ@õ
ˆð Ð(=Õ=Ð=Ð=r   c                   óD   a a€ ] tR t^Kt oV3R lV 3R lltR tRtVtV ;t# )ÚMoEGatec                ó    <€ V ^8„  d   QhRS[ /# ©r"   rP   ©r   )r$   r%   s   "€r   r&   ÚMoEGate.__annotate__L   s   ø€ ÷ Qñ Q™zñ Qr   c                óZ  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! VP                  VP                  R R7      V n        \        P                  ! VP                  3\        P                  R7      V n        R# )FrV   )rJ   N)r*   r+   Ú	moe_top_krn   Úmoe_router_scaling_factorro   Únorm_expert_weightrp   r   rX   r>   Úmoe_num_expertsr   r   Úzerosrh   rm   )r,   rP   r-   s   &&€r   r+   ÚMoEGate.__init__L   sw   ø€ Ü‰ÑÔØ×%Ñ%ˆŒ
Ø%+×%EÑ%EˆÔ"Ø$×7Ñ7ˆÔÜ—I’I˜f×0Ñ0°&×2HÑ2HÈuÔUˆŒ	ÜŸ8š8 V×%;Ñ%;Ð$=ÄRÇZÁZÔPˆÖr   c                óŽ   € \        V P                  V4      V P                  V P                  V P                  V P
                  4      # r)   )ru   r   rm   rn   ro   rp   rK   s   &&r   r0   ÚMoEGate.__call__T   s<   € ÜØ�I‰I�a‹LØ×ÑØ�J‰JØ×&Ñ&Ø×Ñó
ð 	
r   )r   rp   ro   rm   rn   r2   r:   s   @@r   rw   rw   K   s   ù‡ € ÷Qó Q÷
ò 
r   rw   c                   óD   a a€ ] tR t^^t oV3R lV 3R lltR tRtVtV ;t# )ÚMoEc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r"   rP   Ú	layer_idxrR   )r$   r%   s   "€r   r&   ÚMoE.__annotate___   s   ø€ ÷ Oñ O™zð O±cñ Or   c                óB  <€ \         SV `  4        ^ pVP                  '       d8   V\        VP                  4      8  d   VP                  V,          ;'       g    ^ p^ pVP                  '       d8   V\        VP                  4      8  d   VP                  V,          ;'       g    ^ p\        V4      V n        V^ 8”  d   \        V4      M	\        4       p\        VP                  VP                  VP                  VR7      V n        \        WP                  V4      V n        R# )r   )Ú
activationN)r*   r+   Úswiglu_limitsÚlenÚswiglu_limits_sharedrw   r   r    r   r	   r>   Úmoe_intermediate_sizer€   Ú
switch_mlprN   Úshare_expert_dimÚshare_expert)r,   rP   r‰   r\   Úshared_limitrŒ   r-   s   &&&   €r   r+   ÚMoE.__init___   så   ø€ Ü‰ÑÔØˆØ××Ð I´°F×4HÑ4HÓ0IÔ$IØ!×/Ñ/°	Õ:×?Ð?¸aˆLØˆØ×&×&Ð&¨9´s¸6×;VÑ;VÓ7WÔ+WØ!×6Ñ6°yÕA×FÐFÀQˆLä˜F“OˆŒ	Ø4@À1Ô4D”] <Ô0Ì&Ë(ˆ
Ü#Ø×ÑØ×(Ñ(Ø×"Ñ"Ø!ô	
ˆŒô   ×(?Ñ(?ÀÓNˆÖr   c                óè   € V P                  V4      w  r#V P                  W4      pWCR,          ,          P                  RR7      P                  VP                  4      pW@P                  V4      ,           # ).rd   ©.Néþÿÿÿ)r   r‘   rk   rI   rJ   r“   )r,   r   rs   rt   Úrouted_outputs   &&   r   r0   ÚMoE.__call__r   sa   € Ø%)§Y¡Y¨q£\Ñ"ˆØŸ™¨Ó8ˆà¨)Õ4Õ4ß‰S�bˆS‹\ß‰V�M×'Ñ'Ó(ð 	ð
 ×0Ñ0°Ó3Õ3Ð3r   )r   r“   r‘   r2   r:   s   @@r   r†   r†   ^   s   ù‡ € ÷Oó O÷&4ò 4r   r†   c                   óT   a a€ ] tR t^}t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# rˆ   rR   )r$   r%   s   "€r   r&   ÚAttention.__annotate__~   s   ø€ ÷ 8
ñ 8
™zð 8
±cñ 8
r   c                ó$  <€ \         SV `  4        VP                  ;'       g    . pV'       d   W2,          R 8H  MV^,          ^ 8H  V n        V P                  '       dD   VP                  '       d2   VP                  R,          V n        VP                  R,          V n        M"VP                  V n        VP                  V n        VP                  V n	        V P                  R	,          V n
        \        P                  ! VP                  V P
                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  V P                  ,          RR7      V n        \        P                  ! V P
                  V P                  ,          VP                  RR7      V n        \%        V P                  VP&                  4      V n        \%        V P                  VP&                  4      V n        VP,                  V n        V P,                  '       d3   \        P                  ! VP                  V P
                  RR7      V n        \1        VP2                  \4        4      '       d   VP2                  V,          MVP2                  pRpVP6                  '       d.   V\9        VP6                  4      8  d   VP6                  V,          pV'       d	   W2,          MRpVP:                  '       d   WaP:                  9  d   RMVP<                  p\?        \A        V P                  V,          4      VRVVPB                  R7      V n"        R# )
Úsliding_attentionÚnum_attention_headsÚnum_attention_groupsFrV   g      ð?Úfull_attentionN)ÚdimsÚbaseÚtraditionalÚscaling_configÚmax_position_embeddingsg      à¿)#r*   r+   Úlayer_typesÚ
is_slidingÚattention_other_settingÚn_headsÚ
n_kv_headsr¡   r¢   Úhead_dimÚscaler   rX   r>   Úq_projÚk_projÚv_projÚo_projr<   Úrms_norm_epsÚq_normÚk_normÚuse_head_wise_attn_gateÚg_projÚ
isinstanceÚ
rope_thetaÚlistÚpartial_rotary_factorsrŽ   Úyarn_only_typesÚrope_scalingr   r@   r¨   Úrope)	r,   rP   r‰   r©   Úthetar   Ú
layer_typer¾   r-   s	   &&&     €r   r+   ÚAttention.__init__~   s©  ø€ Ü‰ÑÔØ×(Ñ(×.Ð.¨Bˆ÷ ð Õ"Ð&9Ò9à˜Q• !Ñ#ð 	Œð
 �?�?ˆ?˜v×=×=Ð=Ø!×9Ñ9Ð:OÕPˆDŒLØ$×<Ñ<Ð=SÕTˆD�Oà!×5Ñ5ˆDŒLØ$×9Ñ9ˆDŒOØŸ™ˆŒØ—]‘] DÕ(ˆŒ
Ü—i’iØ×Ñ §¡¨t¯}©}Õ <À5ô
ˆŒô —i’iØ×Ñ §¡°$·-±-Õ ?Àeô
ˆŒô —i’iØ×Ñ §¡°$·-±-Õ ?Àeô
ˆŒô —i’iØ�L‰L˜4Ÿ=™=Õ(¨&×*<Ñ*<À5ô
ˆŒô " $§-¡-°×1DÑ1DÓEˆŒÜ! $§-¡-°×1DÑ1DÓEˆŒØ'-×'EÑ'EˆÔ$Ø×'×'Ð'ÜŸ)š) F×$6Ñ$6¸¿¹È5ÔQˆDŒKô ˜&×+Ñ+¬T×2Ò2ð ×Ñ˜iÖ(à×"Ñ"ð 	ð
 ˆØ×(×(Ð(¨Y¼Ø×)Ñ)ó:
ô .
ð ×3Ñ3°IÕ>ˆGß/:�[Ö+Ð@Pˆ
ð ×%×%Ð%¨*×<RÑ<RÔ*Rñ à×$Ñ$ð 	ô
 $Ü�T—]‘] WÕ,Ó-ØØØ'Ø$*×$BÑ$Bô
ˆŽ	r   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# )r"   Úcache)r   r   )r$   r%   s   "€r   r&   rž   ¸   s   ø€ ÷ 0ñ 0©H±S­Mñ 0r   c           	     óÚ  € VP                   w  rEpV P                  V P                  V4      P                  WEV P                  R4      4      P                  ^ ^^^4      pV P                  V P                  V4      P                  WEV P                  R4      4      P                  ^ ^^^4      pV P                  V4      P                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      p
V P                  '       d4   V
\         P"                  ! V P%                  V4      4      R,          ,          p
V P'                  V
P                  WER4      4      # )ra   )Úoffset)rÄ   r¯   Úmaskrf   r—   )Úshaperµ   r°   Úreshaper¬   Ú	transposer¶   r±   r­   r²   r¿   rÆ   Úupdate_and_fetchr   r¯   r·   r   rg   r¸   r³   )r,   r   rÇ   rÄ   ÚbÚlÚ_ÚqÚkÚvÚys   &&&&       r   r0   ÚAttention.__call__¸   sŽ  € Ø—'‘'‰ˆˆaØ�K‰K˜Ÿ™ A›×.Ñ.¨q°T·\±\À2ÓFÓG×QÑQØˆq�!�Qó
ˆð �K‰K˜Ÿ™ A›×.Ñ.¨q°T·_±_ÀbÓIÓJ×TÑTØˆq�!�Qó
ˆð �K‰K˜‹N×"Ñ" 1¨¯©¸"Ó=×GÑGÈÈ1ÈaÐQRÓSˆØÒØ—	‘	˜!§L¡L�	Ó1ˆAØ—	‘	˜!§L¡L�	Ó1ˆAØ×)Ñ)¨!Ó/‰DˆAˆqà—	‘	˜!“ˆAØ—	‘	˜!“ˆAÜ(Ø�!¨¯
©
¸ô
ˆð �K‰K˜˜1˜a Ó#ˆØ×'×'Ð'Ø”B—J’J˜tŸ{™{¨1›~Ó.¨yÕ9Õ9ˆAØ�{‰{˜1Ÿ9™9 Q¨2Ó.Ó/Ð/r   )r¸   r®   rª   r¶   r±   r¬   r­   r³   rµ   r°   r¿   r¯   r·   r²   ©NNr2   r:   s   @@r   rœ   rœ   }   s   ù‡ € ÷8
ó 8
÷t0÷ 0ò 0r   rœ   c                   óH   a a€ ] tR t^Ñt oV3R lV 3R lltRR ltRtVtV ;t# )ÚDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# rˆ   rR   )r$   r%   s   "€r   r&   ÚDecoderLayer.__annotate__Ò   s   ø€ ÷ 
ñ 
™zð 
±cñ 
r   c                ó4  <€ \         SV `  4        \        W4      V n        V P                  P                  V n        VP
                  p\        V\        4      '       dP   VP                  4       P                  R 4       Uu0 uF%  qDP                  4       '       g   K  \        V4      kK'  	  ppM\        V4      pW#9   V n        V P                  '       d   \        W4      V n        Mf^ pVP                  '       d8   V\!        VP                  4      8  d   VP                  V,          ;'       g    ^ p\#        WP$                  V4      V n        \'        VP(                  VP*                  4      V n        \'        VP(                  VP*                  4      V n        R# u upi )Ú,N)r*   r+   rœ   Ú	self_attnrª   Úmoe_layers_enumr¹   ÚstrÚstripÚsplitr@   ÚsetÚis_moe_layerr†   Úmlpr�   rŽ   rN   rQ   r<   r>   r´   Úinput_layernormÚpost_attention_layernorm)r,   rP   r‰   Ú
moe_layersÚir”   r-   s   &&&   €r   r+   ÚDecoderLayer.__init__Ò   s<  ø€ Ü‰ÑÔÜ" 6Ó5ˆŒØŸ.™.×3Ñ3ˆŒØ×+Ñ+ˆ
Ü�j¤#×&Ò&Ø*4×*:Ñ*:Ó*<×*BÑ*BÀ3Ô*GÓUÑ*G QÏ7É7Ï9œ&œ#˜až&Ñ*GˆJÐUˆJä˜Z›ˆJØ%Ñ3ˆÔØ××ÐÜ˜6Ó-ˆD�HàˆLØ×*×*Ð*¨y¼3Ø×+Ñ+ó<ô 0ð  &×:Ñ:¸9ÕE×JÐJÈ�Ü˜6×#;Ñ#;¸\ÓJˆDŒHÜ*¨6×+=Ñ+=¸v×?RÑ?RÓSˆÔÜ(3Ø×Ñ × 3Ñ 3ó)
ˆÖ%ùò Vs   Á>FÂFc                óž   € WP                  V P                  V4      W#R 7      ,           pW@P                  V P                  V4      4      ,           # )©rÇ   rÄ   )rÛ   rã   râ   rä   )r,   r   rÇ   rÄ   Úhs   &&&& r   r0   ÚDecoderLayer.__call__ê   s?   € Ø—‘˜t×3Ñ3°AÓ6¸T�ÓOÕOˆØ—8‘8˜D×9Ñ9¸!Ó<Ó=Õ=Ð=r   )rã   rá   rª   râ   rä   rÛ   rÔ   r2   r:   s   @@r   rÖ   rÖ   Ñ   s   ù‡ € ÷
ó 
÷0>ô >r   rÖ   c                   óH   a a€ ] tR t^ït oV3R lV 3R lltRR ltRtVtV ;t# )ÚStepTextModelc                ó    <€ V ^8„  d   QhRS[ /# ry   rz   )r$   r%   s   "€r   r&   ÚStepTextModel.__annotate__ð   s   ø€ ÷ 
ñ 
™zñ 
r   c                óö  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP                  4      V n        \        VP                  4       Uu. uF  p\        W4      NK  	  upV n        \        VP                  VP                  4      V n        \        R  \!        V P                  4       4       R4      V n        \        R \!        V P                  4       4       R4      V n        R# u upi )c              3   óR   "  € T F  w  rVP                   '       g   K  Vx € K  	  R # 5ir)   ©rª   ©Ú.0ræ   Úlayers   &  r   Ú	<genexpr>Ú)StepTextModel.__init__.<locals>.<genexpr>÷   s   é € ÐJÑ5‘8�1¸×9IÕ9I�QŠQÓ5ùó   ‚'�
'Nc              3   óR   "  € T F  w  rVP                   '       d   K  Vx € K  	  R # 5ir)   rò   ró   s   &  r   rö   r÷   ú   s   é € ÐNÑ5‘8�1¸U×=MÕ=M�QŠQÓ5ùrø   )r*   r+   Úargsr   Ú	EmbeddingÚ
vocab_sizer>   Úembed_tokensÚrangeÚnum_hidden_layersrÖ   Úlayersr<   r´   ÚnormÚnextÚ	enumerateÚ_swa_idxÚ	_full_idx)r,   rP   ræ   r-   s   && €r   r+   ÚStepTextModel.__init__ð   s»   ø€ Ü‰ÑÔØŒ	ÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔÜ8=¸f×>VÑ>VÔ8WÓXÑ8W°1”| FÖ.Ñ8WÑXˆŒÜ × 2Ñ 2°F×4GÑ4GÓHˆŒ	ÜÙJœy¨¯©Ô5ÓJÈDó
ˆŒô ÙNœy¨¯©Ô5ÓNÐPTó
ˆŽùò Ys   ÁC6c                óî  € Vf   V P                  V4      MTpVf   R .\        V P                  4      ,          pV P                  e   \	        WCV P                  ,          4      MR pV P
                  e3   \	        WCV P
                  ,          V P                  P                  R7      MR p\        V P                  V4       F%  w  rxVP                  '       d   TMTp	V! WIVR7      pK'  	  V P                  V4      # )N)Úwindow_sizeré   )rý   rŽ   r   r  r   r  rú   Úsliding_windowÚziprª   r  )
r,   Ú	input_idsÚinputs_embedsrÄ   rê   Ú	full_maskÚswa_maskrõ   ÚcrÇ   s
   &&&&      r   r0   ÚStepTextModel.__call__ý   sÙ   € Ø,9Ò,AˆD×Ñ˜iÔ(À}ˆØŠ=Ø�FœS §¡Ó-Õ-ˆEð �~‰~Ò)ô " !¨4¯>©>Õ%:Ô;àð 	ð �}‰}Ò(ô "Ø˜Ÿ™Õ'°T·Y±Y×5MÑ5Mõð ð 	ô ˜DŸK™K¨Ö/‰HˆEØ$×/×/Ð/‘8°YˆDÙ�a¨!Ô,ŠAñ 0ð �y‰y˜‹|Ðr   )r  r  rú   rý   r   r  rÔ   r2   r:   s   @@r   rí   rí   ï   s   ù‡ € ÷
ó 
÷ô r   rí   c                   ó¤   a a€ ] tR tRt oRV3R lV 3R llltRR ltR t]R 4       t]R 4       t	]R 4       t
]R	 4       tV3R
 lR ltRtVtV ;t# )ÚLanguageModeli  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r"   rú   rP   )r   r   )r$   r%   s   "€r   r&   ÚLanguageModel.__annotate__  s   ø€ ÷ Pñ P™Zð P±ñ Pr   c                ó´   <€ \         SV `  4        Wn        \        V4      V n        \
        P                  ! VP                  VP                  R R7      V n	        R# rU   )
r*   r+   rú   rí   Úmodelr   rX   r>   rü   Úlm_head)r,   rú   rP   r-   s   &&&€r   r+   ÚLanguageModel.__init__  s=   ø€ Ü‰ÑÔØŒ	Ü" 4Ó(ˆŒ
Ü—y’y ×!1Ñ!1°4·?±?ÈÔOˆŽr   c                ó¨   € Vf   VP                  RR 4      pVf   \        R4      hV P                  WVR7      p\        V P	                  V4      R7      # )NÚinputszinput_ids must be provided)r  rÄ   )Úlogits)ÚpopÚ
ValueErrorr  r
   r  )r,   r  rÄ   r  Úkwargsrê   s   &&&&, r   r0   ÚLanguageModel.__call__  sO   € ØÒØŸ
™
 8¨TÓ2ˆIØÒÜÐ9Ó:Ð:Ø�J‰J�yÀUˆJÓKˆÜ"¨$¯,©,°q«/Ô:Ð:r   c                ó¸   € V P                    Uu. uF?  pVP                  '       d!   \        V P                  P                  R 7      M	\        4       NKA  	  up# u upi ))Úmax_size)r   rª   r   rú   r	  r   )r,   rõ   s   & r   Ú
make_cacheÚLanguageModel.make_cache"  sX   € ð Ÿšó
ñ %�ð ×#×#Ð#ô  ¨¯©×)AÑ)AÕBä“Yòñ %ñ
ð 	
ùò 
s
   �A¦.Ac                ó.   € V P                   P                  # r)   )r  r   ©r,   s   &r   r   ÚLanguageModel.layers,  s   € à�z‰z× Ñ Ð r   c                ó.   € V P                   P                  # r)   )rú   r®   r%  s   &r   r®   ÚLanguageModel.head_dim0  s   € à�y‰y×!Ñ!Ð!r   c                ó.   € V P                   P                  # r)   )rú   r¢   r%  s   &r   r­   ÚLanguageModel.n_kv_heads4  s   € à�y‰y×-Ñ-Ð-r   c                ó   € R  pV# )c                 ó    € R V 9   d   R^@R^/# R# )zmlp.gate.gateÚ
group_sizeÚbitsT© )ÚpathrÎ   s   &&r   Ú	predicateÚ0LanguageModel.quant_predicate.<locals>.predicate:  s   € Ø $Ô&Ø$ b¨&°!Ð4Ð4Ùr   r/  )r,   r1  s   & r   Úquant_predicateÚLanguageModel.quant_predicate8  s   € ò	ð
 Ðr   c                óJ   <€ V ^8„  d   QhRS[ S[S[P                  3,          /# )r"   Úweights)r   rÝ   r   Úarray)r$   r%   s   "€r   r&   r  A  s!   ø€ ÷ !ñ !¡¡S©"¯(©( ]Õ 3ñ !r   c                óÈ  a
€ . R
Oo
\         ;QJ d    V
3R lV 4       F  '       g   K   RM	  RM! V
3R lV 4       4      p/ pVP                  4        EF  w  rERV9   d   K  RV9   d—   VP                  R4      pRV9   d   VP                  R4      pV^,           \	        V4      8  dW   Wg^,           ,          P                  4       '       d4   \        Wg^,           ,          4      V P                  P                  8¼  d   K¬  S
 F&  w  r‰W„9   g   K  W”9  g   K  VP                  W‰4      p M	  V'       d(   VP                  R4      '       d   R	V9   d
   V^,           pWSV&   EK  	  V# )ú.moe.gate_proj.c              3   óZ   <"  € T F   pS F  w  r#W!9   ;'       d    W19  x € K  	  K"  	  R # 5ir)   r/  )rô   rÐ   ÚsrcÚdstÚ
remappingss   &   €r   rö   Ú)LanguageModel.sanitize.<locals>.<genexpr>J  s.   øé € ð 
Ù/6¨!Ãz¹8¸3ˆC‰H×%Ð%˜™Ô%ÁzÑ%«wùs   ƒ+š+TFz.mtpzmodel.layers.Ú.r   z.weightr  ))r9  z.mlp.switch_mlp.gate_proj.)z.moe.up_proj.z.mlp.switch_mlp.up_proj.)z.moe.down_proj.z.mlp.switch_mlp.down_proj.)z
.moe.gate.z.mlp.gate.gate.)z.moe.router_biasz.mlp.gate.router_bias)z.share_expert.z.mlp.share_expert.)ÚanyÚitemsrß   ÚindexrŽ   Úisdigitr@   rú   rÿ   ÚreplaceÚendswith)r,   r6  Ú
is_vanillaÚ	sanitizedÚkeyÚvalueÚpartsÚ	layer_posr;  r<  r=  s   &&        @r   ÚsanitizeÚLanguageModel.sanitizeA  s$  ø€ ò
ˆ
÷ “Sô 
Ù/6ó
—S—S’Sô 
Ù/6ó
ó 
ˆ
ð ˆ	Ø!Ÿ-™-Ÿ/‰JˆCØ˜Œ}ÙØ #Ô%ØŸ	™	 #›�Ø˜uÔ$Ø %§¡¨HÓ 5�Ià! A�¬¨E«
Ô2Ø!¨a¥-Õ0×8Ñ8×:Ò:Ü °!¥mÕ 4Ó5¸¿¹×9TÑ9TÔTá Û&‘�Ø–: #¦.ØŸ+™+ cÓ/�CÙñ '÷ ˜cŸl™l¨9×5Ò5¸&ÀC¼-Ø �	�Ø"�cŒNñ' *ð( Ðr   )rú   r  r  r)   )NNN)r3   r4   r5   r6   r+   r0   r"  Úpropertyr   r®   r­   r3  rL  r7   r8   r9   r:   s   @@r   r  r    sz   ù‡ € ÷Põ Pô;ò
ð ñ!ó ð!ð ñ"ó ð"ð ñ.ó ð.ð ñó ð÷!÷ !ð !r   r  )'Ú	functoolsr   Útypingr   r   r   Úmlx.coreÚcorer   Úmlx.nnr   Úmlx_lm.models.rope_utilsr   Úmlx_lm.models.switch_layersr   r	   r¥   r
   r   r   rÄ   r   r   rP   r   r   Úcompiler   ÚModuler    r<   rN   ru   rw   r†   rœ   rÖ   rí   r  r/  r   r   Ú<module>rX     sý   ðÝ ß &Ñ &å Ý Ý 4ß 9÷ñ ÷
 -ß +ñ 	ˆ�‰˜tÔ$ñó %ðô4�B—I‘Iô 4ôJ�"—)‘)ô JôLˆ"�)‰)ô Lð  ‡�ñ>ó ð>ô
ˆb�i‰iô 
ô&4ˆ"�)‰)ô 4ô>Q0�—	‘	ô Q0ôh>�2—9‘9ô >ô<!�B—I‘Iô !ôHO�B—I‘Iö Or   