+
    TV-j"  ã                   óh  € ^ RI Ht ^ RIHtHtHt ^ RIHt ^ RI	H
t
 ^RIHt ^RIHtHtHt ^RIHtHtHtHt ] ! R R]4      4       t ! R	 R
]
P.                  4      t ! R R]
P.                  4      t ! R R]
P.                  4      t ! R R]
P.                  4      t ! R R]
P.                  4      tR# )é    )Ú	dataclass)ÚAnyÚListÚOptionalN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚArraysCacheÚ	CacheListÚKVCacheÚRotatingKVCachec                   ó<   a € ] tR t^t o RtRtRtRtV 3R ltRt	V t
R# )Ú	ModelArgsÚbaichuan_m1NFc                óþ   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[ ;R&   S[S[ ,          ;R	&   S[ ;R
&   S[;R&   S[;R&   S[S[ ,          ;R&   S[S[ ,          ;R&   S[;R&   # )é   Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚ
rope_thetaÚsliding_windowÚsliding_window_layersÚconv_windowÚrms_norm_epsÚ
model_typeÚnum_swa_attention_headsÚnum_swa_key_value_headsÚtie_word_embeddings)ÚintÚfloatr   Ústrr   Úbool)ÚformatÚ__classdict__s   "€Új/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/baichuan_m1.pyÚ__annotate__ÚModelArgs.__annotate__   sÑ   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ  ¡�9Ñ$ñ ñ Ññ ñ Ññ ñ Ñ#ñ ñ &¡c�]Ñ1ñ ñ &¡c�]Ñ1ñ ñ  Ñ%ò! ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r    r!   r"   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r(   s   @r)   r   r      s%   ø‡ € ð $€JØ-1ÐØ-1ÐØ %Ð÷! ƒ r,   r   c                   ób   a a€ ] tR t^!t oRV3R lV 3R llltRR ltRV3R lR lltRtVtV ;t	# )	Ú	Attentionc                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# ©r   ÚconfigÚ	layer_idx)r   r   r#   )r'   r(   s   "€r)   r*   ÚAttention.__annotate__"   s#   ø€ ÷ 'Oñ 'O™yð 'O±X¹cµ]ñ 'Or,   c                ó.  <€ \         SV `  4        Wn        W n        Vf   \	        R4      hW!P
                  9   V n        V P                  '       d   VP                  '       d   VP                  MVP                  V n	        V P                  '       d   VP                  '       d   VP                  MVP                  V n        VP                  V n        V P                  V P                  ,          V n        V P                  V P                  ,          V P                  8X  g   Q hV P                  R,          V n        \         P"                  ! VP                  V P                  ^V P                  ,          V P                  ,          ,           RR7      V n        \         P"                  ! V P                  V P                  ,          VP                  RR7      V n        \         P(                  ! V P                  RVP*                  R7      V n        VP.                  V n        V P.                  ^8X  g   Q h\0        P2                  ! ^^V P                  ^V P.                  34      V n        \0        P2                  ! ^^V P                  ^V P.                  34      V n        R # )Nz1Layer index must be provided to Attention module.F©Úbias)ÚtraditionalÚbaseg      à¿)ÚsuperÚ__init__r9   r:   Ú
ValueErrorr   Úis_swar    r   Ú	num_headsr!   r   Únum_kv_headsr   Úhead_dimÚscaleÚnnÚLinearÚW_packÚo_projÚRoPEr   Úroper   ÚmxÚzerosÚconv_kÚconv_v©Úselfr9   r:   Ú	__class__s   &&&€r)   rB   ÚAttention.__init__"   sî  ø€ Ü‰ÑÔØŒØ"ŒØÒÜÐPÓQÐQà×#?Ñ#?Ñ?ˆŒð �{�{ˆ{˜v×=×=Ð=ð ×*Ò*à×+Ñ+ð 	Œð �{�{ˆ{˜v×=×=Ð=ð ×*Ò*à×+Ñ+ð 	Ôð "×-Ñ-ˆÔØ×(Ñ(¨D¯N©NÕ:ˆŒØ�}‰}˜tŸ~™~Õ-°×1AÑ1AÔAÐAÐAà—]‘] DÕ(ˆŒ
ä—i’iØ×ÑØ×Ñ˜q 4×#4Ñ#4Õ4°t·}±}ÕDÕDØô
ˆŒô
 —i’iØ�N‰N˜TŸ]™]Õ*¨F×,>Ñ,>ÀUô
ˆŒô —G’G˜DŸM™M°uÀ6×CTÑCTÔUˆŒ	à!×-Ñ-ˆÔØ×Ñ 1Ô$Ð$Ð$Ü—h’h  1 d×&7Ñ&7¸¸D×<LÑ<LÐMÓNˆŒÜ—h’h  1 d×&7Ñ&7¸¸D×<LÑ<LÐMÓNˆŽr,   c                óZ  € VP                   w  rErgVP                  ^WPP                  ^^34      pVR,          pVR,          p	Vf%   \        P                  ! WE^V3VP
                  4      pV^8”  d'   \        P                  ! W1RRRR13,          .^R7      p
MTp
W¨,          W,          ,           # )é   ºNNNN©Úaxis)rY   rY   r   )rY   rY   rX   éÿÿÿÿ)ÚshapeÚreshaper   rO   rP   ÚdtypeÚconcatenate)rT   ÚuÚweightsÚstateÚBÚHÚLÚDÚw0Úw1Úu_prevs   &&&&       r)   Ú_custom_convolutionÚAttention._custom_convolutionK   s›   € Ø—W‘W‰
ˆˆaØ—/‘/ 1 a×)9Ñ)9¸1¸aÐ"@ÓAˆØ�WÕˆØ�WÕˆØŠ=Ü—H’H˜a A q˜\¨1¯7©7Ó3ˆEØˆqŒ5Ü—^’^ U¨a°°C°R°C¨i­LÐ$9ÀÔB‰FàˆFØ�{˜Q�VÕ#Ð#r,   c                ón   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©rO   Úarrayr   )r'   r(   s   "€r)   r*   r;   X   s9   ø€ ÷ ' ñ ' Ù—‘ð' Ù!#§¡ð' Ù9<ð' á	�‰ñ' r,   c           	     óÐ  € VP                   w  rEpV P                  V4      p\        P                  ! WvW`P                  V P
                  ,          ,           3RR7      w  r‰p
VP                  WEV P                  V P
                  4      P                  ^ ^^^4      pV	P                  WEV P                  V P
                  4      P                  ^ ^^^4      p	V
P                  WEV P                  V P
                  4      P                  ^ ^^^4      p
Vf   RpV^ ,          e4   V^,          P                  pV^ ,          ^ ,          V^ ,          ^,          rÜM^ pRRrÜT	pT
pV P                  W�P                  VR7      p	V P                  W P                  VR7      p
V P                  W‹R7      pV P                  W›R7      p	V^ ,          eT   V^,          P                  Wš4      w  ršV^ 8”  d3   VRRRR1R3,          V^ ,          ^ &   VRRRR1R3,          V^ ,          ^&   \        W‰W£^,          V P                   VR7      pVP                  ^ ^^^4      P                  WER4      pV P#                  V4      # )	rX   rZ   N)rc   )ÚoffsetrY   )rq   rH   rp   r\   ©NN)r]   rK   rO   ÚsplitrF   rG   r^   rE   Ú	transposerv   rk   rQ   rR   rN   Úupdate_and_fetchr
   rH   rL   )rT   ro   rp   rq   rd   rf   rg   ÚprojÚqÚkÚvrv   Úlast_kÚlast_vÚk_initÚv_initÚouts   &&&&             r)   Ú__call__ÚAttention.__call__X   s  € ð —'‘'‰ˆˆaà�{‰{˜1‹~ˆÜ—(’(˜4 Q×):Ñ):¸T¿]¹]Õ)JÕ%JÐ!KÐRTÔU‰ˆˆaà�I‰I�a˜DŸN™N¨D¯M©MÓ:×DÑDÀQÈÈ1ÈaÓPˆØ�I‰I�a˜D×-Ñ-¨t¯}©}Ó=×GÑGÈÈ1ÈaÐQRÓSˆØ�I‰I�a˜D×-Ñ-¨t¯}©}Ó=×GÑGÈÈ1ÈaÐQRÓSˆàŠ=Ø ˆEà��8ÒØ˜1•X—_‘_ˆFØ" 1�X a�[¨%°­(°1­+‘FàˆFØ! 4�FàˆØˆØ×$Ñ$ Q¯©¸6Ð$ÓBˆØ×$Ñ$ Q¯©¸6Ð$ÓBˆØ�I‰I�aˆIÓ'ˆØ�I‰I�aˆIÓ'ˆà��8ÒØ˜•8×,Ñ,¨QÓ2‰DˆAØ�1ŒuØ$ Q¨¨2©3° \Õ2��a•˜‘Ø$ Q¨¨2©3° \Õ2��a•˜‘ä*Ø�! �8¨4¯:©:¸Dô
ˆð �m‰m˜A˜q ! QÓ'×/Ñ/°°bÓ9ˆØ�{‰{˜3ÓÐr,   )rK   r9   rQ   rR   r   rG   r   rD   r:   rE   rF   rL   rN   rH   ©Nrw   )
r.   r/   r0   r1   rB   rk   r„   r3   r4   Ú__classcell__©rU   r(   s   @@r)   r6   r6   !   s&   ù‡ € ÷'Oõ 'OôR$÷' ÷ ' ò ' r,   r6   c                   óP   a a€ ] tR t^‚t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPc                ó    <€ V ^8„  d   QhRS[ /# ©r   r9   ©r   )r'   r(   s   "€r)   r*   ÚMLP.__annotate__ƒ   s   ø€ ÷ 

ñ 

™yñ 

r,   c                óP  <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP
                  VP                  R R7      V n        R# ©Fr=   N)	rA   rB   rI   rJ   r   r   Ú	gate_projÚup_projÚ	down_proj©rT   r9   rU   s   &&€r)   rB   ÚMLP.__init__ƒ   s{   ø€ Ü‰ÑÔÜŸšØ×Ñ × 8Ñ 8¸uô
ˆŒô —y’yØ×Ñ × 8Ñ 8¸uô
ˆŒô ŸšØ×$Ñ$ f×&8Ñ&8¸uô
ˆŽr,   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   ro   rr   )rO   rt   )r'   r(   s   "€r)   r*   rŽ   �   s'   ø€ ÷ Jñ J™"Ÿ(™(ð J¡r§x¡xñ Jr,   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # r†   )r“   r   r‘   r’   )rT   ro   s   &&r)   r„   ÚMLP.__call__�   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr,   )r“   r‘   r’   ©	r.   r/   r0   r1   rB   r„   r3   r4   r‡   rˆ   s   @@r)   rŠ   rŠ   ‚   s!   ù‡ € ÷

ó 

÷J÷ Jð Jr,   rŠ   c                   óT   a a€ ] tR t^“t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# r8   )r   r#   )r'   r(   s   "€r)   r*   ÚDecoderLayer.__annotate__”   s   ø€ ÷ 
ñ 
™yð 
±Sñ 
r,   c                ó(  <€ \         SV `  4        \        W4      V n        \	        V4      V n        \        P                  ! VP                  VP                  R 7      V n
        \        P                  ! VP                  VP                  R 7      V n        R# ©)ÚepsN)rA   rB   r6   Ú	self_attnrŠ   ÚmlprI   ÚRMSNormr   r   Úinput_layernormÚpost_attention_layernormrS   s   &&&€r)   rB   ÚDecoderLayer.__init__”   sf   ø€ Ü‰ÑÔÜ" 6Ó5ˆŒÜ�v“;ˆŒÜ!Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÖ%r,   c                ón   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[RS[ P                  /# rn   rs   )r'   r(   s   "€r)   r*   r�   �   s9   ø€ ÷ ñ Ù—‘ðÙ!#§¡ðÙ9<ðá	�‰ñr,   c                ó¤   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pW,           # r†   )r¡   r¤   r¢   r¥   )rT   ro   rp   rq   Úrs   &&&& r)   r„   ÚDecoderLayer.__call__�   sF   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�uˆr,   )r¤   r¢   r¥   r¡   rw   r™   rˆ   s   @@r)   r›   r›   “   s   ù‡ € ÷
ó 
÷÷ ò r,   r›   c                   óT   a a€ ] tR t^¦t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚBaichuanModelc                ó    <€ V ^8„  d   QhRS[ /# rŒ   r�   )r'   r(   s   "€r)   r*   ÚBaichuanModel.__annotate__§   s   ø€ ÷ ñ ™yñ r,   c                ó’  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP                  4      V n        \        VP                  4       Uu. uF  p\        W4      NK  	  upV n        \        P                  ! VP                  VP                  R 7      V n        VP                  V n        RV n        VP"                  '       d   VP"                  ^ ,          V n        RV n        \'        VP"                  4      V n        \        VP                  4       F  pW P(                  9   d   K  W n         R# 	  R# u upi rŸ   )rA   rB   ÚargsrI   Ú	Embeddingr   r   Úembed_tokensÚranger   r›   Úlayersr£   r   Únormr   Úfirst_swa_idxr   Úfirst_global_idxÚsetÚ
swa_layers)rT   r9   ÚirU   s   && €r)   rB   ÚBaichuanModel.__init__§   sø   ø€ Ü‰ÑÔØŒ	ÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔÜ8=¸f×>VÑ>VÔ8WÓXÑ8W°1”| FÖ.Ñ8WÑXˆŒÜ—J’J˜v×1Ñ1°v×7JÑ7JÔKˆŒ	à$×3Ñ3ˆÔØ!ˆÔØ×'×'Ð'Ø!'×!=Ñ!=¸aÕ!@ˆDÔà $ˆÔÜ˜f×:Ñ:Ó;ˆŒÜ�v×/Ñ/Ö0ˆAØ—O‘OÔ#ÙØ$%Ô!Úó	 1ùò Ys   ÁEc                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# ©r   Úinputsrq   rr   rs   )r'   r(   s   "€r)   r*   r®   »   s*   ø€ ÷ ñ ™rŸx™xð ±ð ¹r¿x¹xñ r,   c                ó  € V P                  V4      pVf   R.\        V P                  4      ,          pV P                  f   R pMW P                  ,          ^,          pV P                  f   R pMW P                  ,          ^,          p\        W44      p\        W5V P                  R7      p\        \        V P                  V4      4       F%  w  pw  ršW€P                  9   d   TMTpV	! W;V
4      pK'  	  V P                  V4      # )N)Úwindow_sizerw   )r²   Úlenr´   r·   r¶   r	   r   Ú	enumerateÚzipr¹   rµ   )rT   r¾   rq   ro   Úc_globalÚc_swaÚglobal_maskÚswa_maskÚlÚlayerÚcrp   s   &&&         r)   r„   ÚBaichuanModel.__call__»   sÝ   € Ø×Ñ˜fÓ%ˆàŠ=Ø!�N¤S¨¯©Ó%5Õ5ˆEà× Ñ Ò(Ø‰Hà×2Ñ2Õ3°AÕ6ˆHà×ÑÒ%Ø‰Eà×,Ñ,Õ-¨aÕ0ˆEä+¨AÓ8ˆÜ(¨¸t×?RÑ?RÔSˆä&¤s¨4¯;©;¸Ó'>Ö?‰MˆA‰z�Ø §O¡OÔ3‘8¸ˆDÙ�a˜qÓ!ŠAñ @ð �y‰y˜‹|Ðr,   )r°   r²   r·   r¶   r´   rµ   r   r¹   r†   r™   rˆ   s   @@r)   r¬   r¬   ¦   s   ù‡ € ÷ó ÷(÷ ò r,   r¬   c                   ó”   a a€ ] tR t^Ôt oV3R lV 3R lltV3R lR ltV3R lR ltRV3R lR llt]V3R	 lR
 l4       t	Rt
VtV ;t# )ÚModelc                ó    <€ V ^8„  d   QhRS[ /# rŒ   r�   )r'   r(   s   "€r)   r*   ÚModel.__annotate__Õ   s   ø€ ÷ Xñ X™yñ Xr,   c                ó   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# r�   )rA   rB   r9   r   r¬   Úmodelr"   rI   rJ   r   r   Úlm_headr”   s   &&€r)   rB   ÚModel.__init__Õ   sl   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒÜ" 6Ó*ˆŒ
Ø#)×#=Ñ#=ˆÔ Ø×)×)Ð)ÜŸ9š9 V×%7Ñ%7¸×9JÑ9JÐQVÔWˆDŽLñ *r,   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©r   rr   )r   r   )r'   r(   s   "€r)   r*   rÏ   Þ   s   ø€ ÷ 
ñ 
™D¡�Iñ 
r,   c                ó<  € . p\        V P                  P                  4       Fv  w  r#W P                  P                  9   p\        ^R7      pV'       d"   \        V P                  P                  R7      pM
\        4       pVP                  \        WV4      4       Kx  	  V# )r   )Úsize)Úmax_size)rÂ   rÑ   r´   r9   r   r   r   r   r   Úappendr   )rT   Úcachesrº   rÉ   rD   Ú
conv_cacheÚkv_caches   &      r)   Ú
make_cacheÚModel.make_cacheÞ   st   € ØˆÜ! $§*¡*×"3Ñ"3Ö4‰HˆAØŸ+™+×;Ñ;Ñ;ˆFÜ$¨!Ô,ˆJßÜ*°D·K±K×4NÑ4NÔO‘ä"›9�Ø�M‰Mœ) JÓ9Ö:ñ 5ð ˆr,   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   rb   rr   )Údict)r'   r(   s   "€r)   r*   rÏ   ê   s   ø€ ÷ 	ñ 	¡ð 	©ñ 	r,   c                ó  € R V9   pV'       g€   RV9   dy   VR,          pVP                   pVP                  \        P                  4      p\        P                  P                  VRRR7      pW5R,           ,          P                  V4      pW1R&   V# )zlm_head.scaleszlm_head.weightT)r[   ÚkeepdimsgH¯¼šò×z>r\   )r_   ÚastyperO   Úfloat32Úlinalgrµ   )rT   rb   Úis_quantizedÚwr_   rµ   s   &&    r)   ÚsanitizeÚModel.sanitizeê   sz   € Ø'¨7Ñ2ˆßÐ 0°GÔ ;ØÐ(Õ)ˆAØ—G‘GˆEØ—‘œŸ™Ó$ˆAÜ—9‘9—>‘> !¨"°t�>Ó<ˆDØ˜T•kÕ"×*Ñ*¨5Ó1ˆAØ()Ð$Ñ%Øˆr,   c                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# r½   rs   )r'   r(   s   "€r)   r*   rÏ   õ   s*   ø€ ÷ %ñ %™rŸx™xð %±ð %¹r¿x¹xñ %r,   c                óF   € V P                  W4      pV P                  V4      # r†   )rÑ   rÒ   )rT   r¾   rq   Úoutputss   &&& r)   r„   ÚModel.__call__õ   s   € Ø—*‘*˜VÓ+ˆØ�|‰|˜GÓ$Ð$r,   c                óD   <€ V ^8„  d   QhRS[ S[P                  ,          /# rÕ   )r   rI   ÚModule)r'   r(   s   "€r)   r*   rÏ   ú   s   ø€ ÷ !ñ !™™RŸY™Y�ñ !r,   c                ó.   € V P                   P                  # r†   )rÑ   r´   )rT   s   &r)   r´   ÚModel.layersù   s   € à�z‰z× Ñ Ð r,   )r9   rÒ   rÑ   r   r"   r†   )r.   r/   r0   r1   rB   rÝ   rè   r„   Úpropertyr´   r3   r4   r‡   rˆ   s   @@r)   rÍ   rÍ   Ô   sH   ù‡ € ÷Xó X÷
ð 
÷	ð 	÷%ò %ð ÷!ó ÷!ð !r,   rÍ   )Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorerO   Úmlx.nnrI   Úactivationsr   r@   r   r	   r
   rq   r   r   r   r   r   rï   r6   rŠ   r›   r¬   rÍ   r-   r,   r)   Ú<module>rù      s–   ðõ "ß &Ñ &å Ý å ß TÑ Tß CÓ Cð ô&�ó &ó ð&ô$^ �—	‘	ô ^ ôBJˆ"�)‰)ô Jô"�2—9‘9ô ô&+�B—I‘Iô +ô\'!ˆB�I‰Iö '!r,   