+
    TV-j|  ã                   ól  € ^ RI Ht ^ RIHt ^ RIHtHtHtHt ^ RI	H
t ^ RIHt ^RIHtHtHt ^RIHt ^RIHt ] ! R R	]4      4       t ! R
 R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      tR# )é    )Ú	dataclass)Úpartial)ÚAnyÚDictÚOptionalÚUnionN)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	BitLinear)Úinitialize_ropec                   óL   a € ] tR t^t o RtRtRtRtRtRt	Rt
RtV 3R ltRtV tR# )Ú	ModelArgsNFi'  Tc                ó6  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R	&   S[S[,          ;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   S[;R&   # )é   Ú
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚnum_key_value_headsÚrms_norm_epsÚ
vocab_sizeÚhead_dimÚmax_position_embeddingsÚattention_biasÚmlp_biasÚ
rope_thetaÚrope_traditionalÚrope_scalingÚtie_word_embeddings)ÚstrÚintÚfloatr   Úboolr   r   )ÚformatÚ__classdict__s   "€Úe/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/bitnet.pyÚ__annotate__ÚModelArgs.__annotate__   sñ   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ññ ñ �Oñ ñ ‘s�mÑ"ñ ñ &¡c�]Ñ1ñ ñ Ñ ñ ñ Ññ ñ Ññ ñ Ñ"ñ ñ  ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?ñ! ñ" Ñ$ò# ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r    r!   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r'   s   @r(   r   r      s9   ø‡ € ð #€HØ-1ÐØ €NØ€HØ€JØ"ÐØ;?€LØ $Ð÷# ƒ r+   r   c                   óT   a a€ ] tR t^#t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r&   r'   s   "€r(   r)   ÚAttention.__annotate__$   s   ø€ ÷ Qñ Q™Yñ Qr+   c                óÌ  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pVP                  ;'       g    VP                  V,          ;V n        pVR,          V n        VP                  p\        W#V,          VR7      V n        \        W$V,          VR7      V n        \        W$V,          VR7      V n        \        W5,          W&R7      V n        \        V P                  VP                   VP"                  VP$                  VP&                  4      V n        \*        P,                  ! VP                  VP.                  R7      V n        R# )g      à?©Úbias©ÚepsNg      à¿)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr   Úscaler   r   Úq_projÚk_projÚv_projÚo_projr   r   r   r    r   ÚropeÚnnÚRMSNormr   Úattn_sub_norm)Úselfr8   ÚdimrB   rC   r   r   Ú	__class__s   &&     €r(   rA   ÚAttention.__init__$   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'§=¡=×#OÐ#O°D×4DÑ4DÈÕ4OÐOˆŒ˜à˜t•^ˆŒ
Ø×,Ñ,ˆä ¨xÕ%7¸nÔMˆŒÜ °(Õ%:ÀÔPˆŒÜ °(Õ%:ÀÔPˆŒÜ Õ 2°CÔMˆŒä#Ø�M‰MØ�O‰OØ×!Ñ!Ø×ÑØ×(Ñ(ó
ˆŒ	ô  ŸZšZ¨×(8Ñ(8¸d×>OÑ>OÔPˆÖr+   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r&   r'   s   "€r(   r)   r:   >   sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr+   c           	     óD  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      p
V P                  V
4      p
V
# )é   )Úoffset)rU   rD   rT   éÿÿÿÿ)ÚshaperE   rF   rG   ÚreshaperB   Ú	transposerC   rI   r\   Úupdate_and_fetchr   rD   rL   rH   )rM   rS   rT   rU   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r(   Ú__call__ÚAttention.__call__>   sg  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆð —/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ×#Ñ# FÓ+ˆØ—‘˜VÓ$ˆàˆr+   )
rL   r   rF   rB   rC   rH   rE   rI   rD   rG   ©NN©	r-   r.   r/   r0   rA   ri   r2   r3   Ú__classcell__©rO   r'   s   @@r(   r5   r5   #   s    ù‡ € ÷Qó Q÷4÷ ò r+   r5   c                   óP   a a€ ] tR t^`t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPc                ó    <€ V ^8„  d   QhRS[ /# r7   r9   )r&   r'   s   "€r(   r)   ÚMLP.__annotate__a   s   ø€ ÷ Vñ V™Yñ Vr+   c                ód  <€ \         SV `  4        VP                  pVP                  p\	        VR 4      '       d   VP
                  pMRp\        W#VR7      V n        \        W2VR7      V n        \        W#VR7      V n	        \        P                  ! VP                  VP                  R7      V n        R# )r   Fr<   r>   N)r@   rA   r   r   Úhasattrr   r   Ú	gate_projÚ	down_projÚup_projrJ   rK   r   Úffn_sub_norm)rM   r8   rN   Ú
hidden_dimr   rO   s   &&   €r(   rA   ÚMLP.__init__a   sˆ   ø€ Ü‰ÑÔà×ÑˆØ×+Ñ+ˆ
Ü�4˜×$Ò$Ø—}‘}‰HàˆHä" 3¸ÔBˆŒÜ" :¸ÔBˆŒÜ  °xÔ@ˆŒäŸJšJ t×'=Ñ'=À4×CTÑCTÔUˆÖr+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rV   ©rX   rY   )r&   r'   s   "€r(   r)   rr   q   s   ø€ ÷ ñ ™RŸX™Xñ r+   c                óÀ   € \         P                  ! V P                  V4      4      V P                  V4      ,          pV P	                  V4      pV P                  V4      pV# ©N)rJ   Úrelu2ru   rw   rx   rv   )rM   rS   s   &&r(   ri   ÚMLP.__call__q   sI   € Ü�HŠH�T—^‘^ AÓ&Ó'¨$¯,©,°q«/Õ9ˆØ×Ñ˜aÓ ˆØ�N‰N˜1ÓˆØˆr+   )rv   rx   ru   rw   rl   rn   s   @@r(   rp   rp   `   s    ù‡ € ÷Vó V÷ ÷ ð r+   rp   c                   óT   a a€ ] tR t^xt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó    <€ V ^8„  d   QhRS[ /# r7   r9   )r&   r'   s   "€r(   r)   ÚTransformerBlock.__annotate__y   s   ø€ ÷ 	
ñ 	
™Yñ 	
r+   c                ól  <€ \         SV `  4        VP                  V n        VP                  V n        \	        V4      V n        \        V4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        R# )r>   N)r@   rA   r   r   r5   Ú	self_attnrp   ÚmlprJ   rK   r   Úinput_layernormÚpost_attention_layernorm©rM   r8   rO   s   &&€r(   rA   ÚTransformerBlock.__init__y   s�   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø×+Ñ+ˆÔÜ" 4›ˆŒÜ�t“9ˆŒÜ!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÖ%r+   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rR   rW   )r&   r'   s   "€r(   r)   r„   „   sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr+   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# r~   )r†   rˆ   r‡   r‰   )rM   rS   rT   rU   ÚrÚhÚouts   &&&&   r(   ri   ÚTransformerBlock.__call__„   sK   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØˆ
r+   )r   rˆ   r‡   r   r‰   r†   rk   rl   rn   s   @@r(   r‚   r‚   x   s   ù‡ € ÷	
ó 	
÷÷ ò r+   r‚   c                   óT   a a€ ] tR t^’t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú
LlamaModelc                ó    <€ V ^8„  d   QhRS[ /# r7   r9   )r&   r'   s   "€r(   r)   ÚLlamaModel.__annotate__“   s   ø€ ÷ 	Hñ 	H™Yñ 	Hr+   c                ó¨  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        VR 7      NK  	  upV n        \
        P                  ! VP                  VP                  R7      V n        R# u upi ))r8   r>   N)r@   rA   r8   r   r   rJ   Ú	Embeddingr   Úembed_tokensÚranger‚   ÚlayersrK   r   Únorm)rM   r8   Ú_rO   s   && €r(   rA   ÚLlamaModel.__init__“   s›   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔä16°t×7MÑ7MÔ1Nó
Ù1N¨AÔ $×'Ñ1Nñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŽ	ùò
s   Á?Cc                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   Úinputsr|   )r&   r'   s   "€r(   r)   r•   ž   s   ø€ ÷ ñ á—‘ñr+   c                óþ   € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2^ ,          4      p\	        V P                  V4       F  w  rVV! W4VR7      pK  	  V P                  V4      # )N)rU   )r˜   Úlenrš   r
   Úzipr›   )rM   r    rU   r�   rT   ÚlayerÚcs   &&&    r(   ri   ÚLlamaModel.__call__ž   so   € ð
 ×Ñ˜fÓ%ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a QÔ'ŠAñ 0ð �y‰y˜‹|Ðr+   )r8   r˜   rš   r›   r   r   r~   rl   rn   s   @@r(   r“   r“   ’   s    ù‡ € ÷	Hó 	H÷÷ ò r+   r“   c                   ój   a a€ ] tR t^°t oV3R lV 3R lltRV3R lR lltR t]R 4       tRt	Vt
V ;t# )	ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r7   r9   )r&   r'   s   "€r(   r)   ÚModel.__annotate__±   s   ø€ ÷ Tñ T™Yñ Tr+   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# )Fr<   N)r@   rA   r8   r   r“   Úmodelr!   rJ   ÚLinearr   r   Úlm_headrŠ   s   &&€r(   rA   ÚModel.__init__±   sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ Ó%ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rŸ   r|   )r&   r'   s   "€r(   r)   rª   ¹   s   ø€ ÷ 
ñ 
á—‘ñ
r+   c                óÐ   € V P                  W4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# r~   )r¬   r8   r!   r˜   Ú	as_linearr®   )rM   r    rU   r�   s   &&& r(   ri   ÚModel.__call__¹   sX   € ð
 �j‰j˜Ó'ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r+   c                óÆ   € VP                  4        UUu/ uF  w  r#R V9  g   K  W#bK  	  pppV P                  P                  '       d   VP                  RR4       V# u uppi )zself_attn.rotary_emb.inv_freqzlm_head.weightN)Úitemsr8   r!   Úpop)rM   ÚweightsÚkÚvs   &&  r(   ÚsanitizeÚModel.sanitizeÅ   s`   € ð %Ÿ]™]œ_ô
Ù,‘T�QÐ0OÐWXÑ0XŒDˆAŠD™_ð 	ñ 
ð �9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/Øˆùó
s
   ”A¤Ac                ó.   € V P                   P                  # r~   )r¬   rš   )rM   s   &r(   rš   ÚModel.layersÎ   s   € à�z‰z× Ñ Ð r+   )r8   r®   r¬   r   r~   )r-   r.   r/   r0   rA   ri   rº   Úpropertyrš   r2   r3   rm   rn   s   @@r(   r¨   r¨   °   s9   ù‡ € ÷Tó T÷
ò 
òð ñ!ó ÷!ð !r+   r¨   )Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcorerX   Úmlx.nnrJ   Úbaser	   r
   r   Úbitlinear_layersr   Ú
rope_utilsr   r   ÚModuler5   rp   r‚   r“   r¨   r,   r+   r(   Ú<module>rÉ      s‘   ðõ "Ý ß -Ó -å Ý ç TÑ TÝ 'Ý 'ð ô%�ó %ó ð%ô&:�—	‘	ô :ôzˆ"�)‰)ô ô0�r—y‘yô ô4�—‘ô ô< !ˆB�I‰Iö  !r+   