+
    TV-jL  ã                   óœ  € ^ RI t ^ RIHt ^ RIHt ^ RIHtHt ^ RIH	t
 ^ RIHt ^RIHtHtHt ] ! R R]4      4       tR tRR	 lt ! R
 R]P(                  4      t ! R R]P(                  4      t ! R R]P(                  4      t ! R R]P(                  4      t]! ]
P2                  RR7      RR l4       t ! R R]P(                  4      tR# )é    N)Ú	dataclass)Úpartial)ÚAnyÚOptional)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   óP   a € ] tR t^t o RtRt^t^
t^
tRt	Rt
RtRtV 3R ltRtV tR	# )
Ú	ModelArgsÚnanochati   i   i   i   ç     ˆÃ@c                ó†   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   # )
é   Ú
model_typeÚhidden_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚ
vocab_sizeÚmax_position_embeddingsÚintermediate_sizeÚ
rope_theta)ÚstrÚintÚfloat)ÚformatÚ__classdict__s   "€Úg/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/nanochat.pyÚ__annotate__ÚModelArgs.__annotate__   sx   ø‡ ‚ áÑ ñ ñ Ññ ñ Ññ	 ñ
 Ñ!ñ ñ Ñ!ñ ñ Ññ ñ !Ñ'ñ ñ Ñ!ñ ñ Ñò ó    © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r   r   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r   s   @r   r   r      s>   ø‡ € à €JØ€KØÐØ!ÐØ!ÐØ€JØ#'ÐØ!ÐØ€J÷ ƒ r!   r   c                óD   € \         P                  P                  V RR4      # )z0Functional RMSNorm with no learnable parameters.Ngñhãˆµøä>)ÚmxÚfastÚrms_norm)Úxs   &r   r-   r-      s   € ä�7‰7×Ñ˜A˜t TÓ*Ð*r!   c           
     óJ  € V P                   R,          pVfh   V^,          p\        P                  ! \        P                  ! RV\        P                  R7      \
        P                  ! V4      V,          ,          4      ) p\        P                  P                  V VRRVRVR7      # )a   Apply RoPE with blocked layout.


Args:
    x: Input tensor in (B, H, T, D) format
    offset: Position offset for KV caching
    base: RoPE base frequency (default 10000.0)
    freqs: Precomputed negated frequencies (optional)

Returns:
    Tensor with RoPE applied, same shape as input
Nç        ©ÚdtypeFg      ð?)ÚdimsÚtraditionalÚbaseÚfreqsÚscaleÚoffsetéÿÿÿÿ)	Úshaper+   ÚexpÚarangeÚfloat32ÚmathÚlogr,   Úrope)r.   r8   r5   r6   Úhead_dimÚhalf_Ds   &&&&  r   Úapply_rotary_embrC       s‰   € ð �w‰w�r�{€Hà‚}à˜Q•ˆÜ—’Ü�IŠI�c˜6¬¯©Ô4¼¿ºÀ»ÈÕ8OÕPó
ð 
ˆô
 �7‰7�<‰<Ø	ØØØØØØð ó ð r!   c                   óT   a a€ ] tR t^Bt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r   r   s   "€r   r   ÚAttention.__annotate__C   s   ø€ ÷ 
ñ 
™Yñ 
r!   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        V P                  R,          V n        VP                  V n	        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  RR7      V n        V P                  ^,          p\         P"                  ! \         P$                  ! RV\         P&                  R7      \(        P*                  ! V P                  4      V,          ,          4      ) V n        R# )g      à?F©Úbiasr0   r1   Ng      à¿)ÚsuperÚ__init__r   r   Ú	num_headsr   Únum_kv_headsrA   r7   r   ÚnnÚLinearÚc_qÚc_kÚc_vÚc_projr+   r;   r<   r=   r>   r?   Ú_rope_freqs)ÚselfrH   rB   Ú	__class__s   && €r   rO   ÚAttention.__init__C   sd  ø€ Ü‰ÑÔà×+Ñ+ˆÔØ×1Ñ1ˆŒØ ×4Ñ4ˆÔØ×(Ñ(¨D¯N©NÕ:ˆŒØ—]‘] DÕ(ˆŒ
ØŸ/™/ˆŒä—9’9Ø×Ñ˜dŸn™n¨t¯}©}Õ<À5ô
ˆŒô —9’9Ø×Ñ˜d×/Ñ/°$·-±-Õ?Àeô
ˆŒô —9’9Ø×Ñ˜d×/Ñ/°$·-±-Õ?Àeô
ˆŒô —i’i × 0Ñ 0°$×2BÑ2BÈÔOˆŒð —‘ !Õ#ˆÜŸFšFÜ�IŠI�c˜6¬¯©Ô4Ü�xŠx˜Ÿ™Ó(¨6Õ1õ3ó
ð 
ˆÖr!   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   r.   ÚmaskÚcacheÚreturn©r+   Úarrayr   r   )r   r   s   "€r   r   rJ   _   sH   ø€ ÷ .#ñ .#á�8‰8ð.#ñ ‘r—x‘xÕ ð.#ñ ™�}ð	.#ñ
 
�‰ñ.#r!   c           	     ó’  € VP                   w  rEpV P                  V4      pV P                  V4      pV P                  V4      p	VP	                  WEV P
                  V P                  4      P                  ^ ^^^4      pVP	                  WEV P                  V P                  4      P                  ^ ^^^4      pV	P	                  WEV P                  V P                  4      P                  ^ ^^^4      p	Ve   VP                  M^ p
\        WzV P                  V P                  R7      p\        WŠV P                  V P                  R7      p\        V4      p\        V4      pVe   VP                  W‰4      w  r‰\        WxW“V P                   VR7      pVP                  ^ ^^^4      P	                  WEV P"                  4      pV P%                  V4      # )r   )r8   r5   r6   )r_   r7   r^   )r:   rT   rU   rV   ÚreshaperP   rA   Ú	transposerQ   r8   rC   r   rX   r-   Úupdate_and_fetchr	   r7   r   rW   )rY   r.   r^   r_   ÚBÚLÚ_ÚqueriesÚkeysÚvaluesr8   Úoutputs   &&&&        r   Ú__call__ÚAttention.__call___   s–  € ð —'‘'‰ˆˆaà—(‘(˜1“+ˆØ�x‰x˜‹{ˆØ—‘˜!“ˆð —/‘/ !¨¯©¸¿¹ÓF×PÑPØˆq�!�Qó
ˆð �|‰|˜A $×"3Ñ"3°T·]±]ÓC×MÑMØˆq�!�Qó
ˆð —‘  d×&7Ñ&7¸¿¹ÓG×QÑQØˆq�!�Qó
ˆð
 "'Ò!2�—’¸ˆÜ"Ø¨¯©À×@PÑ@Pô
ˆô  Ø d§o¡o¸T×=MÑ=Mô
ˆô
 ˜7Ó#ˆÜ˜‹~ˆð ÒØ ×1Ñ1°$Ó?‰LˆDä-Ø˜6°d·j±jÀtô
ˆð
 ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸D×<LÑ<LÓMˆØ�{‰{˜6Ó"Ð"r!   )rX   rU   rW   rT   rV   rA   r   rP   rQ   r   r7   ©NN©	r#   r$   r%   r&   rO   rn   r(   r)   Ú__classcell__©rZ   r   s   @@r   rE   rE   B   s   ù‡ € ÷
ó 
÷8.#÷ .#ò .#r!   rE   c                   óP   a a€ ] tR t^�t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPc                ó    <€ V ^8„  d   QhRS[ /# rG   rI   )r   r   s   "€r   r   ÚMLP.__annotate__‘   s   ø€ ÷ Vñ V™Yñ Vr!   c                óì   <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP
                  VP                  R R7      V n        R# ©FrL   N)rN   rO   rR   rS   r   r   Úc_fcrW   ©rY   rH   rZ   s   &&€r   rO   ÚMLP.__init__‘   sN   ø€ Ü‰ÑÔÜ—I’I˜d×.Ñ.°×0FÑ0FÈUÔSˆŒ	Ü—i’i × 6Ñ 6¸×8HÑ8HÈuÔUˆŽr!   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r.   r`   ©r+   rb   )r   r   s   "€r   r   rw   –   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r!   c                ór   € V P                  V4      p\        P                  ! V4      pV P                  V4      # ©N)rz   rR   Úrelu2rW   )rY   r.   s   &&r   rn   ÚMLP.__call__–   s*   € à�I‰I�a‹LˆÜ�HŠH�Q‹KˆØ�{‰{˜1‹~Ðr!   )rz   rW   rq   rs   s   @@r   ru   ru   �   s    ù‡ € ÷Vó V÷
÷ ð r!   ru   c                   óT   a a€ ] tR t^�t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó    <€ V ^8„  d   QhRS[ /# rG   rI   )r   r   s   "€r   r   ÚTransformerBlock.__annotate__ž   s   ø€ ÷ ñ ™Yñ r!   c                ód   <€ \         SV `  4        \        V4      V n        \	        V4      V n        R # r€   )rN   rO   rE   Úattnru   Úmlpr{   s   &&€r   rO   ÚTransformerBlock.__init__ž   s$   ø€ Ü‰ÑÔÜ˜d“OˆŒ	Ü�t“9ˆŽr!   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r]   ra   )r   r   s   "€r   r   r†   £   sH   ø€ ÷ 	ñ 	á�8‰8ð	ñ ‘r—x‘xÕ ð	ñ ™�}ð		ñ
 
�‰ñ	r!   c                óŠ   € WP                  \        V4      W#R 7      ,           pW@P                  \        V4      4      ,           pV# )©r^   r_   )rˆ   r-   r‰   )rY   r.   r^   r_   ÚhÚouts   &&&&  r   rn   ÚTransformerBlock.__call__£   s7   € ð —	‘	œ( 1›+¨D�	Ó>Õ>ˆØ—(‘(œ8 A›;Ó'Õ'ˆØˆ
r!   )rˆ   r‰   rp   rq   rs   s   @@r   r„   r„   �   s   ù‡ € ÷ó ÷
	÷ 	ò 	r!   r„   c                   óT   a a€ ] tR t^¯t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚNanoChatModelc                ó    <€ V ^8„  d   QhRS[ /# rG   rI   )r   r   s   "€r   r   ÚNanoChatModel.__annotate__°   s   ø€ ÷ Qñ Q™Yñ Qr!   c                ó   <€ \         SV `  4        Wn        \        P                  ! VP
                  VP                  4      V n        \        VP                  4       Uu. uF  p\        V4      NK  	  upV n        R # u upi r€   )rN   rO   rH   rR   Ú	Embeddingr   r   ÚwteÚranger   r„   rŽ   )rY   rH   ri   rZ   s   && €r   rO   ÚNanoChatModel.__init__°   s[   ø€ Ü‰ÑÔØŒ	Ü—<’< §¡°×1AÑ1AÓBˆŒÜ27¸×8NÑ8NÔ2OÓPÑ2O¨QÔ" 4Ö(Ñ2OÑPˆŽùÒPs   ÁA;c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   Úinputsr`   r~   )r   r   s   "€r   r   r”   ¶   s'   ø€ ÷ ñ á—‘ðñ 
�‰ñ	r!   c                ó  € V P                  V4      p\        V4      pVf   R .\        V P                  4      ,          p\	        W2^ ,          4      p\        V P                  V4       F  w  rVV! W4VR7      pK  	  \        V4      pV# )Nr�   )r—   r-   ÚlenrŽ   r   Úzip)rY   rœ   r_   rŽ   r^   ÚlayerÚcs   &&&    r   rn   ÚNanoChatModel.__call__¶   su   € ð
 �H‰H�VÓˆä�Q‹KˆàŠ=Ø�FœS §¡›[Õ(ˆEä$ Q¨a­Ó1ˆä˜DŸF™F EÖ*‰HˆEÙ�a¨!Ô,ŠAñ +ô �Q‹Kˆàˆr!   )rH   rŽ   r—   r€   rq   rs   s   @@r   r’   r’   ¯   s    ù‡ € ÷Qó Q÷÷ ò r!   r’   T)Ú	shapelessc                 óH   € V\         P                  ! W,          4      ,          # r€   )r+   Útanh)ÚlogitsÚcaps   &&r   Úsoftcapr¨   Í   s   € à”—’˜�Ó&Õ&Ð&r!   c                   ód   a a€ ] tR t^Òt oV3R lV 3R lltRV3R lR llt]R 4       tRtVt	V ;t
# )ÚModelc                ó    <€ V ^8„  d   QhRS[ /# rG   rI   )r   r   s   "€r   r   ÚModel.__annotate__Ó   s   ø€ ÷ Pñ P™Yñ Pr!   c                óÖ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# ry   )rN   rO   rH   r   r’   ÚtransformerrR   rS   r   r   Úlm_headr{   s   &&€r   rO   ÚModel.__init__Ó   sI   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ(¨Ó.ˆÔÜ—y’y ×!1Ñ!1°4·?±?ÈÔOˆŽr!   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r›   r~   )r   r   s   "€r   r   r¬   Ú   s'   ø€ ÷ ñ á—‘ðñ 
�‰ñ	r!   c                ób   € V P                  WR 7      pV P                  V4      p\        V4      pV# ))r_   )r®   r¯   r¨   )rY   rœ   r_   r�   r¦   s   &&&  r   rn   ÚModel.__call__Ú   s5   € ð
 ×Ñ˜vÐÓ3ˆØ—‘˜cÓ"ˆô ˜“ˆàˆr!   c                ó.   € V P                   P                  # r€   )r®   rŽ   )rY   s   &r   ÚlayersÚModel.layersç   s   € à×Ñ×!Ñ!Ð!r!   )rH   r¯   r   r®   r€   )r#   r$   r%   r&   rO   rn   Úpropertyrµ   r(   r)   rr   rs   s   @@r   rª   rª   Ò   s4   ù‡ € ÷Pó P÷ò ð ñ"ó ÷"ð "r!   rª   )r   N)g      .@)r>   Údataclassesr   Ú	functoolsr   Útypingr   r   Úmlx.coreÚcorer+   Úmlx.nnrR   r5   r   r   r	   r   r-   rC   ÚModulerE   ru   r„   r’   Úcompiler¨   rª   r"   r!   r   Ú<module>rÀ      s·   ðó Ý !Ý ß  å Ý ç TÑ Tð ô	 �ó 	 ó ð	 ò+ô
ôDK#�—	‘	ô K#ô\
ˆ"�)‰)ô 
ô�r—y‘yô ô$�B—I‘Iô ñ< 	ˆ�‰˜tÔ$ó'ó %ð'ô"ˆB�I‰Iö "r!   