+
    TV-jÊ'  ã                   óŽ  € ^ RI t ^ RIHt ^ RIHt ^ RIHtHt ^ RIH	t
 ^ RIHt ^RIHtHtHt ] ! R R]4      4       t]! ]
P$                  RR	7      R
 4       tR t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      tR# )é    N)Ú	dataclass)Úpartial)ÚAnyÚOptional)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   óP   a € ] tR t^t o RtRtRtRtRtRt	^@t
^t^tV 3R ltRtV tR# )	Ú	ModelArgsç      ð?Tg      $@g       @i@B c                óþ   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )é   Ú
model_typeÚhidden_sizeÚdense_attention_every_n_layersÚff_intermediate_sizeÚgegelu_limitÚnum_hidden_layersÚnum_attention_headsÚlayer_norm_epsilonÚ
vocab_sizeÚnum_key_value_headsÚmup_attn_multiplierÚmup_use_scalingÚmup_embedding_multiplierÚmup_width_multiplierÚrope_embedding_baseÚrope_position_scaleÚblocksparse_block_sizeÚblocksparse_num_local_blocksÚblocksparse_vert_stride)ÚstrÚintÚfloatÚbool)ÚformatÚ__classdict__s   "€Úh/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/phi3small.pyÚ__annotate__ÚModelArgs.__annotate__   sø   ø‡ ‚ á�Oñ ñ Ññ ñ %(Ñ'ñ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ �Oñ ñ Ññ ñ Ñ$ñ ñ Ñ ñ ñ $Ñ*ñ ñ  Ñ%ñ ñ  Ñ(ñ! ñ" Ñ$ñ# ñ$  Ñ$ñ% ñ& #&Ñ*ñ' ñ( !Ñ$ò) ó    © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r   r    r!   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r'   s   @r(   r   r      sC   ø‡ € ð "%ÐØ €OØ&*ÐØ"%ÐØ!(ÐØ!$ÐØ"$ÐØ(*Ð Ø#$Ð÷) ƒ r+   r   T)Ú	shapelessc           
      ót  € \         P                  ! \         P                  ! V 4      V \         P                  ! V R VR7      4      p \         P                  ! \         P                  ! V4      V\         P                  ! W) VR7      4      pV \         P                  ! RV ,          4      ,          pW1R,           ,          # )N)Úa_minÚa_maxg¬Zd;û?r   )ÚmxÚwhereÚisinfÚclipÚsigmoid)Úa_geluÚa_linearÚlimitÚout_gelus   &&& r(   Úgegelu_implrA   %   s„   € ä�XŠXÜ
�Š�ÓØÜ
�Š�˜d¨%Ô0ó€Fô
 �xŠxÜ
�Š�ÓØÜ
�Š� ¨eÔ4ó€Hð
 œŸ
š
 5¨6¥>Ó2Õ2€HØ #•~Õ&Ð&r+   c                 ó<   € V R,          V R,          r2\        W#V4      # ).).:NNr   ).:é   Nr   )rA   )Úxr?   r=   r>   s   &&  r(   ÚgegelurE   5   s   € Ø˜•{ A i¥LˆHÜ�v¨Ó/Ð/r+   c                   ó`   a a€ ] tR t^:t oV3R lV 3R lltR tR tRV3R lR lltRtVt	V ;t
# )	Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r&   r'   s   "€r(   r)   ÚAttention.__annotate__;   s   ø€ ÷ &&ñ &&™Yñ &&r+   c                ó’  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pWE,          V n        VP                  V,          ;V n        p\        P                  ! W0P                  ^V P                  ,          ,           V,          4      V n        \        P                  ! W34      V n        VP                  '       d   WaP                  ,          pM\        P                   ! V4      pRV,          V n        \        P$                  ! VRVP&                  VP(                  R7      V n        W!P,                  ,          ^ 8X  df   RV n        VP0                  V n        V P0                  R9  d   \3        RV P0                   24      hVP4                  V n        VP6                  V n        R# RV n        R# )r   r   F)ÚtraditionalÚbaseÚscaleTzUnsupported block size N)é    é@   )ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚ
n_q_per_kvÚhead_dimÚnnÚLinearÚquery_key_valueÚdenser   r   ÚmathÚsqrtrP   ÚRoPEr   r   Úroper   Úblock_sparser   Ú
ValueErrorr    r!   )	ÚselfrJ   Ú	layer_idxÚdimrU   rV   rX   Únorm_factorÚ	__class__s	   &&&     €r(   rT   ÚAttention.__init__;   sr  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*Ø!Õ/ˆŒà#'×#3Ñ#3°wÕ#>Ð>ˆŒ˜ä!ŸyšyØ—,‘,  T§_¡_Õ!4Õ4¸Õ@ó 
ˆÔô —Y’Y˜sÓ(ˆŒ
à××ÐØ"×%=Ñ%=Õ=‰KäŸ)š) HÓ-ˆKØ˜;Õ&ˆŒ
ä—G’GØØØ×)Ñ)Ø×*Ñ*ô	
ˆŒ	ð ×:Ñ:Õ:¸aÔ?Ø $ˆDÔØ*.×*EÑ*EˆDÔ'Ø×*Ñ*°(Ô:Ü Ø-¨d×.IÑ.IÐ-JÐKóð ð 15×0QÑ0QˆDÔ-Ø+/×+GÑ+GˆDÖ(à %ˆDÖr+   c                ó  € V P                   pV P                  pV P                  pV P                  pW%,           ^,
          V,          pW,           ^,
          V,          p\        P
                  ! Wx,
          V4      R,          p	\        P
                  ! V4      R,          p
\        P
                  ! V4      R,          \        P
                  ! ^V^,           4      R,          ,           V,          pV^ 8H  R,          pWš8¬  Wš,
          V8  V,          ,          pVP                  ! V P                  V P                  .VP                  R	R O5!  p\        P                  ! \        P                  ! WÅR
R7      VR	R7      pWÍRV) R1RV13,          3# )rC   N)Úaxis.)NºNNNN©NN)Nrk   )rk   N)rk   Nrk   éþÿÿÿéÿÿÿÿ)r!   r    r   rU   r8   ÚarangeÚreshaperV   rW   ÚshapeÚrepeat)rc   Úq_lenÚkv_lenÚvert_strideÚlocal_blocksÚ
block_sizerU   Ú	kv_blocksÚq_blocksÚq_posÚk_posÚmask_vert_stridedÚ
block_maskÚ
dense_masks   &&&           r(   Ú_block_sparse_maskÚAttention._block_sparse_maskc   sU  € Ø×2Ñ2ˆØ×8Ñ8ˆØ×0Ñ0ˆ
Ø—,‘,ˆàÕ(¨1Õ,°Õ;ˆ	ØÕ&¨Õ*¨zÕ9ˆÜ—	’	˜)Õ.°	Ó:¸=ÕIˆÜ—	’	˜)Ó$ ZÕ0ˆô �IŠI�iÓ  Õ)¬B¯IªI°a¸À1½Ó,EÀgÕ,NÕNØõÐð /°!Ñ3°ZÕ@Ðà‘nØ�]˜\Ñ)Ð->Õ>õ
ˆ
ð  ×'Ò'Ø�O‰O˜TŸ_™_ð
Ø/9×/?Ñ/?ÀÀÐ/Dó
ˆ
ô —Y’YÜ�IŠI�j°2Ô6¸
Èô
ˆ
ð  c¨E¨6©7°G°V°GÐ&;Õ<Ð<Ð<r+   c           
     ó  € WA,          pVP                   ^ ,          pVP                   ^,          p\        P                  ! WV P                  V P                  VR34      p\        P
                  ! V^4      p\        P
                  ! V^4      pV P                  WrP                   R,          4      w  r‰V\        P                  ! VRR4      ,          p
Ve	   W¥,           p
V
\        P                  ! V	\        P                  ! ^ V
P                  4      \        P                  ! \        R4      ) V
P                  4      4      ,           p
\        P                  ! V
RRR7      p
W£,          p\        P                  ! W¶V P                  VR34      # )r   ÚinfT)rj   Úprecisern   rm   )rq   r8   rp   rV   rW   Úexpand_dimsr   Úswapaxesr9   ÚarrayÚdtyper$   ÚsoftmaxrU   )rc   ÚqueriesÚkeysÚvaluesrP   ÚmaskÚBÚLr}   r~   ÚscoresÚoutputs   &&&&&&      r(   Ú_block_sparse_attentionÚ!Attention._block_sparse_attention~   s"  € Ø•/ˆØ�M‰M˜!ÕˆØ�M‰M˜!ÕˆÜ—*’*˜W¨$¯/©/¸4¿?¹?ÈAÈrÐ&RÓSˆÜ�~Š~˜d AÓ&ˆÜ—’ ¨Ó*ˆð "&×!8Ñ!8¸¿J¹JÀr½NÓ!KÑˆ
Øœ2Ÿ;š; t¨R°Ó4Õ4ˆð ÒØ•]ˆFØœ"Ÿ(š(ØœŸš  F§L¡LÓ1´2·8²8¼UÀ5»\¸MÈ6Ï<É<Ó3Xó
õ 
ˆô —’˜F¨°TÔ:ˆà•ˆô
 �zŠz˜& d§l¡l°A°rÐ":Ó;Ð;r+   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   rD   rŒ   ÚcacheÚreturn©r8   r†   r   r   )r&   r'   s   "€r(   r)   rL   Ÿ   sH   ø€ ÷ $"ñ $"á�8‰8ð$"ñ ‘r—x‘xÕ ð$"ñ ™�}ð	$"ñ
 
�‰ñ$"r+   c           	     óZ  € VP                   w  rEpV P                  V4      pVP                  WERV P                  ^,           V P                  4      pVRRR1R3,          P                  R	R4      pVR
,          p	VR,          p
VP                  ^ ^^^4      pV	P                  ^ ^^^4      p	V
P                  ^ ^^^4      p
VeM   V P                  WƒP                  R7      pV P                  W“P                  R7      p	VP                  Wš4      w  ršM"V P                  V4      pV P                  V	4      p	V P                  '       d    V P                  W‰W P                  VR7      pM\        W‰W£V P                  VR7      pVP                  ^ ^^^4      P                  WER4      pV P                  V4      # )rC   .Nrk   )Úoffset)rP   rŒ   )r•   rP   rŒ   rn   rm   éýÿÿÿ).rm   rk   ).rn   rk   )rq   r[   rp   rW   rX   ÚflattenÚ	transposer`   r™   Úupdate_and_fetchra   r‘   rP   r	   r\   )rc   rD   rŒ   r•   r�   rŽ   ÚDÚqkvr‰   rŠ   r‹   r�   s   &&&&        r(   Ú__call__ÚAttention.__call__Ÿ   s�  € ð —'‘'‰ˆˆaà×"Ñ" 1Ó%ˆØ�k‰k˜!  D§O¡O°aÕ$7¸¿¹ÓGˆØ�c˜3˜B˜3 �kÕ"×*Ñ*¨2¨rÓ2ˆØ�:�ˆØ�Z•ˆð ×#Ñ# A q¨!¨QÓ/ˆØ�~‰~˜a  A qÓ)ˆØ×!Ñ! ! Q¨¨1Ó-ˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDà××ÐØ×1Ñ1Ø˜v¯Z©Z¸dð 2ó ‰Fô 2Ø˜v¸$¿*¹*È4ôˆFð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�z‰z˜&Ó!Ð!r+   )ra   r   r    r!   r\   rX   rU   rV   rW   r[   r`   rP   rl   )r-   r.   r/   r0   rT   r   r‘   r    r2   r3   Ú__classcell__©rg   r'   s   @@r(   rG   rG   :   s*   ù‡ € ÷&&ó &&òP=ò6<÷B$"÷ $"ò $"r+   rG   c                   óD   a a€ ] tR t^Æt oV 3R ltV3R lR ltRtVtV ;t# )ÚMLPc                óò   <€ \         SV `  4        VP                  pVP                  pVP                  V n        \
        P                  ! V^V,          4      V n        \
        P                  ! W24      V n        R# )r   N)	rS   rT   r   r   r   rY   rZ   Úup_projÚ	down_proj)rc   rJ   re   Ú
hidden_dimrg   s   &&  €r(   rT   ÚMLP.__init__Ç   sX   ø€ Ü‰ÑÔØ×ÑˆØ×.Ñ.ˆ
Ø ×-Ñ-ˆÔÜ—y’y  a¨*¥nÓ5ˆŒÜŸš :Ó3ˆŽr+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r–   ©r8   r†   )r&   r'   s   "€r(   r)   ÚMLP.__annotate__Ï   s   ø€ ÷ <ñ <™RŸX™Xñ <r+   c                ól   € V P                  V4      pV P                  \        WP                  4      4      # ©N)r§   r¨   rE   r   )rc   rD   s   &&r(   r    ÚMLP.__call__Ï   s)   € Ø�L‰L˜‹OˆØ�~‰~œf Q×(9Ñ(9Ó:Ó;Ð;r+   )r¨   r   r§   ©	r-   r.   r/   r0   rT   r    r2   r3   r¢   r£   s   @@r(   r¥   r¥   Æ   s   ù‡ € õ4÷<÷ <ð <r+   r¥   c                   óT   a a€ ] tR t^Ôt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó    <€ V ^8„  d   QhRS[ /# rI   rK   )r&   r'   s   "€r(   r)   ÚTransformerBlock.__annotate__Õ   s   ø€ ÷ ñ ™Yñ r+   c                óx  <€ \         SV `  4        VP                  V n        VP                  V n        \	        W4      V n        \        V4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        Wn        R# )©ÚepsN)rS   rT   r   r   rG   Ú	self_attnr¥   ÚmlprY   Ú	LayerNormr   Úinput_layernormÚpost_attention_layernormrJ   )rc   rJ   rd   rg   s   &&&€r(   rT   ÚTransformerBlock.__init__Õ   s�   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø×+Ñ+ˆÔÜ" 4Ó3ˆŒÜ�t“9ˆŒÜ!Ÿ|š|Ø×Ñ $×"9Ñ"9ô 
ˆÔô )+¯ªØ×ÑØ×'Ñ'ô)
ˆÔ%ð Ž	r+   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r”   r—   )r&   r'   s   "€r(   r)   rµ   ä   sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r+   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# r¯   )r¹   r¼   rº   r½   )rc   rD   rŒ   r•   ÚrÚhÚouts   &&&&   r(   r    ÚTransformerBlock.__call__ä   sK   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØˆ
r+   )rJ   r   r¼   rº   r   r½   r¹   rl   r±   r£   s   @@r(   r³   r³   Ô   s   ù‡ € ÷ó ÷
÷ 
ò 
r+   r³   c                   óT   a a€ ] tR t^ñt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Phi3Modelc                ó    <€ V ^8„  d   QhRS[ /# rI   rK   )r&   r'   s   "€r(   r)   ÚPhi3Model.__annotate__ò   s   ø€ ÷ 
ñ 
™Yñ 
r+   c                óð  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        V P                  ^ 8”  g   Q hVP
                  V n        \        P                  ! VP                  VP                  4      V n	        \        VP                  4       Uu. uF  p\        WR7      NK  	  upV n        \        P                  ! VP                  VP                  R7      V n        R# u upi )r   )rJ   rd   r·   N)rS   rT   rJ   r   r   r   rY   Ú	Embeddingr   Úembed_tokensÚranger³   Úlayersr»   r   Úfinal_layernorm)rc   rJ   Úlrg   s   && €r(   rT   ÚPhi3Model.__init__ò   sÃ   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ô"Ð"Ð"Ø(,×(EÑ(EˆÔ%ÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô  $×4Ù2ñ
ˆŒô  "Ÿ|š|Ø×Ñ $×"9Ñ"9ô 
ˆÖùò	
s   Â#C3c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   Úinputsr¬   )r&   r'   s   "€r(   r)   rÈ     s   ø€ ÷ 'ñ 'á—‘ñ'r+   c                óJ  € V P                  V4      pV P                  '       d   V P                  V,          pVf   R .\        V P                  4      ,          p\	        W2^ ,          RR7      p\        V P                  V4       F  w  rVV! W4V4      pK  	  V P                  V4      # )NT)Úreturn_array)rË   r   ÚlenrÍ   r   ÚziprÎ   )rc   rÓ   r•   rÂ   rŒ   ÚlayerÚcs   &&&    r(   r    ÚPhi3Model.__call__  s�   € ð
 ×Ñ˜fÓ%ˆØ×(×(Ð(Ø×-Ñ-°Õ1ˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­¸tÔDˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð ×#Ñ# AÓ&Ð&r+   )rJ   rË   rÎ   rÍ   r   r   r   r¯   r±   r£   s   @@r(   rÆ   rÆ   ñ   s   ù‡ € ÷
ó 
÷ '÷ 'ò 'r+   rÆ   c                   ój   a a€ ] tR tRt oV3R lV 3R lltR	V3R lR llt]R 4       tR tRt	Vt
V ;t# )
ÚModeli  c                ó    <€ V ^8„  d   QhRS[ /# rI   rK   )r&   r'   s   "€r(   r)   ÚModel.__annotate__  s   ø€ ÷ 	
ñ 	
™Yñ 	
r+   c           	     ó  <€ \         SV `  4        VP                  V n        \        V4      V n        Wn        VP                  V n        \        P                  ! . RO\        \        RR4      4      ,           4      V n        R# )é ‡ i«‡ i ˆ N)rà   i¢‡ i£‡ i¤‡ i¨‡ i©‡ )rS   rT   r   rÆ   ÚmodelrJ   r   r8   r†   ÚlistrÌ   Ú_dummy_tokenizer_ids)rc   rJ   rg   s   &&€r(   rT   ÚModel.__init__  s_   ø€ Ü‰ÑÔØŸ/™/ˆŒÜ˜t“_ˆŒ
ØŒ	Ø$(×$=Ñ$=ˆÔ!Ü$&§H¢HÚ<Ü”5˜ Ó(Ó)õ*ó%
ˆÖ!r+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rÒ   r¬   )r&   r'   s   "€r(   r)   rÞ   "  s   ø€ ÷ 
ñ 
á—‘ñ
r+   c                óê   € V P                  W4      pV P                   P                  P                  V4      pV P                  '       d   W0P                  ,          p\	        R 4      ) W0P
                  &   V# )r‚   )rá   rË   Ú	as_linearr   r$   rã   )rc   rÓ   r•   rÃ   s   &&& r(   r    ÚModel.__call__"  s_   € ð
 �j‰j˜Ó'ˆØ�j‰j×%Ñ%×/Ñ/°Ó4ˆØ×$×$Ð$Ø×1Ñ1Õ1ˆCÜ*/°«,¨ˆ×%Ñ%Ñ&Øˆ
r+   c                ó.   € V P                   P                  # r¯   )rá   rÍ   )rc   s   &r(   rÍ   ÚModel.layers.  s   € à�z‰z× Ñ Ð r+   c                óh   € VP                  4        UUu/ uF  w  r#R V9  g   K  W#bK  	  upp# u uppi )zself_attn.rotary_emb.inv_freq)Úitems)rc   ÚweightsÚkÚvs   &&  r(   ÚsanitizeÚModel.sanitize2  s:   € ð %Ÿ]™]œ_ô
Ù,‘T�QÐ0OÐWXÑ0XŒDˆAŠD™_ò
ð 	
ùó 
s   ”.¤.)rã   rJ   rá   r   r   r¯   )r-   r.   r/   r0   rT   r    ÚpropertyrÍ   rð   r2   r3   r¢   r£   s   @@r(   rÜ   rÜ     s7   ù‡ € ÷	
ó 	
÷
ò 
ð ñ!ó ð!÷
ò 
r+   rÜ   )r]   Údataclassesr   Ú	functoolsr   Útypingr   r   Úmlx.coreÚcorer8   Úmlx.nnrY   rO   r   r   r	   r   ÚcompilerA   rE   ÚModulerG   r¥   r³   rÆ   rÜ   r,   r+   r(   Ú<module>rû      s²   ðó Ý !Ý ß  å Ý ç TÑ Tð ô%�ó %ó ð%ñ, 	ˆ�‰˜tÔ$ñ'ó %ð'ò0ô
I"�—	‘	ô I"ôX<ˆ"�)‰)ô <ô�r—y‘yô ô:"'�—	‘	ô "'ôJ 
ˆB�I‰Iö  
r+   