+
    TV-j½  ã                   ó8  € ^ RI t ^ RIHt ^ RIHt ^ RIHt ^RIH	t	H
t
Ht ] ! R R]	4      4       t ! R R]P                  4      t ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR# )é    N)Ú	dataclass)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   ó^   a € ] tR t^t o RtRtRtRt^ t^ t	^ t
RtRtRtRtR	 tV 3R
 ltRtV tR# )Ú	ModelArgsÚphii   i È  i 
  gš™™™™™Ù?i (  gñhãˆµøä>g     ˆÃ@c                óH   € V P                   f   V P                  V n         R # R # ©N)Únum_key_value_headsÚnum_attention_heads©Úselfs   &Úb/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/phi.pyÚ__post_init__ÚModelArgs.__post_init__   s!   € Ø×#Ñ#Ò+Ø'+×'?Ñ'?ˆDÖ$ñ ,ó    c                óž   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   # )é   Ú
model_typeÚmax_position_embeddingsÚ
vocab_sizeÚhidden_sizer   Únum_hidden_layersr   Úpartial_rotary_factorÚintermediate_sizeÚlayer_norm_epsÚ
rope_theta)ÚstrÚintÚfloat)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚModelArgs.__annotate__   s’   ø‡ ‚ áÑñ ñ !Ñ'ñ ñ Ññ	 ñ
 Ññ ñ Ñ!ñ ñ Ññ ñ Ñ!ñ ñ !Ñ&ñ ñ Ñ"ñ ñ Ñ ñ ñ Ñò r   )r   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r   r   r   r   r   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r#   s   @r   r   r      sO   ø‡ € à€JØ#'ÐØ€JØ€KØ!ÐØÐØ!ÐØ#&ÐØ"ÐØ €NØ€Jò@÷ ƒ r   r   c                   óH   a a€ ] tR t^t oV3R lV 3R lltRR ltRtVtV ;t# )ÚPhiAttentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfig©r   )r"   r#   s   "€r   r$   ÚPhiAttention.__annotate__    s   ø€ ÷ "
ñ "
™yñ "
r   c                óª  <€ \         SV `  4        VP                  V n        VP                  V n        V P                  V P                  ,          V n        VP                  V n        V P                  V P                  ,          V n        VP                  V n        VP                  V n	        V P
                  V P                  ,          V P                  8w  d'   \        R V P                   RV P                   R24      h\        P                  ! V P                  V P                  V P
                  ,          RR7      V n        \        P                  ! V P                  V P                  V P
                  ,          RR7      V n        \        P                  ! V P                  V P                  V P
                  ,          RR7      V n        \        P                  ! V P                  V P
                  ,          V P                  RR7      V n        \        P"                  ! \%        V P                  V P
                  ,          4      RV P                  R7      V n        R# )z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).T©ÚbiasF)ÚtraditionalÚbaseN)ÚsuperÚ__init__r   r   Ú	num_headsÚhead_dimr   Úrepeatsr   r   Ú
ValueErrorÚnnÚLinearÚq_projÚk_projÚv_projÚdenseÚRoPEr    Úrope©r   r1   Ú	__class__s   &&€r   r:   ÚPhiAttention.__init__    s¯  ø€ Ü‰ÑÔà!×-Ñ-ˆÔØ×3Ñ3ˆŒØ×(Ñ(¨D¯N©NÕ:ˆŒØ#)×#=Ñ#=ˆÔ Ø—~‘~¨×)AÑ)AÕAˆŒØ ×+Ñ+ˆŒØ%+×%AÑ%AˆÔ"à�M‰M˜DŸN™NÕ*¨t×/?Ñ/?Ô?ÜØQÐRV×RbÑRbÐQcØ$ T§^¡^Ð$4°Bð8óð ô
 —i’iØ×Ñ˜dŸn™n¨t¯}©}Õ<À4ô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈTô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈTô
ˆŒô —Y’YØ�N‰N˜TŸ]™]Õ*¨D×,<Ñ,<À4ô
ˆŒ
ô —G’GÜ�×*Ñ*¨T¯]©]Õ:Ó;ØØ—‘ô
ˆŽ	r   c           	     óº  € V P                  V4      V P                  V4      V P                  V4      repVP                  w  rxp	V P                  V P
                  rºVP                  VVV
R4      P                  ^^4      pVP                  WxVR4      P                  ^^4      pVP                  WxVR4      P                  ^^4      pVeM   V P                  WCP                  R7      pV P                  WSP                  R7      pVP                  WV4      w  rVM"V P                  V4      pV P                  V4      p\        P                  ! ^VP                  R,          ,          4      p\        VP                  \        P                   4      VVVVVR7      P                  VP"                  4      pVP                  ^^4      P                  WxR4      pV P%                  V4      # )é   )Úoffset)ÚcacheÚscaleÚmaskéÿÿÿÿ)rA   rB   rC   Úshaper;   r   ÚreshapeÚmoveaxisrF   rL   Úupdate_and_fetchÚmathÚsqrtr   ÚastypeÚmxÚfloat32ÚdtyperD   )r   ÚxrO   rM   ÚqueriesÚkeysÚvaluesÚBÚLÚDÚn_headsÚ
n_kv_headsrN   Úoutputs   &&&&          r   Ú__call__ÚPhiAttention.__call__D   s–  € Ø $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆð —-‘-‰ˆˆaØ"Ÿn™n¨d×.FÑ.F�ð —/‘/ØØØØó	
÷
 ‰(�1�a‹.ð 	ð �|‰|˜A *¨bÓ1×:Ñ:¸1¸aÓ@ˆØ—‘  j°"Ó5×>Ñ>¸qÀ!ÓDˆð ÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä—	’	˜!˜gŸm™m¨BÕ/Õ/Ó0ˆÜ-Ø�N‰Nœ2Ÿ:™:Ó&ØØØØØô
÷ ‰&�—‘Ó
ð 	ð —‘  AÓ&×.Ñ.¨q°RÓ8ˆà�z‰z˜&Ó!Ð!r   )rD   r<   r   rB   r;   r   r   rA   r=   rF   r   rC   )NN©	r&   r'   r(   r)   r:   re   r+   r,   Ú__classcell__©rH   r#   s   @@r   r.   r.      s   ù‡ € ÷"
ó "
÷H&"ô &"r   r.   c                   óP   a a€ ] tR t^mt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚPhiMLPc                ó    <€ V ^8„  d   QhRS[ /# r0   r2   )r"   r#   s   "€r   r$   ÚPhiMLP.__annotate__n   s   ø€ ÷ Kñ K™yñ Kr   c                óä   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  4      V n        R # r   )r9   r:   r?   r@   r   r   Úfc1Úfc2rG   s   &&€r   r:   ÚPhiMLP.__init__n   sJ   ø€ Ü‰ÑÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   Úreturn©rX   Úarray)r"   r#   s   "€r   r$   rm   s   s   ø€ ÷ 5ñ 5™RŸX™Xñ 5r   c                ój   € V P                  \        P                  ! V P                  V4      4      4      # r   )rp   r?   Úgelu_approxro   )r   r[   s   &&r   re   ÚPhiMLP.__call__s   s"   € Ø�x‰xœŸš t§x¡x°£{Ó3Ó4Ð4r   )ro   rp   rg   ri   s   @@r   rk   rk   m   s    ù‡ € ÷Kó K÷
5÷ 5ð 5r   rk   c                   óD   a a€ ] tR t^wt oV3R lV 3R lltR tRtVtV ;t# )ÚPhiDecoderLayerc                ó    <€ V ^8„  d   QhRS[ /# r0   r2   )r"   r#   s   "€r   r$   ÚPhiDecoderLayer.__annotate__x   s   ø€ ÷ "ñ "™yñ "r   c                óÈ   <€ \         SV `  4        \        VR 7      V n        \        P
                  ! VP                  VP                  R7      V n        \        V4      V n
        R# ))r1   ©ÚepsN)r9   r:   r.   Ú	self_attnr?   Ú	LayerNormr   r   Úinput_layernormrk   ÚmlprG   s   &&€r   r:   ÚPhiDecoderLayer.__init__x   sI   ø€ Ü‰ÑÔÜ%¨VÔ4ˆŒÜ!Ÿ|š|Ø×Ñ F×$9Ñ$9ô 
ˆÔô ˜&“>ˆŽr   c                óˆ   € V P                  V4      pV P                  WBV4      pV P                  V4      pWV,           V,           # r   )r‚   r€   rƒ   )r   r[   rO   rM   ÚhÚattn_hÚff_hs   &&&&   r   re   ÚPhiDecoderLayer.__call__€   s<   € Ø× Ñ  Ó#ˆØ—‘ ¨Ó/ˆØ�x‰x˜‹{ˆØ�}˜qÕ Ð r   )r‚   rƒ   r€   rg   ri   s   @@r   rz   rz   w   s   ù‡ € ÷"ó "÷!ò !r   rz   c                   óD   a a€ ] tR t^‡t oV3R lV 3R lltR tRtVtV ;t# )ÚPhiModelc                ó    <€ V ^8„  d   QhRS[ /# r0   r2   )r"   r#   s   "€r   r$   ÚPhiModel.__annotate__ˆ   s   ø€ ÷ 
ñ 
™yñ 
r   c                óV  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        V4      NK  	  upV n
        \        P                  ! VP
                  VP                  R 7      V n        R# u upi )r~   N)r9   r:   r?   Ú	Embeddingr   r   Úembed_tokensÚranger   rz   Úlayersr�   r   Úfinal_layernorm)r   r1   ÚirH   s   && €r   r:   ÚPhiModel.__init__ˆ   s}   ø€ Ü‰ÑÔÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔÜ8=¸f×>VÑ>VÔ8WÓXÑ8W°1” vÖ.Ñ8WÑXˆŒÜ!Ÿ|š|Ø×Ñ F×$9Ñ$9ô 
ˆÖùò Ys   ÁB&c                óü   € V P                  V4      pVf   R .\        V P                  4      ,          p\        W^ ,          4      p\	        V P                  V4       F  w  rEV! WV4      pK  	  V P                  V4      # r   )r�   Úlenr’   r   Úzipr“   )r   r[   rM   rO   ÚlayerÚcs   &&&   r   re   ÚPhiModel.__call__�   sn   € Ø×Ñ˜aÓ ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEÜ$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0à×#Ñ# AÓ&Ð&r   )r�   r“   r’   rg   ri   s   @@r   r‹   r‹   ‡   s   ù‡ € ÷
ó 
÷	'ò 	'r   r‹   c                   ód   a a€ ] tR t^œt oV3R lV 3R lltRV3R lR llt]R 4       tRtVt	V ;t
# )ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r0   r2   )r"   r#   s   "€r   r$   ÚModel.__annotate__�   s   ø€ ÷ ñ ™yñ r   c                óÖ   <€ \         SV `  4        VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  R R7      V n	        Wn
        R# )Tr5   N)r9   r:   r   r‹   Úmodelr?   r@   r   r   Úlm_headÚargsrG   s   &&€r   r:   ÚModel.__init__�   sL   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ˜fÓ%ˆŒ
Ü—y’y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒØŽ	r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r[   rs   rt   )r"   r#   s   "€r   r$   rŸ   ¤   s'   ø€ ÷ ñ á�8‰8ðñ 
�‰ñ	r   c                óF   € V P                  W4      pV P                  V4      # r   )r¡   r¢   )r   r[   rM   Úys   &&& r   re   ÚModel.__call__¤   s    € ð
 �J‰J�qÓ ˆØ�|‰|˜A‹Ðr   c                ó.   € V P                   P                  # r   )r¡   r’   r   s   &r   r’   ÚModel.layers¬   s   € à�z‰z× Ñ Ð r   )r£   r¢   r¡   r   r   )r&   r'   r(   r)   r:   re   Úpropertyr’   r+   r,   rh   ri   s   @@r   r�   r�   œ   s2   ù‡ € ÷ó ÷ò ð ñ!ó ÷!ð !r   r�   )rU   Údataclassesr   Úmlx.coreÚcorerX   Úmlx.nnr?   r8   r   r   r   r   ÚModuler.   rk   rz   r‹   r�   © r   r   Ú<module>r²      sŠ   ðó Ý !å Ý ç TÑ Tð ô@�ó @ó ð@ô$K"�2—9‘9ô K"ô\5ˆR�Y‰Yô 5ô!�b—i‘iô !ô 'ˆr�y‰yô 'ô*!ˆB�I‰Iö !r   