+
    TV-j�  ã                   óh  € ^ RI Ht ^ RIHtHtHtHtHt ^ RIH	t
 ^ RIHt ^RIHt ^RIHtHtHt ] ! R R]4      4       tRR R	 llt ! R
 R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      tR# )é    )Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnionN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   ó@   a € ] tR t^t o RtRtRtRtRtV 3R lt	Rt
V tR# )Ú	ModelArgsTg�íµ ÷Æ°>i'  c                óÂ   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   # )é   Ú
model_typeÚhead_dimÚnum_transformer_layersÚ	model_dimÚ
vocab_sizeÚffn_dim_divisorÚnum_query_headsÚnum_kv_headsÚffn_multipliersÚffn_with_gluÚnormalize_qk_projectionsÚshare_input_output_layersÚrms_norm_epsÚrope_freq_constant)ÚstrÚintr   ÚboolÚfloat)ÚformatÚ__classdict__s   "€Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/openelm.pyÚ__annotate__ÚModelArgs.__annotate__   sµ   ø‡ ‚ á�Oñ ñ �Mñ ñ  Ññ	 ñ
 �Nñ ñ �Oñ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ #Ñ)ñ ñ  $Ñ*ñ ñ Ññ ñ Ñ%ò ó    © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r$   s   @r%   r   r      s*   ø‡ € ð €LØ%)ÐØ&*ÐØ€LØ %Ð÷ ƒ r(   r   c          
      óÚ   € V ^8„  d   QhR\         \        \        3,          R\        \        ,          R\        \         \        \        3,          ,          R\         \        \        3,          /# )r   ÚvÚdivisorÚ	min_valueÚreturn)r   r"   r    r   )r#   s   "r%   r&   r&      sT   € ÷ ñ ÜŒU”CˆZÕðä”c�]ðô œœe¤S˜jÕ)Õ*ðô Œ5”#ˆ:Õñ	r(   c                óž   € Vf   Tp\        V\        W^,          ,           4      V,          V,          4      pVRV ,          8  d	   W1,          pV# )a™  
This function is taken from the original tf repo.
It ensures that all layers have a channel number that is divisible by the divisor
It can be seen at:
https://github.com/tensorflow/models/blob/2cfc99eff5e5eb729c6793d2f3d03aa1c9be2b15/research/slim/nets/mobilenet/mobilenet.py#L62
Args:
    v: input value
    divisor: default to 8
    min_value: minimum divisor value
Returns:
    new_v: new divisible value
gÍÌÌÌÌÌì?)Úmaxr    )r2   r3   r4   Únew_vs   &&& r%   Úmake_divisibler9      sH   € ð" ÒØˆ	Ü�	œ3˜q¨Q¥;�Ó/°7Õ:¸WÕDÓE€Eàˆs�Q�w„ØÕˆØ€Lr(   c                   óT   a a€ ] tR t^9t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   ÚargsÚlayer_id©r   r    )r#   r$   s   "€r%   r&   ÚAttention.__annotate__:   s   ø€ ÷ Wñ W™Yð W±#ñ Wr(   c                óÜ  <€ \         SV `  4        VP                  ;V n        pW n        VP                  ;V n        pVP
                  V,          ;V n        pVP                  V,          ;V n        pVR,          V n	        WV^,          ,           V,          p\        P                  ! WGRR7      V n        \        P                  ! WS,          VRR7      V n        VP                  V n        V P                  '       dM   \        P                  ! W1P                   R7      V n        \        P                  ! W1P                   R7      V n        \        P&                  ! VRVP(                  R7      V n        R# )g      à?F©Úbias©Úeps)ÚtraditionalÚbaseNg      à¿)ÚsuperÚ__init__r   r?   r   r   Ún_headsr   Ú
n_kv_headsÚscaleÚnnÚLinearÚqkv_projÚout_projr   ÚRMSNormr   Úq_normÚk_normÚRoPEr   Úrope)	Úselfr>   r?   r   r   rK   rL   Úop_sizeÚ	__class__s	   &&&     €r%   rJ   ÚAttention.__init__:   s  ø€ Ü‰ÑÔØ#'§=¡=Ð0ˆŒ˜Ø ŒØ%)§^¡^Ð3ˆŒ˜à!%×!5Ñ!5°hÕ!?Ð?ˆŒ�wØ'+×'8Ñ'8¸Õ'BÐBˆŒ˜*Ø˜t•^ˆŒ
à¨1�nÕ-°Õ9ˆÜŸ	š	 )¸5ÔAˆŒÜŸ	š	 'Õ"4°iÀeÔLˆŒà(,×(EÑ(EˆÔ%à×(×(Ð(ÜŸ*š* X×3DÑ3DÔEˆDŒKÜŸ*š* X×3DÑ3DÔEˆDŒKä—G’G˜H°%¸d×>UÑ>UÔVˆŽ	r(   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacher5   ©ÚmxÚarrayr   r   )r#   r$   s   "€r%   r&   rA   P   sH   ø€ ÷ %%ñ %%á�8‰8ð%%ñ ‘r—x‘xÕ ð%%ñ ™�}ð	%%ñ
 
�‰ñ%%r(   c           	     óP  € VP                   w  rEpV P                  V4      pVP                  WEV P                  V P                  ^,          ,           V P
                  4      P                  ^ ^^^4      p\        P                  ! WpP                  V P                  V P                  ,           .^R7      w  r‰p
V P                  '       d#   V P                  V4      pV P                  V	4      p	VeM   V P                  WƒP                  R7      pV P                  W“P                  R7      p	VP                  Wš4      w  ršM"V P                  V4      pV P                  V	4      p	\        W‰W£V P                   VR7      pVP                  ^ ^^^4      P                  WER4      pV P#                  V4      # )r   ©Úaxis)Úoffset)r_   rM   r^   éÿÿÿÿ)ÚshaperP   ÚreshaperK   rL   r   Ú	transposera   Úsplitr   rS   rT   rV   rf   Úupdate_and_fetchr   rM   rQ   )rW   r]   r^   r_   ÚBÚLÚDÚqkvÚqueriesÚkeysÚvaluesÚoutputs   &&&&        r%   Ú__call__ÚAttention.__call__P   s_  € ð —'‘'‰ˆˆaà�m‰m˜AÓˆà�k‰kØ�$—,‘, $§/¡/°AÕ"5Õ6¸¿¹ó
ç
‰)�A�q˜!˜QÓ
ð 	ô !#§¢Ø—,‘, §¡¨t¯©Õ >Ð?Àaô!
Ñˆ�vð
 ×(×(Ð(Ø—k‘k 'Ó*ˆGØ—;‘;˜tÓ$ˆDàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆà�}‰}˜VÓ$Ð$r(   )r   rT   r?   r   rK   rL   r   rQ   rS   rP   rV   rM   ©NN©	r*   r+   r,   r-   rJ   ru   r/   r0   Ú__classcell__©rY   r$   s   @@r%   r;   r;   9   s    ù‡ € ÷Wó W÷,%%÷ %%ò %%r(   r;   c                   óP   a a€ ] tR t^xt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPc                ó&   <€ V ^8„  d   QhRS[ RS[/# r=   r@   )r#   r$   s   "€r%   r&   ÚMLP.__annotate__y   s   ø€ ÷ Cñ C™Yð C±#ñ Cr(   c                óR  <€ \         SV `  4        Wn        VP                  pVP                  V,          p\        \        WAP                  ,          VP                  R 7      4      p\        P                  ! V^V,          RR7      V n
        \        P                  ! WSRR7      V n        R# ))r3   FrC   N)rI   rJ   r>   r   r   r    r9   r   rN   rO   Úproj_1Úproj_2)rW   r>   r?   ÚdimÚffn_multiplierÚintermediate_dimrY   s   &&&   €r%   rJ   ÚMLP.__init__y   s�   ø€ Ü‰ÑÔØŒ	Ø�n‰nˆØ×-Ñ-¨hÕ7ˆäÜØ§¡Õ/Ø×,Ñ,ôó
Ðô —i’i  QÐ)9Õ%9ÀÔFˆŒÜ—i’iÐ 0¸EÔBˆŽr(   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r5   ©ra   rb   )r#   r$   s   "€r%   r&   r~   ‰   s   ø€ ÷ ,ñ ,™RŸX™Xñ ,r(   c                óŽ   € V P                  V4      p\        P                  ! V^RR7      w  r!V P                  \	        W!4      4      # )r   rd   rg   )r€   ra   rk   r�   r	   )rW   r]   Úgates   && r%   ru   ÚMLP.__call__‰   s6   € Ø�K‰K˜‹NˆÜ—(’(˜1˜a bÔ)‰ˆØ�{‰{œ6 $›?Ó+Ð+r(   )r>   r€   r�   rx   rz   s   @@r%   r|   r|   x   s    ù‡ € ÷Có C÷ ,÷ ,ð ,r(   r|   c                   óT   a a€ ] tR t^�t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó&   <€ V ^8„  d   QhRS[ RS[/# r=   r@   )r#   r$   s   "€r%   r&   ÚTransformerBlock.__annotate__�   s   ø€ ÷ @ñ @™Yð @±#ñ @r(   c                ó  <€ \         SV `  4        VP                  p\        WR 7      V n        \        WR 7      V n        \        P                  ! W1P                  R7      V n
        \        P                  ! W1P                  R7      V n        R# )©r?   rE   N)rI   rJ   r   r;   Úattnr|   ÚffnrN   rR   r   Úffn_normÚ	attn_norm)rW   r>   r?   r‚   rY   s   &&& €r%   rJ   ÚTransformerBlock.__init__�   s[   ø€ Ü‰ÑÔØ�n‰nˆÜ˜dÔ6ˆŒ	Ü�tÔ/ˆŒÜŸ
š
 3×,=Ñ,=Ô>ˆŒÜŸš C×->Ñ->Ô?ˆŽr(   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r\   r`   )r#   r$   s   "€r%   r&   rŽ   ˜   sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r(   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# ©N©r‘   r”   r’   r“   )rW   r]   r^   r_   ÚrÚhÚouts   &&&&   r%   ru   ÚTransformerBlock.__call__˜   sG   € ð �I‰I�d—n‘n QÓ'¨Ó5ˆØ�EˆØ�H‰H�T—]‘] 1Ó%Ó&ˆØ�eˆØˆ
r(   r™   rw   rx   rz   s   @@r%   rŒ   rŒ   �   s    ù‡ € ÷@ó @÷
÷ 
ò 
r(   rŒ   c                   óT   a a€ ] tR t^¥t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚOpenELMModelc                ó    <€ V ^8„  d   QhRS[ /# ©r   r>   ©r   )r#   r$   s   "€r%   r&   ÚOpenELMModel.__annotate__¦   s   ø€ ÷ Fñ F™Yñ Fr(   c                óÎ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        V P                  ^ 8”  g   Q h\
        P                  ! VP                  VP                  4      V n        \        V P                  4       Uu. uF  p\        WR7      NK  	  upV n        \
        P                  ! VP                  VP                  R7      V n        R# u upi )r   r�   rE   N)rI   rJ   r>   r   r   rN   Ú	Embeddingr   Útoken_embeddingsÚrangerŒ   ÚlayersrR   r   Únorm)rW   r>   r?   rY   s   && €r%   rJ   ÚOpenELMModel.__init__¦   s­   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ&*×&AÑ&AˆÔ#Ø�‰ Ô"Ð"Ð"Ü "§¢¨T¯_©_¸d¿n¹nÓ MˆÔô " $×"=Ñ"=Ô>ó
á>�ô ˜T×5Ù>ñ
ˆŒô —J’J˜tŸ~™~°4×3DÑ3DÔEˆŽ	ùò	
s   ÂC"c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   Úinputsr‡   )r#   r$   s   "€r%   r&   r£   ³   s   ø€ ÷ ñ á—‘ñr(   c                óþ   € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2^ ,          4      p\	        V P                  V4       F  w  rVV! W4VR7      pK  	  V P                  V4      # )N)r_   )r¦   Úlenr¨   r   Úzipr©   )rW   r­   r_   r›   r^   ÚlayerÚcs   &&&    r%   ru   ÚOpenELMModel.__call__³   so   € ð
 ×!Ñ! &Ó)ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆÜ˜DŸK™K¨Ö/‰HˆEÙ�a QÔ'ŠAñ 0ð �y‰y˜‹|Ðr(   )r>   r¨   r©   r   r¦   r   r˜   rx   rz   s   @@r%   rŸ   rŸ   ¥   s    ù‡ € ÷Fó F÷÷ ò r(   rŸ   c                   ód   a a€ ] tR t^Ät oV3R lV 3R lltRV3R lR llt]R 4       tRtVt	V ;t
# )ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r¡   r¢   )r#   r$   s   "€r%   r&   ÚModel.__annotate__Å   s   ø€ ÷ Rñ R™Yñ Rr(   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# )FrC   N)rI   rJ   r>   r   rŸ   Útransformerr   rN   rO   r   r   Úlm_head)rW   r>   rY   s   &&€r%   rJ   ÚModel.__init__Å   sX   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ'¨Ó-ˆÔØ×-×-Ð-ÜŸ9š9 T§^¡^°T·_±_È5ÔQˆDŽLñ .r(   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r¬   r‡   )r#   r$   s   "€r%   r&   r·   Í   s   ø€ ÷ ñ á—‘ñr(   c                óÐ   € V P                  W4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# r˜   )r¹   r>   r   r¦   Ú	as_linearrº   )rW   r­   r_   rœ   s   &&& r%   ru   ÚModel.__call__Í   s\   € ð
 ×Ñ˜vÓ-ˆØ�9‰9×.×.Ð.Ø×"Ñ"×3Ñ3×=Ñ=¸cÓBˆCð ˆ
ð —,‘,˜sÓ#ˆCàˆ
r(   c                ó.   € V P                   P                  # r˜   )r¹   r¨   )rW   s   &r%   r¨   ÚModel.layersÚ   s   € à×Ñ×&Ñ&Ð&r(   )r>   rº   r   r¹   r˜   )r*   r+   r,   r-   rJ   ru   Úpropertyr¨   r/   r0   ry   rz   s   @@r%   rµ   rµ   Ä   s4   ù‡ € ÷Ró R÷ò ð ñ'ó ÷'ð 'r(   rµ   )é   N)Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorera   Úmlx.nnrN   Úactivationsr	   rH   r
   r   r   r   r9   ÚModuler;   r|   rŒ   rŸ   rµ   r)   r(   r%   Ú<module>rË      s�   ðõ "ß 3Õ 3å Ý å ß TÑ Tð ô&�ó &ó ð&÷"ô4<%�—	‘	ô <%ô~,ˆ"�)‰)ô ,ô.�r—y‘yô ô,�2—9‘9ô ô>'ˆB�I‰Iö 'r(   