+
    UV-jó6  ã                   ó€  € ^ RI HtHt ^ RIHt ^ RIHt ^RIH	t	H
t
Ht ^RIHt ^RIHt  ! R R]P                   4      t ! R R	]P                   4      t ! R
 R]P                   4      t ! R R]P                   4      t ! R R]P                   4      t ! R R]P                   4      t ! R R]P                   4      tR# )é    )ÚOptionalÚTupleN)ÚLanguageModelOutputÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCache©Ú
TextConfigc                   óX   a a€ ] tR t^t oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚMllamaTextCrossAttentionc                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# ©é   ÚconfigÚ	layer_idx)r
   r   Úint)ÚformatÚ__classdict__s   "€Úo/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/mllama/language.pyÚ__annotate__Ú%MllamaTextCrossAttention.__annotate__   s#   ø€ ÷ Iñ I™zð I±h¹sµmñ Ió    c                ó6  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        V P                  V P
                  ,          V n        VP                  V n        V P
                  V P                  ,          V n        W n	        V P                  R,          V n
        \        P                  ! V P                  V P
                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P
                  V P                  ,          V P                  RR7      V n        \        P"                  ! V P                  VP$                  R7      V n        \        P"                  ! V P                  VP$                  R7      V n        R# )ç      à?F©Úbias©ÚepsNç      à¿)ÚsuperÚ__init__r   Úhidden_sizeÚnum_attention_headsÚ	num_headsÚhead_dimÚnum_key_value_headsÚnum_key_value_groupsr   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRMSNormÚrms_norm_epsÚq_normÚk_norm©Úselfr   r   Ú	__class__s   &&&€r   r!   Ú!MllamaTextCrossAttention.__init__   sj  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ×(Ñ(¨D¯N©NÕ:ˆŒØ#)×#=Ñ#=ˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø"ŒØ—]‘] DÕ(ˆŒ
Ü—i’iØ×Ñ˜dŸn™n¨t¯}©}Õ<À5ô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ�N‰N˜TŸ]™]Õ*¨D×,<Ñ,<À5ô
ˆŒô —j’j §¡°F×4GÑ4GÔHˆŒÜ—j’j §¡°F×4GÑ4GÔHˆŽr   c          
      ó¸   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r   Úhidden_statesÚcross_attention_statesÚattention_maskÚcacheÚreturn©ÚmxÚarrayr   r   )r   r   s   "€r   r   r   *   s\   ø€ ÷ 0(ñ 0(á—x‘xð0(ñ !)©¯©Õ 2ð0(ñ !¡§¡Õ*ð	0(ñ
 ™Õ ð0(ñ 
�‰ñ0(r   c           	     ó€  € VP                   w  rVpV P                  V4      P                  WVV P                  V P                  4      P                  ^ ^^^4      pV P                  V4      p	Ve¥   V P                  V4      P                  VRV P                  V P                  4      P                  ^ ^^^4      p
V P                  V4      P                  VRV P                  V P                  4      P                  ^ ^^^4      pV P                  V
4      p
MÕVe¦   \        VP                  \        P                  4      '       d.   VP                  P                  4       P!                  4       ^ 8”  g<   \        VP                  \        P                  4      '       g%   VP                  ^ 8”  d   VP#                  4       w  r«M,\        P$                  ! V^^R7      w  r«V P                  V
4      p
\'        V	V
VVV P(                  VR7      pVP                  ^ ^^^4      P                  WVV P*                  4      pV P-                  V4      # )r   ©Úaxis©r(   Úmaskéÿÿÿÿ)Úshaper+   Úreshaper$   r%   Ú	transposer1   r,   r&   r-   r2   Ú
isinstanceÚoffsetr>   r?   ÚmaxÚitemÚfetchÚsplitr   r(   r"   r.   )r4   r8   r9   r:   r;   ÚbszÚq_lenÚ_ÚqueryÚquery_statesÚ
key_statesÚvalue_statesÚattn_outputs   &&&&&        r   Ú__call__Ú!MllamaTextCrossAttention.__call__*   sÕ  € ð &×+Ñ+‰ˆ�Aà�K‰K˜Ó&ß‰W�S §¡°·±Ó?ß‰Y�q˜!˜Q Ó"ð 	ð
 —{‘{ 5Ó)ˆà!Ò-à—‘Ð2Ó3ß‘˜˜b $×":Ñ":¸D¿M¹MÓJß‘˜1˜a  AÓ&ð ð —‘Ð2Ó3ß‘˜˜b $×":Ñ":¸D¿M¹MÓJß‘˜1˜a  AÓ&ð ð
 Ÿ™ ZÓ0‰JØÒÜ˜Ÿ™¤b§h¡h×/Ò/°E·L±L×4DÑ4DÓ4F×4KÑ4KÓ4MÐPQÔ4QÜ˜uŸ|™|¬R¯X©X×6Ò6¸5¿<¹<È!Ô;Kà',§{¡{£}Ñ$ˆJ˜ä')§x¢x°°q¸qÔ'AÑ$ˆJØŸ™ ZÓ0ˆJä2ØØØØØ—*‘*Øô
ˆð "×+Ñ+¨A¨q°!°QÓ7×?Ñ?Ø˜×(Ñ(ó
ˆð �{‰{˜;Ó'Ð'r   )r   r%   r"   r2   r,   r   r$   r'   r&   r.   r1   r+   r(   r-   ©N©NNN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r!   rW   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r5   r   s   @@r   r   r      s    ù‡ € ÷Iõ I÷40(÷ 0(ò 0(r   r   c                   óT   a a€ ] tR t^]t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMllamaTextSelfAttentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# r   ©r
   r   )r   r   s   "€r   r   Ú$MllamaTextSelfAttention.__annotate__^   s   ø€ ÷ 
ñ 
™zð 
±cñ 
r   c                óì  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        V P                  V P
                  ,          V n        VP                  V n        V P
                  V P                  ,          V n        V P                  R,          V n	        W n
        \        P                  ! V P                  V P
                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P
                  V P                  ,          V P                  RR7      V n        \        P"                  ! V P                  VP$                  VP&                  ^R7      V n        R# )r   Fr   )ÚtraditionalÚbaser(   Nr   )r    r!   r   r"   r#   r$   r%   r&   r'   r(   r   r)   r*   r+   r,   r-   r.   ÚRoPEÚrope_traditionalÚ
rope_thetaÚroper3   s   &&&€r   r!   Ú MllamaTextSelfAttention.__init__^   s^  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ×(Ñ(¨D¯N©NÕ:ˆŒØ#)×#=Ñ#=ˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø—]‘] DÕ(ˆŒ
Ø"Œä—i’iØ×Ñ˜dŸn™n¨t¯}©}Õ<À5ô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ�N‰N˜TŸ]™]Õ*¨D×,<Ñ,<À5ô
ˆŒô —G’GØ�M‰MØ×/Ñ/Ø×"Ñ"Øô	
ˆŽ	r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r   ÚxrD   r;   r<   r=   )r   r   s   "€r   r   rh   }   sI   ø€ ÷ .(ñ .(á�8‰8ð.(ñ ‘r—x‘xÕ ð.(ñ ™Õ ð	.(ñ
 
�‰ñ.(r   c           	     óJ  € VP                   w  rEpV P                  V4      P                  WEV P                  R4      P	                  ^ ^^^4      pV P                  V4      P                  WEV P                  R4      P	                  ^ ^^^4      pV P                  V4      P                  WEV P                  R4      P	                  ^ ^^^4      p	VeÞ   \        VR4      '       d   \        VP                  4      p
M}VP                  p
\        V
\        P                  4      '       dR   V
P                  ^8”  d(   \        V
P!                  4       P#                  4       4      M\        V
P#                  4       4      p
V P%                  WzR7      pV P%                  WŠR7      pVP'                  W‰4      w  r‰M"V P%                  V4      pV P%                  V4      p\)        WxW“V P*                  VR7      pVP	                  ^ ^^^4      P                  WEV P,                  4      pV P/                  V4      # )é   Ú_idx)rJ   rC   rE   )rF   r+   rG   r$   rH   r,   r&   r-   Úhasattrr   ru   rJ   rI   r>   r?   ÚsizerK   rL   ro   Úupdate_and_fetchr   r(   r"   r.   )r4   rr   rD   r;   rO   rP   rQ   rS   rT   rU   rJ   rV   s   &&&&        r   rW   Ú MllamaTextSelfAttention.__call__}   sÓ  € ð Ÿ™‰ˆ�Aà�K‰K˜‹N×"Ñ" 3¨t¯~©~¸rÓB×LÑLÈQÐPQÐSTÐVWÓXð 	ð �K‰K˜‹Nß‰W�S ×!9Ñ!9¸2Ó>ß‰Y�q˜!˜Q Ó"ð 	ð �K‰K˜‹Nß‰W�S ×!9Ñ!9¸2Ó>ß‰Y�q˜!˜Q Ó"ð 	ð Òä�u˜f×%Ò%Ü˜UŸZ™Z›‘àŸ™�Ü˜f¤b§h¡h×/Ò/ð "Ÿ;™;¨œ?ô ˜FŸJ™J›L×-Ñ-Ó/Ô0ä  §¡£Ó/ð ð
  Ÿ9™9 \˜9ÓAˆLØŸ™ :˜Ó=ˆJØ',×'=Ñ'=¸jÓ'WÑ$ˆJ˜àŸ9™9 \Ó2ˆLØŸ™ :Ó.ˆJä2Ø lÀÇÁÐRVô
ˆð "×+Ñ+¨A¨q°!°QÓ7×?Ñ?Ø˜×(Ñ(ó
ˆð �{‰{˜;Ó'Ð'r   )r   r%   r"   r,   r   r$   r'   r&   r.   r+   ro   r(   r-   ©NNr[   rc   s   @@r   re   re   ]   s   ù‡ € ÷
ó 
÷>.(÷ .(ò .(r   re   c                   óD   a a€ ] tR t^®t oV3R lV 3R lltR tRtVtV ;t# )ÚMllamaTextMLPc                ó    <€ V ^8„  d   QhRS[ /# ©r   r   r	   )r   r   s   "€r   r   ÚMllamaTextMLP.__annotate__¯   s   ø€ ÷ 2ñ 2™zñ 2r   c                ó`  <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP
                  VP                  R R7      V n        R V n	        R# )Fr   c                 ó<   € V \         P                  ! V 4      ,          # rY   )r>   Úsigmoid)rr   s   &r   Ú<lambda>Ú(MllamaTextMLP.__init__.<locals>.<lambda>º   s   €  ¤B§J¢J¨q£MÖ 1r   N)
r    r!   r)   r*   r"   Úintermediate_sizeÚ	gate_projÚup_projÚ	down_projÚact_fn©r4   r   r5   s   &&€r   r!   ÚMllamaTextMLP.__init__¯   s„   ø€ Ü‰ÑÔÜŸšØ×Ñ × 8Ñ 8¸uô
ˆŒô —y’yØ×Ñ × 8Ñ 8¸uô
ˆŒô ŸšØ×$Ñ$ f×&8Ñ&8¸uô
ˆŒñ 2ˆŽr   c                óŒ   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      # rY   )rˆ   r‰   r†   r‡   )r4   rr   s   &&r   rW   ÚMllamaTextMLP.__call__¼   s0   € Ø�~‰~˜dŸk™k¨$¯.©.¸Ó*;Ó<¸t¿|¹|ÈA»ÕNÓOÐOr   )r‰   rˆ   r†   r‡   r[   rc   s   @@r   r|   r|   ®   s   ù‡ € ÷2ó 2÷Pò Pr   r|   c                   óT   a a€ ] tR t^Àt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMllamaSelfAttentionDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# r   rg   )r   r   s   "€r   r   Ú,MllamaSelfAttentionDecoderLayer.__annotate__Á   s   ø€ ÷ 
ñ 
™zð 
±cñ 
r   c                óL  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        P                  ! VP                  VP                  R7      V n
        \        P                  ! VP                  VP                  R7      V n        R# ©)r   r   N)r    r!   r"   re   Ú	self_attnr|   Úmlpr)   r/   r0   Úinput_layernormÚpost_attention_layernormr3   s   &&&€r   r!   Ú(MllamaSelfAttentionDecoderLayer.__init__Á   su   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ0°ÔMˆŒÜ  Ó(ˆŒÜ!Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÖ%r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r   r8   rD   r;   r<   r=   )r   r   s   "€r   r   r‘   Ë   sI   ø€ ÷ ñ á—x‘xðñ ‘r—x‘xÕ ðñ ™Õ ð	ñ
 
�‰ñr   c                ó¼   € TpV P                  V4      pV P                  VVVR 7      pWA,           pTpV P                  V4      pV P                  V4      pWA,           pV# ))rr   rD   r;   )r–   r”   r—   r•   )r4   r8   rD   r;   Úresiduals   &&&& r   rW   Ú(MllamaSelfAttentionDecoderLayer.__call__Ë   sp   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ™ØØØð 'ó 
ˆð
 !Õ0ˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆàÐr   )r"   r–   r•   r—   r”   rz   r[   rc   s   @@r   r�   r�   À   s   ù‡ € ÷
ó 
÷÷ ò r   r�   c                   óT   a a€ ] tR t^ât oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú MllamaCrossAttentionDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# r   rg   )r   r   s   "€r   r   Ú-MllamaCrossAttentionDecoderLayer.__annotate__ã   s   ø€ ÷ 
/ñ 
/™zð 
/±cñ 
/r   c                ó¸  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        P                  ! VP                  VP                  R7      V n
        \        P                  ! VP                  VP                  R7      V n        \        P                  ! ^4      V n        \        P                  ! ^4      V n        R# r“   )r    r!   r"   r   Ú
cross_attnr|   r•   r)   r/   r0   r–   r—   r>   ÚzerosÚcross_attn_attn_gateÚcross_attn_mlp_gater3   s   &&&€r   r!   Ú)MllamaCrossAttentionDecoderLayer.__init__ã   s—   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ2°6ÔOˆŒÜ  Ó(ˆŒÜ!Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÔ%ô %'§H¢H¨Q£KˆÔ!Ü#%§8¢8¨A£;ˆÖ r   c                óÒ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r   r8   r9   r:   Úfull_text_row_masked_out_maskr;   r<   r=   )r   r   s   "€r   r   r    ï   sj   ø€ ÷ ñ á—x‘xðñ !#§¡ðñ !¡§¡Õ*ð	ñ
 (0±·±Õ'9ðñ ™Õ ðñ 
�‰ñr   c                ó~  € TpV P                  V4      pV P                  VVVVR 7      pV\        P                  ! V P                  4      V,          ,           pTpV P                  V4      pV P                  V4      pVe   VR,          V,          pV\        P                  ! V P                  4      V,          ,           pV# ))r8   r9   r:   r;   ):NNNr   )r–   r¢   r>   Útanhr¤   r—   r•   r¥   )r4   r8   r9   r:   r¨   r;   r›   s   &&&&&& r   rW   Ú)MllamaCrossAttentionDecoderLayer.__call__ï   s´   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ™Ø'Ø#9Ø)Øð	 (ó 
ˆð !¤2§7¢7¨4×+DÑ+DÓ#EÈÕ#UÕUˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ(Ò4Ø9¸$Õ?À-ÕOˆMØ ¤2§7¢7¨4×+CÑ+CÓ#DÀ}Õ#TÕTˆàÐr   )r¢   r¤   r¥   r"   r–   r•   r—   rZ   r[   rc   s   @@r   rž   rž   â   s   ù‡ € ÷
/ó 
/÷÷ ò r   rž   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMllamaTextModeli  c                ó    <€ V ^8„  d   QhRS[ /# r~   r	   )r   r   s   "€r   r   ÚMllamaTextModel.__annotate__  s   ø€ ÷ Lñ L™zñ Lr   c                óê  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! VP                  ^,           VP                  4      V n        \        VP                  4       Uu. uF)  pW!P                  9   d   \        W4      M
\        W4      NK+  	  upV n        \
        P                  ! VP                  VP                  R7      V n        R# u upi )é   r   N)r    r!   r   Ú
vocab_sizer"   r)   Ú	EmbeddingÚembed_tokensÚrangeÚnum_hidden_layersÚcross_attention_layersrž   r�   Úlayersr/   r0   Únormr3   s   && €r   r!   ÚMllamaTextModel.__init__  sÈ   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒØ!×-Ñ-ˆÔäŸLšL¨×):Ñ):¸QÕ)>À×@RÑ@RÓSˆÔô # 6×#;Ñ#;Ô<ó
ñ =�	ð × =Ñ =Ô=ô 1°ÔCä4°VÓGòHñ =ñ
ˆŒô —J’J˜v×1Ñ1°v×7JÑ7JÔKˆŽ	ùò
s   Â/C0c                óp  <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[,          R	S[P                  /	# )
r   Ú	input_idsrD   Úposition_idsr9   Úcross_attention_maskr¨   Úinputs_embedsr;   r<   )r   r>   r?   r   )r   r   s   "€r   r   r¯     s­   ø€ ÷ 3ñ 3á™BŸH™HÕ%ð3ñ ‘r—x‘xÕ ð3ñ ™rŸx™xÕ(ð	3ñ
 !)©¯©Õ 2ð3ñ '¡r§x¡xÕ0ð3ñ (0±·±Õ'9ð3ñ  ¡§¡Õ)ð3ñ ™Õ ð3ñ 
�‰ñ3r   c	           	     ó`  € Ve   VP                   w  ršpM/Ve!   VP                   w  ršV P                  V4      pM\        R4      hVfD   \        P                  ! \        P
                  ! V
4      ^ 4      p\        P                  ! W9^ R7      pTpVf   R .\        V P                  4      ,          pVf   \        WÈ4      p\        \        V P                  V4      4       F=  w  pw  rïWÐP                  P                  9   d   V! VVVVVR7      pMV! VVVR7      pTpK?  	  V P                  V4      pV# )Nz5You have to specify either input_ids or inputs_embedsrA   )r9   r:   r¨   r;   )rD   r;   )rF   r´   Ú
ValueErrorr>   Úexpand_dimsÚarangeÚrepeatÚlenr¸   r   Ú	enumerateÚzipr   r·   r¹   )r4   r¼   rD   r½   r9   r¾   r¨   r¿   r;   Ú
batch_sizeÚ
seq_lengthrQ   r8   ÚidxÚdecoder_layerÚcÚlayer_outputss   &&&&&&&&&        r   rW   ÚMllamaTextModel.__call__  s#  € ð Ò$Ø(5×(;Ñ(;Ñ%ˆJ¡AØÒ"Ø%.§_¡_Ñ"ˆJØ ×-Ñ-¨iÓ8‰MäÐTÓUÐUàÒÜŸ>š>¬"¯)ª)°JÓ*?ÀÓCˆLÜŸ9š9 \ÀAÔFˆLà%ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEàŠ<Ü(¨Ó>ˆDä'0´°T·[±[À%Ó1HÖ'IÑ#ˆCÑ#�-Ø—k‘k×8Ñ8Ô8Ù -Ø!Ø+AØ#7Ø2OØô!‘ñ !.Ø!ØØô!�ð
 *ŠMñ (Jð" Ÿ	™	 -Ó0ˆàÐr   )r   r´   r"   r¸   r¹   r²   )NNNNNNNNr[   rc   s   @@r   r­   r­     s    ù‡ € ÷Ló L÷"3÷ 3ò 3r   r­   c                   ó”   a a€ ] tR tRt oV3R lV 3R lltRV3R lR llt]R 4       t]R 4       t	]R 4       t
]R	 4       tR
tVtV ;t# )ÚLanguageModeliS  c                ó    <€ V ^8„  d   QhRS[ /# r~   r	   )r   r   s   "€r   r   ÚLanguageModel.__annotate__T  s   ø€ ÷ Tñ T™zñ Tr   c                ó´   <€ \         SV `  4        Wn        \        V4      V n        \
        P                  ! VP                  VP                  R R7      V n	        R# )Fr   N)
r    r!   r   r­   Úmodelr)   r*   r"   r²   Úlm_headrŠ   s   &&€r   r!   ÚLanguageModel.__init__T  s?   ø€ Ü‰ÑÔØŒÜ$ VÓ,ˆŒ
Ü—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŽr   c                ó€  <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[S[P                  S[ S[P                  ,          3,          /# )	r   Úinputsr¿   rD   r;   r9   r¾   r¨   r<   )r   r>   r?   r   r   )r   r   s   "€r   r   rÒ   Z  s­   ø€ ÷ 
ñ 
á™Ÿ™Õ"ð
ñ  ¡§¡Õ)ð
ñ ‘r—x‘xÕ ð	
ñ
 ™Õ ð
ñ !)©¯©Õ 2ð
ñ '¡r§x¡xÕ0ð
ñ (0±·±Õ'9ð
ñ 
‰r�x‰x™¡"§(¡(Õ+Ð+Õ	,ñ
r   c           
     ól   € V P                  VVVVVVVR 7      p	V P                  V	4      p
\        W¥R7      # ))r¼   rD   r9   r¾   r¨   r¿   r;   )Úlogitsr9   )rÔ   rÕ   r   )r4   rØ   r¿   rD   r;   r9   r¾   r¨   Úkwargsr8   rÚ   s   &&&&&&&&,  r   rW   ÚLanguageModel.__call__Z  sL   € ð Ÿ
™
ØØØ#9Ø!5Ø*GØ'Øð #ó 
ˆð —‘˜mÓ,ˆä"Øô
ð 	
r   c                óh   € V P                  4        UUu/ uF  w  rR V9  g   K  WbK  	  upp# u uppi )zself_attn.rotary_emb.inv_freq)Úitems)ÚweightsÚkÚvs   &  r   ÚsanitizeÚLanguageModel.sanitizev  s:   € ð %Ÿ]™]œ_ô
Ù,‘T�QÐ0OÐWXÑ0XŒDˆAŠD™_ò
ð 	
ùó 
s   ”.¤.c                ó.   € V P                   P                  # rY   )rÔ   r¸   ©r4   s   &r   r¸   ÚLanguageModel.layers}  s   € à�z‰z× Ñ Ð r   c                ód   € V P                   P                  V P                   P                  ,          # rY   )r   r"   r#   rå   s   &r   r%   ÚLanguageModel.head_dim�  s!   € à�{‰{×&Ñ&¨$¯+©+×*IÑ*IÕIÐIr   c                ó.   € V P                   P                  # rY   )r   r&   rå   s   &r   Ú
n_kv_headsÚLanguageModel.n_kv_heads…  s   € à�{‰{×.Ñ.Ð.r   )r   rÕ   rÔ   )NNNNNNN)r\   r]   r^   r_   r!   rW   Ústaticmethodrâ   Úpropertyr¸   r%   rê   r`   ra   rb   rc   s   @@r   rÐ   rÐ   S  sr   ù‡ € ÷Tó T÷
ò 
ð8 ñ
ó ð
ð ñ!ó ð!ð ñJó ðJð ñ/ó ÷/ð /r   rÐ   )Útypingr   r   Úmlx.coreÚcorer>   Úmlx.nnr)   rk   r   r   r   r;   r   r   r
   ÚModuler   re   r|   r�   rž   r­   rÐ   © r   r   Ú<module>rô      s�   ðß "å Ý ÷ñ õ
 Ý ôK(˜rŸy™yô K(ô\N(˜bŸi™iô N(ôbP�B—I‘Iô Pô$ b§i¡iô ôD& r§y¡yô &ôRE�b—i‘iô EôP4/�B—I‘Iö 4/r   