+
    UV-j9  ã                   óV  € ^ RI Ht ^ RIHt ^ RIHt ^RIHtH	t	H
t
 ^RIHt ^RIHtHt  ! R R]P                   4      t ! R R	]P                   4      t ! R
 R]P                   4      t ! R R]P                   4      t ! R R]P                   4      t ! R R]P                   4      tR# )é    )ÚOptionalN)ÚLanguageModelOutputÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCache)ÚModelConfigÚ
TextConfigc                   óP   a a€ ] tR t^t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMolmo2Embeddingc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )é   Únum_embeddingsÚnum_new_embeddingsÚfeatures©Úint)ÚformatÚ__classdict__s   "€Úo/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/molmo2/language.pyÚ__annotate__ÚMolmo2Embedding.__annotate__   s.   ø€ ÷ Fñ FáðFñ  ðFñ ñ	Fó    c                ó”   <€ \         SV `  4        \        P                  ! W34      V n        \        P                  ! W#34      V n        R # ©N)ÚsuperÚ__init__ÚmxÚzerosÚ	embeddingÚnew_embedding)Úselfr   r   r   Ú	__class__s   &&&&€r   r   ÚMolmo2Embedding.__init__   s8   ø€ ô 	‰ÑÔÜŸš >Ð"<Ó=ˆŒÜŸXšXÐ'9Ð&DÓEˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   ÚxÚreturn©r   Úarray)r   r   s   "€r   r   r      s'   ø€ ÷ Oñ O™"Ÿ(™(ð O¡r§x¡xñ Or   c                ól   € \         P                  ! V P                  V P                  .^ R7      V,          # )r   ©Úaxis)r   Úconcatenater   r    )r!   r&   s   &&r   Ú__call__ÚMolmo2Embedding.__call__   s'   € Ü�~Š~˜tŸ~™~¨t×/AÑ/AÐBÈÔKÈAÕNÐNr   )r   r    ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r.   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r"   r   s   @@r   r   r      s#   ù‡ € ÷Fó F÷O÷ Oð Or   r   c                   óP   a a€ ] tR t^t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚLanguageModelMLPc                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Ú	input_dimÚintermediate_sizer   )r   r   s   "€r   r   ÚLanguageModelMLP.__annotate__   s   ø€ ÷ Jñ J¡#ð J¹#ñ Jr   c                ó¦   <€ \         SV `  4        \        P                  ! W^,          RR7      V n        \        P                  ! W!RR7      V n        R# )r   F©ÚbiasN)r   r   ÚnnÚLinearÚff_projÚff_out)r!   r<   r=   r"   s   &&&€r   r   ÚLanguageModelMLP.__init__   s8   ø€ Ü‰ÑÔÜ—y’y ÀÕ,AÈÔNˆŒÜ—i’iÐ 1À5ÔIˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r%   r(   )r   r   s   "€r   r   r>   $   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ó¶   € V P                  V4      p\        P                  ! V^RR7      w  r\        P                  ! V4      V,          pV P                  V4      # )r   r+   éÿÿÿÿ)rD   r   ÚsplitrB   ÚsilurE   )r!   r&   Úgates   && r   r.   ÚLanguageModelMLP.__call__$   sC   € Ø�L‰L˜‹OˆÜ—(’(˜1˜a bÔ)‰ˆÜ�GŠG�D‹M˜AÕˆØ�{‰{˜1‹~Ðr   )rE   rD   r0   r8   s   @@r   r:   r:      s    ù‡ € ÷Jó J÷
÷ ð r   r:   c                   óT   a a€ ] tR t^+t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMolmo2Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfig©r	   )r   r   s   "€r   r   ÚMolmo2Attention.__annotate__,   s   ø€ ÷ Iñ I™zñ Ir   c                ó²  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        V P                  R,          V n        VP                  VP                  ,          VP                  VP
                  ,          VP                  VP
                  ,          3V n        \        P                  ! VP                  \        V P                  4      VP                  R7      V n        \        P                  ! VP                  VP                   R7      V n        \        P                  ! VP                  VP                   R7      V n        \        P                  ! VP                  VP                  ,          VP                  RR7      V n        \        P(                  ! V P                  VP*                  R7      V n        R# )g      à?r@   )ÚdimsÚepsF)ÚbaseNg      à¿)r   r   rR   Únum_attention_headsÚ	num_headsÚnum_key_value_headsÚhead_dimÚscaleÚ
fused_dimsrB   rC   Úhidden_sizeÚsumÚqkv_biasÚatt_projÚRMSNormÚlayer_norm_epsÚq_normÚk_normÚattn_outÚRoPEÚ
rope_thetaÚ
rotary_emb©r!   rR   r"   s   &&€r   r   ÚMolmo2Attention.__init__,   s>  ø€ Ü‰ÑÔØŒØ×3Ñ3ˆŒØ#)×#=Ñ#=ˆÔ ØŸ™ˆŒØ—]‘] DÕ(ˆŒ
ð ×&Ñ&¨¯©Õ8Ø�O‰O˜f×8Ñ8Õ8Ø�O‰O˜f×8Ñ8Õ8ð
ˆŒô Ÿ	š	Ø×ÑÜ�—‘Ó Ø—‘ô
ˆŒô —j’j f§o¡o¸6×;PÑ;PÔQˆŒÜ—j’j f§o¡o¸6×;PÑ;PÔQˆŒäŸ	š	Ø�O‰O˜f×8Ñ8Õ8Ø×ÑØô
ˆŒô Ÿ'š' $§-¡-°f×6GÑ6GÔHˆŽr   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   Úhidden_statesÚmaskÚcacher'   ©r   r)   r   r   )r   r   s   "€r   r   rT   K   sI   ø€ ÷ #"ñ #"á—x‘xð#"ñ ‘r—x‘xÕ ð#"ñ ™Õ ð	#"ñ
 
�‰ñ#"r   c           	     ó  € VP                   w  rEpV P                  V4      p\        P                  ! VV P                  ^ ,          V P                  ^ ,          V P                  ^,          ,           .RR7      w  r‰p
V P                  VP                  WEV P                  V P                  4      4      pV P                  V	P                  WEV P                  V P                  4      4      p	V
P                  WEV P                  V P                  4      p
VP                  ^ ^^^4      pV	P                  ^ ^^^4      p	V
P                  ^ ^^^4      p
VeM   V P                  WƒP                  R7      pV P                  W“P                  R7      p	VP                  Wš4      w  ršM"V P                  V4      pV P                  V	4      p	\        W‰W£V P                   VR7      pVP                  ^ ^^^4      P                  WER4      pV P#                  V4      # )r   r+   )Úoffset)r]   rp   rI   )Úshaperb   r   rJ   r^   re   ÚreshaperZ   r\   rf   r[   Ú	transposerj   rt   Úupdate_and_fetchr   r]   rg   )r!   ro   rp   rq   Ú
batch_sizeÚseq_lenÚ_ÚqkvÚqÚkÚvÚatts   &&&&        r   r.   ÚMolmo2Attention.__call__K   s¬  € ð "/×!4Ñ!4Ñˆ
˜Qà�m‰m˜MÓ*ˆÜ—(’(ØØ�_‰_˜QÕ §¡°Õ!3°d·o±oÀaÕ6HÕ!HÐIØô
‰ˆˆað �K‰K˜Ÿ	™	 *°t·~±~ÀtÇ}Á}ÓUÓVˆØ�K‰KØ�I‰I�j¨4×+CÑ+CÀTÇ]Á]ÓSó
ˆð �I‰I�j¨4×+CÑ+CÀTÇ]Á]ÓSˆà�K‰K˜˜1˜a Ó#ˆØ�K‰K˜˜1˜a Ó#ˆØ�K‰K˜˜1˜a Ó#ˆàÒØ—‘ ¯,©,�Ó7ˆAØ—‘ ¯,©,�Ó7ˆAØ×)Ñ)¨!Ó/‰DˆAˆqà—‘ Ó"ˆAØ—‘ Ó"ˆAä*¨1°ÀÇÁÐRVÔWˆØ�m‰m˜A˜q ! QÓ'×/Ñ/°
ÀRÓHˆØ�}‰}˜SÓ!Ð!r   )rb   rg   rR   r^   r\   rf   rZ   r[   re   rj   r]   ©NNr0   r8   s   @@r   rO   rO   +   s    ù‡ € ÷Ió I÷>#"÷ #"ò #"r   rO   c                   óT   a a€ ] tR t^qt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMolmo2DecoderLayerc                ó    <€ V ^8„  d   QhRS[ /# rQ   rS   )r   r   s   "€r   r   ÚMolmo2DecoderLayer.__annotate__r   s   ø€ ÷ Rñ R™zñ Rr   c                óR  <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  VP                  R 7      V n        \        P
                  ! VP                  VP                  R 7      V n	        \        VP                  VP                  4      V n        R# ©)rW   N)r   r   rO   Ú	self_attnrB   rc   r_   rd   Ú	attn_normÚff_normr:   r=   Úmlprk   s   &&€r   r   ÚMolmo2DecoderLayer.__init__r   sp   ø€ Ü‰ÑÔÜ(¨Ó0ˆŒÜŸš F×$6Ñ$6¸F×<QÑ<QÔRˆŒÜ—z’z &×"4Ñ"4¸&×:OÑ:OÔPˆŒÜ# F×$6Ñ$6¸×8PÑ8PÓQˆŽr   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rn   rr   )r   r   s   "€r   r   r†   y   sI   ø€ ÷ ñ á—x‘xðñ ‘r—x‘xÕ ðñ ™Õ ð	ñ
 
�‰ñr   c                ó°   € TpV P                  V4      pW@P                  WV4      ,           pTpV P                  V4      pW@P                  V4      ,           pV# r   )rŠ   r‰   r‹   rŒ   )r!   ro   rp   rq   Úresiduals   &&&& r   r.   ÚMolmo2DecoderLayer.__call__y   sT   € ð !ˆØŸ™ }Ó5ˆØ §>¡>°-ÀuÓ#MÕMˆà ˆØŸ™ ]Ó3ˆØ §8¡8¨MÓ#:Õ:ˆØÐr   )rŠ   r‹   rŒ   r‰   r‚   r0   r8   s   @@r   r„   r„   q   s    ù‡ € ÷Ró R÷÷ ò r   r„   c                   óT   a a€ ] tR t^‰t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMolmo2Transformerc                ó    <€ V ^8„  d   QhRS[ /# rQ   rS   )r   r   s   "€r   r   ÚMolmo2Transformer.__annotate__Š   s   ø€ ÷ =ñ =™zñ =r   c                ó¬  <€ \         SV `  4        Wn        \        VP                  VP
                  VP                  4      V n        \        VP                  4       Uu. uF  p\        V4      NK  	  upV n        \        P                  ! VP                  VP                  R 7      V n        \        P                   ! VP"                  4      V n        R# u upi rˆ   )r   r   rR   r   Ú
vocab_sizeÚadditional_vocab_sizer_   ÚwteÚrangeÚnum_hidden_layersr„   ÚblocksrB   rc   rd   Úln_fÚDropoutÚembedding_dropoutÚemb_drop)r!   rR   r{   r"   s   && €r   r   ÚMolmo2Transformer.__init__Š   s¢   ø€ Ü‰ÑÔØŒä"Ø×Ñ˜v×;Ñ;¸V×=OÑ=Oó
ˆŒô 16°f×6NÑ6NÔ0Oó
Ù0O¨1Ô˜vÖ&Ñ0Oñ
ˆŒô —J’J˜v×1Ñ1°v×7LÑ7LÔMˆŒ	äŸ
š
 6×#;Ñ#;Ó<ˆŽùò
s   ÁCc          
      óÈ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[S[,          ,          RS[ P                  /# )r   Ú	input_idsÚinputs_embedsrp   rq   r'   )r   r)   r   Úlistr   )r   r   s   "€r   r   r•   ˜   s`   ø€ ÷ (ñ (á—8‘8ð(ñ  ¡§¡Õ)ð(ñ ‘r—x‘xÕ ð	(ñ
 ™™W�Õ&ð(ñ 
�‰ñ(r   c                ó&  € Vf   V P                  V4      pMTpVf   R .\        V P                  4      ,          pVf   \        WT4      pV P	                  V4      p\        V P                  V4       F  w  rgV! WSV4      pK  	  V P                  V4      # r   )r™   Úlenrœ   r   r    Úzipr�   )r!   r£   r¤   rp   rq   ro   ÚblockÚcs   &&&&&   r   r.   ÚMolmo2Transformer.__call__˜   sˆ   € ð Ò Ø ŸH™H YÓ/‰Mà)ˆMàŠ=Ø�FœS §¡Ó-Õ-ˆEàŠ<Ü(¨Ó>ˆDàŸ™ mÓ4ˆä˜DŸK™K¨Ö/‰HˆEÙ! -°qÓ9ŠMñ 0ð �y‰y˜Ó'Ð'r   )rœ   rR   r    r�   r™   ©NNNr0   r8   s   @@r   r“   r“   ‰   s   ù‡ € ÷=ó =÷(÷ (ò (r   r“   c                   óx   a a€ ] tR t^²t oRV3R lV 3R llltR	V3R lR llt]R 4       t]R 4       t	Rt
VtV ;t# )
ÚLanguageModelc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   ÚargsrR   )r	   r   )r   r   s   "€r   r   ÚLanguageModel.__annotate__³   s   ø€ ÷ Pñ P™Zð P±ñ Pr   c                óâ   <€ \         SV `  4        Wn        W n        VP                  V n        \        V4      V n        \        P                  ! VP                  VP                  R R7      V n        R# )Fr@   N)r   r   r°   rR   Ú
model_typer“   ÚmodelrB   rC   r_   r—   Úlm_head)r!   r°   rR   r"   s   &&&€r   r   ÚLanguageModel.__init__³   sM   ø€ Ü‰ÑÔØŒ	ØŒØŸ/™/ˆŒÜ& tÓ,ˆŒ
Ü—y’y ×!1Ñ!1°4·?±?ÈÔOˆŽr   c          
      ó´   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[S[,          ,          RS[/# )r   Úinputsr¤   rp   rq   r'   )r   r)   r   r¥   r   r   )r   r   s   "€r   r   r±   »   s\   ø€ ÷ 
2ñ 
2á—‘ð
2ñ  ¡§¡Õ)ð
2ñ ‘r—x‘xÕ ð	
2ñ
 ™™W�Õ&ð
2ñ 
ñ
2r   c                ó`   € V P                  WW44      pV P                  V4      p\        VR 7      # ))Úlogits)r´   rµ   r   )r!   r¸   r¤   rp   rq   Úkwargsro   rº   s   &&&&&,  r   r.   ÚLanguageModel.__call__»   s-   € ð Ÿ
™
 6¸$ÓFˆØ—‘˜mÓ,ˆÜ"¨&Ô1Ð1r   c                óh   € V P                  4        UUu/ uF  w  rR V9  g   K  WbK  	  upp# u uppi )zrotary_emb.inv_freq)Úitems)Úweightsr~   r   s   &  r   ÚsanitizeÚLanguageModel.sanitizeÇ   s1   € ð ")§¡¤ÔS¡™˜Ð4IÐQRÑ4R”�’¡ÒSÐSùÓSs   ”.¤.c                ó.   € V P                   P                  # r   )r´   rœ   )r!   s   &r   ÚlayersÚLanguageModel.layersÌ   s   € à�z‰z× Ñ Ð r   )r°   rR   rµ   r´   r³   r   r¬   )r1   r2   r3   r4   r   r.   ÚstaticmethodrÀ   ÚpropertyrÃ   r5   r6   r7   r8   s   @@r   r®   r®   ²   sJ   ù‡ € ÷Põ P÷
2ò 
2ð ñTó ðTð ñ!ó ÷!ð !r   r®   )Útypingr   Úmlx.coreÚcorer   Úmlx.nnrB   rX   r   r   r   rq   r   rR   r   r	   ÚModuler   r:   rO   r„   r“   r®   © r   r   Ú<module>rÍ      s†   ðÝ å Ý ÷ñ õ
 ß +ôO�b—i‘iô Oô
�r—y‘yô 
ôC"�b—i‘iô C"ôL˜Ÿ™ô ô0&(˜Ÿ	™	ô &(ôR!�B—I‘Iö !r   