+
    UV-jå  ã                   ó(  € ^ RI Ht ^ RIHt ^ RIHt ^RIHtH	t	H
t
 ^RIHt ^RIHt  ! R R]P                  4      t ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR# )é    )ÚOptionalN)ÚLanguageModelOutputÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCache©Ú
TextConfigc                   ó2   a € ] tR t^t o V 3R lR ltRtV tR# )ÚSwiGLUc                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©é   ÚxÚreturn©ÚmxÚarray)ÚformatÚ__classdict__s   "€Ún/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/molmo/language.pyÚ__annotate__ÚSwiGLU.__annotate__   s#   ø€ ÷ !ñ !™"Ÿ(™(ð !¡r§x¡xñ !ó    c                ór   € \         P                  ! V^RR7      w  r\        P                  ! V4      V,          # )r   ©Úaxiséÿÿÿÿ)r   ÚsplitÚnnÚsilu)Úselfr   Úgates   && r   Ú__call__ÚSwiGLU.__call__   s)   € Ü—(’(˜1˜a bÔ)‰ˆÜ�wŠw�t‹}˜qÕ Ð r   © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r#   Ú__static_attributes__Ú__classdictcell__)r   s   @r   r   r      s   ø‡ € ÷!ö !r   r   c                   óH   a a€ ] tR t^t oV3R lV 3R lltRR ltRtVtV ;t# )Ú
MolmoBlockc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfigr   )r   r   s   "€r   r   ÚMolmoBlock.__annotate__   s   ø€ ÷ ñ ™zñ r   c                ó.  <€ \         SV `  4        \        P                  ! VP                  VP                  R R7      V n        \        P                  ! \        VP                  VP                  ,          4      VP                  R R7      V n	        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R R7      V n        VP                  VP                  ,          p\        P                   ! W!P"                  R7      V n        VR,          V n        VP                  V n        VP(                  V n        VP                  VP(                  V,          VP(                  V,          3V n        \        P                  ! VP                  \-        V P*                  4      RR7      V n        \1        4       V n        R# )F©Úbias©Úeps)ÚbaseTNg      à¿)ÚsuperÚ__init__r   ÚLinearÚd_modelÚattn_outÚintÚact_output_multiplierÚmlp_hidden_sizeÚff_outÚRMSNormÚlayer_norm_epsÚ	attn_normÚff_normÚff_projÚn_headsÚRoPEÚ
rope_thetaÚ
rotary_embÚscaleÚ
n_kv_headsÚ
fused_dimsÚsumÚatt_projr   Úact)r!   r0   Úhead_dimÚ	__class__s   && €r   r9   ÚMolmoBlock.__init__   sX  ø€ Ü‰ÑÔÜŸ	š	 &§.¡.°&·.±.ÀuÔMˆŒÜ—i’iÜ�×,Ñ,¨v×/EÑ/EÕEÓFØ�N‰NØô
ˆŒô
 Ÿš F§N¡N¸×8MÑ8MÔNˆŒÜ—z’z &§.¡.°f×6KÑ6KÔLˆŒÜ—y’y §¡°×1GÑ1GÈeÔTˆŒØ—>‘> V§^¡^Õ3ˆÜŸ'š' (×1BÑ1BÔCˆŒØ˜t•^ˆŒ
Ø—~‘~ˆŒØ ×+Ñ+ˆŒà�N‰NØ×Ñ Õ(Ø×Ñ Õ(ð
ˆŒô
 Ÿ	š	 &§.¡.´#°d·o±oÓ2FÈTÔRˆŒÜ“8ˆŽr   c           	     ó¼  € VP                   w  rEpV P                  V4      pV P                  V4      p\        P                  ! W€P
                  ^ ,          V P
                  ^ ,          V P
                  ^,          ,           .RR7      w  ršpV	P                  WEV P                  W`P                  ,          4      P                  ^ ^^^4      p	V
P                  WEV P                  W`P                  ,          4      P                  ^ ^^^4      p
VP                  WEV P                  W`P                  ,          4      P                  ^ ^^^4      pVeM   V P                  W“P                  R7      p	V P                  W£P                  R7      p
VP                  W«4      w  r«M"V P                  V	4      p	V P                  V
4      p
\        WšW³V P                  VR7      pVP                  ^ ^^^4      P                  WEV4      pV P                  V4      pW,           pTpV P!                  V4      pV P#                  V4      pV P%                  V4      pV P'                  V4      pWÑ,           pV# )r   r   )Úoffset)rJ   Úmaskr   )ÚshaperC   rN   r   r   rL   ÚreshaperF   Ú	transposerK   rI   rT   Úupdate_and_fetchr   rJ   r<   rD   rE   rO   r@   )r!   r   rU   ÚcacheÚ
batch_sizeÚseq_lenÚDÚattn_inÚqkvÚqÚkÚvÚattÚog_xs   &&&&          r   r#   ÚMolmoBlock.__call__.   sü  € Ø!"§¡Ñˆ
˜QØ—.‘. Ó#ˆà�m‰m˜GÓ$ˆä—(’(Ø—/‘/ !Õ$ d§o¡o°aÕ&8¸4¿?¹?È1Õ;MÕ&MÐNÐUWô
‰ˆˆað �I‰I�j¨4¯<©<¸¿l¹lÕ9JÓK×UÑUØˆq�!�Qó
ˆð �I‰IØ §¡°!·|±|Õ2Có
ç
‰)�A�q˜!˜QÓ
ð 	
ð �I‰IØ §¡°!·|±|Õ2Có
ç
‰)�A�q˜!˜QÓ
ð 	
ð ÒØ—‘ ¯,©,�Ó7ˆAØ—‘ ¯,©,�Ó7ˆAØ×)Ñ)¨!Ó/‰DˆAˆqà—‘ Ó"ˆAØ—‘ Ó"ˆAô +¨1°ÀÇÁÐRVÔWˆØ�m‰m˜A˜q ! QÓ'×/Ñ/°
ÀQÓGˆØ�m‰m˜CÓ ˆð �Gˆð ˆØ�L‰L˜‹OˆØ�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�K‰K˜‹NˆØ�Hˆàˆr   )rO   rN   rC   r<   rD   r@   rE   rL   rF   rK   rI   rJ   )NN©	r&   r'   r(   r)   r9   r#   r*   r+   Ú__classcell__©rQ   r   s   @@r   r-   r-      s   ù‡ € ÷ó ÷0-ô -r   r-   c                   óT   a a€ ] tR t^^t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )Ú	Embeddingc          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[/# )r   Únum_embeddingsÚnum_new_embeddingsÚfeaturesÚinitializer_rangeÚnew_embed_initializer_range)r=   Úfloat)r   r   s   "€r   r   ÚEmbedding.__annotate___   s=   ø€ ÷ 
ñ 
áð
ñ  ð
ñ ð	
ñ
 !ð
ñ &+ñ
r   c                ó   <€ \         SV `  4        W@n        WPn        \        P
                  P                  W3V P                  R 7      V n        \        P
                  P                  W#3V P                  R 7      V n        R# ))rJ   N)	r8   r9   ro   rp   r   ÚrandomÚnormalÚ	embeddingÚnew_embedding)r!   rl   rm   rn   ro   rp   rQ   s   &&&&&&€r   r9   ÚEmbedding.__init___   ss   ø€ ô 	‰ÑÔØ!2ÔØ+FÔ(ô Ÿ™×)Ñ)ØÐ&¨d×.DÑ.Dð *ó 
ˆŒô  ŸY™Y×-Ñ-ØÐ*°$×2RÑ2Rð .ó 
ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   rr   s   s'   ø€ ÷ Jñ J™"Ÿ(™(ð J¡r§x¡xñ Jr   c                ól   € \         P                  ! V P                  V P                  .^ R7      V,          # )r   r   )r   Úconcatrv   rw   )r!   r   s   &&r   r#   ÚEmbedding.__call__s   s'   € Ü�yŠy˜$Ÿ.™.¨$×*<Ñ*<Ð=ÀAÔFÀqÕIÐIr   )rv   ro   rp   rw   )ç{®Gáz”?r}   rf   rh   s   @@r   rj   rj   ^   s!   ù‡ € ÷
õ 
÷(J÷ Jð Jr   rj   c                   óT   a a€ ] tR t^wt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMolmoc                ó    <€ V ^8„  d   QhRS[ /# r/   r   )r   r   s   "€r   r   ÚMolmo.__annotate__x   s   ø€ ÷ Sñ S™zñ Sr   c                óî  <€ \         SV `  4        Wn        \        VP                  VP
                  VP                  4      V n        \        VP                  4       Uu. uF  p\        V4      NK  	  upV n        \        P                  ! VP                  VP                  R 7      V n        VP                   '       g5   \        P"                  ! VP                  VP$                  RR7      V n        R# R# u upi )r5   Fr3   N)r8   r9   r0   rj   Úembedding_sizeÚadditional_vocab_sizer;   ÚwteÚrangeÚn_layersr-   Úblocksr   rA   rB   Úln_fÚweight_tyingr:   Ú
vocab_sizer@   )r!   r0   Ú_rQ   s   && €r   r9   ÚMolmo.__init__x   s¯   ø€ Ü‰ÑÔØŒäØ×!Ñ! 6×#?Ñ#?ÀÇÁó
ˆŒô 49¸¿¹Ô3IÓJÑ3I¨a”z &Ö)Ñ3IÑJˆŒä—J’J˜vŸ~™~°6×3HÑ3HÔIˆŒ	à×"×"Ð"ÜŸ)š) F§N¡N°F×4EÑ4EÈEÔRˆDŽKñ #ùò	 Ks   ÁC2c          
      ó¤   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          RS[/# )r   Ú	input_idsÚinputs_embedsrU   rZ   r   ©r   r   r   r   r   )r   r   s   "€r   r   r�   ‡   sX   ø€ ÷ !2ñ !2á—8‘8ð!2ñ  ¡§¡Õ)ð!2ñ ‘r—x‘xÕ ð	!2ñ
 ™Õ ð!2ñ 
ñ!2r   c                ó–  € Vf   V P                  V4      pMTpVf   R .V P                  P                  ,          pVf   \        WT4      pMYVP                  R,          p\
        P                  ! \
        P                  ! Wf3\
        P                  R7      4      pW7,          ^ 8„  p\        V P                  V4       F  w  r‰V! WSV	4      pK  	  V P                  V4      pV P                  P                  '       d6   \
        P                  ! WPP                   P                  P                  4      p
MV P!                  V4      p
\#        V
R7      # )N)Údtype)Úlogitséþÿÿÿ)r…   r0   r‡   r   rV   r   ÚtrilÚonesÚbool_Úziprˆ   r‰   rŠ   ÚmatmulÚweightÚTr@   r   )r!   r�   r�   rU   rZ   Úhr\   Úcausal_maskÚblockÚcr”   s   &&&&&      r   r#   ÚMolmo.__call__‡   s÷   € ð Ò Ø—‘˜Ó#‰AàˆAàŠ=Ø�F˜TŸ[™[×1Ñ1Õ1ˆEàŠ<Ü(¨Ó2‰Dà—g‘g˜b•kˆGÜŸ'š'¤"§'¢'¨7Ð*<ÄBÇHÁHÔ"MÓNˆKØÕ&¨!Ñ+ˆDä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �I‰I�a‹Lˆà�;‰;×#×#Ð#Ü—Y’Y˜q§(¡(§/¡/×"3Ñ"3Ó4‰Fà—[‘[ “^ˆFä"¨&Ô1Ð1r   )rˆ   r0   r@   r‰   r…   ©NNNrf   rh   s   @@r   r   r   w   s    ù‡ € ÷Só S÷!2÷ !2ò !2r   r   c                   ó”   a a€ ] tR t^«t oV3R lV 3R lltR
V3R lR llt]R 4       t]R 4       t	]R 4       t
]R 4       tR	tVtV ;t# )ÚLanguageModelc                ó    <€ V ^8„  d   QhRS[ /# r/   r   )r   r   s   "€r   r   ÚLanguageModel.__annotate__¬   s   ø€ ÷ #ñ #™zñ #r   c                óÆ   <€ \         SV `  4        Wn        VP                  V n        V P                  R 8w  d   \	        RV P                   R24      h\        V4      V n        R# )ÚmolmozModel type z3 not supported. Currently only 'molmo' is supportedN)r8   r9   r0   Ú
model_typeÚ
ValueErrorr   Úmodel)r!   r0   rQ   s   &&€r   r9   ÚLanguageModel.__init__¬   sX   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒØ�?‰?˜gÔ%ÜØ˜dŸo™oÐ.Ð.aÐbóð ô ˜6“]ˆŽ
r   c          
      ó¤   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          RS[/# )r   Úinputsr�   rU   rZ   r   r‘   )r   r   s   "€r   r   r¦   ¶   sX   ø€ ÷ 	ñ 	á—‘ð	ñ  ¡§¡Õ)ð	ñ ‘r—x‘xÕ ð		ñ
 ™Õ ð	ñ 
ñ	r   c                ó*   € V P                  WW44      pV# ©N)r«   )r!   r®   r�   rU   rZ   ÚkwargsÚoutputss   &&&&&, r   r#   ÚLanguageModel.__call__¶   s   € ð —*‘*˜V°DÓ@ˆØˆr   c                óh   € V P                  4        UUu/ uF  w  rR V9  g   K  WbK  	  upp# u uppi )zrotary_emb.inv_freq)Úitems)Úweightsra   rb   s   &  r   ÚsanitizeÚLanguageModel.sanitizeÁ   s1   € ð ")§¡¤ÔS¡™˜Ð4IÐQRÑ4R”�’¡ÒSÐSùÓSs   ”.¤.c                ó.   € V P                   P                  # r°   )r«   rˆ   ©r!   s   &r   ÚlayersÚLanguageModel.layersÆ   s   € à�z‰z× Ñ Ð r   c                ód   € V P                   P                  V P                   P                  ,          # r°   )r0   r;   rF   rº   s   &r   rP   ÚLanguageModel.head_dimÊ   s!   € à�{‰{×"Ñ" d§k¡k×&9Ñ&9Õ9Ð9r   c                ó.   € V P                   P                  # r°   )r0   rK   rº   s   &r   rK   ÚLanguageModel.n_kv_headsÎ   s   € à�{‰{×%Ñ%Ð%r   )r0   r«   r©   r¢   )r&   r'   r(   r)   r9   r#   Ústaticmethodr·   Úpropertyr»   rP   rK   r*   r+   rg   rh   s   @@r   r¤   r¤   «   sp   ù‡ € ÷#ó #÷	ò 	ð ñTó ðTð ñ!ó ð!ð ñ:ó ð:ð ñ&ó ÷&ð &r   r¤   )Útypingr   Úmlx.coreÚcorer   Úmlx.nnr   r7   r   r   r   rZ   r   r0   r	   ÚModuler   r-   rj   r   r¤   r%   r   r   Ú<module>rÈ      sv   ðÝ å Ý ÷ñ õ
 Ý ô!ˆR�Y‰Yô !ôF�—‘ô FôRJ�—	‘	ô Jô212ˆB�I‰Iô 12ôh%&�B—I‘Iö %&r   