+
    UV-j.)  ã                   óÄ   € ^ RI Ht ^ RIHt ^ RIHt ^RIHt ^RI	H
t
 ^RIHt ^RIHtHt ^RIHt R t ! R	 R
]P&                  4      t ! R R]P&                  4      tR# )é    )ÚOptionalN)ÚInputEmbeddingsFeatures)ÚAudioEncoder©ÚModelConfig)ÚLanguageModelÚRMSNormNoScale)ÚVisionModelc                 ó`  € VP                  4       P                  \        P                  4      p\        P                  ! V4      ^,
          pVP                  4       WBP
                  ,          ,          p\        P                  ! W5V P                  4       4      P                  V P                  4      # )é   )	ÚflattenÚastypeÚmxÚint32ÚcumsumÚsizeÚwhereÚreshapeÚshape)Úinput_tensorÚmaskÚsourceÚ	mask_flatÚindicesÚaligneds   &&&   Úm/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/gemma4/gemma4.pyÚmasked_scatterr      sw   € Ø—‘“×%Ñ%¤b§h¡hÓ/€IÜ�iŠi˜	Ó" QÕ&€GØ�n‰nÓ˜w¯©Õ4Õ5€GÜ�8Š8�I¨×(<Ñ(<Ó(>Ó?×GÑGØ×Ñóð ó    c                   óX   a a€ ] tR t^t oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚMultimodalEmbedderzAProjects soft tokens from vision/audio into language model space.c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )é   Úembedding_dimÚtext_hidden_sizeÚeps)ÚintÚfloat)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚMultimodalEmbedder.__annotate__   s'   ø€ ÷ Tñ T¡cð T¹Sð TÁuñ Tr   c                ó€   <€ \         SV `  4        \        P                  ! WR R7      V n        \        WR7      V n        R# )F)Úbias)r%   N)ÚsuperÚ__init__ÚnnÚLinearÚembedding_projectionr	   Úembedding_pre_projection_norm)Úselfr#   r$   r%   Ú	__class__s   &&&&€r   r/   ÚMultimodalEmbedder.__init__   s3   ø€ Ü‰ÑÔÜ$&§I¢IØ°%ô%
ˆÔ!ô .<¸MÔ-SˆÖ*r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r"   Úinputs_embedsÚreturn©r   Úarray)r(   r)   s   "€r   r*   r+       s#   ø€ ÷ 1ñ 1¡b§h¡hð 1±2·8±8ñ 1r   c                óF   € V P                  V4      pV P                  V4      # ©N©r3   r2   )r4   r8   Únormeds   && r   Ú__call__ÚMultimodalEmbedder.__call__    s#   € Ø×3Ñ3°MÓBˆØ×(Ñ(¨Ó0Ð0r   r>   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r/   r@   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r5   r)   s   @@r   r    r       s#   ù‡ € ÙK÷Tõ T÷1÷ 1ð 1r   r    c                   ó¨   a a€ ] tR t^%t oV3R lV 3R lltRV3R lR lltV3R lR ltRV3R lR lltR	 t]	R
 4       t
]	R 4       tR tRtVtV ;t# )ÚModelc                ó    <€ V ^8„  d   QhRS[ /# )r"   Úconfigr   )r(   r)   s   "€r   r*   ÚModel.__annotate__&   s   ø€ ÷ $ñ $™{ñ $r   c                óÜ  <€ \         SV `  4        VP                  V n        Wn        \	        VP
                  4      V n        VP
                  P                  V n        \        VP                  4      V n
        \        VP                  P                  VP
                  P                  VP                  P                  R 7      V n        VP                  eŒ   \!        VP                  4      V n        VP                  P$                  ;'       g    VP                  P                  p\        VVP
                  P                  VP                  P                  R 7      V n        R# RV n        RV n        R# ))r#   r$   r%   N)r.   r/   Ú
model_typerN   r   Útext_configÚlanguage_modelÚ
vocab_sizer
   Úvision_configÚvision_towerr    Úhidden_sizeÚrms_norm_epsÚembed_visionÚaudio_configr   Úaudio_towerÚoutput_proj_dimsÚembed_audio)r4   rN   Úaudio_output_dimr5   s   && €r   r/   ÚModel.__init__&   s"  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØŒô ,¨F×,>Ñ,>Ó?ˆÔØ ×,Ñ,×7Ñ7ˆŒô (¨×(<Ñ(<Ó=ˆÔÜ.Ø ×.Ñ.×:Ñ:Ø#×/Ñ/×;Ñ;Ø×$Ñ$×1Ñ1ô
ˆÔð ×ÑÒ*Ü+¨F×,?Ñ,?Ó@ˆDÔà×#Ñ#×4Ñ4×WÐW¸×8KÑ8K×8WÑ8Wð ô  2Ø.Ø!'×!3Ñ!3×!?Ñ!?Ø×'Ñ'×4Ñ4ô ˆDÖð  $ˆDÔØ#ˆDÖr   c                ó@  <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          RS[ S[P                  ,          /# )r"   Ú	input_idsÚpixel_valuesÚpixel_values_videosÚaudio_featuresÚ
audio_maskÚinput_featuresÚinput_features_mask)r   r   r;   )r(   r)   s   "€r   r*   rO   F   s™   ø€ ÷ ]
ñ ]
á™BŸH™HÕ%ð]
ñ ™rŸx™xÕ(ð]
ñ &¡b§h¡hÕ/ð	]
ñ
 !¡§¡Õ*ð]
ñ ™RŸX™XÕ&ð]
ñ !¡§¡Õ*ð]
ñ &¡b§h¡hÕ/ñ]
r   c                óÖ  a aaaa€ Ve   Vf   TpVe%   Sf!   VP                  \        P                  4      ( oS P                  P                  P                  S4      oSS P                  P                  P                  ,          o\        S P                  RR 4      p	R p
S P                  P                  P                  '       dŸ   SS P                  P                  8H  pSS P                  P                  8H  pW¼,          pV	e   VSV	8H  ,          pV( p\        P                  ! VS\        P                  ! S4      4      pS P                  P                  P                  V4      p
R
VVV3R llpV 3R lpV! VS P                  P                  VRR4      oV! VV	VRR4      oS P                  e4   S P                   e&   VV 3R lpV! VS P                  P                  V4      o\#        SV
R	7      # )NÚvideo_token_idc                 óZ  <€ V e   Vf   S# V'       d   SP                  RR 4      MR pV'       d   SP                  V4      MR pV'       d   SP                  VR 4      MR pVf   Ve   VP                  V4      pVe   VP                  SP                  4      pMPV! V 4      P                  SP                  4      pVe,   Ve(   \        P                  ! V4       VP                  Wh4       \        P                  ! \        P                  ! S
V8H  R4      SP                  4      p	\        SW˜4      # )NÚvision_cacheéÿÿÿÿ)
Úgetr   Údtyper   ÚevalÚputÚbroadcast_toÚexpand_dimsr   r   )r   Útoken_idÚencodeÚ	cached_kwÚkey_kwrk   Ú	cache_keyÚcachedÚfeaturesÚmask_expandedra   r8   Úkwargss   &&&&&     €€€r   Ú_scatterÚ,Model.get_input_embeddings.<locals>._scatteri   sø   ø€ ØŠ~ Ò!1Ø$Ð$ß?E˜6Ÿ:™: n°dÔ;È4ˆLß.4˜Ÿ
™
 6Ô*¸$ˆIß4=�V—Z‘Z 	¨4Ô0À4ˆFØŠ~ ,Ò":Ø%×)Ñ)¨)Ó4�ØÒ!Ø!Ÿ=™=¨×)<Ñ)<Ó=‘á! &›>×0Ñ0°×1DÑ1DÓE�ØÒ+°	Ò0EÜ—G’G˜HÔ%Ø ×$Ñ$ YÔ9äŸOšOÜ—’˜y¨HÑ4°bÓ9¸=×;NÑ;NóˆMô " -°ÓIÐIr   c                 óD   <€ SP                  SP                  V 4      4      # r=   )rY   rV   )Úpixelsr4   s   &€r   Ú_encode_visionÚ2Model.get_input_embeddings.<locals>._encode_vision~   s   ø€ Ø×$Ñ$ T×%6Ñ%6°vÓ%>Ó?Ð?r   Úcached_image_featuresÚ
_image_keyÚcached_video_featuresÚ
_video_keyc                 óÆ   <€ Se   SM6\         P                  ! V P                  R,          \         P                  R7      pSP	                  W4      w  r#SP                  V4      # )N:Nr"   N)rn   )r   Úzerosr   Úbool_r[   r]   )Úfeatr   ÚencÚ_re   r4   s   &   €€r   Ú_encode_audioÚ1Model.get_input_embeddings.<locals>._encode_audio’   sT   ø€ ð "Ò-ñ äŸš $§*¡*¨R¥.¼¿¹ÔAð ð
 ×)Ñ)¨$Ó5‘�Ø×'Ñ'¨Ó,Ð,r   )r8   Úper_layer_inputs)NN)r   r   rˆ   rS   ÚmodelÚembed_tokensÚembed_scaleÚgetattrrN   Úhidden_size_per_layer_inputÚimage_token_idÚaudio_token_idr   Ú
zeros_likeÚget_per_layer_inputsr[   r]   r   )r4   ra   rb   rc   rd   re   rf   rg   r{   ri   rŽ   Úimage_mask_idsÚaudio_mask_idsÚmm_maskÚ	text_maskÚper_layer_inputs_tokensr|   r€   rŒ   r8   s   ff&&&f&&l          @r   Úget_input_embeddingsÚModel.get_input_embeddingsF   sÑ  ü€ ð Ò%¨.Ò*@Ø+ˆNØÒ*¨zÒ/AØ-×4Ñ4´R·X±XÓ>Ð>ˆJØ×+Ñ+×1Ñ1×>Ñ>¸yÓIˆØ%¨×(;Ñ(;×(AÑ(A×(MÑ(MÕMˆä  §¡Ð.>ÀÓEˆàÐØ×Ñ×$Ñ$×@×@Ð@Ø&¨$¯+©+×*DÑ*DÑDˆNØ&¨$¯+©+×*DÑ*DÑDˆNØ$Õ5ˆGØÒ)Ø! Y°.Ñ%@ÕA�Ø ˜ˆIÜ&(§h¢hØ˜9¤b§m¢m°IÓ&>ó'Ð#ð  $×2Ñ2×8Ñ8×MÑMØ'ó Ð÷	Jñ 	Jõ*	@ñ !ØØ�K‰K×&Ñ&ØØ#Øó
ˆñ !ØØØØ#Øó
ˆð ×ÑÒ'¨D×,<Ñ,<Ò,Hö-ñ %ØØ—‘×*Ñ*ØóˆMô 'Ø'Ð:Jô
ð 	
r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r"   rb   r9   r:   )r(   r)   s   "€r   r*   rO   ¥   s#   ø€ ÷ ñ ©¯©ð ±b·h±hñ r   c                óJ   € V P                  V4      pV P                  V4      pV# )z¢Encode pixel_values through vision_tower + embed_vision.

Returns projected image features suitable for passing as
cached_image_features to get_input_embeddings.
)rV   rY   )r4   rb   Úimage_featuress   && r   Úencode_imageÚModel.encode_image¥   s*   € ð ×*Ñ*¨<Ó8ˆØ×*Ñ*¨>Ó:ˆØÐr   c          	      ó²   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[ P                  ,          /# )r"   ra   rb   r   Úcache)r   r;   r   )r(   r)   s   "€r   r*   rO   ¯   sR   ø€ ÷ "ñ "á—8‘8ð"ñ ™rŸx™xÕ(ð"ñ ‘r—x‘xÕ ð	"ñ
 ™Ÿ™Õ!ñ"r   c                óØ   € V P                   ! RR VRV/VB pR Uu/ uF  pWu9   g   K  WuV,          bK  	  ppV P                  ! RR RRVRVP                  RVP                  /VB p	V	# u upi )ra   rb   Nr¥   r8   rŽ   © )Úcapture_layer_idsÚhidden_sinkÚshared_kv_sinkÚreturn_hiddenÚreturn_shared_kv)r�   rS   r8   rŽ   )
r4   ra   rb   r   r¥   r{   Úinput_embeddings_featuresÚkÚ	lm_kwargsÚlogitss
   &&&&&,    r   r@   ÚModel.__call__¯   s¼   € ð %)×$=Ò$=ñ %
Øð%
à%ð%
ð ñ%
Ð!ñó

ñ�ð ‰{ô ˆA�a�yŠLñð 	ð 

ð ×$Ò$ñ 
Øð
àð
ð 4×AÑAð
ð 7×GÑGð	
ð
 ñ
ˆð ˆùò'

s
   ›A'¨A'c                óä  a
€ \        V P                  P                  R R4      p/ pVP                  4        EF6  w  o
p\        ;QJ d    V
3R lR 4       F  '       g   K   RM	  RM! V
3R lR 4       4      '       d"   RS
9   d   V'       g   KX  RS
9  d
   RS
9  d   Kh  RS
9   g   RS
9   d   Kx  V P
                  f   RS
9   g   R	S
9   d   K–  S
P                  R
4      '       d   S
\        R
4      R pMS
pVP                  R4      '       d/   VP                  R4      '       g   V\        R4      R pRV,           pRV9   d-   RV9   d&   VP                  ^8X  d   VP                  ^ ^^^4      pRV9   d%   VP                  ^8X  d   VP                  ^ ^^4      pVP                  R4      '       d   VP                  RR4      pVP                  R4      '       d’   VP                  RR4      pVP                  RR4      pVP                  RR4      pVP                  R,          ^,          p	VRRV	13,          P                  RR4      W7&   VRV	R13,          P                  RR4      W8&   EK2  WCV&   EK9  	  V# )Úuse_clipped_linearsFc              3   ó,   <"  € T F	  qS9   x € K  	  R # 5ir=   r§   )Ú.0Úsr®   s   & €r   Ú	<genexpr>Ú!Model.sanitize.<locals>.<genexpr>Ø   s   øé € ð Ù V˜1�Q–Ó Vùs   ƒTrV   r[   zrotary_emb.inv_freqÚ
rotary_embNr]   zmodel.zlanguage_model.zlanguage_model.model.Úsubsample_conv_projectionzconv.weightzdepthwise_conv1d.weightz.experts.down_projz$.experts.switch_glu.down_proj.weightz.experts.gate_up_projz$.experts.switch_glu.gate_proj.weightz".experts.switch_glu.up_proj.weight.)Ú	input_maxÚ	input_minÚ
output_maxÚ
output_minrl   éþÿÿÿ)r’   rN   rU   ÚitemsÚanyr[   Ú
startswithÚlenÚndimÚ	transposeÚendswithÚreplaceÚswapaxesr   )r4   ÚweightsÚuse_clippedÚ	sanitizedÚvÚnew_keyÚrestÚgate_keyÚup_keyÚmid_dimr®   s   &&        @r   ÚsanitizeÚModel.sanitizeÓ   sV  ø€ Ü˜dŸk™k×7Ñ7Ð9NÐPUÓVˆØˆ	Ø—M‘M—O‰DˆAˆqç‹sô Ù Vó�s�sŠsô Ù Vó÷ ò ð " QÔ&¯{ÙØ!¨Ô*¨}ÀAÔ/EÙØ$¨Ô)¨\¸QÔ->ÙØ×ÑÒ'¨]¸aÔ-?À=ÐTUÔCUÙà�|‰|˜H×%Ò%ØœC ›M˜OÐ,‘à�à×!Ñ!Ð"3×4Ò4¸W×=OÑ=OØ'÷>ò >ð œsÐ#4Ó5Ð7Ð8�Ø1°DÕ8�ð ,¨wÔ6Ø! WÔ,Ø—F‘F˜a”Kà—K‘K  1 a¨Ó+�à(¨GÔ3¸¿¹À!¼Ø—K‘K  1 aÓ(�ð ×ÑÐ 4×5Ò5Ø!Ÿ/™/Ø(Ð*Pó�ð ×ÑÐ 7×8Ò8Ø"Ÿ?™?Ø+Ø:ó�ð !Ÿ™Ø+Ø8ó�ð
 —J‘J˜r 2Ó&�ØŸ'™' "�+¨Õ*�Ø&'¨¨X¨g¨X¨Õ&6×&?Ñ&?ÀÀBÓ&G�	Ñ#Ø$% c¨7©8 mÕ$4×$=Ñ$=¸bÀ"Ó$E�	Ñ!Úà!"�gÔñu $ðv Ðr   c                ó.   € V P                   P                  # r=   )rS   Úquant_predicate©r4   s   &r   rÕ   ÚModel.quant_predicate  s   € à×"Ñ"×2Ñ2Ð2r   c                óB   € V P                   P                  P                  # r=   )rS   r�   ÚlayersrÖ   s   &r   rÙ   ÚModel.layers  s   € à×"Ñ"×(Ñ(×/Ñ/Ð/r   c                ó6   € V P                   P                  4       # r=   )rS   Ú
make_cacherÖ   s   &r   rÜ   ÚModel.make_cache  s   € Ø×"Ñ"×-Ñ-Ó/Ð/r   )r[   rN   r]   rY   rS   rQ   rV   rT   )NNNNNNN)NNN)rB   rC   rD   rE   r/   r�   r¢   r@   rÒ   ÚpropertyrÕ   rÙ   rÜ   rG   rH   rI   rJ   s   @@r   rL   rL   %   sj   ù‡ € ÷$ó $÷@]
ò ]
÷~ð ÷"ò "òH>ð@ ñ3ó ð3ð ñ0ó ð0÷0ò 0r   rL   )Útypingr   Úmlx.coreÚcorer   Úmlx.nnr0   Úbaser   Úaudior   rN   r   Úlanguager   r	   Úvisionr
   r   ÚModuler    rL   r§   r   r   Ú<module>rè      sD   ðÝ å Ý å *Ý Ý ß 3Ý òô1˜Ÿ™ô 1ôw0ˆB�I‰Iö w0r   