+
    UV-jlI  ã                   ó¼  € R t ^ RIt^ RIHt ^ RIHt ^RIHt ^RI	H
t
  ! R R]P                  4      t ! R R]P                  4      t ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR# )aV  Gemma 4 Conformer audio encoder, ported from PyTorch to MLX.

Architecture:
- SSCP (SubSample Conv Projection): 2x Conv2d blocks + Linear
- 12 Conformer blocks: FFW -> Attention -> LightConv1d -> FFW -> Clamp -> RMSNorm
- Chunked local attention with relative position embeddings and logit softcapping
- Output projection to text_hidden_size
N©ÚAudioConfig)ÚClippableLinearc                   óX   a a€ ] tR t^t oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚAudioRMSNormz1RMSNorm with weight applied directly (no offset).c                ó&   <€ V ^8„  d   QhRS[ RS[/# )é   ÚdimÚeps)ÚintÚfloat)ÚformatÚ__classdict__s   "€Úl/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/gemma4/audio.pyÚ__annotate__ÚAudioRMSNorm.__annotate__   s   ø€ ÷ ñ ™Cð ¡eñ ó    c                óh   <€ \         SV `  4        \        P                  ! V34      V n        W n        R # ©N)ÚsuperÚ__init__ÚmxÚonesÚweightr
   )Úselfr	   r
   Ú	__class__s   &&&€r   r   ÚAudioRMSNorm.__init__   s$   ø€ Ü‰ÑÔÜ—g’g˜s˜f“oˆŒØŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   ÚxÚreturn©r   Úarray)r   r   s   "€r   r   r      s#   ø€ ÷ :ñ :™"Ÿ(™(ð :¡r§x¡xñ :r   c                ój   € \         P                  P                  WP                  V P                  4      # r   )r   ÚfastÚrms_normr   r
   )r   r   s   &&r   Ú__call__ÚAudioRMSNorm.__call__   s!   € Ü�w‰w×Ñ §;¡;°·±Ó9Ð9r   )r
   r   )g�íµ ÷Æ°>©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r&   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   r   s   @@r   r   r      s!   ù‡ € Ù;÷õ ÷
:÷ :ð :r   r   c                   óT   a a€ ] tR t^t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚSSCPConvBlockz=Conv2d -> LayerNorm(channels) -> ReLU with symmetric padding.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   ÚconfigÚidx)r   r   )r   r   s   "€r   r   ÚSSCPConvBlock.__annotate__"   s   ø€ ÷ Tñ T™{ð T±ñ Tr   c           	     ó<  <€ \         SV `  4        V^ 8X  d   ^MVP                  V^,
          ,          pVP                  V,          p^V n        RV n        \
        P                  ! VVRR^ RR7      V n        \
        P                  ! WAP                  RR7      V n
        R# )é    F©Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingÚbias)r
   r@   N)é   rA   rA   rA   )é   rB   )r   r   )r   r   Úsubsampling_conv_channelsÚtime_strider?   ÚnnÚConv2dÚconvÚ	LayerNormÚrms_norm_epsÚnorm)r   r5   r6   r;   r<   r   s   &&&  €r   r   ÚSSCPConvBlock.__init__"   sˆ   ø€ Ü‰ÑÔØ !œ8‘a¨×)IÑ)IÈ#ÐPQÍ'Õ)RˆØ×7Ñ7¸Õ<ˆØˆÔØ#ˆŒô —I’IØ#Ø%ØØØØô
ˆŒ	ô —L’L ×3FÑ3FÈUÔSˆŽ	r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r   Úmaskr!   )r   r   s   "€r   r   r7   6   s#   ø€ ÷ ñ ™"Ÿ(™(ð ©"¯(©(ñ r   c                óz  € \         P                  ! VR,          RV4      pV P                  w  r4rV\         P                  ! VRW43WV3R.4      pV P	                  V4      pVP
                  ^,          pVR RRV P                  13,          R RV13,          pV P                  V4      p\        P                  ! V4      pW3# )ºNNNNç        )rO   rO   NN©r9   r9   )
r   Úwherer?   ÚpadrG   ÚshaperD   rJ   rE   Úrelu)	r   r   rM   Ú	pad_t_topÚpad_t_bottomÚ
pad_f_leftÚpad_f_rightÚt_outÚoutput_masks	   &&&      r   r&   ÚSSCPConvBlock.__call__6   s¿   € ô
 �HŠH�TÐ*Õ+¨S°!Ó4ˆð <@¿<¹<Ñ8ˆ	 Ü�FŠFØàØÐ)ØÐ)Øð	ó
ˆð �I‰I�a‹Lˆð —‘˜•
ˆØ˜1Ñ1 ×!1Ñ!1Ð1Ð1Õ2°1°f°u°f°9Õ=ˆð �I‰I�a‹LˆÜ�GŠG�A‹Jˆàˆ~Ðr   )rG   rJ   r?   rD   r(   r1   s   @@r   r3   r3      s#   ù‡ € ÙG÷Tó T÷(÷ ð r   r3   c                   óX   a a€ ] tR t^Vt oRt^€tV3R lV 3R lltV3R lR ltRtVt	V ;t
# )ÚSubSampleConvProjectionzKSSCP: 2 Conv2d blocks -> flatten(F, C) -> Linear projection to hidden_size.c                ó    <€ V ^8„  d   QhRS[ /# ©r   r5   r   )r   r   s   "€r   r   Ú$SubSampleConvProjection.__annotate__[   s   ø€ ÷ 
ñ 
™{ñ 
r   c                ód  <€ \         SV `  4        \        V^ R7      V n        \        V^R7      V n        V P
                  p\        ^4       F!  pV^,           ^,
          ^,          ^,           pK#  	  W!P                  R,          ,          p\        P                  ! WAP                  RR7      V n        R# )r9   )r6   F©r@   Néÿÿÿÿ)r   r   r3   Úlayer0Úlayer1ÚINPUT_FEAT_SIZEÚrangerC   rE   ÚLinearÚhidden_sizeÚinput_proj_linear)r   r5   ÚfreqÚ_Úproj_input_dimr   s   &&   €r   r   Ú SubSampleConvProjection.__init__[   s‰   ø€ Ü‰ÑÔä# F°Ô2ˆŒÜ# F°Ô2ˆŒà×#Ñ#ˆÜ�q–ˆAØ˜1•H˜q•L QÕ&¨Õ*ŠDñ à× @Ñ @ÀÕ DÕDˆÜ!#§¢Ø×.Ñ.°Uô"
ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Ú	audio_melrM   r!   )r   r   s   "€r   r   ra   i   s#   ø€ ÷ ñ ¡"§(¡(ð ±"·(±(ñ r   c                óò   € \         P                  ! VR4      pV P                  W24      w  r2V P                  W24      w  r2VP                  w  rErgVP                  WEWg,          4      pV P                  V4      pW23# )rA   rd   )r   Úexpand_dimsre   rf   rT   Úreshaperk   )r   rq   rM   r   ÚBÚTÚFÚCs   &&&     r   r&   Ú SubSampleConvProjection.__call__i   sn   € ô �NŠN˜9 bÓ)ˆà—+‘+˜aÓ&‰ˆØ—+‘+˜aÓ&‰ˆð —W‘W‰
ˆˆaØ�I‰I�a˜A�EÓ"ˆð ×"Ñ" 1Ó%ˆàˆwˆr   )rk   re   rf   )r)   r*   r+   r,   r-   rg   r   r&   r.   r/   r0   r1   s   @@r   r^   r^   V   s&   ù‡ € ÙUà€O÷
ó 
÷÷ ð r   r^   c                   óT   a a€ ] tR t^{t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚConformerFeedForwardz(Macaron-style FFW with residual scaling.c                ó    <€ V ^8„  d   QhRS[ /# r`   r   )r   r   s   "€r   r   Ú!ConformerFeedForward.__annotate__~   s   ø€ ÷ @ñ @™{ñ @r   c                óˆ  <€ \         SV `  4        VP                  V n        VP                  V n        \	        VP
                  4      V n        \        VP
                  VP
                  ^,          RR7      V n        \        VP
                  ^,          VP
                  RR7      V n	        \	        VP
                  4      V n
        R# )é   Frc   N)r   r   Úgradient_clippingÚresidual_weightr   rj   Úpre_layer_normr   Úffw_layer_1Úffw_layer_2Úpost_layer_norm©r   r5   r   s   &&€r   r   ÚConformerFeedForward.__init__~   sž   ø€ Ü‰ÑÔØ!'×!9Ñ!9ˆÔØ%×5Ñ5ˆÔä*¨6×+=Ñ+=Ó>ˆÔÜ*Ø×Ñ × 2Ñ 2°QÕ 6¸Uô
ˆÔô +Ø×Ñ Õ" F×$6Ñ$6¸Uô
ˆÔô  ,¨F×,>Ñ,>Ó?ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r!   )r   r   s   "€r   r   r}   Œ   s#   ø€ ÷ 	3ñ 	3™"Ÿ(™(ð 	3¡r§x¡xñ 	3r   c                óœ  € Tp\         P                  ! WP                  ) V P                  4      pV P                  V4      pV P	                  V4      p\
        P                  ! V4      pV P                  V4      p\         P                  ! WP                  ) V P                  4      pV P                  V4      pW!V P                  ,          ,           # r   )
r   Úclipr€   r‚   rƒ   rE   Úsilur„   r…   r�   )r   r   Úresiduals   && r   r&   ÚConformerFeedForward.__call__Œ   s¦   € ØˆÜ�GŠG�A×.Ñ.Ð.°×0FÑ0FÓGˆØ×Ñ Ó"ˆØ×Ñ˜QÓˆÜ�GŠG�A‹JˆØ×Ñ˜QÓˆÜ�GŠG�A×.Ñ.Ð.°×0FÑ0FÓGˆØ× Ñ  Ó#ˆØ˜d×2Ñ2Õ2Õ2Ð2r   )rƒ   r„   r€   r…   r‚   r�   r(   r1   s   @@r   r{   r{   {   s#   ù‡ € Ù2÷@ó @÷	3÷ 	3ð 	3r   r{   c                   ól   a a€ ] tR t^˜t oRtV3R lV 3R lltV3R lR ltR tV3R lR ltR	t	Vt
V ;t# )
ÚAudioRelativePositionEmbeddingz=Sinusoidal relative position embedding for chunked attention.c                ó    <€ V ^8„  d   QhRS[ /# r`   r   )r   r   s   "€r   r   Ú+AudioRelativePositionEmbedding.__annotate__›   s   ø€ ÷ Lñ L™{ñ Lr   c                óÐ  <€ \         SV `  4        VP                  V n        VP                  V n        V P
                  V P                  ,          V n        \        ^ VP                  ^,
          4      V n	        VP                  V n        \        P                  ! V P
                  V P                  V P                  ,          RR7      V n        RpRpV P
                  ^,          p\        P                   ! W2,          4      \        V^,
          ^4      ,          pV\"        P$                  ! \"        P&                  ! V4      V) ,          4      ,          pVP)                  ^^V4      V n        R# )r9   Frc   ç      ð?ç     ˆÃ@N)r   r   Únum_attention_headsÚ	num_headsrj   ÚchannelsÚhead_dimÚmaxÚattention_context_leftÚmax_backwardÚattention_context_rightÚmax_forwardrE   ri   Úpos_projÚmathÚlogr   ÚexpÚarangert   Ú_inv_timescales©r   r5   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementÚinv_timescalesr   s   &&     €r   r   Ú'AudioRelativePositionEmbedding.__init__›   s  ø€ Ü‰ÑÔØ×3Ñ3ˆŒØ×*Ñ*ˆŒØŸ™¨¯©Õ7ˆŒÜ  6×#@Ñ#@À1Õ#DÓEˆÔØ!×9Ñ9ˆÔäŸ	š	Ø�M‰M˜4Ÿ>™>¨D¯M©MÕ9Àô
ˆŒð ˆØˆØŸ™¨!Õ+ˆÜ"&§(¢(¨=Õ+HÓ"IÌCØ˜QÕ óM
õ #
Ðð '¬¯ªÜ�IŠI�nÓ%Ð)@Ð(@Õ@ó*
õ 
ˆð  .×5Ñ5°a¸¸NÓKˆÖr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   ÚpositionÚdtyper    )r   r"   ÚDtype)r   r   s   "€r   r   r‘   ²   s.   ø€ ÷ $ñ $©2¯8©8ð $¹B¿H¹Hð $ÉÏÉñ $r   c                ó  € VP                  \        P                  4      R,          pWP                  ,          p\        P                  ! \        P
                  ! V4      \        P                  ! V4      .RR7      pVP                  V4      # ).©Úaxis).Nrd   )Úastyper   Úfloat32r£   ÚconcatenateÚsinÚcos)r   r¬   r­   Úscaled_timeÚsignals   &&&  r   Ú_get_timing_signalÚ1AudioRelativePositionEmbedding._get_timing_signal²   s^   € Ø—?‘?¤2§:¡:Ó.¨yÕ9ˆØ×!5Ñ!5Õ5ˆÜ—’¤§¢¨Ó!4´b·f²f¸[Ó6IÐ JÐQSÔTˆØ�}‰}˜UÓ#Ð#r   c           	     óò   € V^,           V,
          p\         P                  ! VRRRR^ V3.4      pVP                  W#WEV^,           ,          4      pVRRRRWV,          13,          pVP                  W#WEV4      pV# )rA   rO   NrQ   )r   rS   rt   )	r   Úterm_bdÚ
batch_sizer–   Ú
num_blocksÚ
block_sizeÚcontext_sizeÚmax_span_plus_1Ú
pad_amounts	   &&&&&&&& r   Ú_relative_shiftÚ.AudioRelativePositionEmbedding._relative_shift¸   s‰   € ð # QÕ&¨/Õ9ˆ
Ü—&’&˜ 6¨6°6¸6ÀAÀzÀ?Ð"SÓTˆØ—/‘/Ø :¸\ÈAÕ=MÕ/Nó
ˆð ˜!˜Q Ð#> ZÕ%>Ð#>Ð>Õ?ˆØ—/‘/Ø :¸<ó
ˆð ˆr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   ÚqueriesÚkeysr    r!   )r   r   s   "€r   r   r‘   Í   s.   ø€ ÷ !ñ !¡§¡ð !±·±ð !¹R¿X¹Xñ !r   c           	     óB  € VP                   w  r4rVpVP                   ^,          p\        P                  ! V P                  V P                  ) ^,
          R4      R,          p	V	P                   ^,          p
V P                  W‘P                  4      pV P                  VP                  V P                  P                  P                  4      4      pVP                  W P                  V P                  4      pVP                  VP                  4      pVP                  ^ ^^^^4      pVP                  ^ ^^^^4      pWÍ,          pVP                  ^^^ 4      pVP                  W6WE,          V4      pVV,          P                  W6WEV
4      pV P                  VW6WEWŠ4      pVV,           # )r   Nrd   )rT   r   r¢   r›   r�   r¹   r­   rž   r²   r   rt   r–   r˜   Ú	transposerÃ   )r   rÆ   rÇ   ru   ÚUÚWÚNÚHrx   Úpos_indicesrÁ   Úsin_embÚ	queries_pÚkeys_pÚterm_acÚ	sin_emb_tÚ
q_reshapedr¼   s   &&&               r   r&   Ú'AudioRelativePositionEmbedding.__call__Í   sX  € ØŸ™‰ˆˆa�AØ�J‰J�q�Mˆä—i’i × 1Ñ 1°D×4DÑ4DÐ3DÀqÕ3HÈ"ÓMÈdÕSˆØ%×+Ñ+¨AÕ.ˆà×)Ñ)¨+·}±}ÓEˆØ—-‘- §¡¨t¯}©}×/CÑ/C×/IÑ/IÓ JÓKˆØ—/‘/ /·>±>À4Ç=Á=ÓQˆØ—.‘. §¡Ó/ˆà×%Ñ% a¨¨A¨q°!Ó4ˆ	Ø—‘  1 a¨¨AÓ.ˆØÕ$ˆà×%Ñ% a¨¨AÓ.ˆ	Ø×&Ñ& q¨Q­U°AÓ6ˆ
Ø 	Õ)×2Ñ2°1¸¸ÓOˆà×&Ñ& w°°a¸AÓOˆà˜Õ Ð r   )r£   r—   r˜   r›   r�   r–   rž   )r)   r*   r+   r,   r-   r   r¹   rÃ   r&   r.   r/   r0   r1   s   @@r   r�   r�   ˜   s2   ù‡ € ÙG÷Ló L÷.$ð $ò÷*!÷ !ð !r   r�   c                   óŠ   a a€ ] tR t^æt oRtV3R lV 3R lltV3R lR ltV3R lR ltV3R lR	 ltV3R
 lR lt	Rt
VtV ;t# )ÚAudioAttentionz³Chunked local attention with relative position embeddings and logit softcapping.

Attribute names match checkpoint keys: self_attn.{q_proj,k_proj,v_proj,post,relative_k_proj,...}
c                ó    <€ V ^8„  d   QhRS[ /# r`   r   )r   r   s   "€r   r   ÚAudioAttention.__annotate__ì   s   ø€ ÷ 66ñ 66™{ñ 66r   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        V P                  V P                  ,          V n        VP                  V n        VP                  V n	        \        ^ VP                  ^,
          4      V n        V P                  V P                  ,           V P                  ,           V n        VP                  V n        VP                   V n        \$        P&                  ! V P                  V P                  V P
                  ,          RR7      V n        \*        P,                  ! V P
                  34      V n        \1        V P                  V P                  V P
                  ,          RR7      V n        \1        V P                  V P                  V P
                  ,          RR7      V n        \1        V P                  V P                  V P
                  ,          RR7      V n        \1        V P                  V P                  RR7      V n        V P
                  R,          \:        P<                  ! ^4      ,          V n        \:        P<                  ! ^\:        P@                  ,           4      \:        P<                  ! ^4      ,          V n!        \D        PG                  \D        4      V n$        V P                  V PH                  n        V P                  V PH                  n%        V P
                  V PH                  n        V P                  V PH                  n&        V P                  V PH                  n'        RpRpV P                  ^,          p\:        P<                  ! W2,          4      \        V^,
          ^4      ,          pV\*        PP                  ! \*        PR                  ! V4      V) ,          4      ,          pVPU                  ^^V4      V PH                  n+        V P(                  V PH                  n,        R# )r9   Frc   r“   r”   Ng      à¿)-r   r   r•   r–   rj   r˜   Úattention_chunk_sizeÚ
chunk_sizerœ   Úmax_future_horizonr™   rš   Úmax_past_horizonrÀ   Úattention_invalid_logits_valueÚinvalid_logits_valueÚattention_logit_capÚsoftcaprE   ri   Úrelative_k_projr   ÚzerosÚper_dim_scaler   Úq_projÚk_projÚv_projÚpostrŸ   r    Úq_scaleÚeÚk_scaler�   Ú__new__Ú_rel_posr—   r›   r�   r¡   r¢   rt   r£   rž   r¤   s   &&     €r   r   ÚAudioAttention.__init__ì   sÞ  ø€ Ü‰ÑÔØ×3Ñ3ˆŒØ!×-Ñ-ˆÔØ×(Ñ(¨D¯N©NÕ:ˆŒà ×5Ñ5ˆŒØ"(×"@Ñ"@ˆÔÜ # A v×'DÑ'DÀqÕ'HÓ IˆÔà�O‰O˜d×3Ñ3Õ3°d×6MÑ6MÕMð 	Ôð %+×$IÑ$IˆÔ!Ø×1Ñ1ˆŒä!ŸyšyØ×Ñ˜dŸn™n¨t¯}©}Õ<À5ô 
ˆÔô  ŸXšX t§}¡}Ð&6Ó7ˆÔä%Ø×Ñ˜dŸn™n¨t¯}©}Õ<À5ô
ˆŒô &Ø×Ñ˜dŸn™n¨t¯}©}Õ<À5ô
ˆŒô &Ø×Ñ˜dŸn™n¨t¯}©}Õ<À5ô
ˆŒô $ D×$4Ñ$4°d×6FÑ6FÈUÔSˆŒ	àŸ™ tÕ+¬t¯xªx¸«{Õ:ˆŒÜ—x’x ¤D§F¡F¥
Ó+¬d¯hªh°q«kÕ9ˆŒô 7×>Ñ>Ü*ó
ˆŒð #'§.¡.ˆ�‰ÔØ!%×!1Ñ!1ˆ�‰ÔØ!%§¡ˆ�‰ÔØ%)×%:Ñ%:ˆ�‰Ô"Ø$(×$;Ñ$;ˆ�‰Ô!ØˆØˆØ×)Ñ)¨QÕ.ˆÜ"&§(¢(¨=Õ+HÓ"IÌCØ˜QÕ óM
õ #
Ðð '¬¯ªÜ�IŠI�nÓ%Ð)@Ð(@Õ@ó*
õ 
ˆð )7×(>Ñ(>¸qÀ!À^Ó(Tˆ�‰Ô%à!%×!5Ñ!5ˆ�‰Ör   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r   r   Úpad_leftÚ	pad_rightr    )r   r"   r   )r   r   s   "€r   r   rÙ   $  s1   ø€ ÷ ñ ™2Ÿ8™8ð ©sð ¹sð ÁrÇxÁxñ r   c                ób   € R.VP                   ,          pW#3V^&   \        P                  ! W4      # )r9   rQ   )Úndimr   rS   )r   r   rñ   rò   Úpadss   &&&& r   Ú	_pad_dim1ÚAudioAttention._pad_dim1$  s,   € Øˆx˜!Ÿ&™&Õ ˆØÐ'ˆˆQ‰Ü�vŠv�a‹Ðr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r!   )r   r   s   "€r   r   rÙ   )  s'   ø€ ÷ @ñ @¡2§8¡8ð @±·±ñ @r   c                ój  € VP                   ^ ,          VP                   ^,          r2VP                   R,          pW0P                  ,           ^,
          V P                  ,          pWPP                  ,          V,
          pV^ 8”  d   V P                  V^ V4      pVP                  ! W%V P                  .VO5!  # )z/[B, T, ...] -> [B, num_blocks, chunk_size, ...]:r   NN)rT   rÜ   rö   rt   )r   r   ru   rv   Úrestr¾   Úpad_lens   &&     r   Ú_convert_to_blockÚ AudioAttention._convert_to_block)  s„   € à�w‰w�q�z˜1Ÿ7™7 1�:ˆ1Ø�w‰w�r�{ˆØŸ/™/Õ)¨AÕ-°$·/±/ÕAˆ
ØŸ™Õ.°Õ2ˆØ�QŒ;Ø—‘˜q ! WÓ-ˆAØ�yŠy˜¨¯©Ð?¸$Ó?Ð?r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r!   )r   r   s   "€r   r   rÙ   3  s#   ø€ ÷ 
ñ 
©¯©ð 
±R·X±Xñ 
r   c                óÒ  € V P                   pV P                  V P                  ,           ^,
          pV P                  WV4      pVP                  ^,          pW@P
                  ,
          V P                  ,          ^,           p\        P                  ! V4      V P                  ,          p\        P                  ! V P
                  4      pVR,          VR,          ,           pVRV3,          # )z1[B, T, ...] -> [B, num_blocks, context_size, ...]rO   )rO   N)NrO   )rÞ   rÝ   rÜ   rö   rT   rÀ   r   r¢   )	r   r   rñ   rò   ÚT_paddedr¾   ÚstartsÚoffsetsÚindicess	   &&       r   Ú_extract_block_contextÚ%AudioAttention._extract_block_context3  s«   € à×(Ñ(ˆØ×+Ñ+¨d¯o©oÕ=ÀÕAˆ	Ø�N‰N˜1¨	Ó2ˆØ—7‘7˜1•:ˆØ×!2Ñ!2Õ2°t·±ÕFÈÕJˆ
Ü—’˜:Ó&¨¯©Õ8ˆÜ—)’)˜D×-Ñ-Ó.ˆØ˜•/ G¨GÕ$4Õ4ˆØ��G��}Ðr   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r   Úhidden_statesrM   Úcausal_valid_maskr    r!   )r   r   s   "€r   r   rÙ   ?  s=   ø€ ÷ &"ñ &"ÙŸX™Xð&"Ù-/¯X©Xð&"ÙJLÏ(É(ð&"á	�‰ñ&"r   c                ó|  € VP                   R ,          w  rEWEV P                  V P                  3pV P                  V4      P	                  \
        P                  4      P                  V4      pV P                  V4      P	                  \
        P                  4      P                  V4      pV P                  V4      P	                  \
        P                  4      P                  V4      p	\        P                  ! V P                  4      p
WpP                  V
,          ,          pW€P                  ,          pV P                  V4      pV P!                  V4      pV P!                  V	4      pVP                   ^,          pV( pV P!                  V4      pVR,          VR,          ,          pV P#                  W¼4      p\
        P$                  ! VV P&                  ,          4      V P&                  ,          p\
        P(                  ! VVV P*                  4      p\
        P,                  ! VRR7      p\
        P.                  ! RVV4      pVP                  WNV P0                  ,          V P                  V P                  4      pVRRV13,          pVP                   w  ppppVP                  VVVV,          4      pV P3                  V4      # ):Nr   NrO   Nr°   zbnuwc,bucnh->buwnh)rO   NrO   NrO   )NNNrO   rO   rd   )rT   r–   r˜   ræ   r²   r   r³   rt   rç   rè   rE   Úsoftplusrå   rê   rì   rü   r  rî   Útanhrâ   rR   rà   ÚsoftmaxÚeinsumrÜ   ré   )r   r  rM   r  ru   rv   Ú	qkv_shapeÚqÚkÚvrå   Úquery_blocksÚ
key_blocksÚvalue_blocksrÊ   Ú
valid_maskÚextracted_validÚ	conditionÚlogitsÚprobsÚcontextÚB_outÚT_outrÌ   rÍ   s   &&&&                     r   r&   ÚAudioAttention.__call__?  s  € ð ×"Ñ" 2Õ&‰ˆØ˜4Ÿ>™>¨4¯=©=Ð9ˆ	à�K‰K˜Ó&×-Ñ-¬b¯j©jÓ9×AÑAÀ)ÓLˆØ�K‰K˜Ó&×-Ñ-¬b¯j©jÓ9×AÑAÀ)ÓLˆØ�K‰K˜Ó&×-Ñ-¬b¯j©jÓ9×AÑAÀ)ÓLˆäŸš D×$6Ñ$6Ó7ˆØ—‘ Õ-Õ.ˆØ—‘Õˆà×-Ñ-¨aÓ0ˆØ×0Ñ0°Ó3ˆ
Ø×2Ñ2°1Ó5ˆØ×Ñ˜qÕ!ˆà�Uˆ
Ø×5Ñ5°jÓAˆàÐ/Õ0ØÐ 6Õ7õ8ð 	ð
 —‘˜|Ó8ˆÜ—’˜ $§,¡,Õ.Ó/°$·,±,Õ>ˆÜ—’˜) V¨T×-FÑ-FÓGˆä—
’
˜6¨Ô+ˆÜ—)’)Ð0°%¸ÓFˆØ—/‘/ !¨¯©Õ%8¸$¿.¹.È$Ï-É-ÓXˆØ˜!˜R˜a˜R˜%•.ˆð %Ÿ]™]Ñˆˆu�a˜Ø—/‘/ %¨°°AµÓ6ˆØ�y‰y˜Ó!Ð!r   )rî   rÜ   rÀ   r˜   rj   rà   rç   rì   rÝ   rÞ   r–   rå   ré   ræ   rê   rã   râ   rè   )r)   r*   r+   r,   r-   r   rö   rü   r  r&   r.   r/   r0   r1   s   @@r   r×   r×   æ   sD   ù‡ € ñ÷
66ó 66÷pð ÷
@ð @÷
ð 
÷&"÷ &"ð &"r   r×   c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚConformerLightConv1dih  znLight convolution: norm -> linear(2x) -> GLU -> depthwise_conv1d(causal) -> norm -> SiLU -> linear + residual.c                ó    <€ V ^8„  d   QhRS[ /# r`   r   )r   r   s   "€r   r   Ú!ConformerLightConv1d.__annotate__k  s   ø€ ÷ 
ñ 
™{ñ 
r   c           	     ó®  <€ \         SV `  4        VP                  V n        VP                  ^,
          V n        \        VP                  VP                  R7      V n        \        VP                  VP                  ^,          RR7      V n
        \        P                  ! VP                  VP                  VP                  ^^ RR7      V n        \        P                  ! VP                  VP                  ^34      V P                  n        \        VP                  VP                  R7      V n        \        VP                  VP                  RR7      V n        R# )rA   )r
   Frc   r:   N)r   r   r€   Úconv_kernel_sizeÚcausal_paddingr   rj   rI   r‚   r   Úlinear_startrE   ÚConv1dÚdepthwise_conv1dr   rä   r   Ú	conv_normÚ
linear_endr†   s   &&€r   r   ÚConformerLightConv1d.__init__k  s  ø€ Ü‰ÑÔØ!'×!9Ñ!9ˆÔØ$×5Ñ5¸Õ9ˆÔä*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ+Ø×Ñ × 2Ñ 2°QÕ 6¸Uô
ˆÔô !#§	¢	Ø×*Ñ*Ø×+Ñ+Ø×/Ñ/ØØØô!
ˆÔô (*§x¢xØ×Ñ ×!8Ñ!8¸!Ð<ó(
ˆ×ÑÔ$ô & f×&8Ñ&8¸f×>QÑ>QÔRˆŒÜ)Ø×Ñ × 2Ñ 2¸ô
ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r!   )r   r   s   "€r   r   r!  †  s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ó^  € TpV P                  V4      pV P                  V4      p\        P                  ! V^RR7      w  r4V\        P                  ! V4      ,          p\        P
                  ! VRV P                  ^ 3R.4      p\        P                  ! VV P                  P                  ^^ VP                  R,          R7      p\        P                  ! WP                  ) V P                  4      pV P                  V4      p\        P                  ! V4      pV P!                  V4      pW,           # )r   r°   )r>   r?   Úgroupsrd   rQ   )r‚   r%  r   ÚsplitÚsigmoidrS   r$  Úconv_generalr'  r   rT   rŠ   r€   r(  rE   r‹   r)  )r   r   rŒ   Úx1Úx2s   &&   r   r&   ÚConformerLightConv1d.__call__†  sø   € Øˆà×Ñ Ó"ˆØ×Ñ˜aÓ ˆô —’˜!˜Q RÔ(‰ˆØ”—’˜B“Õˆô �FŠF�1�v × 3Ñ 3°QÐ7¸Ð@ÓAˆô �OŠOØØ×!Ñ!×(Ñ(ØØØ—7‘7˜2•;ô
ˆô �GŠG�A×.Ñ.Ð.°×0FÑ0FÓGˆØ�N‰N˜1ÓˆÜ�GŠG�A‹JˆØ�O‰O˜AÓˆà�|Ðr   )r$  r(  r'  r€   r)  r%  r‚   r(   r1   s   @@r   r  r  h  s!   ù‡ € Ùx÷
ó 
÷6÷ ð r   r  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚConformerBlocki¤  z‚Macaron-style Conformer block.

Attribute names match checkpoint: layers.X.{feed_forward1,self_attn,lconv1d,feed_forward2,norm_*}
c                ó    <€ V ^8„  d   QhRS[ /# r`   r   )r   r   s   "€r   r   ÚConformerBlock.__annotate__ª  s   ø€ ÷ 	9ñ 	9™{ñ 	9r   c                ób  <€ \         SV `  4        VP                  V n        \        V4      V n        \        V4      V n        \        V4      V n        \        V4      V n	        \        VP                  4      V n        \        VP                  4      V n        \        VP                  4      V n        R # r   )r   r   r€   r{   Úfeed_forward1r×   Ú	self_attnr  Úlconv1dÚfeed_forward2r   rj   Únorm_pre_attnÚnorm_post_attnÚnorm_outr†   s   &&€r   r   ÚConformerBlock.__init__ª  s†   ø€ Ü‰ÑÔØ!'×!9Ñ!9ˆÔÜ1°&Ó9ˆÔÜ'¨Ó/ˆŒÜ+¨FÓ3ˆŒÜ1°&Ó9ˆÔÜ)¨&×*<Ñ*<Ó=ˆÔÜ*¨6×+=Ñ+=Ó>ˆÔÜ$ V×%7Ñ%7Ó8ˆŽr   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r   r   rM   r  r    r!   )r   r   s   "€r   r   r7  µ  s=   ø€ ÷  ñ  Ù—‘ð Ù!#§¡ð Ù>@¿h¹hð á	�‰ñ r   c                ó`  € V P                  V4      pTp\        P                  ! WP                  ) V P                  4      pV P	                  V4      pV P                  WV4      p\        P                  ! WP                  ) V P                  4      pW@P                  V4      ,           pV( R,          P                  VP                  4      pW,          pV P                  V4      pV P                  V4      p\        P                  ! WP                  ) V P                  4      pV P                  V4      # )rO   ©rO   rO   N)r9  r   rŠ   r€   r=  r:  r>  r²   r­   r;  r<  r?  )r   r   rM   r  rŒ   Úvalidity_masks   &&&&  r   r&   ÚConformerBlock.__call__µ  sý   € ð ×Ñ˜qÓ!ˆð ˆÜ�GŠG�A×.Ñ.Ð.°×0FÑ0FÓGˆØ×Ñ˜qÓ!ˆØ�N‰N˜1Ð$5Ó6ˆÜ�GŠG�A×.Ñ.Ð.°×0FÑ0FÓGˆØ×*Ñ*¨1Ó-Õ-ˆð ˜ 
Õ+×2Ñ2°1·7±7Ó;ˆØÕˆà�L‰L˜‹OˆØ×Ñ˜qÓ!ˆÜ�GŠG�A×.Ñ.Ð.°×0FÑ0FÓGˆØ�}‰}˜QÓÐr   )r9  r<  r€   r;  r?  r>  r=  r:  r(   r1   s   @@r   r5  r5  ¤  s#   ù‡ € ñ÷
	9ó 	9÷ ÷  ð  r   r5  c                   óf   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltV3R lR ltR	tVt	V ;t
# )
ÚAudioEncoderiÌ  zGGemma 4 audio encoder based on the Universal Speech Model architecture.c                ó    <€ V ^8„  d   QhRS[ /# r`   r   )r   r   s   "€r   r   ÚAudioEncoder.__annotate__Ï  s   ø€ ÷ $ñ $™{ñ $r   c                óR  <€ \         SV `  4        Wn        \        V4      V n        \        VP                  4       Uu. uF  p\        V4      NK  	  upV n        VP                  e5   \        P                  ! VP                  VP                  RR7      V n        R # R V n        R # u upi )NTrc   )r   r   r5   r^   Úsubsample_conv_projectionrh   Únum_hidden_layersr5  ÚlayersÚoutput_proj_dimsrE   ri   rj   Úoutput_proj)r   r5   rm   r   s   && €r   r   ÚAudioEncoder.__init__Ï  sˆ   ø€ Ü‰ÑÔØŒä)@ÀÓ)HˆÔ&Ü7<¸V×=UÑ=UÔ7VÓWÑ7V°!”~ fÖ-Ñ7VÑWˆŒà×"Ñ"Ò.Ü!ŸyšyØ×"Ñ" F×$;Ñ$;À$ô ˆDÖð  $ˆDÖùò Xs   ½B$c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r    r!   )r   r   s   "€r   r   rI  Ý  s   ø€ ÷ ñ ©"¯(©(ñ r   c                óè  € V P                   P                  pV P                   P                  p\        ^ V P                   P                  ^,
          4      pW2,           pW,           V,           p\
        P                  ! \
        P                  ! WQ34      4      P                  p\
        P                  ! \
        P                  ! W34      VR7      pWg,          P                  \
        P                  4      pV# )z;Build the local causal+validity mask for chunked attention.)r  )r5   rÛ   rœ   r™   rš   r   Útrilr   rv   r²   Úbool_)	r   rÜ   rÝ   rÞ   Úupper_diagonalrÀ   Úlower_causalÚupper_causalrM   s	   &        r   Ú_build_causal_valid_maskÚ%AudioEncoder._build_causal_valid_maskÝ  s«   € à—[‘[×5Ñ5ˆ
Ø!Ÿ[™[×@Ñ@ÐÜ˜q $§+¡+×"DÑ"DÀqÕ"HÓIÐØ)Õ>ˆØ!Õ4Ð7IÕIˆä—w’wœrŸwšw¨Ð'AÓBÓC×EÑEˆÜ—w’wœrŸwšw¨
Ð'AÓBÀnÔUˆØÕ+×3Ñ3´B·H±HÓ=ˆØˆr   c                óT   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[/# )r   rq   Úaudio_mel_maskr    )r   r"   Útuple)r   r   s   "€r   r   rI  ê  s*   ø€ ÷ -ñ -¡"§(¡(ð -¹B¿H¹Hð -Éñ -r   c                ó”  € V P                  W4      w  r4V P                  4       pV P                   F  pV! W4V4      pK  	  V P                  e   V P                  V4      pVP                  ^,          VP                  ^,          8w  d!   VP                  ^,          pVRR V13,          p\
        P                  ! VR,          RV4      pW43# )NrO   rP   rC  )rK  rX  rM  rO  rT   r   rR   )r   rq   r[  Úaudio_encodingsÚcurrent_maskr  ÚblockÚ
target_lens   &&&     r   r&   ÚAudioEncoder.__call__ê  sÂ   € Ø(,×(FÑ(FØó)
Ñ%ˆð !×9Ñ9Ó;Ðà—[”[ˆEÙ# OÐCTÓUŠOñ !ð ×ÑÒ'Ø"×.Ñ.¨Ó?ˆOà×Ñ˜aÕ  O×$9Ñ$9¸!Õ$<Ô<Ø(×.Ñ.¨qÕ1ˆJØ'¨¨;¨J¨;¨Õ7ˆLäŸ(š( <°
Õ#;¸SÀ/ÓRˆàÐ,Ð,r   )r5   rM  rO  rK  )r)   r*   r+   r,   r-   r   rX  r&   r.   r/   r0   r1   s   @@r   rG  rG  Ì  s+   ù‡ € ÙQ÷$ó $÷ð ÷-÷ -ð -r   rG  )r-   rŸ   Úmlx.coreÚcorer   Úmlx.nnrE   r5   r   Úvisionr   ÚModuler   r3   r^   r{   r�   r×   r  r5  rG  © r   r   Ú<module>ri     sµ   ðñó å Ý å Ý #ô	:�2—9‘9ô 	:ô4�B—I‘Iô 4ôn"˜bŸi™iô "ôJ3˜2Ÿ9™9ô 3ô:K! R§Y¡Yô K!ô\"�R—Y‘Yô "ôD9˜2Ÿ9™9ô 9ôx% �R—Y‘Yô % ôP1-�2—9‘9ö 1-r   