+
    UV-jh  ã                   ó$  € R t ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RI	t
^RIHt  ! R R]P                  4      tR(R R llt ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R  R!]P                  4      t ! R" R#]P                  4      t ! R$ R%]P                  4      t ! R& R']P                  4      tR# ))a/  
Cascades Conformer audio encoder for Phi-4 Multimodal, ported to MLX.

Architecture:
- NeMo Conv Subsampling (time_reduction=8, depthwise separable)
- 24 Conformer blocks (attention_dim=1024, 16 heads)
- T5 relative attention bias
- Absolute positional encoding
- Mean-variance normalization embedding
N)ÚOptional©ÚAudioConfigc                   ó2   a € ] tR t^t o V 3R lR ltRtV tR# )ÚSwishc                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©é   ÚxÚreturn©ÚmxÚarray)ÚformatÚ__classdict__s   "€Úl/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/phi4mm/audio.pyÚ__annotate__ÚSwish.__annotate__   s#   ø€ ÷ !ñ !™"Ÿ(™(ð !¡r§x¡xñ !ó    c                ó<   € V\         P                  ! V4      ,          # ©N)r   Úsigmoid©Úselfr
   s   &&r   Ú__call__ÚSwish.__call__   s   € Ø”2—:’:˜a“=Õ Ð r   © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Ú__static_attributes__Ú__classdictcell__)r   s   @r   r   r      s   ø‡ € ÷!ö !r   r   c                ó$   € V ^8„  d   QhR\         /# )r	   Úname)Ústr)r   s   "r   r   r      s   € ÷ 
ñ 
œñ 
r   c                 óò   € V P                  4       p V R 8X  d   \        P                  ! 4       # V R8X  d   \        P                  ! 4       # V R8X  d   \	        4       # V R8X  d   \        P
                  ! 4       # R # )ÚreluÚgeluÚswishr   c                 ó   € V # r   r   )r
   s   &r   Ú<lambda>Ú get_activation.<locals>.<lambda>)   s   € ‘Qr   )ÚlowerÚnnÚReLUÚGELUr   ÚSigmoid)r$   s   &r   Úget_activationr2      s[   € Ø�:‰:‹<€DØˆv„~Ü�wŠw‹yÐØˆv„~Ü�wŠw‹yÐØˆw„Ü‹wˆØˆyÔÜ�zŠz‹|ÐÙÐr   c                   óX   a a€ ] tR t^,t oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚGLUz?Gated Linear Unit: splits input in half, gates with activation.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r	   ÚdimÚact_name)Úintr%   )r   r   s   "€r   r   ÚGLU.__annotate__/   s   ø€ ÷ ,ñ ,™Cð ,±ñ ,r   c                óP   <€ \         SV `  4        Wn        \        V4      V n        R # r   )ÚsuperÚ__init__r6   r2   Úact)r   r6   r7   Ú	__class__s   &&&€r   r<   ÚGLU.__init__/   s   ø€ Ü‰ÑÔØŒÜ! (Ó+ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   r9   4   s#   ø€ ÷ 'ñ '™"Ÿ(™(ð '¡r§x¡xñ 'r   c                óâ  € VP                   V P                  ,          ^,          pV P                  R8X  d   VRRV13,          pVRVR13,          pM‰\        P                  ! V\        P                  ! V4      V P                  R7      p\        P                  ! V\        P                  ! W!P                   V P                  ,          4      V P                  R7      pW0P                  V4      ,          # )r	   .N©Úaxiséÿÿÿÿ)Úshaper6   r   ÚtakeÚaranger=   )r   r
   ÚhalfÚhalf_xÚgates   &&   r   r   ÚGLU.__call__4   sŸ   € Ø�w‰w�t—x‘xÕ  AÕ%ˆØ�8‰8�rŒ>Ø�s˜E˜T˜E�z•]ˆFØ�S˜$™%�Z•=‰Dä—W’W˜Q¤§	¢	¨$£°d·h±hÔ?ˆFÜ—7’7˜1œbŸiši¨¯g©g°d·h±hÕ.?Ó@ÀtÇxÁxÔPˆDØŸ™ ›Õ&Ð&r   )r=   r6   )rD   r   ©
r   r   r   r    Ú__doc__r<   r   r!   r"   Ú__classcell__©r>   r   s   @@r   r4   r4   ,   s!   ù‡ € ÙI÷,õ ,÷
'÷ 'ð 'r   r4   c                   óL   a a€ ] tR t^?t oRtRV 3R lltV3R lR ltRtVtV ;t	# )Ú	GLULinearzLinear + GLU.c                óŽ   <€ \         SV `  4        \        P                  ! W^,          VR7      V n        \        RV4      V n        R# )r	   ©ÚbiasNrD   )r;   r<   r.   ÚLinearÚlinearr4   Úglu)r   Ú	input_dimÚ
output_dimÚglu_typerT   r>   s   &&&&&€r   r<   ÚGLULinear.__init__B   s2   ø€ Ü‰ÑÔÜ—i’i 	¸­>ÀÔEˆŒÜ�r˜8Ó$ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   ÚGLULinear.__annotate__G   s#   ø€ ÷ (ñ (™"Ÿ(™(ð (¡r§x¡xñ (r   c                óB   € V P                  V P                  V4      4      # r   ©rW   rV   r   s   &&r   r   ÚGLULinear.__call__G   s   € Ø�x‰x˜Ÿ™ A›Ó'Ð'r   r_   ©r   TrL   rO   s   @@r   rQ   rQ   ?   s   ù‡ € Ù÷%÷
(÷ (ð (r   rQ   c                   óL   a a€ ] tR t^Kt oRtRV 3R lltV3R lR ltRtVtV ;t	# )ÚGLUPointWiseConva  GLU with pointwise Conv1D for the Conformer conv module.

In the checkpoint:
- ext_pw_conv_1d: Conv1d(input_dim, output_dim*2, kernel_size=1)
- b1, b2: bias parameters (1, output_dim, 1)

We implement this with a Linear (equivalent to Conv1d with kernel=1).
c                ó(  <€ \         SV `  4        W n        W@n        \        P
                  ! W^,          RR7      V n        \        V4      V n        V'       d;   \        P                  ! V34      V n        \        P                  ! V34      V n        R# R# )r	   TrS   N)r;   r<   rY   Úbias_in_glur.   rU   Úext_pw_conv_1dr2   Úglu_actr   ÚzerosÚb1Úb2)r   rX   rY   rZ   re   r>   s   &&&&&€r   r<   ÚGLUPointWiseConv.__init__U   sh   ø€ Ü‰ÑÔØ$ŒØ&Ôä Ÿiši¨	Àµ>ÈÔMˆÔÜ% hÓ/ˆŒßÜ—h’h 
˜}Ó-ˆDŒGÜ—h’h 
˜}Ó-ˆDŽGñ r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   ÚGLUPointWiseConv.__annotate__`   s#   ø€ ÷ 	ñ 	™"Ÿ(™(ð 	¡r§x¡xñ 	r   c                óJ  € V P                  V4      pVR RV P                  13,          pVR V P                  R13,          pV P                  '       d;   W P                  ,           V P	                  W0P
                  ,           4      ,          pV# W P	                  V4      ,          pV# ).N)rf   rY   re   ri   rg   rj   )r   r
   Úx1Úx2s   &&  r   r   ÚGLUPointWiseConv.__call__`   s‹   € à×Ñ Ó"ˆØˆsÐ%�d—o‘oÐ%Ð%Õ&ˆØˆs�D—O‘OÑ%Ð%Õ&ˆØ××ÐØ—g‘g• §¡¨b·7±7­lÓ!;Õ;ˆAð ˆð —\‘\ "Ó%Õ%ˆAØˆr   )ri   rj   re   rf   rg   rY   ra   rL   rO   s   @@r   rc   rc   K   s   ù‡ € ñ÷	.÷	÷ 	ð 	r   rc   c                   óL   a a€ ] tR t^qt oRtRV 3R lltV3R lR ltRtVtV ;t	# )ÚFeedForwardzrFeed Forward module with GLU.

Architecture: LayerNorm -> GLULinear(d_model, d_inner) -> Linear(d_inner, d_model)
c                ó¸   <€ \         SV `  4        \        P                  ! V4      V n        \        WW4R 7      V n        \        P                  ! W!RR 7      V n        R# )rS   TN)	r;   r<   r.   Ú	LayerNormÚ
layer_normrQ   Únet_0rU   Únet_2)r   Úd_modelÚd_innerÚ
activationre   r>   s   &&&&&€r   r<   ÚFeedForward.__init__w   s@   ø€ Ü‰ÑÔÜŸ,š, wÓ/ˆŒô ˜w°ÔNˆŒ
Ü—Y’Y˜w°dÔ;ˆŽ
r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   ÚFeedForward.__annotate__   ó#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r   ©rv   rw   rx   r   s   &&r   r   ÚFeedForward.__call__   s/   € Ø�O‰O˜AÓˆØ�J‰J�q‹MˆØ�J‰J�q‹MˆØˆr   r�   ra   rL   rO   s   @@r   rs   rs   q   s   ù‡ € ñ÷
<÷÷ ð r   rs   c                   óL   a a€ ] tR t^‹t oRtRV 3R lltV3R lR ltRtVtV ;t	# )ÚDepthWiseSeparableConv1dzøDepthwise separable Conv1D.

- dw_conv: depthwise Conv1d (groups=input_dim)
- pw_conv: pointwise Conv1d (1x1)

MLX Conv1d weight shape: (out_channels, kernel_size, in_channels)
For depthwise: groups=input_dim, so each filter operates on 1 channel.
c           	     óä   <€ \         SV `  4        WPn        \        P                  ! VW,          V^VVR7      V n        V^ 8w  d'   \        P                  ! W,          V^^^ R7      V n        W n        R# )é   )ÚstrideÚpaddingÚgroups©Úkernel_sizer‡   rˆ   N)r;   r<   rˆ   r.   ÚConv1dÚdw_convÚpw_convÚout_channel)r   rX   r�   r‹   Údepthwise_multiplierrˆ   r>   s   &&&&&&€r   r<   Ú!DepthWiseSeparableConv1d.__init__•   sn   ø€ ô 	‰ÑÔØŒÜ—y’yØØÕ,ØØØØô
ˆŒð ˜!ÔÜŸ9š9ØÕ0ØØØØôˆDŒLð 'Ör   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   Ú%DepthWiseSeparableConv1d.__annotate__¬   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ól   € V P                  V4      pV P                  ^ 8w  d   V P                  V4      pV# )é    )r�   r�   rŽ   r   s   &&r   r   Ú!DepthWiseSeparableConv1d.__call__¬   s/   € à�L‰L˜‹OˆØ×Ñ˜qÔ Ø—‘˜Q“ˆAØˆr   )r�   r�   rˆ   rŽ   )r†   r•   rL   rO   s   @@r   r„   r„   ‹   s   ù‡ € ñ÷'÷.÷ ð r   r„   c                   óL   a a€ ] tR t^¹t oRtRV 3R lltV3R lR ltRtVtV ;t	# )Ú
ConvModulea  Conformer convolution module.

Architecture:
1. LayerNorm
2. GLU pointwise conv (input_dim -> input_dim, via 2*input_dim with GLU)
3. Depthwise separable Conv1D
4. LayerNorm (when cnn_layer_norm=True)
5. Swish activation
6. Pointwise Conv1D (ext_pw_conv_1d)
c                ó”  <€ \         SV `  4        \        P                  ! V4      V n        Wn        W n        Wpn        WPn        V^ 8w  d   \        WW¼4      V n
        V'       d   V^,
          pMV^,
          ^,          p\        VVVVVR7      V n        \        V
4      V n        V^ 8w  d    \        P                  ! WRR7      V n        R# R# )r•   )rˆ   TrS   N)r;   r<   r.   ru   rv   rX   Úext_pw_out_channelÚcausalr‹   rc   rW   r„   Údw_sep_conv_1dr2   r=   rU   rf   )r   rX   rš   Údepthwise_seperable_out_channelÚext_pw_kernel_sizer‹   r�   r›   Ú
batch_normÚcnn_layer_normr{   rZ   re   rˆ   r>   s   &&&&&&&&&&&&& €r   r<   ÚConvModule.__init__Å   s¶   ø€ ô 	‰ÑÔÜŸ,š, yÓ1ˆŒØ"ŒØ"4ÔØŒØ&Ôà Ô"Ü'Ø¨xóˆDŒH÷ Ø! A•o‰Gà" Q•¨1Õ,ˆGä6ØØ+ØØ Øô
ˆÔô " *Ó-ˆŒà Ô"ä"$§)¢)¨IÐPTÔ"UˆDÖñ #r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   ÚConvModule.__annotate__ó   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                óz  € V P                  V4      pV P                  ^ 8w  d   V P                  V4      pV P                  V4      pV P                  '       d2   V P
                  ^8”  d!   VRRV P
                  ^,
          ) 1R3,          pV P                  V4      pV P                  ^ 8w  d   V P                  V4      pV# )r•   ºNNNN)rv   rš   rW   rœ   r›   r‹   r=   rf   r   s   &&r   r   ÚConvModule.__call__ó   s¬   € Ø�O‰O˜AÓˆà×"Ñ" aÔ'Ø—‘˜“ˆAð ×Ñ Ó"ˆð �;�;ˆ;˜4×+Ñ+¨aÔ/Ø�!Ð.˜×)Ñ)¨AÕ-Ð.Ð.°Ð1Õ2ˆAà�H‰H�Q‹Kˆð ×"Ñ" aÔ'Ø×#Ñ# AÓ&ˆAàˆr   )	r=   r›   rœ   rf   rš   rW   rX   r‹   rv   )FFTr'   r   TrL   rO   s   @@r   r˜   r˜   ¹   s    ù‡ € ñ	÷,V÷\÷ ð r   r˜   c                   óL   a a€ ] tR tRt oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚMultiHeadedAttentioni  z4Multi-Head Attention with optional T5 relative bias.c                ób  <€ \         SV `  4        W!,          V n        Wn        V P                  R,          V n        \
        P                  ! W"RR7      V n        \
        P                  ! W"RR7      V n        \
        P                  ! W"RR7      V n	        \
        P                  ! W"RR7      V n
        R# )ç      à?TrS   Ng      à¿)r;   r<   Úd_kÚhÚscaler.   rU   Úlinear_qÚlinear_kÚlinear_vÚ
linear_out)r   Ún_headÚn_featr>   s   &&&€r   r<   ÚMultiHeadedAttention.__init__  st   ø€ Ü‰ÑÔØÕ#ˆŒØŒØ—X‘X˜t•^ˆŒ
äŸ	š	 &°tÔ<ˆŒÜŸ	š	 &°tÔ<ˆŒÜŸ	š	 &°tÔ<ˆŒÜŸ)š) F¸Ô>ˆŽr   c                óÖ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  /# )r	   ÚqueryÚkeyÚvalueÚmaskÚrelative_attention_biasr   ©r   r   r   )r   r   s   "€r   r   Ú!MultiHeadedAttention.__annotate__  si   ø€ ÷ "ñ "á�x‰xð"ñ �X‰Xð"ñ �x‰xð	"ñ
 ‘r—x‘xÕ ð"ñ "*©"¯(©(Õ!3ð"ñ 
�‰ñ"r   c           	     óü  € VP                   ^ ,          pV P                  V4      P                  VRV P                  V P                  4      P                  ^ ^^^4      pV P                  V4      P                  VRV P                  V P                  4      P                  ^ ^^^4      pV P                  V4      P                  VRV P                  V P                  4      P                  ^ ^^^4      p	WpP                  ,          VP                  ^ ^^^4      ,          p
Ve	   W¥,           p
Ve5   \        P                  ! WJ\        P                  ! \        R4      4      4      p
\        P                  ! V
RR7      pVe,   \        P                  ! WK\        P                  ! R4      4      pW¹,          pVP                  ^ ^^^4      P                  VRV P                  V P                  ,          4      pV P                  V4      # )r•   z-infrB   g        rD   )rE   r®   Úreshaper¬   r«   Ú	transposer¯   r°   r­   r   Úwherer   ÚfloatÚsoftmaxr±   )r   r¶   r·   r¸   r¹   rº   ÚBÚqÚkÚvÚscoresÚattnr
   s   &&&&&&       r   r   ÚMultiHeadedAttention.__call__  s…  € ð �K‰K˜�Nˆà�M‰M˜%Ó ×(Ñ(¨¨B°·±¸¿¹ÓA×KÑKÈAÈqÐRSÐUVÓWˆØ�M‰M˜#Ó×&Ñ& q¨"¨d¯f©f°d·h±hÓ?×IÑIÈ!ÈQÐPQÐSTÓUˆØ�M‰M˜%Ó ×(Ñ(¨¨B°·±¸¿¹ÓA×KÑKÈAÈqÐRSÐUVÓWˆð —j‘j•. A§K¡K°°1°a¸Ó$;Õ;ˆà"Ò.ØÕ5ˆFàÒÜ—X’X˜d¬B¯HªH´U¸6³]Ó,CÓDˆFä�zŠz˜& rÔ*ˆØÒÜ—8’8˜D¬¯ª°«Ó6ˆDà�HˆØ�K‰K˜˜1˜a Ó#×+Ñ+¨A¨r°4·6±6¸D¿H¹HÕ3DÓEˆà�‰˜qÓ!Ð!r   )r«   r¬   r¯   r±   r®   r°   r­   ©NNrL   rO   s   @@r   r¨   r¨     s   ù‡ € Ù>õ	?÷"÷ "ò "r   r¨   c                   óL   a a€ ] tR tRt oRtRV 3R lltV3R lR ltRtVtV ;t	# )ÚT5RelativeAttentionLogitBiasiB  z<T5-style relative attention bias (asymmetric, no bucketing).c                ó¤   <€ \         SV `  4        Wn        W n        V^,          V n        \
        P                  ! V P                  V4      V n        R# )r	   N)r;   r<   Ú	num_headsÚmax_distanceÚnum_bucketsr.   Ú	EmbeddingÚbias_values)r   rÎ   rÏ   r>   s   &&&€r   r<   Ú%T5RelativeAttentionLogitBias.__init__E  s?   ø€ Ü‰ÑÔØ"ŒØ(Ôà'¨!Õ+ˆÔÜŸ<š<¨×(8Ñ(8¸)ÓDˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   Ú)T5RelativeAttentionLogitBias.__annotate__M  s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                óž  € VP                   ^,          p\        P                  ! V4      R,          p\        P                  ! V4      R,          pWC,
          p\        P                  ! WPP                  ) V P                  ^,
          4      pWPP
                  ^,          ,           pV P                  V4      pVP                  ^^ ^4      R,          pV# )r†   ©r¥   N©Nr¥   )Nr¥   r¥   r¥   )rE   r   rG   ÚcliprÏ   rÐ   rÒ   r¿   )r   r
   ÚmaxposÚcontext_positionÚmemory_positionÚrelative_positionÚbias_idxÚt5_biass   &&      r   r   Ú%T5RelativeAttentionLogitBias.__call__M  s¬   € Ø—‘˜•ˆÜŸ9š9 VÓ,¨WÕ5ÐÜŸ)š) FÓ+¨GÕ4ˆØ+Õ>Ðô ŸGšGØ× 1Ñ 1Ð1°4×3DÑ3DÀqÕ3Hó
Ðð
 %×'7Ñ'7¸1Õ'<Õ<ˆà×"Ñ" 8Ó,ˆØ×#Ñ# A q¨!Ó,¨]Õ;ˆàˆr   )rÒ   rÏ   rÐ   rÎ   )iè  rL   rO   s   @@r   rÌ   rÌ   B  s   ù‡ € ÙF÷E÷÷ ð r   rÌ   c                   óR   a a€ ] tR tRt oRtRV 3R lltR tV3R lR ltRtVt	V ;t
# )	ÚAbsolutePositionalEncodingif  z(Sinusoidal absolute positional encoding.c                óˆ   <€ \         SV `  4        Wn        \        P                  ! V4      V n        V P                  V4       R # r   )r;   r<   ry   ÚmathÚsqrtÚxscaleÚ	_build_pe)r   ry   Úmax_lenr>   s   &&&€r   r<   Ú#AbsolutePositionalEncoding.__init__i  s/   ø€ Ü‰ÑÔØŒÜ—i’i Ó(ˆŒØ�‰�wÖr   c           	     ój  € \         P                  ! WP                  3\         P                  R 7      p\         P                  ! ^ V\         P                  R 7      R,          p\         P
                  ! \         P                  ! ^ V P                  ^\         P                  R 7      \        P                  ! R4      V P                  ,          ) ,          4      p\         P                  ! W4,          4      VR&   \         P                  ! W4,          4      VR&   \        P                  ! VR,          4      V n        R# ))ÚdtypeNg     ˆÃ@r×   )r¥   :r•   Nr	   )r¥   :r†   Nr	   )Nr¥   r¥   )Únprh   ry   Úfloat32rG   Úexprä   ÚlogÚsinÚcosr   r   Úpe)r   rè   rò   ÚpositionÚdiv_terms   &&   r   rç   Ú$AbsolutePositionalEncoding._build_peo  s½   € Ü�XŠX�w§¡Ð-´R·Z±ZÔ@ˆÜ—9’9˜Q ¬r¯z©zÔ:¸7ÕCˆÜ—6’6Ü�IŠI�a˜Ÿ™ q´·
±
Ô;Ü—’˜Ó! D§L¡LÕ0Ð1õ2ó
ˆô —f’f˜XÕ0Ó1ˆˆ7‰Ü—f’f˜XÕ0Ó1ˆˆ7‰Ü—(’(˜2˜j�>Ó*ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   Ú'AbsolutePositionalEncoding.__annotate__z  s#   ø€ ÷ 0ñ 0™"Ÿ(™(ð 0¡r§x¡xñ 0r   c                óè   € VP                   ^,          pW P                  P                   ^,          8”  d   V P                  V4       WP                  ,          V P                  RRV13,          ,           # )r†   r¥   N)rE   rò   rç   ræ   )r   r
   ÚTs   && r   r   Ú#AbsolutePositionalEncoding.__call__z  sO   € Ø�G‰G�A�JˆØ�w‰w�}‰}˜QÕÔØ�N‰N˜1ÔØ—;‘;� §¡¨¨B¨Q¨B¨¥Õ/Ð/r   )ry   rò   ræ   )iˆ  )r   r   r   r    rM   r<   rç   r   r!   r"   rN   rO   s   @@r   râ   râ   f  s!   ù‡ € Ù2÷ ò	+÷0÷ 0ð 0r   râ   c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚMeanVarianceNormLayeri†  z6Global mean/variance normalization for input features.c                ó”   <€ \         SV `  4        \        P                  ! V34      V n        \        P
                  ! V34      V n        R # r   )r;   r<   r   rh   Úglobal_meanÚonesÚglobal_invstd)r   Ú
input_sizer>   s   &&€r   r<   ÚMeanVarianceNormLayer.__init__‰  s4   ø€ Ü‰ÑÔÜŸ8š8 Z MÓ2ˆÔÜŸWšW j ]Ó3ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   Ú"MeanVarianceNormLayer.__annotate__Ž  s#   ø€ ÷ ;ñ ;™"Ÿ(™(ð ;¡r§x¡xñ ;r   c                óH   € WP                   ,
          V P                  ,          # r   )rþ   r   r   s   &&r   r   ÚMeanVarianceNormLayer.__call__Ž  s   € Ø×$Ñ$Õ$¨×(:Ñ(:Õ:Ð:r   )r   rþ   rL   rO   s   @@r   rü   rü   †  s   ù‡ € Ù@õ4÷
;÷ ;ð ;r   rü   c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚDWPWConvPairi—  z.A pair of depthwise + pointwise Conv2d layers.c           	     ó¦   <€ \         SV `  4        \        P                  ! VVVVVVR 7      V n        \        P                  ! VV^^^ R7      V n        R# ))r‹   r‡   rˆ   r‰   rŠ   N)r;   r<   r.   ÚConv2dÚdwÚpw)r   Úchannelsr‹   r‡   rˆ   r>   s   &&&&&€r   r<   ÚDWPWConvPair.__init__š  sQ   ø€ Ü‰ÑÔÜ—)’)ØØØ#ØØØô
ˆŒô —)’)ØØØØØô
ˆŽr   c                ór   € V P                  V4      p\        P                  ! V P                  V4      4      pV# r   )r  r.   r'   r  r   s   &&r   r   ÚDWPWConvPair.__call__¬  s*   € Ø�G‰G�A‹JˆÜ�GŠG�D—G‘G˜A“JÓˆØˆr   )r  r  rL   rO   s   @@r   r  r  —  s   ù‡ € Ù8õ
÷$ò r   r  c                   óP   a a€ ] tR tRt oRtRV 3R lltRV3R lR lltRtVtV ;t	# )	ÚNemoConvSubsamplingi²  ah  NeMo-style convolutional subsampling (dw_striding).

For time_reduction=8:
- Layer 0: Conv2d(1, conv_ch, 3x3, stride=2)  -> T/2, F/2
- Layer 1: ReLU
- DW/PW pair 0: DW Conv2d + PW Conv2d -> T/4, F/4
- DW/PW pair 1: DW Conv2d + PW Conv2d -> T/8, F/8
- out: Linear(conv_ch * (F/8), feat_out)

MLX Conv2d expects: (B, H, W, C_in) and weight (C_out, kH, kW, C_in)
c           	     ó  <€ \         SV `  4        W0n        WPn        \	        \
        P                  ! V^4      4      p^p^pV^,
          ^,          p	\        P                  ! ^WGW‰R7      V n	        \        V^,
          4       U
u. uF  p
\        WGW‰4      NK  	  up
V n        Tp\        V4       F(  p
V^V	,          ,           V,
          V,          ^,           pK*  	  \        P                  ! WK,          VRR7      V n        RV n        R# u up
i )r	   rŠ   TrS   N)r;   r<   Útime_reductionr›   r8   rä   rï   r.   r
  Úconv_0Úranger  Údw_pw_layersrU   ÚoutÚconv2d_subsampling)r   Úfeat_inÚfeat_outr  Úconv_channelsr›   Úsampling_numr‹   r‡   rˆ   Ú_Úfreq_outr>   s   &&&&&&      €r   r<   ÚNemoConvSubsampling.__init__¿  sí   ø€ ô 	‰ÑÔØ,ÔØŒÜœ4Ÿ8š8 N°AÓ6Ó7ˆØˆØˆØ •? qÕ(ˆô —i’iØˆ}¸fô
ˆŒô ˜<¨!Õ+Ô,ó
á,�ô ˜°VÖEÙ,ñ
ˆÔð ˆÜ�|Ö$ˆAØ  1 w¥;Õ.°Õ<ÀÕGÈ!ÕKŠHñ %ô —9’9˜]Õ5°xÀdÔKˆŒØ"&ˆÖùò
s   ÂDc                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# )r	   r
   r¹   r»   )r   r   s   "€r   r   Ú NemoConvSubsampling.__annotate__Ý  s(   ø€ ÷ (ñ (™"Ÿ(™(ð (©(±2·8±8Õ*<ñ (r   c                ód  € VP                   w  r4pVR,          p\        P                  ! V P                  V4      4      pV P                   F  pV! V4      pK  	  VP                   w  r7r‰VP                  ^ ^^^4      P                  W7W˜,          4      pV P                  V4      pVe‰   VP                  ^R7      p
\        P                  ! W P                  ,          4      P                  \        P                  4      pTp\        P                  ! V4      R,          pWÛR,          8  pVR,          pW3# )z¨
Args:
    x: (B, T, feat_in) mel spectrogram
    mask: (B, T) validity mask
Returns:
    x: (B, T', feat_out) subsampled features
    mask: (B, 1, T') subsampled mask
rB   )r¥   r¥   r¥   NrØ   r×   )r¥   Nr¥   )rE   r.   r'   r  r  r¿   r¾   r  Úsumr   Úceilr  ÚastypeÚint32rG   )r   r
   r¹   rÃ   rù   ÚFÚpairÚT_outÚF_outÚCÚfeature_lensÚpadding_lengthÚmax_audio_lengthÚindicesÚpad_masks   &&&            r   r   ÚNemoConvSubsampling.__call__Ý  s  € ð —'‘'‰ˆˆaàˆmÕˆô �GŠG�D—K‘K “NÓ#ˆð ×%Ô%ˆDÙ�Q“ŠAñ &ð ŸW™WÑˆ�%Ø�K‰K˜˜1˜a Ó#×+Ñ+¨A°aµiÓ@ˆð �H‰H�Q‹Kˆð ÒØŸ8™8¨˜8Ó+ˆLÜŸWšW \×4GÑ4GÕ%GÓH×OÑOÜ—‘óˆNð  %ÐÜ—i’iÐ 0Ó1°'Õ:ˆGØ°Õ!8Ñ8ˆHØ˜JÕ'ˆDàˆwˆr   )r›   r  r  r  r  r  )é   i   Fr   rL   rO   s   @@r   r  r  ²  s   ù‡ € ñ
÷'÷<(÷ (ò (r   r  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚConformerEncoderLayeri  z•Single Conformer block.

Forward: x += 0.5*FFN_in(x)
         x += Attn(LN(x))
         x += Conv(x)
         x += 0.5*FFN_out(x)
         x = LN(x)
c                ó    <€ V ^8„  d   QhRS[ /# ©r	   Úconfigr   )r   r   s   "€r   r   Ú"ConformerEncoderLayer.__annotate__  s   ø€ ÷ 0ñ 0™{ñ 0r   c                ó¢  <€ \         SV `  4        VP                  pVP                  p\	        W#VP
                  VP                  4      V n        \        VP                  V4      V n
        \        VVP                  VP                  VP                  VP                  VP                   VP"                  VP$                  VP&                  VP(                  VP*                  VP                  R 7      V n        \	        W#VP
                  VP                  4      V n        \0        P2                  ! V4      V n        \0        P2                  ! V4      V n        R# ))r›   rŸ   r    r{   rZ   re   N)r;   r<   Úattention_dimÚlinear_unitsrs   r{   re   Úfeed_forward_inr¨   Úattention_headsÚ	self_attnr˜   rš   r�   rž   r‹   r�   r›   rŸ   r    Úconv_activationÚconv_glu_typeÚconvÚfeed_forward_outr.   ru   Úlayer_norm_attrv   )r   r8  ry   Úd_ffnr>   s   &&  €r   r<   ÚConformerEncoderLayer.__init__  s  ø€ Ü‰ÑÔØ×&Ñ&ˆØ×#Ñ#ˆä*Ø˜F×-Ñ-¨v×/AÑ/Aó 
ˆÔô .¨f×.DÑ.DÀgÓNˆŒäØØ×%Ñ%Ø×2Ñ2Ø×%Ñ%Ø×ÑØ×'Ñ'Ø—=‘=Ø×(Ñ(Ø!×0Ñ0Ø×-Ñ-Ø×)Ñ)Ø×*Ñ*ô
ˆŒ	ô !,Ø˜F×-Ñ-¨v×/AÑ/Aó!
ˆÔô !Ÿlšl¨7Ó3ˆÔÜŸ,š, wÓ/ˆŽr   c                ó¢   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  /# )r	   r
   r¹   rº   r   r»   )r   r   s   "€r   r   r9  8  sM   ø€ ÷ "ñ "á�8‰8ð"ñ ‘r—x‘xÕ ð"ñ "*©"¯(©(Õ!3ð	"ñ
 
�‰ñ"r   c           	     ó(  € VR V P                  V4      ,          ,           pV P                  V4      pWP                  VVVVVR7      ,           pWP                  V4      ,           pVR V P	                  V4      ,          ,           pV P                  V4      # )rª   ©r¹   rº   )r=  rD  r?  rB  rC  rv   )r   r
   r¹   rº   Únorm_xs   &&&& r   r   ÚConformerEncoderLayer.__call__8  s�   € ð ��d×*Ñ*¨1Ó-Õ-Õ-ˆØ×$Ñ$ QÓ'ˆØ—‘ØØØØØ$;ð ó 
õ 
ˆð —	‘	˜!“ÕˆØ��d×+Ñ+¨AÓ.Õ.Õ.ˆØ�‰˜qÓ!Ð!r   )rB  r=  rC  rv   rD  r?  rÊ   rL   rO   s   @@r   r5  r5    s$   ù‡ € ñ÷0ó 0÷B"÷ "ò "r   r5  c                   ó^   a a€ ] tR tRt oRtV3R lV 3R lltR	V3R lR lltR tRtVt	V ;t
# )
ÚConformerEncoderiQ  zÚCascades Conformer encoder.

Pipeline:
1. MeanVarianceNormLayer (global normalization)
2. NemoConvSubsampling (time reduction)
3. AbsolutePositionalEncoding
4. T5RelativeAttentionLogitBias
5. 24x ConformerEncoderLayer
c                ó    <€ V ^8„  d   QhRS[ /# r7  r   )r   r   s   "€r   r   ÚConformerEncoder.__annotate__\  s   ø€ ÷ 
ñ 
™{ñ 
r   c                ó²  <€ \         SV `  4        Wn        VP                  p\	        VP
                  4      V n        \        VP
                  VVP                  VP                  VP                  R 7      V n        \        VP                  VP                  R7      V n        \!        VP"                  4       Uu. uF  p\%        V4      NK  	  upV n        R# u upi ))r  r  r  r  r›   )rÎ   rÏ   N)r;   r<   r8  r;  rü   r  Úencoder_embeddingr  r  r  r›   ÚembedrÌ   r>  Út5_bias_max_distanceÚrelative_attention_bias_layerr  Ú
num_blocksr5  Úencoders)r   r8  Údr  r>   s   &&  €r   r<   ÚConformerEncoder.__init__\  s¸   ø€ Ü‰ÑÔØŒØ× Ñ ˆä!6°v×7HÑ7HÓ!IˆÔä(Ø×%Ñ%ØØ!×0Ñ0Ø ×.Ñ.Ø—=‘=ô
ˆŒ
ô .JØ×,Ñ,Ø×4Ñ4ô.
ˆÔ*ô 49¸×9JÑ9JÔ3Kó
Ù3K¨aÔ! &Ö)Ñ3Kñ
ˆŽùò 
s   Â6Cc                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# )r	   Úxs_padÚmasksr»   )r   r   s   "€r   r   rO  t  s,   ø€ ÷ ?#ñ ?#á—‘ð?#ñ ™Ÿ™Õ!ñ?#r   c                óê  € V P                  V4      pV P                  W4      w  r2RpVP                  ^,          pRpWT8”  d¾   RpVP                  ^ ,          pWT,          ^ 8”  d   WEV,          ,
          pM^ pV^ 8”  dT   \        P                  ! VP                  ^ ,          WƒP                  ^,          34      p	\        P
                  ! W9.^R7      pVP                  w  r«pW´,          pVP                  W­,          WL4      pV P                  V4      pV P                   F  pV! VRVR7      pK  	  V'       d=   VP                  R,          pVP                  XRV4      pX^ 8”  d   VRRV) 1R3,          pW23# )	zË
Args:
    xs_pad: (B, T, 80) mel spectrogram features
    masks: (B, T) validity mask

Returns:
    output: (B, T', attention_dim) encoded features
    masks: (B, 1, T') validity mask after subsampling
iô  FTrB   NrI  r¥   rD   )	rQ  rR  rE   r   rh   Úconcatenater¾   rT  rV  )r   rZ  r[  Úinput_tensorÚmax_seq_lenÚseq_lenÚunfoldedÚori_bzÚchunk_pad_sizeÚpadrÃ   ÚT_paddedÚDÚ
num_chunksrº   Úlayers   &&&             r   r   ÚConformerEncoder.__call__t  sz  € ð ×'Ñ'¨Ó/ˆð #Ÿj™j¨Ó7Ñˆð ˆØ×$Ñ$ QÕ'ˆØˆàÔ ØˆHØ!×'Ñ'¨Õ*ˆFàÕ$ qÔ(Ø!,¸+Õ0EÕ!F‘à!"�à Ô!Ü—h’hØ!×'Ñ'¨Õ*¨N×<NÑ<NÈqÕ<QÐRó�ô  "Ÿ~š~¨|Ð.AÈÔJ�ð *×/Ñ/‰NˆA˜Ø!Õ0ˆJØ'×/Ñ/°µÀÓOˆLð #'×"DÑ"DÀ\Ó"RÐð —]”]ˆEÙ ØØØ(?ôŠLñ #÷ Ø×"Ñ" 2Õ&ˆAØ'×/Ñ/°¸¸AÓ>ˆLØ Ô!Ø+¨AÐ/?°°Ð/?ÀÐ,BÕC�àÐ"Ð"r   c                ó¶  a€ / pRR0pVP                  4        EF<  w  rEVoRV9   Ed   VP                  R4      pV^,          pVP                  R4      ^ ,          pVP                  R^4      ^,          p	\        V4      p
V
^ 8X  d   V^ ,          R,           V	,           oM˜V
^8X  d   KŽ  V
^8X  d   V^ ,          R,           V	,           oMpV
^8X  d   V^ ,          R,           V	,           oMQV
^8X  d   KÕ  V
^8X  d   V^ ,          R,           V	,           oM)V
^8X  d   V^ ,          R	,           V	,           oM
V
^8X  d   EK  R
S9   d   SP                  R
R4      oMRS9   d   SP                  RR4      oRS9   g   RS9   d*   VP                  ^8X  d   VP                  R4      pWRS&   EK‡  VP                  ^8X  d#   RS9   d   VP                  ^ ^^^4      pWRS&   EKº  VP                  ^8X  dn   RS9   dg   \        ;QJ d    V3R lV 4       F  '       g   K   RM	  RM! V3R lV 4       4      pV'       d   VR,          pMVP                  ^ ^^4      pWRS&   EK8  WRS&   EK?  	  V# )a~  Sanitize checkpoint weights for the audio encoder.

Key transformations:
- Conv2d weights: PyTorch (out, in, kH, kW) -> MLX (out, kH, kW, in)
- Conv1d weights for actual convolutions: PyTorch (out, in, kW) -> MLX (out, kW, in)
- Conv1d k=1 weights mapped to Linear: (out, in, 1) -> (out, in)
- GLUPointWiseConv b1/b2: (1, C, 1) -> (C,)
- Map sequential conv indices to named layers
zglu.ext_pw_conv_1dzconv.ext_pw_conv_1dzembed.conv.Ú.zembed.conv_0.zembed.dw_pw_layers.0.dw.zembed.dw_pw_layers.0.pw.zembed.dw_pw_layers.1.dw.zembed.dw_pw_layers.1.pw.z.net.0.linear.z.net_0.linear.z.net.2.z.net_2.zglu.b1zglu.b2Úweightc              3   ó,   <"  € T F	  qS9   x € K  	  R # 5ir   r   )Ú.0ÚlkÚnew_keys   & €r   Ú	<genexpr>Ú,ConformerEncoder.sanitize.<locals>.<genexpr>÷  s   øé € Ð%OÑ>N¸¨G¦mÓ>Nùs   ƒTFrD   )r¥   r¥   r•   )ÚitemsÚsplitr8   ÚreplaceÚndimr¾   r¿   Úany)r   ÚweightsÚ	sanitizedÚlinear_conv_keysrÅ   rÆ   ÚpartsÚrestÚidx_strÚparamÚidxÚis_k1_to_linearrp  s   &&          @r   ÚsanitizeÚConformerEncoder.sanitizeµ  s1  ø€ ð ˆ	ð 1Ð2GÐHÐà—M‘M—O‰DˆAØˆGð  Õ!ØŸ™ Ó.�Ø˜Q•x�ØŸ*™* S›/¨!Õ,�ØŸ
™
 3¨Ó*¨1Õ-�Ü˜'“l�à˜!”8Ø# A�h¨Õ8¸5Õ@‘GØ˜A”XÙØ˜A”XØ# A�hÐ)CÕCÀeÕK‘GØ˜A”XØ# A�hÐ)CÕCÀeÕK‘GØ˜A”XÙØ˜A”XØ# A�hÐ)CÕCÀeÕK‘GØ˜A”XØ# A�hÐ)CÕCÀeÕK‘GØ˜A”XÚð   7Ô*Ø!Ÿ/™/Ð*:Ð<LÓM‘Ø˜gÔ%Ø!Ÿ/™/¨)°YÓ?�ð
 ˜GÔ# x°7Ô':ÀÇÁÈ!ÄØ—I‘I˜b“M�Ø%&˜'Ñ"Úð �v‰v˜Œ{˜x¨7Ô2Ø—K‘K  1 a¨Ó+�Ø%&˜'Ñ"Úð �v‰v˜Œ{˜x¨7Ô2ß"%£#Ô%OÑ>NÓ%O§#§#¢#Ô%OÑ>NÓ%OÓ"O�ç"à˜'�
‘Að Ÿ™ A q¨!Ó,�Aà%&˜'Ñ"Úà!"�gÔñ $ðB Ðr   )r8  rR  rQ  rV  rT  r   )r   r   r   r    rM   r<   r   r�  r!   r"   rN   rO   s   @@r   rM  rM  Q  s+   ù‡ € ñ÷
ó 
÷0?#ò ?#÷BPò Pr   rM  c                   óL   a a€ ] tR tRt oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚAudioProjectioni  z¢Projects audio features to LM hidden size.

Two branches: 'speech' and 'vision', each:
Linear(audio_dim, hidden_size) -> GELU -> Linear(hidden_size, hidden_size)
c                ód   <€ \         SV `  4        \        W4      V n        \        W4      V n        R # r   )r;   r<   ÚAudioProjectionBranchÚspeechÚvision©r   Ú	audio_dimÚhidden_sizer>   s   &&&€r   r<   ÚAudioProjection.__init__  s&   ø€ Ü‰ÑÔÜ+¨IÓCˆŒÜ+¨IÓCˆŽr   c                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# )r	   r
   Úmoder   )r   r   r%   )r   r   s   "€r   r   ÚAudioProjection.__annotate__  s/   ø€ ÷ Añ A™"Ÿ(™(ð A©#ð A¹R¿X¹Xñ Ar   c                ó~   € VR 8X  d   V P                  V4      # VR8X  d   V P                  V4      # \        RV 24      h)r‡  rˆ  zUnknown projection mode: )r‡  rˆ  Ú
ValueError)r   r
   rŽ  s   &&&r   r   ÚAudioProjection.__call__  sA   € Ø�8ÔØ—;‘;˜q“>Ð!Ø�XÔØ—;‘;˜q“>Ð!äÐ8¸¸Ð?Ó@Ð@r   )r‡  rˆ  )r‡  rL   rO   s   @@r   r„  r„    s"   ù‡ € ñõD÷
A÷ Aò Ar   r„  c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )r†  i"  z<Single branch of audio projection: Linear -> GELU -> Linear.c                ó˜   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W"R R7      V n        R# )TrS   N)r;   r<   r.   rU   Úproj_0Úproj_2r‰  s   &&&€r   r<   ÚAudioProjectionBranch.__init__%  s2   ø€ Ü‰ÑÔä—i’i 	¸TÔBˆŒÜ—i’i ¸tÔDˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r   r   )r   r   s   "€r   r   Ú"AudioProjectionBranch.__annotate__+  r   r   c                óv   € V P                  V4      p\        P                  ! V4      pV P                  V4      pV# r   )r•  r.   r(   r–  r   s   &&r   r   ÚAudioProjectionBranch.__call__+  s.   € Ø�K‰K˜‹NˆÜ�GŠG�A‹JˆØ�K‰K˜‹NˆØˆr   )r•  r–  rL   rO   s   @@r   r†  r†  "  s   ù‡ € ÙFõE÷÷ ð r   r†  )r'   ) rM   rä   Útypingr   Úmlx.coreÚcorer   Úmlx.nnr.   Únumpyrì   r8  r   ÚModuler   r2   r4   rQ   rc   rs   r„   r˜   r¨   rÌ   râ   rü   r  r  r5  rM  r„  r†  r   r   r   Ú<module>r¢     sD  ðñ	ó Ý å Ý Û å ô!ˆB�I‰Iô !÷

ô'ˆ"�)‰)ô 'ô&	(�—	‘	ô 	(ô�r—y‘yô ôL�"—)‘)ô ô4&˜rŸy™yô &ô\M�—‘ô Môj,"˜2Ÿ9™9ô ,"ôh 2§9¡9ô ôH0 §¡ô 0ô@	;˜BŸI™Iô 	;ô"�2—9‘9ô ô6S˜"Ÿ)™)ô Sôv<"˜BŸI™Iô <"ôHt�r—y‘yô tôxA�b—i‘iô Aô*˜BŸI™Iö r   