iddlZddlmZmZmZddlZdgZdejdejfdZ ddejdejd ejdejd ejd eejdeejfd Z d e dejjfdZ dee dedeeefdZdeedeededej$dejf dZGddejjZGddejjZGddejjZGddeZy)N)ListOptionalTupleEmformerlengthsreturnc|jd}ttj|j }tj ||j |jj|||jdk\}|S)Nr)devicedtype) shapeinttorchmaxitemaranger r expand unsqueeze)r batch_size max_length padding_masks o/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/torchaudio/models/emformer.py_lengths_to_padding_maskr sqq!JUYYw',,./J<< 7>>W^^J   1 L  utterance right_contextsummarymemsleft_context_keyc|jd|jdz|jdz}|jd}|dk(rd}|S|tj|jz |jdz } ||jdnd} ||jdz| z| z} t | }|S)Nrr )r)sizerrrr) rrrrrrTBrright_context_blocks_lengthleft_context_blocks_lengthklengthss r_gen_padding_maskr's 1 q 11GLLOCA1AAv  '(%))G*<*@*@*B&BW\\RS_&T#AQA]%5%:%:1%=cd"TYYq\),GGJdd/A r activationc|dk(rtjjS|dk(rtjjS|dk(rtjj St d|)NrelugelusiluzUnsupported activation )rnnReLUGELUSiLU ValueError)r(s r_get_activation_moduler2's]Vxx}} v xx}} v xx}}2:,?@@rweight_init_scale_strategy num_layerscH|t|Dcgc]}dc}S|dk(r2t|Dcgc]}dtj|dzz c}S|dk(r/t|Dcgc]}dtjdz c}Std|cc}wcc}wcc}w)N depthwiseg?r constantz-Unsupported weight_init_scale_strategy value )rangemathsqrtr1)r3r4_ layer_idxs r_get_weight_init_gainsr>2s!)#J/0//00 #{ 2@Ej@QR@Q9dii A ..@QRR #z 149*4EF4Eydiil"4EFFHIcHdeff 1RFs B"B%B col_widthscol_masknum_rowsr c t|t|k7r tdt||Dcgc]7\}}|rtj|||ntj |||9}}}tj |dScc}}w)Nz0Length of col_widths must match that of col_maskr r dim)lenr1ziproneszeroscat)r?r@rAr col_width is_ones_col mask_blocks r_gen_attention_mask_blockrN=s :#h-'KLL '**h&? '@ "I{   8Yv6 [[9V < ='@  99ZQ '' sA,>$?!).D:S3T)U&**+IJPPXY_`Pa UEzrattention_weightsattention_maskrc |j}|j|jd|j}|j d}|j d|j z}||j ||j |d}|j|jdjdjtj|j}|j ||j z|d}tjjj|dj|}tjjj|t|j|jS)Nrr r8rD)ptraining)float masked_fillrrVr!rRviewtorboolr- functionalsoftmaxtype_asrSrv)rdrqrrrattention_weights_floatr"r#attention_probss r_gen_attention_probsz'_EmformerAttention._gen_attention_probssG #4"9"9";"9"E"EnF^F^_`Facgctct"u  " "1 %  " "1 % 7  #&=&B&B1dnnVWY[&\ #&=&I&I&&q)33A699%**EtGXGX' #'>&B&B1t~~CUWXZ\&] #((--556MSU5V^^_pqxx""**?eDLL>Q\`\i\i*jjrrrrrrleft_context_valc d|jd} |jd|jdz|jdz} |jtj|||g} |j tj|||gj dd\} } ||| tj |jz |jdz }tj| d|jd|z|| |jd|zdg} tj| d|jd|z|| |jd|zdg} | | | fDcgc]W}|jjd| |jz|j|jzjddYc}\}}}tj||jz|jdd}t||||||}|j!|||}tj||}|j"| |jz| |j|jzfk7r t%d|jddjj| | |j}|j'|}|jd}|d| |z }|| |z d}|j(rtj*|}ntj,|dd }||| | fScc}w) Nr rr8rhrtz+Computed attention has incorrect dimensionsi )minr)r!r_rrJr^rjrr contiguousryrRrQ transposebmmr\r'rr AssertionErrorr`rUtanhclamp)rdrrrrrrrrrr#r"queryrnror$tensorreshaped_query reshaped_keyreshaped_valuerqrr attentionoutput_right_context_memsrkoutput_right_context output_memss r _forward_implz _EmformerAttention._forward_impls$ NN1    q !INN1$5 5 Q G!!%))]Iw,O"PQ**599dM95U+VW]]eflm]n U  ',<,H*+eii.@.D.D.F*FVW*X '))D$))A,)DDE$ ! 'BBDECIIFDIIaL+FFG$$))A,)DDFGE!#u-8 -     $ $RT^^);T^^t~~=] ^ h hijlm n-8 4 n"IInt||&C\E[E[\]_`Eab)M7GUY[kl 334E~WcdIIo~> ??   NNdnn ,  !!NO O''1-88:??1dnnU %)MM)$<! a89M1~;MN/N0B0DE   **[1K++ksCK#[#u<TYYq\>)**.*<*<      --+= + ' S%    ! }11!446 7 $))A,!3!3A!668 9   r)NFח)NN)__name__ __module__ __qualname____doc__rrwrr{r[rTensorrrprrrjitexportr __classcell__res@rrPrPLs  ,0!"[[[ [ #5/ [  [[<ELL u||]b]i]iOiIjk <<k ku||, k  k64837G=<<G=G=|| G=  G= ll G= G=#5<<0G=#5<<0G= u||U\\5<<E FG=R#(<<#(#(|| #(  #( ll #( #( u||U\\) *#(J YY; <<; ; || ;  ; ll ;  ,,;  ,,;  u||U\\5<<E F; ; rrPceZdZdZ d%dededededededed ed eed ed effd Z dedee jde e jfdZde e jdee je je jffdZde jde jdede jde e jde e jf dZde jde jde jde jfdZde jde jdee je jffdZde jde jde jdee je jffdZde jde jde jde jd ee jdee je jff d!Zde jde jde jde jdee e jdee je je e jff d"Zde jde jde jde jd e jdee je je jff d#Ze j0j2de jde jde jdee e jde jdee je je e je jff d$ZxZS)&_EmformerLayera$Emformer layer that constitutes Emformer. Args: input_dim (int): input dimension. num_heads (int): number of attention heads. ffn_dim: (int): hidden layer dimension of feedforward network. segment_length (int): length of each input segment. dropout (float, optional): dropout probability. (Default: 0.0) activation (str, optional): activation function to use in feedforward network. Must be one of ("relu", "gelu", "silu"). (Default: "relu") left_context_length (int, optional): length of left context. (Default: 0) max_memory_size (int, optional): maximum number of memory elements to use. (Default: 0) weight_init_gain (float or None, optional): scale factor to apply when initializing attention module parameters. (Default: ``None``) tanh_on_mem (bool, optional): if ``True``, applies tanh to memory elements. (Default: ``False``) negative_inf (float, optional): value to use for negative infinity in attention weights. (Default: -1e8) rQrRffn_dimsegment_lengthrSr(left_context_lengthmax_memory_sizerTrUrVc Nt |t|||| | | |_tj j ||_tj j||d|_ t|} tj jtj j|tj j||| tj j |tj j||tj j ||_tj j||_tj j||_||_||_||_||_|dkD|_y)N)rQrRrSrTrUrVT kernel_sizestride ceil_moder)rZr[rPrrr-DropoutrS AvgPool1d memory_opr2 Sequential LayerNormr]pos_fflayer_norm_inputlayer_norm_outputrrrrQuse_mem)rdrQrRrrrSr(rrrTrUrVactivation_moduleres rr[z_EmformerLayer.__init__Rs= +-#%  xx''0 ++~im+n2:>hh)) HH  y ) HHOOIw /  HH  W % HHOOGY / HH  W %   !& 2 29 =!&!3!3I!>#6 ,."&* rrr rcjtj|j||j|}tj|j||j|}tj|j||j|}tjd|tj |}||||gS)NrCr r)rrIrrQrint32)rdrr empty_memoryrr past_lengths r _init_statez_EmformerLayer._init_states{{4#7#7T^^\bc  ;;t'?'?T^^djk ;;t'?'?T^^djkkk!Zu{{6R .0@+NNrstatecT|dddj}t|j|}t|jt j ||j z }|d|j|z d}|d|j|z d}|d|j|z d}|||fS)Nrr r8)rrrrr:ceilr)rdrrpast_left_context_lengthpast_mem_lengthpre_memslc_keylc_vals r _unpack_statez_EmformerLayer._unpack_statesAhqk!n))+ #&t'?'?#M d22DIIkDL_L_>_4`a8D00?BDEq$225MMOPq$225MMOP''rnext_knext_v update_lengthrcbtj|d|g}tj|d|g}tj|d|g|j d|d<||jd|jz d|d<||jd|jz d|d<|d|z|d<|S)Nr r8rr)rrJrr r)rdrrrrrnew_knew_vs r _pack_statez_EmformerLayer._pack_states 58V,- 58V,-99eAh-.0D0D/D/FGaQ$*B*BBDEaQ$*B*BBDEa8m+a r rc_outputrrc|j|tj||gz}|j||z}|j |}|SN)rSrrJrr)rdrrrresults r_process_attention_outputz(_EmformerLayer._process_attention_outputsM i(599mY5O+PPV$v-''/ rc|jtj||g}||jdd|d|jdfSNr)rrrJr!)rdrrrs r_apply_pre_attention_layer_normz._EmformerLayer._apply_pre_attention_layer_normsY 00M9;U1VW ]//24 5 4}11!4 5  rcx|j|||}||jdd|d|jdfSr)rr!)rdrrrs r_apply_post_attention_ffnz(_EmformerLayer._apply_post_attention_ffnsJ229iW ++A.019=T}?Q?QRS?T3UUUrrrrcL| td|jr4|j|jdddjddd}n:t j dj |j|j}|j||||||\}}||fS)Nz;attention_mask must be not None when for_inference is Falser r8rr)rrrrrrr) r1rrpermuteremptyrzr r r) rdrrrrrrrrnext_ms r_apply_attention_forwardz'_EmformerLayer._apply_attention_forwards  !Z[ [ <<nnY%6%6q!Q%?@HHAqQGkk!n''iooiFVFV'WG NN') +  6&  rc &|,|j|jd|j}|j|\}}}|jr9|j |j dddj ddd} | dd} n:tjdj|j|j} |jj|||| |||\} } } } |j| | |jd||}| | |fS)Nr rCr8rr)rrrrrrr)rr!r rrrrrrrzr rrr)rdrrrrrrrrrrrrrs r_apply_attention_inferz%_EmformerLayer._apply_attention_infers =$$Y^^A%6y?O?O$PE#'#5#5e#< && <<nnY%6%6q!Q%?@HHAqQGbqkGkk!n''iooiFVFV'WG,0NN,@,@'##-A- ) 666  1BD%P&%''rc|j||\}}|j|||||\}} |j|||\} } | | | fS)a1Forward pass for training. B: batch size; D: feature dimension of each frame; T: number of utterance frames; R: number of right context frames; M: number of memory elements. Args: utterance (torch.Tensor): utterance frames, with shape `(T, B, D)`. lengths (torch.Tensor): with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in ``utterance``. right_context (torch.Tensor): right context frames, with shape `(R, B, D)`. mems (torch.Tensor): memory elements, with shape `(M, B, D)`. attention_mask (torch.Tensor): attention mask for underlying attention module. Returns: (Tensor, Tensor, Tensor): Tensor encoded utterance frames, with shape `(T, B, D)`. Tensor updated right context frames, with shape `(R, B, D)`. Tensor updated memory elements, with shape `(M, B, D)`. )rrr) rdrrrrrrlayer_norm_utterancelayer_norm_right_contextrroutput_utterancers rrz_EmformerLayer.forwardsrH  0 0M J $!%!>!>  $   "  ;261O1OPY[dfs1t..!5{BBrc|j||\}}|j|||||\}} } |j|||\} } | | | | fS)a2Forward pass for inference. B: batch size; D: feature dimension of each frame; T: number of utterance frames; R: number of right context frames; M: number of memory elements. Args: utterance (torch.Tensor): utterance frames, with shape `(T, B, D)`. lengths (torch.Tensor): with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in ``utterance``. right_context (torch.Tensor): right context frames, with shape `(R, B, D)`. state (List[torch.Tensor] or None): list of tensors representing layer internal state generated in preceding invocation of ``infer``. mems (torch.Tensor): memory elements, with shape `(M, B, D)`. Returns: (Tensor, Tensor, List[torch.Tensor], Tensor): Tensor encoded utterance frames, with shape `(T, B, D)`. Tensor updated right context frames, with shape `(R, B, D)`. List[Tensor] list of tensors representing layer internal state generated in current invocation of ``infer``. Tensor updated memory elements, with shape `(M, B, D)`. )rrr) rdrrrrrrrrr output_staterrs rrz_EmformerLayer.inferssR  0 0M J $/3/J/J '+CT50 , ; 261O1OPY[dfs1t..!5|[PPr)rr*rrNFr)rrrrrrwstrrr{r[rr rrrrrrrrrrrrrrrrrs@rrr?s0 #$ ,0!",+,+,+ ,+  ,+  ,+,+!,+,+#5/,+,+,+\OcO8ELL3IOdSXS_S_N`O(4 #5(% ell\a\h\h@h:i(    ll  ELL!  ell  << << ||     6;ll u||U\\) * VV27,,VOT||V u||U\\) *V !<<!!|| ! ll ! !. ! u||U\\) *!2(<<((|| ( ll ( U\\*+ ( u||U\\4 +== >(8-C<<-C-C|| -C ll -C  -C u||U\\5<<7 8-C^ YY-Q<<-Q-Q|| -Q U\\*+ -Q ll -Q u||U\\4 +=u||K L-Q-QrrcLeZdZ ddejj dedededef fd Zdejdejfd Z d ed ede efd Z dejdejfd Z dejdejde ejejffdZejj ddejdejdee e ejde ejeje e ejffdZxZS) _EmformerImplemformer_layersrrright_context_lengthrct||dkD|_tjj ||d|_||_||_||_ ||_ ||_ y)NrTr) rZr[rrr-rrrrrrr)rdrrrrrres rr[z_EmformerImpl.__init__Psj &* ++&!,  /#6 $8!,.rrfrc~|jd}tj||jz |jz }g}t |dz D]7}|dz|jz}||jz}|j |||9|j |||jz dtj|SNrr ) r r:rrrr9appendrrJ)rdrfr"num_segsright_context_blocksseg_idxstartends r_gen_right_contextz _EmformerImpl._gen_right_contextfs KKN99a$";";;t?R?RRS!X\*Gq[D$7$77E$333C ' 'eC(8 9+ ##E!d.G.G*G*I$JKyy-..rrutterance_lengthc tj||jz }|j}|j}||z}||z}t ||jz|z d}t |dz|jz|} |j|z} |jr:t ||jz d} |dz } | || z | |z ||| |z || |z || z g } | S||| |z || |z || z g} | Sr) r:rrrrrrrr)rdrrrrclcrc_startrc_end seg_startseg_end rc_lengthm_start mem_lengthr?s r_gen_attention_mask_col_widthsz,_EmformerImpl._gen_attention_mask_col_widthsqs(99-0C0CCD  & &  % %R<B$"5"55:A> w{d&9&99;KL--8 <<'D$8$88!z;XY[`[g[g%h"##$67+'.eii_(U_T4_(UVVZZ[`[e[efGKD H6)Vs G / G GG rc|jddd}|j|}|d|jd|jz }|j |}|j r6|j |jdddjdddddn9tjdj|j|j}|}|jD]}||||||\}}}|jddd|fS)aGForward pass for training and non-streaming inference. B: batch size; T: max number of input frames in batch; D: feature dimension of each frame. Args: input (torch.Tensor): utterance frames right-padded with right context frames, with shape `(B, T + right_context_length, D)`. lengths (torch.Tensor): with shape `(B,)` and i-th element representing number of valid utterance frames for i-th batch element in ``input``. Returns: (Tensor, Tensor): Tensor output frames, with shape `(B, T, D)`. Tensor output lengths, with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in output frames. r rr8Nrtr) rrr!rrrrrrrzr r r) rdrfrrrrrrrlayers rrz_EmformerImpl.forwards* aA&//6 EEJJqMD,E,EEF 11)<|| NN9,,Q15 6 > >q!Q G LQ""U\\"J ))E*/PTVd*e 'FM4*~~aA&//rstatescJ|jd|j|jzk7r8td|j|jzd|jdd|j ddd}|jd|jz }||d}|d|}t j ||jz d}|jr3|j|j dddj dddn9t jdj|j|j }|} g} t|jD]7\} } | j| |||dn|| |\} }} }| j!| 9| j ddd|| fS) aForward pass for streaming inference. B: batch size; D: feature dimension of each frame. Args: input (torch.Tensor): utterance frames right-padded with right context frames, with shape `(B, segment_length + right_context_length, D)`. lengths (torch.Tensor): with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in ``input``. states (List[List[torch.Tensor]] or None, optional): list of lists of tensors representing internal state generated in preceding invocation of ``infer``. (Default: ``None``) Returns: (Tensor, Tensor, List[List[Tensor]]): Tensor output frames, with shape `(B, segment_length, D)`. Tensor output lengths, with shape `(B,)` and i-th element representing number of valid frames for i-th batch element in output frames. List[List[Tensor]] output states; list of lists of tensors representing internal state generated in current invocation of ``infer``. r zIPer configured segment_length and right_context_length, expected size of z# for dimension 1 of input, but got .rr8N)rr)r!rrr1rrrrrrrzr r enumeraterrr)rdrfrrright_context_start_idxrroutput_lengthsrr output_statesr=rrs rrz_EmformerImpl.infers> ::a=D//$2K2KK K&&*&9&9D >q!Q GQ""U\\"J 24 )$*>*> ? Iu8= F9,= 9 5FM<   .!@~~aA& EEr)rrrr)rrrrr- ModuleListrr[rrrrrrrrrrrrrs@rrrOsj $%$% /,,//! / " /  /, / / /"c"S"UYZ]U^"H..%,,.`!0U\\!0ELL!0U5<