+ G-j^RIHt^RIt^RIHt^RIHt^RIH t H t ^RI H t H t HtHt^RIHt]!]4t!RR 4t!R R ]4tR#) ) getLoggerN)FusionAttentionMaskFormat) FusionUtils NumpyHelper) NodeProto TensorProtohelper numpy_helper OnnxModelcfa]tRt^toRtV3RlRltV3RlRltRtRtV3RlR lt R t Vt R #) AttentionMask2 Fuse Attention subgraph into one Attention node. c <V^8dQhRS[/#)modelr )format __classdict__s"z/Volumes/fast/ai/experiments/nudenet-smoke/.venv/lib/python3.14/site-packages/onnxruntime/transformers/fusion_attention.py __annotate__AttentionMask.__annotate__s77i7cWn/Vn/Vn\V4Vn\ P VnVP4Vn R#N) r mask_indice mask_castedrutilsr MaskIndexEnd mask_formatget_opset_version opset_version)selfrs&&r__init__AttentionMask.__init__sB  ' .;;"446rc <V^8dQhRS[/#)rr!r)rrs"rrr s''+>'rcWnR#r)r!)r$r!s&&rset_mask_formatAttentionMask.set_mask_format s&rctWP9dW PV,8XgQhW PV&R#r)r)r$mask mask_indexs&&&rset_mask_indiceAttentionMask.set_mask_indice#s3 ## #!1!1$!77 77!+rcv\VP4^8gQh\\VP44#r)lenrnextiter)r$s&rget_first_maskAttentionMask.get_first_mask(s14##$q(((D))*++rc4<V^8dQhRS[RS[R,/#)rmask_2dreturnNstr)rrs"rrr,s88C8C$J8rc VP\P8XdR#WP9dVPV,#VPP V4'dVP PV4wr#MVP PV4wr4RpV'dW0PV&VP\P8XdW0PV&V#VPPR4pVP^ 8d\P!RV.V.VPPRR4R7pVPP!\P"!R^.4\P"!R^4.4MRpVPP%V4fEVPP'\P(!V\*P,^.^.R R 74\P!RW7.V.VPPRR4R7pVPP!\P"!R^4.4VPP/V4WPPV&V#) NTr- ReduceSum MaskReduceSuminputsoutputsnameaxeskeepdimsort_const_1_reduce_sum_axesFrB data_typedimsvalsraw)r!rNoMaskrrfind_graph_inputrcast_graph_input_to_int32cast_input_to_int32rrcreate_node_namer#r make_node attributeextendmake_attributeget_initializeradd_initializer make_tensorr INT64add_node)r$r8casted input_name _cast_node output_namemask_index_node axes_names&& r process_maskAttentionMask.process_mask,s$   299 9 && &##G, , :: & &w / /!%!E!Eg!N FJ%)ZZ%C%CG%L "JF (2  W %   2@@ @(2  W % jj11,?    "$.."|$ ZZ00oN O  % % , ,f.C.CFQC.PRXRgRghrtuRv-w x6Izz)))4< **&&&"-"3"3SS! %..".$ ZZ00oN O  % % , ,f.C.CJPQ.R-S T O,$/!r)rr!rrr#rN) __name__ __module__ __qualname____firstlineno____doc__r%r)r.r5r___static_attributes____classdictcell__)rs@rrrs277'', ,88rrcaa]tRt^gtoRtRRRRR.3V3RlV3RllltV3RlR ltV3R lR ltV3R lR ltV3RlRlt V3RlRlt V3RlRlt V3RlRlt V3RlRlt RV3RlRlltRV3RlRlltRtRtVtV;t#) FusionAttentionrNFSkipLayerNormalizationLayerNormalizationcb<V^8dQhRS[RS[RS[RS[R,RS[RS[RS[S[,/#) rr hidden_size num_headsattention_maskNuse_multi_head_attention!disable_multi_head_attention_biassearch_op_types)rintrboollistr;)rrs"rrFusionAttention.__annotate__lsZ%%%% % &, % #' %,0%c%rc<V'dRMRp\S V`WV4W nW0nV'dTM \ V4VnWPnW`nRVnRVn RVn RVn RVn R#)MultiHeadAttention AttentionNT) superr%rmrnrrorprqmask_filter_valuenum_heads_warninghidden_size_warning shape_infershape_infer_done) r$rrmrnrorprqrrattention_op_name __class__s &&&&&&&& rr%FusionAttention.__init__lst5M0R] ?C&"0>nMRWDX(@%1R.!%"&#'  $rc<<V^8dQhRS[RS[S[S[3,/#)rconcatr9r tuplers)rrs"rrrvs&00 0eTWY\T\o0rc \VP4^8XdVPPVP^,4pVPPVP^,4p\ V\ P 4'dcVP^8XdR\ V\ P 4'd2VP^8Xd!V^,V^,V^,,3#VPVP3#)a Detect num_heads and hidden_size from Concat node in the following subgraph: SkipLayerNormalization or EmbedLayerNormalization / | MatMul Shape | | Add Gather(indices=0) | | | Unsqueeze | | | Concat (*, -1, 12, 64) | / Reshape | Transpose ) r2inputrget_constant_value isinstancenpndarraysizernrm)r$rrn head_sizes&& r)get_num_heads_and_hidden_size_from_concat9FusionAttention.get_num_heads_and_hidden_size_from_concats$ v||  ! 55fll1oFI 55fll1oFI9bjj11NNa'y"**55NNa' |Yq\IaL%@@@~~t////rc<<V^8dQhRS[RS[S[S[3,/#)r reshape_qr9r)rrs"rrrvs$,&,&y,&U3PS8_,&rc VPPVP^,4pVfVPPV^4pVe#VPR8XdVP V4#\ PRVP^,4VPVP3#\V\P4'd-\V4^8wgV^,^8:gV^,^8:d/\ PRV4VPVP3#V^,pV^,pWE,pVP^8dKW@P8wd;VP'd)\ PRVPV4RVnVP^8dKW`P8wd;VP 'd)\ PRVPV4RVnWF3#)zDetect num_heads and hidden_size from a reshape node. Args: reshape_q (NodeProto): reshape node for Q Returns: Tuple[int, int]: num_heads and hidden_size Concatz%s is not initializer.zGq_shape_value=%s. Expected value are like [0, 0, num_heads, head_size].z>--num_heads is %d. Detected value is %d. Using detected value.Fz@--hidden_size is %d. Detected value is %d. Using detected value.)rrr get_parentop_typerloggerdebugrnrmrrrr2r|warningr})r$r q_shape_valuerrnrrms&& rget_num_heads_and_hidden_size-FusionAttention.get_num_heads_and_hidden_sizes 55iooa6HI  ZZ**9a8F!fnn&@EEfMM LL19??13E F>>4#3#33 3M2::66=!Q&a A%q)9Q)> LLbdq r>>4#3#33 3!!$ !!$ + >>A )~~"=%%%TVZVdVdfo*/&   a K3C3C$C'''VX\XhXhju,1(%%rc <V^8dQhRS[/#radd_qk)r )rrs"rrrvsYrcVP'g)VPPRR7VnRVnVPfR#VPP VP ^,4pVPP VP ^,4pVeVf\ PRV4R#W#8wd\ PRV4R#VP ^,#)T)updateNzone of the inputs of %s is Nonez)the shape of two inputs of %s is not same)rrinfer_runtime_shaper~get_edge_shaperrr)r$r input_0_shape input_1_shapes&& rget_add_qk_strFusionAttention.get_add_qk_strs$$$#zz==T=JD $(D !    #((77 QH ((77 QH  M$9 LL:F C  ) LLDf M||Arc <V^8dQhRS[/#rr:)rrs"rrrvs  S rcaVR,o\\V3RlVP44p\V4^8XdS#\V4^8XgQhVPP R4p\ P!R\VP4Uu.uFqANK upS.V^R7pVPPV4VPVPV&S#uupi)_maskc0<VP^,S8H#r1)output)nodemask_output_names&r0FusionAttention.reshape_add_qk..st{{1~AQ/Qrrr@rArBaxis) rufilter nodes_to_addr2rrOr rPrangernappendthis_graph_namenode_name_to_graph_name)r$r concat_nodeconcat_node_name_concat_add_qk_fp32rs&& @rreshape_add_qkFusionAttention.reshape_add_qks"G+6"QSWSdSdef { q # #;1$$$::66x@#-- $)$..$9:$9qF$9:%&!     !349=9M9M$$%56;s C! c,<V^8dQhRS[RS[RS[/#)rpast_kpast_vr9r:)rrs"rrrvs"555S5S5rc ZVPPR4pVPPR4pVR,PRR4pVR,PRR4p\P!RV.V.V^.R7p\P!RV.V.V^.R7pVP P V4VP P V4VPVPV&VPVPV&VPPR4p VPRR4PRR4PR R 4p \P!RWV.V .V ^R 7p VP P V 4VPVPV &V #) zConcatenate past_k and past_v inputs to create past_kv input. Args: past_k (str): name of past K value past_v (str): name of past V value Returns: kv_output_name (str): name of past KV value Unsqueeze_5d.r)r@rArBrCrz.valuez.kv_value_kvr) rrOreplacer rPrrrr) r$rrunsqueeze_k_nameunsqueeze_v_name k_5d_name v_5d_namek_5dv_5drkv_output_name concat_kvs &&& rrFusionAttention.concat_kvs ::66{C::66{Ce^,,S#6 e^,,S#6  8K!   8K!     &   &9=9M9M$$%569=9M9M$$%56 ::66x@%8@@cJRRS[]bc$$ )#$!     +9=9M9M$$%56rc,<V^8dQhRS[RS[RS[/#)rpresent_k_namepresent_v_namekv_noder:)rrs"rrrv8s'+K+Ks+KC+K#+Krc RRrTVPPV4pVPPV4pVfT\P!\P !^RR7VR7pVPP W`P4VfT\P!\P !^RR7VR7pVPP WpP4VPPR4pVPPR4p \P!RW4.V.V^R7p \P!RW5.V.V ^R7p VPPV 4VPPV 4VPVPV&VPVPV &R#) aSplit kv_node containing present KV values into separate present K and present V values. Args: present_k_name (str): name of output to store present K value in present_v_name (str): name of output to store present V value in kv_node (str): name of present KV values index_0index_1Nint64)dtype)rBGatherr)rrTr from_arrayrarrayrUrrOr rPrrr) r$rrrk_indexv_indexk_dimv_dim gather_k_name gather_v_name present_k present_vs &&&& rsplit_kvFusionAttention.split_kv8ss%i **73 **73 = ++BHHQg,FWUE JJ & &u.B.B C = ++BHHQg,FWUE JJ & &u.B.B C 33H=  33H= $$ %#$  $$ %#$     +   +6:6J6J$$]36:6J6J$$]3rc b<V^8dQhRS[RS[R,RS[R,RS[RS[R,/#)rq_addk_addNv_add name_prefixr9)r r;)rrs"rrrvesL44    T  rcVPPVP^,4;'g-VPPVP^,4p\P!V4p\ P !V4p\ P !V4pVewVPPVP^,4;'g-VPPVP^,4p \P!V 4pVewVPPVP^,4;'g-VPPVP^,4p \P!V 4p\ P!WgV3^R7p ^\ P!VP4,p VR,p VPV VPV .V R7V #)r _qkv_biasrBrGrHrI) rrTrrto_arrayr zeros_likestackprodshaperUrG)r$rrrrq_biasqbkbvbk_biasv_biasqkv_bias qkv_bias_dim bias_names&&&&& rcreate_combined_qkv_bias(FusionAttention.create_combined_qkv_biases++EKKN;iitzz?Y?YZ_ZeZefgZh?i  ! !& ) ]]2  ]]2   ZZ// A?mm4::C]C]^c^i^ijk^lCmF%%f-B  ZZ// A?mm4::C]C]^c^i^ijk^lCmF%%f-B88RRLq1277288,, +-  &&  rcz<V^8dQhRS[RS[RS[RS[RS[R,RS[R,RS[S[S[S[3,/#) rq_matmulk_matmulv_matmulrrNrr9)r r)rrs"rrrvspM,M,M,M, M,  M, 4 M,4M, y)Y. /M,rc VPPR4pVP^,VP^,8Xd*VP^,VP^,8XgQhVPPVP^,4pVPPVP^,4p VPPVP^,4p \P !V4p \P !V 4p \P !V 4p V P V P 8XdV P V P 8XgQhV P ^,p\P!WV 3^R7PV^V,34pVR,pVPVVPVP ^,VP ^,.VR7VR,p\P!RVP^,V.V.VR7pVPVPV&V.pVR,pVPV\ P"^.^.RR 7VR ,pVPV\ P"^.V.RR 7VR ,pVPV\ P"^.^V,.RR 7VR ,pVPV\ P"^.^V,.RR 7VR ,pVPV\ P"^.R.RR 7VR,p\P!RVVVV.V.VPPR4R7pVPVPVP$&VR,p\P!RVVVV.V.VPPR4R7pVPVPVP$&VR,p\P!RVVVV.V.VPPR4R7pVPVPVP$&TpTp Tp!VP'VVV.4VP('EdVeVPPVP^,4'd^M^p"\P*!\P !VPPVPV",444'dMVVP^V", &TpVP-V4VPVPVP$&VeVPPVP^,4'd^M^p"\P*!\P !VPPVPV",444'dMVVP^V", &Tp VP-V4VPVPVP$&VeVPPVP^,4'd^M^p"\P*!\P !VPPVPV",444'dMVVP^V", &Tp!VP-V4VPVPVP$&VP.P'V4VV V!3#)a9Create packed QKV MatMul node before MultiHeadAttention node. This is for the scenario where an Attention node should be created but cannot be created because past_key and past_value are separate inputs and not one concatenated input. Args: q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size) k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) q_add (NodeProto): name of Add from Q path k_add (NodeProto): name of Add from K path v_add (NodeProto): name of Add from V path Returns: q_output (NodeProto): Slice node for Q k_output (NodeProto): Slice node for K v_output (NodeProto): Slice node for V MatMulr _qkv_weightr_qkv_outr?_q_start_indexFrF_k_start_index_v_start_index_end_of_qkv_index_qkv_last_axis_q_outSlice_k_out_v_out)rrOrrTrrrrrreshaperUrGr rPrrr rWrBrRrqanyrr)#r$rrrrrrmatmul_node_nameq_weightk_weightv_weightqwkwvwd qkv_weightqkv_weight_nameqkv_matmul_output qkv_matmul qkv_nodes q_slice_name k_slice_name v_slice_nameend_of_qkv_nameqkv_last_axis_nameq_slice_outputq_slicek_slice_outputk_slicev_slice_outputv_sliceq_outputk_outputv_outputinitializer_inputs#&&&&&&& rcreate_packed_qkv_matmul_node-FusionAttention.create_packed_qkv_matmul_nodes74 ::66x@~~a HNN1$55(..:Kx~~^_O`:```::--hnnQ.?@::--hnnQ.?@::--hnnQ.?@  ! !( +  ! !( +  ! !( +xx288#BHH(<<< HHQKXXrrl3;;QAJG *]:  ((""1%z'7'7':;  "-z9%% NN1%7&'!  :>9M9M$$%56L (*::  ,+:K:KSTRU]^\_ejk'*::  ,+:K:KSTRU]^\_ejk'*::  ,+:K:KSTRU]^ab]b\cino*-@@ /[=N=NVWUX`ade`e_flqr-0@@ "4 @Q@QYZX[cebflqr)H4"" %|\CUV#$,,W5   6:5I5I$$W\\2)H4"" %|\CUV#$,,W5   6:5I5I$$W\\2)H4"" %|_FXY#$,,W5   6:5I5I$$W\\2'7G45  1 1 1 )-)C)CEKKPQN)S)SAYZ!66+..tzz/I/I%++VgJh/ijkk9GEKK$5 56$H$$U+?C?S?SD00< )-)C)CEKKPQN)S)SAYZ!66+..tzz/I/I%++VgJh/ijkk9GEKK$5 56$H$$U+?C?S?SD00< )-)C)CEKKPQN)S)SAYZ!66+..tzz/I/I%++VgJh/ijkk9GEKK$5 56$H$$U+?C?S?SD00<   +8++rc%<V^8dQhRS[RS[S[,R,RS[S[,R,RS[RS[R,RS[R,RS[R S[R S[R S[R S[R S[RS[RS[RS[RS[RS[RS[R,/#)rrrNrrrrrnrmrkey_padding_maskrunidirectionalrrrr packed_qkvr9)r r;rsrt)rrs"rrrvswwwc/D(wc/D( w  w 4 w4wwwwwwwww !w"#w$%w& T 'wrc ~V^8gQhV^8d&W,^8wd\PRW4R#VPP4PUu0uFpVP kK ppVPP R4p.pV'ddVPVVVVVV4wpppVPVP^,VP^,VP^,.4EMf\V\4'd\V\4'dVP'dIVPVP^,VP^,VP^,.4MVPVP^,VP^,VP^,.4M\V\4'd\V\4'dkVV9ddVV9d]VP'd&VPVP^,W#.4M'VPVP^,W#.4MR#VP'g&VPWEVV4pVPV4MVPR4V 'dV'dVPWW.4M"V 'g V 'dVPW.4V .pV'dV'dVPVV.4\ P"!RVVVR7pRVnVP&P\ P(!RV44V 'd:VP&P\ P(!R \+V 444VP-R4V#uupi) aCreate a MultiHeadAttention node. Args: q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size) k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size) v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size) q_add (NodeProto): name of Add from Q path k_add (NodeProto): name of Add from K path v_add (NodeProto): name of Add from V path num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning. output (str): output name of MHA key_padding_mask (str): name of key padding mask add_qk (str): name of add after Q x K' unidirectional (bool): whether to apply causal attention mask automatically or not past_k (str): name of past K value - (batch_size, num_heads, past_sequence_length, head_size) past_v (str): name of past V value - (batch_size, num_heads, past_sequence_length, head_size) present_k (str): name of present K value - (batch_size, num_heads, sequence_length, head_size) present_v (str): name of present V value - (batch_size, num_heads, sequence_length, head_size) packed_qkv (bool): whether to combine MatMuls from Q, K, V paths Note: This is for the scenario where an Attention node should be created but cannot be created because past_key and past_value are separate inputs and not one concatenated input. Returns: Union[NodeProto, None]: the node created or None if failed. 9input hidden size %d is not a multiple of num of heads %dNryrxr? com.microsoftrnr6)rrrgraphrrBrOr2rRrrr rqr;rrr rPdomainrQrSrsincrease_counter)r$rrrrrrrnrmrr5rr6rrrrr7rgraph_input_names mha_node_name mha_inputsr)r+r-r mha_outputsmha_nodes&&&&&&&&&&&&&&&&&& rcreate_multihead_attention_node/FusionAttention.create_multihead_attention_nodes^1}} ? 7A= LLTVa m37::3C3C3E3K3KL3K4TYY3KL 33K@  (,(J(J ) %GWg   w~~a0'..2CW^^TUEVW X ) , ,Hi1P1P555!!5<<?HOOA4F UV"XY!!8??1#5xq7I8??[\K]"^_ x % %8S))----555!!5<<?H"GH!!8??1#5x"JK 55555eE=YI   i (   b ! f   /H I    /8 9h     95 6##    *!!&"7"7 Y"OP     % %f&;&;eEV0P8P;\*P<!R\AVP>44.4V0#)aCreate an Attention node. Args: mask_index (str | None): mask input q_matmul (NodeProto): MatMul node in fully connection for Q k_matmul (NodeProto): MatMul node in fully connection for K v_matmul (NodeProto): MatMul node in fully connection for V q_add (NodeProto): Add bias node in fully connection for Q k_add (NodeProto): Add bias node in fully connection for K v_add (NodeProto): Add bias node in fully connection for V num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning. first_input (str): first input name output (str): output name add_qk_str (str): name of Add node after Q x K' causal: whether it is uni-directional mask. past_k (str): name of input for past K value past_v (str): name of input for past V value present_k (str): name of output to store present K value present_v (str): name of output to store present V value scale: scale before softmax Returns: Union[NodeProto, None]: the node created or None if failed. r9NTFzl is not an initializer. Please set do_constant_folding=True in torch.onnx.export to unblock attention fusionzInput hidden size (%d) is not same as weight matrix dimension of q,k,v (%d). Please provide a correct input hidden size or pass in 0:rNNrryrrrzVMultiHeadAttention does not support relative_position_bias: cannot fuse the attention.rxr?r:z.key_keyrrr;rnr6rJqkv_hidden_sizesr{)!rrrrTrprintrrrrrr concatenaterrOrprUrGrsrrr rPr>rrrr=rQrRrSr{rK)5r$r-rrrrrrrnrmrGrrHrIrrrrrJhas_biasrrrrrrrrr qw_in_size kw_in_size vw_in_sizeis_qkv_diff_dims qw_out_size kw_out_size vw_out_sizeqkv_weight_dimrrrrrr q_bias_shape k_bias_shape v_bias_shapeattention_node_nameattention_inputsattention_node past_existspast_kvattention_outputs present_kvs5&&&&&&&&&&&&&&&&&&& rcreate_attention_node%FusionAttention.create_attention_nodes(\1}} ? 7A= LLTVa m =U]u}H::--hnnQ.?@::--hnnQ.?@::--hnnQ.?@!%tT 8ZZ// A?mm4::C]C]^c^i^ijk^lCmFZZ// A?mm4::C]C]^c^i^ijk^lCmFZZ// A?mm4::C]C]^c^i^ijk^lCmFf   >>!$%&gg   ! !( +  ! !( +  ! !( +xx288###XXa[ XXa[ XXa[ Z5:55555 ?{j8 NNJ  ! 88rxx #  ggbhhrl+ ggbhhrl+ ggbhhrl+  R 1=J(;6DN2r2,Q7J_N &* 8%%f-B%%f-B%%f-B77288,L77288,L77288,L<>;> >> >>;. ..>>2r2,Q?+l:\I 88RRLq9 </ "jj99+F,,,  (=8",, #n"56 !    (;6 **,'( !   ( ( ( uv"""#k1   % '' 3#--$'( N  ! !"6 7#m35=#k12   % '' 3 ''+ ++VK..8 ''0"$++B/ '' 3!' Y&..vr:BB62NVVWZ\_` !((4 iJ?#--')( N  ! !+ . /  '')>)>{I)V(WX   $ $ + +V-B-BCSUV-W,X Y    $ $ + +V-B-B7E-R,S T   $ $ + +&&'9KVa;bcd   ! ! -  $ $ + +V-B-BCVX]^b^t^tXu-v,w xrcpTpTpVPR8XdXVPPVR^4pVeTpM3VPPVP^,4eTpMR#VPP V.RO.RO4pRpVeVwrrM-VPP V.R O.R!O4pVeVwrrMR#.p \ VP4F[wrW9d VPPV4fK*W^,P^,8XdKJV PV4K] \V 4^8wdR#V ^,pVPPVR^4pVeVVP^,,pVeA\V4^8Xd1V^,pVPR8XdVP^,pMuR#Ve%\V4^8XdVP^,pMKR#VPR"9d9VV,pVF)pVPR8XgKVP^,pK+ VV9dHVV,pVPR8Xd.\VP4^8XdVP^,pVV,pVUu.uFpVPNK ppVPR4^8wdR#VPP V .R#O.R$O4pVf\PR4R#Vwp ppRpRpRpRpR .R%O.R&O3R .R'O.R&O3R .R(O.R)O3R .R*O.R+O3R .R,O.R-O3R.R.O.R/O3/pRp VP4F^wp!p"VPP V V"^,V"^,4p V fK7V!R 8XdRpMV!R 8XdRpMV!R 8XdRpM V!R8XdRpM V f\PR4R#Rp#Rp$Rp%Rp&V'd V wp p%p$p M7V'd V wp p#p%p$M'V'dV wp p$MV'd V wp p#p$p&p MV wp p#p p$T&;'gT$p&VPP V&.R#O.R0O4p'V'f=VPP V&.R1O.R2O4p'V'f\PR4R#V'R3,p(V'R4,p)V'R5,p*T$p+V'dBVPP V$RR.^R.4p,V,f\PR4R#V,wp+p VPP T+.R#OV'd^M^^^R.4p-V-f=VPP V$.R6O.R7O4p-V-f\PR4R#V-R4,p.V-R5,p/Rp0Rp1V'd8VPPV%.R8O.R-O3.R9O.R-O3.R:O.R;O3.V4wp p0p MV'dbVPPV%.RO3.R?O.R@O3.RAO.RBO3.RCO.RDO3.V4wp p0p V'gV0f\PR4R#V'g\V04^8dVPP!V0^,4wp p2V2eG\#V2\$P&4'd'V2P(^8XdV2P+4^8dR#V2P+4RE8wdV2P+4VnVP^,V8XEdV*P^,V8XEdV/P^,V8XEdV'g4VP.P1V0R5,P^,4MRp3VfX MT p4VP3V(4wp5p6V5^8:gV6^8:d\P5R4R#VP7V3V*V/VV)V.VV5V6VV4P^,V1R7 p7V7fR#VP8PV74VP:VP<V7P>&VeVP^,p8RV8,p9VPARV8,\BPD^.^^V5\GV6V5, 4.RR7p:VPPI\JPL!RV4P^,V:P>.V9.RV8,4VP:4V9VP^&VPNPQV4W.4VPNPQV 4VPNPQVPR'gT'MV'RR54VPNPQVPR'gT-MV-RR54VPNPQVPR'gTMVRR54RVn*R#R#R#R#uupi)FrkAddNrReshapeMulrjz&fuse_attention: failed to match v pathFpath1path2path3path4path5sdpaSqrtTz'fuse_attention: failed to match qk pathz&fuse_attention: failed to match q pathz/fuse_attention: failed to match mul sqrt q pathz&fuse_attention: failed to match k pathr:z6fuse_attention: failed to verify shape inference of %sz)fuse_attention: failed to match mask pathzmFailed to detect num_heads and hidden_size for Attention fusion. Please specify those parameters in argument.) r-rrrrrrrnrmrGrrHedge_modified_shape_modified_tensorrFreshape_modified_)rgrrh Transposer)NNrrr)rgEinsumrtr)rNrr)rkrj)rtrhrgr)rrrN)SoftmaxrgDivr)rrNr)rvrgrir)rvWhererrw)rrrr)rvrgrxr)rrrr)rvrwr)rrr)rvrgrrirp)rrNrr)rrrN)rwrtrhrgr)rrrrNr)rtrtrhrgr)rrrrN)ExpandrhEqual)r|rr)Castr{rhr|)rrrr)r}r|rr)riSubr}rr)Nrrrr)rir~rr)Nrrr)rxr}r~r{rr)Nrrrrr)rxr}r~r}r{rr)Nrrrrrri)+rr match_parentrLrmatch_parent_path enumeraterrr2countrritemsmatch_parent_pathsrget_constant_inputrrrritemr{ror_rrrdrrrrBrUr rWrsrXr rPnodes_to_removerRrp prune_graph);r$rinput_name_to_nodesoutput_name_to_nodenormalize_node start_nodeadd_before_layernormr" einsum_noder reshape_qkv transpose_qkv matmul_qkv other_inputs_i node_input root_inputmul_before_layernorm mul_childrenlayernorm_nodechildrenchild parent_nodechildren_typesv_nodesadd_vmatmul_v is_distillis_distill_addis_no_mask_attentionis_sdpaqk_pathsqk_nodeskvr matmul_qkwhere_qkafter_qq_nodesradd_qmatmul_qafter_k mul_k_nodesk_nodesadd_kmatmul_k mask_nodesrHmul_valr-attention_last_node q_num_heads q_hidden_sizenew_node unique_indexnew_edge shape_tensors;&&&& rfuseFusionAttention.fusevs #  ! !%9 9#'::#:#:>5RS#T #/1 ,,^-A-A!-DEQ , JJ00  ? !   =F :Q z 44DoI$>G;  ' (8(89NB4::..z:Bq\0033    +: |  ! !!_  $zz66z5!L  +./C/J/J1/MNL'C ,=,B!-a!))-AA!/!6!6q!9J)c,.?1.D188;   # #'W W*:6H!==$88!&aJ" , ,-j9K""&>>3{GYGYCZ^_C_(//2 &z25=>XE%--X>    )Q . **..z;dfuv ? LLA B ")Auh $ 9?K 9?K ;\J ;\J 2I> @BTU  NN$DAqzz33J!adKHG|! g!%g'+$f %   LLB C   *2 'Q)Q /7 ,Q) ! ( Q9 19 .Q 7A(0 %Q9&&Y**..w8acrs ?jj22@"G  EFBK  2; **66y5&/TUW[S\]K" NO&LWa**.. >gSTVWYZ\`@a  ?jj22F"G  EF 2;  #zz<<3Y?8)D;\J $  Az1#zz<<@,O8)D$   Az1!!008 %LL!Y[ab ! #zz<<EGYZ=OQShiY[st $  Az1$ (: LLD E #J!(;66z!}EJAw"7BJJ77GLLA>!  *x~~a/@J/NS[SaSabcSdhrSrZn,,99*R.:N:Nq:QRtxJ1<1D+- )-)K)KI)V &Ka=A#5C11%!!!%)&*11!4%2 H    $ $X .:>:N:ND ( ( 7&*003 +l:#330<?)//Q S1L-MN 4   ##$$!,33A6 8I8IJ! +l:  (((0 !!!$  ' ')@T%U%%400>,&,&\*  255n+K+KZrsD .1== 8 SSlG$fG$r