+
    G-j™( ã                   óL  € R t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIH	t	 ^ RI
Ht ^ RIt^ RIt^ RIt^ RIHtHt ^ RIHt ^ RIHtHtHt ^ RIHt ^ R	IHtHtHtHtHtH t H!t!H"t" ^ R
I#H$t$H%t%H&t&H't' ^ RI(H)t* ^ RI+H,t, ^ RI-H.t/ ^ RI0H1t1H2t2 ]Pf                  ! R4      t4 ! R R]4      t5RTR R llt6R R lt7R R lt8RUR R llt9RUR R llt:R R lt;R R lt<R  R! lt=R" R# lt>RVR$ R% llt?R& R' lt@RWR( R) lltAR* tBR+ tCR, tDR- R. ltER/ R0 ltFR1 R2 ltGR3 R4 ltH^ . 3R5 R6 lltIR7 R8 ltJR9 R: ltKRXR; R< lltLR= R> ltMR? R@ ltNRA RB ltORURC RD lltPRURE RF lltQRG tR]5P¦                  3RH RI lltTRJ RK ltURL tVRYRM RN lltWRTRO RP lltXRZRQ RR llt)]YRS8X  d
   ])! 4        R# R# )[aë  
This converts GPT2 or T5 model to onnx with beam search operator.

Example 1: convert gpt2 model with beam search:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx

Example 2: convert gpt2 model with beam search containing specific cuda optimizations:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx --use_gpu                       --past_present_share_buffer --use_decoder_masked_attention

Example 3: convert gpt2 model with beam search with mixed precision and enable SkipLayerNorm strict mode:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx --use_gpu -p fp16 --use_sln_strict_mode

Example 4: convert T5 model with beam search in two steps:
    python -m models.t5.convert_to_onnx -m t5-small
    python convert_generation.py -m t5-small --model_type t5                     --decoder_onnx ./onnx_models/t5-small_decoder.onnx                       --encoder_decoder_init_onnx ./onnx_models/t5-small_encoder.onnx          --output ./onnx_models/t5_small_beam_search.onnx

Example 5: convert T5 model with beam search. All in one step:
    python convert_generation.py -m t5-small --model_type t5 --output t5_small_beam_search.onnx

Example 6: convert T5 model with beam search containing specific cuda optimizations. All in one step:
    python convert_generation.py -m t5-small --model_type t5 --output t5_small_beam_search.onnx           --use_gpu --past_present_share_buffer --use_decoder_masked_attention

Example 7: convert MT5 model with external data file like mt5-base-beamsearch.onnx.data in below example.
    python convert_generation.py -m google/mt5-base --model_type mt5 --output mt5-base-beamsearch.onnx -e

Example 8: convert gpt2 model with greedy search:
    python convert_generation.py -m gpt2 --output gpt2_greedy_search.onnx --num_beams 1 --num_return_sequences 1

Example 9: convert gpt2 model with sampling:
    python convert_generation.py -m gpt2 --output gpt2_sampling.onnx --num_beams 1 --num_return_sequences 1 --top_p 0.6
N)ÚEnum)ÚPath)ÚAny)Ú	PrecisionÚsetup_logger)ÚNumpyHelper)Ú
GraphProtoÚ
ModelProtoÚTensorProto©Ú	OnnxModel)Ú
GPT2ConfigÚGPT2LMHeadModelÚGPT2TokenizerÚ	MT5ConfigÚMT5ForConditionalGenerationÚT5ConfigÚT5ForConditionalGenerationÚT5Tokenizer)ÚGraphOptimizationLevelÚInferenceSessionÚSessionOptionsÚget_available_providers)Úmain)ÚPRETRAINED_GPT2_MODELS)Úexport_onnx_models)ÚPRETRAINED_MT5_MODELSÚPRETRAINED_T5_MODELSÚ c                   ó2   a € ] tR t^Zt o RtRtRtR tRtV t	R# )ÚGenerationTypeÚbeam_searchÚgreedy_searchÚsamplingc                ó   € V P                   # ©N)Úvalue)Úselfs   &Ú|/Volumes/fast/ai/experiments/nudenet-smoke/.venv/lib/python3.14/site-packages/onnxruntime/transformers/convert_generation.pyÚ__str__ÚGenerationType.__str___   s   € Ø�z‰zÐó    © N)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú
BEAMSEARCHÚGREEDYSEARCHÚSAMPLINGr)   Ú__static_attributes__Ú__classdictcell__)Ú__classdict__s   @r(   r    r    Z   s   ø‡ € Ø€JØ"€LØ€H÷ð r+   r    c                óh   € V ^8„  d   QhR\         \        ,          R,          R\        P                  /# )é   ÚargvNÚreturn)ÚlistÚstrÚargparseÚ	Namespace)Úformats   "r(   Ú__annotate__r@   c   s-   € ÷ Qñ Qœ$œs�) dÕ*ð Q´h×6HÑ6Hñ Qr+   c                ó’  € \         P                  ! 4       pVP                  R4      pVP                  RRR\        RRP                  \        \        ,           \        ,           4      ,           R7       VP                  RR	\        R
. R˜ORRP                  . R˜O4      ,           R7       VP                  RR	\        \        P                  P                  RR4      RR7       VP                  RR	\        RRR7       VP                  RR	\        RRR7       VP                  RR	RRR7       VP                  R	R7       VP                  R4      pVP                  RR\        RR7       VP                  RR R	\        \        P                  P                  \        P                  P                  \        P                  P                  .R!R7       VP                  R"R#R	R$R%.R&R'7       VP                  R(R)R	RR*R7       VP                  R	R+7       VP                  R,R-R	RR.R7       VP                  R	R/7       VP                  R0R1R	RR2R7       VP                  R	R37       VP                  R4R5R	RR6R7       VP                  R	R77       VP                  R8R9R	RR:R7       VP                  R	R;7       VP                  R<R	RR=R7       VP                  R	R>7       VP                  R?4      pVP                  R@R	RRAR7       VP                  R	RB7       VP                  RCR	RRDR7       VP                  R	RE7       VP                  RFR	RRG7       VP                  R	RH7       VP                  RI\         R	^ RJRK7       VP                  RLR	RRMR7       VP                  R	RN7       VP                  ROR	RRPR7       VP                  R	RQ7       VP                  RRR	RRSR7       VP                  R	RT7       VP                  RUR	RRVR7       VP                  R	RW7       VP                  RXR	RRYR7       VP                  R	RZ7       VP                  R[R	RR\R7       VP                  R	R]7       VP                  R^R	RR_R7       VP                  R	R`7       VP                  Ra4      pVP                  Rb\         R	^RcRK7       VP                  Rd\         R	^2ReRK7       VP                  Rf\         R	^RgRK7       VP                  Rh\         R	^RiRK7       VP                  Rj\"        R	^RkRK7       VP                  Rl\"        R	^RmRK7       VP                  Rn\"        R	RoRpRK7       VP                  Rq\"        R	RoRrRK7       VP                  Rs\"        R	\#        Rt4      ) RuRK7       VP                  Rv\         R	^RwRK7       VP                  Rx\"        R	RyRzRK7       VP                  R{\         R	^ R|RK7       VP                  R}\         R	R™R~RK7       VP                  R\         R	R™R€RK7       VP                  R�\         R	R™R‚RK7       VP                  Rƒ4      pVP                  R„R	RR…R7       VP                  R	R†7       VP                  R‡R	RRˆR7       VP                  R	R‰7       VP                  RŠR	RR‹R7       VP                  R	RŒ7       VP                  R�R	RRŽR7       VP                  R	R�7       VP                  R�R	RR‘R7       VP                  R	R’7       VP                  R“R	\         ^R”R7       VP                  R•R	RR–R7       VP                  R	R—7       VP%                  V 4      pV# )šz–Parse arguments

Args:
    argv (Optional[List[str]], optional): _description_. Defaults to None.

Returns:
    argparse.Namespace: Parsed arguments.
zInput optionsz-mú--model_name_or_pathTzEPytorch model checkpoint path, or pretrained model name in the list: ú, )ÚrequiredÚtypeÚhelpz--model_typeFÚgpt2z*Model type (default is gpt2) in the list: )rD   rE   ÚdefaultÚchoicesrF   ú--cache_dirÚ.Úcache_modelsz%Directory to cache pre-trained models)rD   rE   rH   rF   z--decoder_onnxr   zLPath of onnx model for decoder. Specify it when you have exported the model.z--encoder_decoder_init_onnxzgPath of ONNX model for encoder and decoder initialization. Specify it when you have exported the model.z	--verboseÚ
store_truezPrint more information)rD   ÚactionrF   )ÚverbosezOutput optionsú--outputz,Output path for onnx model with beam search.z-pú--precisionzTPrecision of model to run. fp32 for full precision, fp16 for half or mixed precisionz-bú--op_block_listÚ*ÚautozÿDisable certain onnx operators when exporting model to onnx format. When using defaultvalue for gpt2 type of model fp16 precision, it will be set to ["Add", "LayerNormalization", "SkipLayerNormalization", "FastGelu"]. Other situation, it will be set to [])rD   ÚnargsrH   rF   z-eú--use_external_data_formatz!save external data for model > 2G)Úuse_external_data_formatz-sz--run_shape_inferencezrun shape inference)Úrun_shape_inferencez-dpvsz--disable_pad_vocab_sizez³Do not pad logits MatMul weight to be a multiple of 8 along the dimension where dim value is the vocab size. The logits MatMul may hence be of poor performance for fp16 precision.)Údisable_pad_vocab_sizez-dsgdz,--disable_separate_gpt2_decoder_for_init_runz™Do not create separate decoder subgraphs for initial and remaining runs. This does not allow for optimizations based on sequence lengths in each subgraph)Ú*disable_separate_gpt2_decoder_for_init_runz-iz--disable_shared_initializersz™do not share initializers in encoder and decoder for T5 or in the init decoder and decoder for GPT2. It will increase memory usage of t5/mt5/gpt2 models.)Údisable_shared_initializersz--encoder_decoder_initzbAdd decoder initialization to encoder for T5 model. This is legacy format that will be deprecated.)Úencoder_decoder_initz6Beam search parameters that stored in the output modelz--output_sequences_scoreszoutput sequences scores)Úoutput_sequences_scoresz--output_token_scoreszoutput token scores)Úoutput_token_scoresz--early_stopping)rD   rN   )Úearly_stoppingz--no_repeat_ngram_sizezNo repeat ngram size)rE   rD   rH   rF   z--vocab_maskz\Enable vocab_mask. This mask applies only to every generated token to filter some bad words.)Ú
vocab_maskz--past_present_share_bufferzWUse shared buffer for past and present, currently work for gpt2 greedy/sampling search.)Úpast_present_share_bufferz--use_decoder_masked_attentionzëUses `DecoderMaskedSelfAttention` or `DecoderMaskedMultiHeadAttention` to optimize the decoding Attention computation. Must be used with `past_present_share_buffer`. Currently, only Attention head sizes of 32, 64 and 128 are supported.)Úuse_decoder_masked_attentionz--prefix_vocab_maskzeEnable prefix_vocab_mask. This mask can be used to filter bad words in the first generated token only)Úprefix_vocab_maskz--custom_attention_maskz]Enable custom_attention_mask. This mask can be used to replace default encoder attention mask)Úcustom_attention_maskz--presence_maskz!Presence mask for custom sampling)Úpresence_maskz--seedzRandom seed for sampling op)ÚseedzYBeam search parameters not stored in the output model, for testing parity and performancez--min_lengthzMin sequence lengthz--max_lengthzMax sequence lengthz--num_beamsz	Beam sizez--num_return_sequencesz&Number of return sequence <= num_beamsz--length_penaltyz<Positive. >1 to penalize and <1 to encourage short sentence.z--repetition_penaltyz-Positive. >1 to penalize and <1 to encourage.z--temperatureç      ð?z6The value used to module the next token probabilities.z--top_pzTop P for samplingz--filter_valueÚInfzFilter value for Top P samplingz--min_tokens_to_keepzAMinimum number of tokens we keep per batch example in the output.z--presence_penaltyç        z%presence penalty for custom sampling.z--customz&If 1 customized top P logic is appliedz--vocab_sizezIVocab_size of the underlying model used to decide the shape of vocab maskz--eos_token_idzKcustom eos_token_id for generating model with existing onnx encoder/decoderz--pad_token_idzKcustom pad_token_id for generating model with existing onnx encoder/decoderz0Other options for testing parity and performancez--use_sln_strict_modez_Enable strict mode for SLN in CUDA provider. This ensures a better accuracy but will be slower.)Úuse_sln_strict_modeú	--use_gpuz)use GPU for inference. Required for fp16.)Úuse_gpuz--disable_parityzdo not run parity test)Údisable_parityz--disable_perf_testzdo not run perf test)Údisable_perf_testz--torch_performanceztest PyTorch performance)Útorch_performancez--total_runsz4Number of times of inference for latency measurementz--save_test_dataz-save test data for onnxruntime_perf_test tool)Úsave_test_data)rG   Út5Úmt5éÿÿÿÿ)r=   ÚArgumentParserÚadd_argument_groupÚadd_argumentr<   Újoinr   r   r   ÚosÚpathÚset_defaultsr   ÚFLOAT32r&   ÚFLOAT16ÚintÚfloatÚ
parse_args)r9   ÚparserÚinput_groupÚoutput_groupÚmodel_groupÚbeam_parameters_groupÚ
test_groupÚargss   &       r(   Úparse_argumentsr‡   c   sñ  € ô ×$Ò$Ó&€Fà×+Ñ+¨OÓ<€Kà×ÑØØØÜØTØ
�)‰)Ô*Ô-AÕAÔDYÕYÓ
Zõ[ð ô ð ×ÑØØÜØÚ%Ø9¸D¿I¹IÒF[Ó<\Õ\ð ô ð ×ÑØØÜÜ—‘—‘˜S .Ó1Ø4ð ô ð ×ÑØØÜØØ[ð ô ð ×ÑØ%ØÜØØvð ô ð ×ÑØØØØ%ð	 ô ð ×Ñ ÐÔ&à×,Ñ,Ð-=Ó>€Là×ÑØØÜØ;ð	 ô ð ×ÑØØØÜÜ×!Ñ!×'Ñ'Ü×"Ñ"×(Ñ(¬)×*;Ñ*;×*AÑ*AÐBØcð ô ð ×ÑØØØØØ�ðXð ô 	ð ×ÑØØ$ØØØ0ð ô ð ×Ñ°uÐÔ=à×ÑØØØØØ"ð ô ð ×Ñ°%ÐÔ8à×ÑØØ"ØØðbð ô ð ×Ñ°UÐÔ;à×ÑØØ6ØØðGð ô ð ×ÑÈÐÔOà×ÑØØ'ØØðEð ô ð ×Ñ¸%ÐÔ@à×ÑØ ØØØqð	 ô ð ×Ñ°5ÐÔ9à×+Ñ+Ð,dÓe€Kà×ÑØ#ØØØ&ð	 ô ð ×Ñ°UÐÔ;à×ÑØØØØ"ð	 ô ð ×Ñ°ÐÔ7à×ÑÐ/¸%ÈÐÔUØ×Ñ¨EÐÔ2à×ÑØ ÜØØØ#ð ô ð ×ÑØØØØkð	 ô ð ×Ñ¨ÐÔ.à×ÑØ%ØØØfð	 ô ð ×Ñ°uÐÔ=à×ÑØ(ØØðð	 ô ð ×Ñ¸%ÐÔ@à×ÑØØØØtð	 ô ð ×Ñ¨uÐÔ5à×ÑØ!ØØØlð	 ô ð ×Ñ°5ÐÔ9à×ÑØØØØ0ð	 ô ð ×Ñ¨5ÐÔ1à×ÑØØØØ*ð	 ô ð ×Ñ %ÐÔ(à"×5Ñ5ØcóÐð ×&Ñ& ~¼CÈ%ÐYZÐavÐ&Ôwà×&Ñ& ~¼CÈ%ÐY[ÐbwÐ&Ôxà×&Ñ& }¼3ÈÐXYÐ`kÐ&Ôlà×&Ñ&Ø ÜØØØ5ð 'ô ð ×&Ñ&ØÜØØØKð 'ô ð ×&Ñ&ØÜØØØ<ð 'ô ð ×&Ñ&ØÜØØØEð 'ô ð ×&Ñ&ØÜØØØ!ð 'ô ð ×&Ñ&ØÜØÜ�u“�Ø.ð 'ô ð ×&Ñ&ØÜØØØPð 'ô ð ×&Ñ&ØÜØØØ4ð 'ô ð ×&Ñ&ØÜØØØ5ð 'ô ð ×&Ñ&ØÜØØØXð 'ô ð ×&Ñ&ØÜØØØZð 'ô ð ×&Ñ&ØÜØØØZð 'ô ð ×*Ñ*Ð+]Ó^€Jà×ÑØØØØnð	 ô ð ×Ñ°ÐÔ6à×ÑØØØØ8ð	 ô ð ×Ñ EÐÔ*à×ÑØØØØ%ð	 ô ð ×Ñ¨5ÐÔ1à×ÑØØØØ#ð	 ô ð ×Ñ¨eÐÔ4à×ÑØØØØ'ð	 ô ð ×Ñ¨eÐÔ4à×ÑØØÜØØCð ô ð ×ÑØØØØ<ð	 ô ð ×Ñ¨5ÐÔ1à×Ñ˜TÓ"€Dà€Kr+   c                ó8   € V ^8„  d   QhR\         P                  /# ©r8   r†   ©r=   r>   )r?   s   "r(   r@   r@   ÷  s   € ÷ *)ñ *)”x×)Ñ)ñ *)r+   c                óÌ  € V P                   pRVRV P                  RRV P                  RRRRR	.pV P                  '       d   VP	                  R
V P                  .4       V P
                  '       d   VP                  R4       V P                  '       d   VP                  R4       \        V P                  4      '       d.   VP	                  R.4       VP	                  V P                  4       V P                  \        P                  P                  8X  d   V P
                  '       g   Q R4       hV P                  '       d   \        P                  RV 24       \!        VR7       R# )zeConvert GPT-2 model to onnx

Args:
    args (argparse.Namespace): arguments parsed from command line
rB   rP   z--optimize_onnxrQ   z--test_runsÚ1z--test_casesÚ10z--overwriterJ   rk   rV   rR   zEfp16 or mixed precision model cannot run in CPU. Please add --use_gpuzarguments for convert_to_onnx:)r9   N)Úmodel_name_or_pathÚdecoder_onnxÚ	precisionÚ	cache_dirÚextendrl   ÚappendrW   ÚlenÚop_block_listr   r|   r&   rO   ÚloggerÚinfoÚconvert_gpt2_to_onnx)r†   Ú
model_nameÚ	argumentss   &  r(   Úgpt2_to_onnxr›   ÷  s$  € ð ×(Ñ(€Jð 	ØØØ×ÑØØØ�‰ØØØØØð€Ið ‡~‡~€~Ø×Ñ˜-¨¯©Ð8Ô9Ø‡|‡|€|Ø×Ñ˜Ô%Ø×$×$Ð$Ø×ÑÐ5Ô6ä
ˆ4×Ñ×ÒØ×ÑÐ+Ð,Ô-Ø×Ñ˜×+Ñ+Ô,à‡~�~œ×*Ñ*×0Ñ0Ô0Ø�|�|ˆ|ÐdÐdÓdˆ|ð
 ‡|‡|€|Ü�‰Ð4°Y°KÐ@ÔAä˜i×(r+   c                ó8   € V ^8„  d   QhR\         P                  /# r‰   rŠ   )r?   s   "r(   r@   r@   $  s   € ÷ !ñ !”X×'Ñ'ñ !r+   c                óH  € \        V P                  V P                  \        V P                  4      P
                  V P                  V P                  V P                  \        P                  P                  8g  V P                  RRRRRV P                  V P                  V P                  \        P                  P                  8H  R7      p\        P                  RV^ ,           24       \        P                  RV^,           24       V^ ,          V n        V^,          V n        R# )zbConvert T5 model to onnx

Args:
    args (argparse.Namespace): arguments parsed from command line
FT)rŽ   r‘   Ú
output_dirrl   rW   Úoptimize_onnxr�   rO   Úuse_decoder_start_tokenÚ	overwriteÚdisable_auto_mixed_precisionÚuse_int32_inputsÚ
model_typer\   Úforce_fp16_iozonnx model for encoder: zonnx model for decoder: N)Úexport_t5_onnx_modelsrŽ   r‘   r   ÚoutputÚparentrl   rW   r�   r   r|   r&   r¤   r\   r–   ÚdebugÚencoder_decoder_init_onnxr�   )r†   Úpathss   & r(   Ú
t5_to_onnxr¬   $  sà   € ô "Ø×2Ñ2Ø—.‘.Ü˜Ÿ™Ó$×+Ñ+Ø—‘Ø!%×!>Ñ!>Ø—~‘~¬×):Ñ):×)@Ñ)@Ñ@Ø—.‘.ØØ %ØØ%*ØØ—?‘?Ø!×6Ñ6Ø—~‘~¬×):Ñ):×)@Ñ)@Ñ@ô€Eô$ ‡L�LÐ+¨E°!­H¨:Ð6Ô7Ü
‡L�LÐ+¨E°!­H¨:Ð6Ô7Ø%*¨1¥X€DÔ"Ø˜a�€DÖr+   c                ó0   € V ^8„  d   QhR\         R\        /# )r8   Ú	onnx_pathrW   ©r<   Úbool)r?   s   "r(   r@   r@   B  s   € ÷ Oñ Oœsð O¼dñ Or+   c                óØ   € ^ RI Hp \        P                  ! V RR7      pVP	                  VRRR7      pV'       d   \
        P                  ! W@VR7       R# \        P                  R4       R# )	z·Shape inference on an onnx file, which will be overwritten.

Args:
    onnx_path (str): Path of onnx model
    use_external_data_format(bool): output tensors to external data or not.
)ÚSymbolicShapeInferenceT©Úload_external_dataF)Ú
auto_mergeÚguess_output_rank©Úsave_as_external_dataz4Failed to run symbolic shape inference on the model.N)	Ú&onnxruntime.tools.symbolic_shape_inferr²   ÚonnxÚ
load_modelÚinfer_shapesr   Úsaver–   Úwarning)r®   rW   r²   ÚmodelÚouts   &&   r(   Úshape_inferencerÁ   B  sO   € õ Nä�OŠO˜I¸$Ô?€EØ
 ×
-Ñ
-¨eÀÐX]Ð
-Ó
^€Cß
Ü�Š�sÐ=U×Vä�‰ÐMÖNr+   c                ó<   € V ^8„  d   QhR\         R\        R\        /# )r8   r®   rW   r:   r¯   )r?   s   "r(   r@   r@   T  s'   € ÷ Kñ K¬Cð KÌ4ð KÔ[_ñ Kr+   c                ól  € \         P                  ! V RR7      pVP                  P                  ^ ,          P                  p\        V4      pVP                  4       pW59   g   Q hWS,          pVP                  R8w  d   R# RpVP                  VP                  ^,          4      pVfD   VP                  VR^4      p	V	f   R# VP                  V	P                  ^ ,          4      pVf   R# RpVP                  \        P                  P                  8w  d   R# \        VP                   4      ^8w  d   R# VP                   ^,          p
V
^,          ^ 8X  d   R# \"        P$                  ! V
^,          4      ^,          pWº,
          pVP&                  '       Ed	   V'       dw   \(        P*                  ! VP                   ^ ,          V3\(        P,                  R7      p\(        P.                  ! \0        P2                  ! V4      V3^R7      pW¸P                   ^&   Mt\(        P*                  ! WÈP                   ^,          3\(        P,                  R7      p\(        P.                  ! \0        P2                  ! V4      V3^ R7      pW¸P                   ^ &   VP5                  4       Vn        MR# \
        P6                  ! W VR7       R# )	zÒPad the logits MatMul weight in the provided decoder model, which will be overwritten.

Args:
    onnx_path (str): Path of onnx model
    use_external_data_format(bool): output tensors to external data or not.
Tr³   ÚMatMulFÚ	Transpose©Údtype©Úaxisr·   )rº   r»   Úgraphr§   Únamer   Úoutput_name_to_nodeÚop_typeÚget_initializerÚinputÚmatch_parentÚ	data_typer
   ÚDataTyper|   r”   ÚdimsÚmathÚceilÚraw_dataÚnpÚzerosÚfloat16Úconcatenater   Úto_arrayÚtobytesr½   )r®   rW   Údecoder_model_protoÚlogits_output_nameÚdecoder_modelrÌ   Úmatmul_nodeÚpad_along_axis_1Úlogits_weightÚtranspose_before_matmulÚactual_vocab_sizeÚpadded_vocab_sizeÚpaddingÚpadding_dataÚweight_with_paddings   &&             r(   Úpad_weights_of_logits_matmulré   T  sC  € ô Ÿ/š/¨)ÈÔMÐà,×2Ñ2×9Ñ9¸!Õ<×AÑAÐäÐ1Ó2€Mà'×;Ñ;Ó=ÐØÔ4Ð4Ð4à%Õ9€Kà×Ñ˜hÔ&Ùð
 ÐØ!×1Ñ1°+×2CÑ2CÀAÕ2FÓG€MØÒØ"/×"<Ñ"<¸[È+ÐWXÓ"YÐà"Ò*Ùà%×5Ñ5Ð6M×6SÑ6SÐTUÕ6VÓWˆàÒ Ùà Ðð ×Ñ¤+×"6Ñ"6×">Ñ">Ô>Ùô ˆ=×ÑÓ !Ô#Ùð &×*Ñ*¨1Õ-Ðà˜AÕ !Ô#áäŸ	š	Ð"3°aÕ"7Ó8¸1Õ<ÐØÕ3€Gð ××ÑßÜŸ8š8 ]×%7Ñ%7¸Õ%:¸GÐ$DÌBÏJÉJÔWˆLÜ"$§.¢.´+×2FÒ2FÀ}Ó2UÐWcÐ1dÐklÔ"mÐØ$5×Ñ˜qÒ!äŸ8š8 W×.@Ñ.@ÀÕ.CÐ$DÌBÏJÉJÔWˆLÜ"$§.¢.´+×2FÒ2FÀ}Ó2UÐWcÐ1dÐklÔ"mÐØ$5×Ñ˜qÑ!à!4×!<Ñ!<Ó!>ˆÕáô ‡N‚NÐ&ÐIaÕbÙr+   c                óH   € V ^8„  d   QhR\         R\        R\        R\        /# )r8   Ú
model_pathrl   rj   r:   )r<   r°   r   )r?   s   "r(   r@   r@   ¢  s)   € ÷ ñ ¤3ð ´ð ÌDð ÔUeñ r+   c                óV  € \        4       p\        P                  Vn        V'       d   RR.MR.pV'       da   R\	        4       9  d   \        R4      h\        P                  R4       V'       d*   RR/pRV/pV Uu. uF  qwV9   d   WvV,          3MTNK  	  pp\        WVR7      pV# u upi )a`  Create OnnxRuntime session.

Args:
    model_path (str): onnx model path
    use_gpu (bool): use GPU or not
    use_sln_strict_mode (bool): use strict mode for skip layer normalization or not

Raises:
    RuntimeError: CUDAExecutionProvider is not available when --use_gpu is specified.

Returns:
    onnxruntime.InferenceSession: The created session.
ÚCUDAExecutionProviderÚCPUExecutionProviderz5CUDAExecutionProvider is not available for --use_gpu!zuse CUDAExecutionProviderÚ"enable_skip_layer_norm_strict_modeT)Ú	providers)	r   r   ÚORT_DISABLE_ALLÚgraph_optimization_levelr   ÚRuntimeErrorr–   r—   r   )	rë   rl   rj   Úsess_optionsÚexecution_providersÚcuda_provider_optionsÚprovider_optionsrË   Úort_sessions	   &&&      r(   Úcreate_ort_sessionrù   ¢  s½   € ô "Ó#€LÜ,B×,RÑ,R€LÔ)ßOVÐ2Ð4JÑKÐ]sÐ\tÐßØ"Ô*AÓ*CÔCÜÐVÓWÐWä�K‰KÐ3Ô4ßØ%IÈ4Ð$PÐ!Ø 7Ð9NÐOÐáató#ÙatÐY]Ð:JÔ2J�¨Õ-Ñ.ÐPTÒTÑatð  ð #ô # :ÐGZÔ[€KØÐùò#s   Á:B&c                óD   € V ^8„  d   QhR\         P                  R\        /# ©r8   rÊ   r�   ©rº   r   r   )r?   s   "r(   r@   r@   Ã  s   € ÷ 5ñ 5¤§¡ð 5¼Iñ 5r+   c           
     ó^  € V\         P                  P                  8H  p\        V P                  4      pV^,
          pV^8¼  g   Q h. RO\        V4       Uu. uF  pRV 2NK
  	  up,           p\        V P                  4      \        V4      8w  d.   \        R\        V4       R\        V P                  4       24      h\        V4       Fà  w  rWV P                  V,          P                  V8w  d0   \        RV RV RV P                  V,          P                   24      h\        P                  pV^8¼  d)   V'       d   \        P                  M\        P                  pV P                  V,          P                  P                  P                  p	W˜8w  g   KÎ  \        RV RV RV	 24      h	  \        P!                  R4       R.\        V4       Uu. uF  pR	V 2NK
  	  up,           p
\        V P"                  4      \        V
4      8w  d.   \        R
\        V
4       R\        V P"                  4       24      h\        V
4       FÉ  w  r[V P"                  V,          P                  V8w  d0   \        RV RV RV P"                  V,          P                   24      hV'       d   \        P                  M\        P                  pV P"                  V,          P                  P                  P                  pWÈ8w  g   K·  \        RV RV RV 24      h	  \        P!                  R4       R# u upi u upi )aº  Verify GPT-2 subgraph

Args:
    graph (onnx.GraphProto): onnx graph of GPT-2
    precision (Precision): Precision (FLOAT16 or FLOAT32) of the model.

Raises:
    ValueError: Number of inputs not expected.
    ValueError: Input name is not expected.
    ValueError: Input data type is not expected.
    ValueError: Number of outputs not expected.
    ValueError: Output name is not expected.
    ValueError: Output data type is not expected.
Úpast_ú Number of inputs expected to be ú. Got úInput ú is expected to be ú$ is expected to have onnx data type z:Verifying GPT-2 graph inputs: name and data type are good.ÚlogitsÚpresent_ú!Number of outputs expected to be úOutput z;Verifying GPT-2 graph outputs: name and data type are good.N)Ú	input_idsÚposition_idsÚattention_mask)r   r|   r&   r”   rÏ   ÚrangeÚ
ValueErrorÚ	enumeraterË   r
   ÚINT32ÚFLOATrE   Útensor_typeÚ	elem_typer–   r—   r§   )rÊ   r�   Ú
is_float16Úinput_countÚlayer_countÚiÚexpected_inputsÚexpected_inputÚexpected_typeÚ
input_typeÚexpected_outputsÚexpected_outputÚoutput_types   &&           r(   Úverify_gpt2_subgraphr  Ã  sÑ  € ð œi×/Ñ/×5Ñ5Ñ5€Jä�e—k‘kÓ"€KØ •/€KØ˜!ÔÐÐâEÔ^cÐdoÔ^pÓHqÑ^pÐYZÈ5ÐQRÐPSËÑ^pÑHqÕq€OÜ
ˆ5�;‰;Óœ3˜Ó/Ô/ÜÐ;¼CÀÓ<PÐ;QÐQWÔX[Ð\a×\gÑ\gÓXhÐWiÐjÓkÐkä& Ö7ÑˆØ�;‰;�q�>×Ñ .Ô0Ü˜v a SÐ(;¸NÐ;KÈ6ÐRW×R]ÑR]Ð^_ÕR`×ReÑReÐQfÐgÓhÐhä#×)Ñ)ˆØ�Œ6ß3=œK×/Ò/Ä;×CTÑCTˆMà—[‘[ •^×(Ñ(×4Ñ4×>Ñ>ˆ
ØÖ&Ü˜v a SÐ(LÈ]ÈOÐ[aÐblÐamÐnÓoÐoñ 8ô ‡K�KÐLÔMà �z¼UÀ;Ô=OÓ$PÑ=O¸ x°¨s£^Ñ=OÑ$PÕPÐÜ
ˆ5�<‰<ÓœCÐ 0Ó1Ô1ÜÐ<¼SÐAQÓ=RÐ<SÐSYÔZ]Ð^c×^jÑ^jÓZkÐYlÐmÓnÐnä'Ð(8Ö9ÑˆØ�<‰<˜�?×Ñ ?Ô2Ü˜w q cÐ)<¸_Ð<MÈVÐTY×T`ÑT`ÐabÕTc×ThÑThÐSiÐjÓkÐkç/9œ×+Ò+¼{×?PÑ?PˆØ—l‘l 1•o×*Ñ*×6Ñ6×@Ñ@ˆØÖ'Ü˜v a SÐ(LÈ]ÈOÐ[aÐbmÐanÐoÓpÐpñ :ô ‡K�KÐMÔNñ ùòA Irùò" %Qs   ÁL%ÇL*c                óD   € V ^8„  d   QhR\         P                  R\        /# rû   rü   )r?   s   "r(   r@   r@   û  s&   € ÷ Ioñ Io¤d§o¡oð IoÄ)ñ Ior+   c           
     óˆ  € V\         P                  P                  8H  pV'       d   \        P                  M\        P                  p\        V P                  4      pV^,
          ^,          pV^8¼  g   Q hRR.p\        V4       F+  pVP                  RV 24       VP                  RV 24       K-  	  \        V4       F+  pVP                  RV 24       VP                  RV 24       K-  	  \        V P                  4      \        V4      8w  d.   \        R\        V4       R\        V P                  4       24      h\        V4       Fº  w  rxV P                  V,          P                  V8w  d0   \        R	V R
V RV P                  V,          P                   24      hV^8  d   \        P                  MTp	V P                  V,          P                  P                  P                  p
W©8w  g   K¨  \        R	V RV	 RV
 24      h	  R.p\        V4       F+  pVP                  RV 24       VP                  RV 24       K-  	  \        V P                   4      \        V4      8w  d.   \        R\        V4       R\        V P                   4       24      h\        V4       F¡  w  r|V P                   V,          P                  V8w  d0   \        RV R
V RV P                   V,          P                   24      hV P                   V,          P                  P                  P                  pWÓ8w  g   K�  \        RV RV RV 24      h	  R# )áÄ  Verify T5 decoder subgraph

Args:
    graph (onnx.GraphProto): onnx graph of T5 decoder
    precision (Precision): Precision (FLOAT16 or FLOAT32) of the model.

Raises:
    ValueError: Number of inputs not expected.
    ValueError: Input name is not expected.
    ValueError: Input data type is not expected.
    ValueError: Number of outputs not expected.
    ValueError: Output name is not expected.
    ValueError: Output data type is not expected.
r  Úencoder_attention_maskÚpast_key_self_Úpast_value_self_Úpast_key_cross_Úpast_value_cross_rÿ   r   r  r  r  r  Úpresent_key_self_Úpresent_value_self_r  r  N)r   r|   r&   r
   r  r”   rÏ   r  r“   r  r  rË   r  rE   r  r  r§   )rÊ   r�   r  Ú
float_typer  r  r  r  r  r  r  r  r  r  s   &&            r(   Úverify_t5_decoder_subgraphr)  û  s	  € ð œi×/Ñ/×5Ñ5Ñ5€Jß(2”×$Ò$¼×8IÑ8I€Jä�e—k‘kÓ"€KØ •? qÕ(€KØ˜!ÔÐÐð #Ð$<Ð=€OÜ�;ÖˆØ×Ñ °¨sÐ3Ô4Ø×ÑÐ!1°!°Ð5Ö6ñ  ô �;ÖˆØ×Ñ °°Ð4Ô5Ø×ÑÐ!2°1°#Ð6Ö7ñ  ô ˆ5�;‰;Óœ3˜Ó/Ô/ÜÐ;¼CÀÓ<PÐ;QÐQWÔX[Ð\a×\gÑ\gÓXhÐWiÐjÓkÐkä& Ö7ÑˆØ�;‰;�q�>×Ñ .Ô0Ü˜v a SÐ(;¸NÐ;KÈ6ÐRW×R]ÑR]Ð^_ÕR`×ReÑReÐQfÐgÓhÐhà-.°¬Uœ×)Ò)¸
ˆØ—[‘[ •^×(Ñ(×4Ñ4×>Ñ>ˆ
ØÖ&Ü˜v a SÐ(LÈ]ÈOÐ[aÐblÐamÐnÓoÐoñ 8ð !�zÐÜ�;ÖˆØ×ÑÐ"3°A°3Ð 7Ô8Ø×ÑÐ"5°a°SÐ 9Ö:ñ  ô ˆ5�<‰<ÓœCÐ 0Ó1Ô1ÜÐ<¼SÐAQÓ=RÐ<SÐSYÔZ]Ð^c×^jÑ^jÓZkÐYlÐmÓnÐnä'Ð(8Ö9ÑˆØ�<‰<˜�?×Ñ ?Ô2Ü˜w q cÐ)<¸_Ð<MÈVÐTY×T`ÑT`ÐabÕTc×ThÑThÐSiÐjÓkÐkØ—l‘l 1•o×*Ñ*×6Ñ6×@Ñ@ˆØÖ$Ü˜w q cÐ)MÈjÈ\ÐY_Ð`kÐ_lÐmÓnÐnó :r+   c                óD   € V ^8„  d   QhR\         P                  R\        /# rû   rü   )r?   s   "r(   r@   r@   G  s&   € ÷ Yañ Ya´4·?±?ð YaÌyñ Yar+   c           
     ó$  € V\         P                  P                  8H  pRV P                  ^ ,          P                  9   p. ROpV'       d
   VR,          p\        V P                  4      \        V4      8w  d.   \        R\        V4       R\        V P                  4       24      h\        V4       F±  w  rVV P                  V,          P                  V8w  d0   \        RV RV RV P                  V,          P                   24      h\        P                  pV P                  V,          P                  P                  P                  pW‡8w  g   KŸ  \        RV RV RV 24      h	  V'       d†   \        V P                  4      ^,          ^ 8X  g   Q h\        V P                  4      ^,          p	V	^8¼  g   Q h. p
\        V	4       F+  pV
P                  RV 24       V
P                  R	V 24       K-  	  Mã\         P#                  R
4       \        V P                  4      ^,
          ^,          ^ 8X  g   Q h\        V P                  4      ^,
          ^,          p	V	^8¼  g   Q hRR.p
\        V	4       F+  pV
P                  RV 24       V
P                  RV 24       K-  	  \        V	4       F+  pV
P                  RV 24       V
P                  R	V 24       K-  	  \        V P                  4      \        V
4      8w  d.   \        R\        V
4       R\        V P                  4       24      h\        V
4       FÉ  w  r[V P                  V,          P                  V8w  d0   \        RV RV RV P                  V,          P                   24      hV'       d   \        P                  M\        P$                  pV P                  V,          P                  P                  P                  pWÇ8w  g   K·  \        RV RV RV 24      h	  \         P'                  R4       R# )r   Úcross:Nr8   Nrÿ   r   r  r  r  Úpresent_key_cross_Úpresent_value_cross_zZThis format is deprecated. Please export T5 encoder in new format with only cross outputs.r  Úencoder_hidden_statesr&  r'  r  r  zMT5 encoder graph verified: name and data type of inputs and outputs are good.N)Úencoder_input_idsr!  Údecoder_input_ids)r   r|   r&   r§   rË   r”   rÏ   r  r  r
   r  rE   r  r  r  r“   r–   r¾   r  r—   )rÊ   r�   r  Ú
new_formatr  r  r  r  r  r  r  r  r  s   &&           r(   Ú'verify_t5_encoder_decoder_init_subgraphr3  G  s«  € ð œi×/Ñ/×5Ñ5Ñ5€JØ˜EŸL™L¨�O×0Ñ0Ñ0€Jò€O÷
 Ø)¨"Õ-ˆÜ
ˆ5�;‰;Óœ3˜Ó/Ô/ÜÐ;¼CÀÓ<PÐ;QÐQWÔX[Ð\a×\gÑ\gÓXhÐWiÐjÓkÐkä& Ö7ÑˆØ�;‰;�q�>×Ñ .Ô0Ü˜v a SÐ(;¸NÐ;KÈ6ÐRW×R]ÑR]Ð^_ÕR`×ReÑReÐQfÐgÓhÐhä#×)Ñ)ˆØ—[‘[ •^×(Ñ(×4Ñ4×>Ñ>ˆ
ØÖ&Ü˜v a SÐ(LÈ]ÈOÐ[aÐblÐamÐnÓoÐoñ 8÷ Ü�5—<‘<Ó  1Õ$¨Ô)Ð)Ð)Ü˜%Ÿ,™,Ó'¨1Õ,ˆØ˜aÔÐÐð ÐÜ�{Ö#ˆAØ×#Ñ#Ð&8¸¸Ð$<Ô=Ø×#Ñ#Ð&:¸1¸#Ð$>Ö?ò $ô 	�‰ÐsÔtÜ�E—L‘LÓ! AÕ%¨Õ*¨aÔ/Ð/Ð/Ü˜5Ÿ<™<Ó(¨1Õ,°Õ2ˆØ˜aÔÐÐð %Ð&=Ð>ÐÜ�{Ö#ˆAØ×#Ñ#Ð&7¸°sÐ$;Ô<Ø×#Ñ#Ð&9¸!¸Ð$=Ö>ñ $ô �{Ö#ˆAØ×#Ñ#Ð&8¸¸Ð$<Ô=Ø×#Ñ#Ð&:¸1¸#Ð$>Ö?ñ $ô ˆ5�<‰<ÓœCÐ 0Ó1Ô1ÜÐ<¼SÐAQÓ=RÐ<SÐSYÔZ]Ð^c×^jÑ^jÓZkÐYlÐmÓnÐnä'Ð(8Ö9ÑˆØ�<‰<˜�?×Ñ ?Ô2Ü˜w q cÐ)<¸_Ð<MÈVÐTY×T`ÑT`ÐabÕTc×ThÑThÐSiÐjÓkÐkç/9œ×+Ò+¼{×?PÑ?PˆØ—l‘l 1•o×*Ñ*×6Ñ6×@Ñ@ˆØÖ'Ü˜w q cÐ)MÈmÈ_Ð\bÐcnÐboÐpÓqÐqñ :ô ‡K�KÐ_Ö`r+   c                ó|   € V ^8„  d   QhR\         R\         R\        R\        R\        R,          R\        R,          /# )r8   Úgraph1Úgraph2Úshared_prefixÚmin_elementsÚsignature_cache1NÚsignature_cache2)r   r<   r}   Údict)r?   s   "r(   r@   r@   £  sT   € ÷ g!ñ g!Üðg!äðg!ô ðg!ô ð	g!ô
 ˜T•kðg!ô ˜T•kñg!r+   c                ó0
  € / p/ p. p. p	. p
V P                    EF  pVP                  '       d   \        VP                  4      V8¼  g   K2  VP                    FÔ  pVP                  '       d   \        VP                  4      V8¼  g   K1  \        P                  ! W¼WE4      '       g   KP  W,P
                  ,           WkP
                  &   VP                  V4       VP
                  V9  dC   W,P
                  ,           pW×VP
                  &   V	P                  V4       V
P                  V4        EK  	  EK  	  \        P                  RV
 24       V P                   F_  p\        \        VP                  4      4       F:  pVP                  V,          V
9   g   K  \        RVP                  V,           24      h	  Ka  	  VP                   F_  p\        \        VP                  4      4       F:  pVP                  V,          V
9   g   K  \        RVP                  V,           24      h	  Ka  	  V	 F  pVP                   P                  V4       K   	  VP                   F.  pVP
                  V9   g   K  VVP
                  ,          Vn        K0  	  VP                   F¦  p\        \        VP                  4      4       F�  pVP                  V,          V9   g   K  W~P                  V,          ,          p\        P                  RVP
                   RV RVP                  V,           RV 24       VVP                  V&   Kƒ  	  K¨  	  V F  pV P                   P                  V4       K   	  V P                   F.  pVP
                  V9   g   K  VVP
                  ,          Vn        K0  	  V P                   F¦  p\        \        VP                  4      4       F�  pVP                  V,          V9   g   K  WnP                  V,          ,          p\        P                  RVP
                   RV RVP                  V,           RV 24       VVP                  V&   Kƒ  	  K¨  	  V	 F  pVVP
                  ,          Vn        K  	  V	 F—  p\         P"                  P%                  V4      P&                  p\         P(                  P+                  VP
                  VP,                  V4      pV P                  P                  V4       VP                  P                  V4       K™  	  V	# )	ae  Remove initializers with same value from two graphs.

Args:
    graph1 (GraphProto): the first graph to process
    graph2 (GraphProto): the second graph to process
    shared_prefix (str): add prefix to the shared initializers among two graphs
    min_elements (int, optional): minimal number of elements for initializers to be considered. Defaults to 1024.
    signature_cache1 (dict): Optional dictionary to store data signatures of tensors in graph1 in order to speed up comparison
    signature_cache2 (dict): Optional dictionary to store data signatures of tensors in graph2 in order to speed up comparison
zshared initializers:zname is found in graph 1: zname is found in graph 2: zgraph 2 rename node z input z from z to zgraph 1 rename node )ÚinitializerrÓ   Úsumr   Úhas_same_valuerË   r“   r–   r©   Únoder  r”   rÏ   ró   ÚremoveÚ
value_inforº   Únumpy_helperrÛ   ÚshapeÚhelperÚmake_tensor_value_inforÑ   )r5  r6  r7  r8  r9  r:  Úmapping_initializers_1Úmapping_initializers_2Úshared_initializers_1Úshared_initializers_2Úshared_initializers_namesÚinitializer1Úinitializer2Úshared_namer@  Újr=  rB  Únew_namerD  s   &&&&&&              r(   Úremove_shared_initializersrQ  £  s  € ð&  ÐØÐØÐØÐØ "Ðà×*Õ*ˆØ×!×!Ð!¤c¨,×*;Ñ*;Ó&<ÀÔ&LÙà"×.Ô.ˆLØ ×%×%Ð%¬#¨l×.?Ñ.?Ó*@ÀLÔ*PÙä×'Ò'¨ÐDT×gÔgØ<I×L]ÑL]Õ<]Ð&×'8Ñ'8Ñ9Ø%×,Ñ,¨\Ô:à×$Ñ$Ð,BÔBØ"/×2CÑ2CÕ"C�KØ@K¨<×+<Ñ+<Ñ=Ø)×0Ñ0°Ô>Ø-×4Ñ4°[ÔAÛô /ñ	 +ô& ‡L�LÐ'Ð(AÐ'BÐCÔDð —”ˆÜ”s˜4Ÿ:™:“Ö'ˆAØ�z‰z˜!�}Ð 9Ö9Ü"Ð%?ÀÇ
Á
È1Å¸Ð#OÓPÐPó (ñ ð —”ˆÜ”s˜4Ÿ:™:“Ö'ˆAØ�z‰z˜!�}Ð 9Ö9Ü"Ð%?ÀÇ
Á
È1Å¸Ð#OÓPÐPó (ñ ó -ˆØ×Ñ×!Ñ! +Ö.ñ -ð ×'Ô'ˆ
Ø�?‰?Ð4Ö4Ø4°Z·_±_ÕEˆJŽOñ (ð
 —”ˆÜ”s˜4Ÿ:™:“Ö'ˆAØ�z‰z˜!�}Ð 6Ö6Ø1·*±*¸Qµ-Õ@�Ü—‘Ð3°D·I±I°;¸gÀaÀSÈÈtÏzÉzÐZ[Í}ÈoÐ]aÐbjÐakÐlÔmØ (�—
‘
˜1“ó	 (ñ ó -ˆØ×Ñ×!Ñ! +Ö.ñ -ð ×'Ô'ˆ
Ø�?‰?Ð4Ö4Ø4°Z·_±_ÕEˆJŽOñ (ð
 —”ˆÜ”s˜4Ÿ:™:“Ö'ˆAØ�z‰z˜!�}Ð 6Ö6Ø1·*±*¸Qµ-Õ@�Ü—‘Ð3°D·I±I°;¸gÀaÀSÈÈtÏzÉzÐZ[Í}ÈoÐ]aÐbjÐakÐlÔmØ (�—
‘
˜1“ó	 (ñ ó -ˆØ1°+×2BÑ2BÕCˆÖñ -ó -ˆÜ×!Ñ!×*Ñ*¨;Ó7×=Ñ=ˆÜ—[‘[×7Ñ7¸×8HÑ8HÈ+×J_ÑJ_ÐafÓgˆ
à×Ñ× Ñ  Ô,Ø×Ñ× Ñ  Ö,ñ -ð !Ð r+   c                ó0   € V ^8„  d   QhR\         R\         /# )r8   Úencoder_modelrß   ©r	   )r?   s   "r(   r@   r@     s   € ÷ ñ ¬:ð Äjñ r+   c                 ó0  € \        V 4      p\        V4      pVP                  R 4       VP                  R4       / / rTVP                  V4       VP                  V4       \        VP                  P
                  VP                  P
                  RVVR7      pV# )Úe_Úd_Ús_)r7  r9  r:  )r   Úadd_prefix_to_namesÚremove_duplicated_initializerrQ  r¿   rÊ   )rS  rß   ÚencoderÚdecoderr9  r:  Úinitializerss   &&     r(   Úget_shared_initializersr^    s�   € Ü˜Ó&€GÜ˜Ó&€GØ×Ñ Ô%Ø×Ñ Ô%Ø)+¨RÐ&Ø×)Ñ)Ð*:Ô;Ø×)Ñ)Ð*:Ô;Ü-Ø�‰×ÑØ�‰×ÑØØ)Ø)ô€Lð Ðr+   c                óR   € V ^8„  d   QhR\         R\        R\        \        ,          /# )r8   rÊ   r8  r:   )r   r}   r;   r
   )r?   s   "r(   r@   r@     s+   € ÷ ñ Üðäðô 
Œ+Õñr+   c                óú  € . pV P                    FB  pVP                  '       d   \        VP                  4      V8¼  g   K1  VP                  V4       KD  	  V F  pV P                   P	                  V4       K   	  V F|  p\
        P                  P                  V4      P                  p\
        P                  P                  VP                  VP                  V4      pV P                  P                  V4       K~  	  V# )aF  Remove initializers of a graph, when they have number of elements larger than a threshold.

Args:
    graph (GraphProto): the graph.
    min_elements (int, optional): minimal number of elements for initializers to be considered. Defaults to 1024.

Returns:
    List[TensorProto]: initializers that are removed from the graph.
)r=  rÓ   r>  r“   rA  rº   rC  rÛ   rD  rE  rF  rË   rÑ   rB  )rÊ   r8  Úmoved_initializersÚtensorr=  rD  rB  s   &&     r(   Úmove_initializersrc    sÍ   € ð ÐØ×#Ô#ˆØ——�¤ F§K¡KÓ 0°LÔ @ÙØ×!Ñ! &Ö)ñ $ó
 *ˆØ×Ñ× Ñ  Ö-ñ *ó *ˆÜ×!Ñ!×*Ñ*¨;Ó7×=Ñ=ˆÜ—[‘[×7Ñ7¸×8HÑ8HÈ+×J_ÑJ_ÐafÓgˆ
Ø×Ñ×Ñ 
Ö+ñ *ð
 Ðr+   c                ó  € V P                   ^ 8X  d   \        RV P                   R24      hV P                   ^8X  d   V P                  pEM5V P                   ^8X  d   V P                  pEMV P                   ^8X  d   V P
                  pMøV P                   ^8X  d   V P                  pMÚV P                   ^8X  d   V P                  pM¼V P                   ^8X  d   V P                  pMžV P                   ^8X  d   V P                  pM€V P                   ^8X  d   V P                  pMbV P                   ^	8X  d   V P                  pMDV P                   ^
8X  d   V P                  pM&\        RV P                   RV P                    R24      hV P                  V3# )z´
Convert attribute to kwarg format for use with onnx.helper.make_node.
    :parameter attribute: attribute in AttributeProto format.
    :return: attribute in {key: value} format.
z
attribute z does not have type specified.z has unsupported type rK   )rE   r  rË   Úfr  ÚsÚtÚgÚfloatsÚintsÚstringsÚtensorsÚgraphs)Ú	attributer&   s   & r(   Ú_attribute_to_pairro  >  s;  € ð ‡~�~˜ÔÜ˜: i§n¡nÐ%5Ð5SÐTÓUÐUð ‡~�~˜ÔØ—‘ŠØ	�‰˜1Ô	Ø—‘ŠØ	�‰˜1Ô	Ø—‘‰Ø	�‰˜1Ô	Ø—‘‰Ø	�‰˜1Ô	Ø—‘‰Ø	�‰˜1Ô	Ø× Ñ ‰Ø	�‰˜1Ô	Ø—‘‰Ø	�‰˜1Ô	Ø×!Ñ!‰Ø	�‰˜1Ô	Ø×!Ñ!‰Ø	�‰˜2Ô	Ø× Ñ ‰ä˜: i§n¡nÐ%5Ð5KÈIÏNÉNÐK[Ð[\Ð]Ó^Ð^à�N‰N˜EÐ"Ð"r+   c                 óÌ   € / pV P                    F"  p\        V4      w  r4VP                  W4/4       K$  	  V P                  '       d   VP                  R V P                  /4       V# )Údomain)rn  ro  Úupdaterq  )r@  ÚkwargsÚattrÚkeyr&   s   &    r(   Ú	kwargs_ofrv  c  sT   € Ø€FØ—”ˆÜ)¨$Ó/‰ˆØ�‰�s�lÖ#ñ ð ‡{‡{€{Ø�‰�x §¡Ð-Ô.Ø€Mr+   c                 óà   € \        V P                  P                  P                  P                   Uu. uF,  qP
                  '       d   VP
                  MVP                  NK.  	  up4      # u upi r%   )ÚtuplerE   r  rD  ÚdimÚ	dim_paramÚ	dim_value)ÚviÚds   & r(   Úshape_ofr~  m  sJ   € ÜÈÏÉ×I\ÑI\×IbÑIb×IfÒIfÓgÑIfÀA§+§+ +�!—+’+°A·K±KÒ?ÑIfÑgÓhÐhùÒgs   ²2A+c                ó$   € V ^8„  d   QhR\         /# ©r8   Úsubg©r   )r?   s   "r(   r@   r@   q  s   € ÷ .ñ .¼Jñ .r+   c                 óJ  € ^p^p. p\        V P                  4       F“  w  rEWA8¼  dw   \        V4      p\        P                  P                  VP                  VP                  P                  P                  V^ ,          V^,          V^,          RV^,          .R7      pVP                  V.4       K•  	  VP                  \        P                  P                  R\        P                  P                  ^.R7      .4       V P                  R4       V P                  P                  V4       . p\        V P                  4       F“  w  rEWB8¼  dw   \        V4      p\        P                  P                  VP                  VP                  P                  P                  V^ ,          V^,          V^,          RV^,          .R7      pVP                  V.4       K•  	  V P                  R4       V P                  P                  V4       . pV P                   Fá  p	T	p
V	P                   R8X  dº   \#        V	4      pVP%                  R^/4       . pVP                  V	P                  4       \'        V4      ^8  d   VP                  R	.4       K$  \'        V4      ^8  d   VP                  R.4       \        P                  P(                  ! RWÉP                  3R
V	P                  /VB p
VP                  V
.4       Kã  	  V P                  R4       V P                  P                  V4       V # )é   Úmax_seq_len©r  rD  Úpast_sequence_length©rD  rÏ   r§   Ú	Attentionra   r   rË   r@  )r  rÏ   r~  rº   rE  rF  rË   rE   r  r  r’   r
   r  Ú
ClearFieldr§   r@  rÍ   rv  rr  r”   Ú	make_node)r�  Úinput_past_0Úoutput_past_0Ú
new_inputsr  r|  rD  Únew_outputsÚ	new_nodesr@  Únew_noders  Úniss   &            r(   Ú1update_decoder_subgraph_past_present_share_bufferr“  q  s†  € Ø€LØ€MØ€JÜ˜4Ÿ:™:Ö&‰ˆØÔÜ˜R“LˆEÜ—‘×3Ñ3Ø—‘ØŸ'™'×-Ñ-×7Ñ7Ø˜Q•x  q¥¨5°­8°]ÀEÈ!ÅHÐMð 4ó ˆBð
 	×Ñ˜2˜$Öñ 'ð ×Ñ”t—{‘{×9Ñ9Ð:PÔRV×RbÑRb×RhÑRhÐqrÐpsÐ9ÓtÐuÔvØ‡O�O�GÔØ‡J�J×Ñ�jÔ!à€KÜ˜4Ÿ;™;Ö'‰ˆØÔÜ˜R“LˆEÜ—‘×3Ñ3Ø—‘ØŸ'™'×-Ñ-×7Ñ7Ø˜Q•x  q¥¨5°­8°]ÀEÈ!ÅHÐMð 4ó ˆBð
 	×Ñ˜B˜4Ö ñ (ð 	‡O�O�HÔØ‡K�K×Ñ�{Ô#à€IØ—	”	ˆØˆØ�<‰<˜;Ô&Ü˜t“_ˆFØ�M‰MÐ6¸Ð:Ô;ØˆCØ�J‰J�t—z‘zÔ"Ü�c“(˜Q”,Ø—
‘
˜B˜4Ö Ü�3‹x˜!Œ|Ø—
‘
Ð2Ð3Ô4Ü—{‘{×,Ò,¨[¸#¿{¹{ÑeÐQU×QZÑQZÐeÐ^dÑeˆHØ×Ñ˜(˜Ö$ñ ð 	‡O�O�FÔØ‡I�I×Ñ�YÔØ€Kr+   c                óH   € V ^8„  d   QhR\         R\        R\        R\        /# )r8   r�  Úis_beam_searchÚswitch_attentionr:   )r   r°   )r?   s   "r(   r@   r@   ¢  s2   € ÷ Sñ SÜ
ðSÜ&*ðSÜ>BðSä	ñSr+   c                ó"  € V'       dö   . p\        V P                  4       F  w  rEVP                  V.4       K  	  VP                  \        P                  P                  R\        P                  P                  ^.R7      .4       VP                  \        P                  P                  R\        P                  P                  . ROR7      .4       V P                  R4       V P                  P                  V4       V'       Ed‰   . ROp. pV P                   EFE  pVP                  R8X  Ed   \        V4      p	V	P                  4        F8  p
V
R8X  d     R# W¦9  g   K  V
R8w  d   \        P                  R	V
 R
24       Wš K:  	  . pVP                  VP                  4       V'       di   \        V4      ^8  d   VP                  R.4       K$  \        V4      ^8  d   VP                  R.4       \        V4      ^	8  d   VP                  R.4       \        P                  P                   ! RVVP"                  3RVP$                  /V	B pVP                  V.4       EKH  	  V P                  R4       V P                  P                  V4       R# )a?  Update the Attention nodes to DecoderMaskedSelfAttention.

Args:
    subg (GraphProto): GraphProto of the decoder subgraph
    is_beam_search (bool): Boolean specifying if the sampling algo is BeamSearch
    switch_attention (bool): Boolean specifying if `Attention` is to be switched with `DecoderMaskedSelfAttention`
Ú
beam_widthrˆ  Úcache_indirectionrÏ   r‰  Úqkv_hidden_sizesFÚunidirectionalzRemoving attribute: zB from Attention node while switching to DecoderMaskedSelfAttentionr   ÚDecoderMaskedSelfAttentionrË   r@  T©Ú
batch_sizer˜  r…  ©ra   Ú	num_headsÚscaleÚmask_filter_valuerq  )r  rÏ   r’   rº   rE  rF  r
   r  rŠ  r@  rÍ   rv  Úcopyr–   r¾   r”   r‹  r§   rË   )r�  r•  r–  rŽ  Ú_ir|  Ú'decoder_masked_attention_supported_attrr�  r@  rs  Úkr’  s   &&&         r(   Ú4update_decoder_subgraph_use_decoder_masked_attentionr§  ¢  s)  € ÷ Øˆ
Ü §
¡
Ö+‰FˆBØ×Ñ˜r˜dÖ#ñ ,ð 	×Ñœ4Ÿ;™;×=Ñ=¸lÌD×L\ÑL\×LbÑLbÐklÐjmÐ=ÓnÐoÔpØ×Ñä—‘×2Ñ2Ø'Ü×$Ñ$×*Ñ*ÚEð 3ó ðô	
ð 	�‰˜Ô Ø�
‰
×Ñ˜*Ô%çÐò3
Ð/ð ˆ	Ø—I•IˆDØ�|‰|˜{Õ*Ü" 4›�ØŸ™ž�Að Ð.Ô.Û$àÖGð Ð 0Ô0Ü"ŸN™NØ"6°q°cÐ9{Ð |ôð #šIñ! 'ð$ �Ø—
‘
˜4Ÿ:™:Ô&÷ "Ü˜c›( Qœ,ØŸ
™
 B 4Ö(Ü˜3“x !”|ØŸ
™
 L >Ô2Ü˜3“x !”|ØŸ
™
Ð$7Ð#8Ô9ä—{‘{×,Ò,Ø0ØØ—K‘Kñð Ÿ™ð	ð
 ñ�ð ×Ñ˜d˜V×$ñQ ðR 	�‰˜ÔØ�	‰	×Ñ˜Ô#ár+   c                ó$   € V ^8„  d   QhR\         /# r€  r‚  )r?   s   "r(   r@   r@   ø  s   € ÷ f3ñ f3¤*ñ f3r+   c                ó
  € \        4       p. p\        V P                  4       UUu/ uF  w  r4VP                  VbK  	  ppp/ p/ pV P                   Fe  pVP                   F1  p	V	'       g   K  W–9  d   V.Wi&   K  Wi,          P                  V4       K3  	  VP                   F  p
V
'       g   K  W‡V
&   K  	  Kg  	  V P                   EF.  pVP                  R8X  g   K  VP                  ^,          '       d   VP                  ^ ,          '       g   KK  VP                  ^ ,          VP                  ^,          rËRpRV9   d_   V P                   FM  pVP                  R8X  g   K  VP                  ^ ,          V8X  g   K0  VP                  ^ ,          P                  p M+	  M(V P                   F  pVP                  V8X  g   K  Tp M	  Vf   EK  \        P                  P                  V4      pVP                  ^8X  g   EK9  VP                  4       R
9   g   EKQ  VP                  ^ ,          V9   g   EKl  W{,          pVP                  R8X  d   VP                  ^ ,          '       g   EK¡  VP                  ^ ,          V9   dÕ   VP                  ^ ,          P!                  R4      '       g)   VP                  ^ ,          P!                  R4      '       d…   VP                  4       ^8X  dp   VP#                  VP                  ^ ,          4       VP                  V4       \%        VVP                  ^ ,          ,          4      ^8X  d   VP                  V4       EK�  VP                  ^ ,          V9  d   EK¨  VVP                  ^ ,          ,          pVP                  R8X  d   VP                  ^ ,          '       g   EKï  VVP                  ^ ,          ,          pVP                  R	8X  d   VP                  ^ ,          '       g   EK6  VP                  ^ ,          V9   g   EKQ  VP                  ^ ,          P!                  R4      '       g,   VP                  ^ ,          P!                  R4      '       g   EK¤  VP                  4       ^8X  g   EK¼  VP#                  VP                  ^ ,          4       VP'                  VVV.4       \%        VVP                  ^ ,          ,          4      ^8X  g   EK  VP                  V4       EK1  	  W3# u uppi )a^  Correct graph which originally use dim of past_seq_len from input_ids's shape which is fixed to max_seq_len after
   shared past/present buffer

Args:
    subg (GraphProto): GraphProto of the decoder subgraph
return:
    tensor_names_to_rename : set of tensor names which is equal to past_sequence_length
    nodes_to_remove : list of node to remove
ÚGatherNÚ	Constant_ÚConstantÚShaper"  r#  ÚReshaperÅ   >   é   r8   )Úsetr  rÏ   rË   r@  r“   r§   rÍ   rn  rg  r=  rº   rC  rÛ   ÚsizeÚitemÚ
startswithÚaddr”   r’   )r�  Útensor_names_to_renameÚnodes_to_removeÚindexÚinpÚgraph_input_namesÚinput_name_to_nodesrÌ   r@  Ú
input_nameÚoutput_nameÚshape_tensor_nameÚshape_index_nameÚini_gather_indicesÚ
const_noderb  Úgather_indices_arrÚ
shape_nodeÚreshape_nodeÚtranspose_nodes   &                   r(   Úfind_past_seq_len_usagerÅ  ø  s  € ô !›UÐØ€Oä;DÀTÇZÁZÔ;PÔQÑ;P©Z¨U˜Ÿ™ 5šÑ;PÐÑQàÐØÐØ—	”	ˆØŸ*œ*ˆJß‰zØÔ8Ø7;°fÐ'Ó3à'Õ3×:Ñ:¸4Ö@ñ %ð  Ÿ;œ;ˆKß‰{Ø37 KÓ0ó 'ñ ð —	•	ˆð �<‰<˜8Ö#Ø—:‘:˜a—=”=¨¯
©
°1¯¬Ùð 48·:±:¸aµ=À$Ç*Á*ÈQÅ-Ð/Ø!%ÐØÐ.Ô.à"&§)¤)�JØ!×)Ñ)¨ZÖ7¸J×<MÑ<MÈaÕ<PÐTdÖ<dØ-7×-AÑ-AÀ!Õ-D×-FÑ-FÐ*Ùò #,ð #×.Ô.�FØ—{‘{Ð&6Ö6Ø-3Ð*Ùñ /ð "Ò)ÚÜ!%×!2Ñ!2×!;Ñ!;Ð<NÓ!OÐð #×'Ñ'¨1×,Ø&×+Ñ+Ó-°×7Ø—J‘J˜q•MÐ%8×8à0ÕC�
Ø"×*Ñ*¨gÔ5¸*×:JÑ:JÈ1×:MÔ:MÚð ×$Ñ$ QÕ'Ð+<Ô<à"×(Ñ(¨Õ+×6Ñ6Ð7G×HÒHØ%×+Ñ+¨AÕ.×9Ñ9Ð:L×MÒMà*×/Ñ/Ó1°QÔ6ð +×.Ñ.¨t¯{©{¸1­~Ô>Ø#×*Ñ*¨4Ô0ÜÐ.¨z×/@Ñ/@ÀÕ/CÕDÓEÈÔJØ'×.Ñ.¨zÔ:Ú à×#Ñ# AÕ&Ð.AÔAÚØ2°:×3CÑ3CÀAÕ3FÕG�Ø$×,Ñ,°	Ô9¸l×>PÑ>PÐQR×>SÔ>SÚØ!4°\×5GÑ5GÈÕ5JÕ!K�Ø&×.Ñ.°+Ô=À.×BVÑBVÐWX×BYÔBYÚð #×(Ñ(¨Õ+Ð/@×@à&×,Ñ,¨QÕ/×:Ñ:Ð;K×LÒLØ)×/Ñ/°Õ2×=Ñ=Ð>P×QÕQà*×/Ñ/Ó1°Q×6ð +×.Ñ.¨t¯{©{¸1­~Ô>Ø#×*Ñ*¨D°*¸lÐ+KÔLÜÐ.¨~×/DÑ/DÀQÕ/GÕHÓIÈQ×NØ'×.Ñ.¨~Ô>Ú ñQ ðT "Ð2Ð2ùós Rs   ¥S;c                ó0   € V ^8„  d   QhR\         R\        /# ©r8   r¿   Úpast_seq_len_name©r   r<   )r?   s   "r(   r@   r@   a  s   € ÷ ñ ¬	ð Äcñ r+   c                 ó<  € R p\        \        R V P                  P                  P                  4      4      pV Fp  p\        VP                  4      ^8  d   VP                  P                  R4       K7  VP                  P                  V4       VP                  P                  V4       Kr  	  V P                  P                  P                  P                  \        P                  P                  V\        P                  . ROR7      4       V P                  4        V # )r™  c                 ó    € V P                   R 8H  # ©ÚMultiHeadAttention©rÍ   ©r@  s   &r(   Ú<lambda>Ú.add_cache_indirection_to_mha.<locals>.<lambda>d  ó   € ¨¯©Ð9MÒ)Mr+   r   rˆ  ©rž  r˜  Úmax_sequence_length)r;   Úfilterr¿   rÊ   r@  r”   rÏ   r“   rº   rE  rF  r
   r  Útopological_sort)r¿   rÈ  Úcache_indirection_nameÚ	mha_nodesr@  s   &&   r(   Úadd_cache_indirection_to_mharÙ  a  sÖ   € à0ÐÜ”VÑMÈuÏ{É{×O`ÑO`×OeÑOeÓfÓg€IÛˆô �$—*‘*‹o Ô!Ø�J‰J×Ñ˜bÖ!Ø�
‰
×ÑÐ+Ô,Ø�
‰
×ÑÐ0Ö1ñ ð 
‡K�K×Ñ×Ñ×"Ñ"Ü�‰×*Ñ*Ø"¤K×$5Ñ$5Ò=pð 	+ó 	
ôð
 
×ÑÔØ€Lr+   c                óR   € V ^8„  d   QhR\         R\        R\        \        ,          /# )r8   r¿   rÇ   Úskip_node_idxs)r   r}   r;   )r?   s   "r(   r@   r@   v  s&   € ÷ 1ñ 1¤	ð 1´#ð 1Ì4ÔPSÍ9ñ 1r+   c                 ó’  € R p. p\        \        R V P                  P                  P                  4      4      p\        V4       EF¿  w  rgWb9   d   K  ^ pVP                   F"  p	V	P                  R8X  g   K  V	P                  p M	  Tp
V
^ 8X  dX   V P                  P                  P                   F3  pVP                  VP                  ^,          8X  g   K'  VP                  p
 M	  RpV P                  P                  P                   Fb  pVP                  VP                  ^,          8X  g   K'  VP                  P                  P                  P                  ^,          P                   p M	  \#        VP$                  4      ^8  d   VP$                  P'                  R4       K7  V RV^,           2pVP$                  P'                  V4       VP'                  \(        P*                  P-                  VV
RVRV.R7      4       EKÂ  	  V P                  P                  P$                  P/                  V4       V P1                  4        V # )	Úoutput_cross_qkc                 ó    € V P                   R 8H  # rÌ  rÎ  rÏ  s   &r(   rÐ  Ú&add_output_qk_to_mha.<locals>.<lambda>z  rÒ  r+   r   Útarget_sequence_lengthr   Ú_rž  Úsequence_lengthrˆ  )r;   rÕ  r¿   rÊ   r@  r  rn  rË   r  r=  rÏ   rÑ   rE   r  rD  ry  r{  r”   r§   r“   rº   rE  rF  r’   rÖ  )r¿   rÇ   rÛ  Úoutput_qk_basenameÚ
output_qksrØ  Úidxr@  r   ÚattÚoutput_qk_dtyper  rà  Úoutput_qk_names   &&&           r(   Úadd_output_qk_to_mharé  v  s×  € à*ÐØ€JÜ”VÑMÈuÏ{É{×O`ÑO`×OeÑOeÓfÓg€IÜ˜y×)‰	ˆàÔ Ùð ˆ	Ø—>”>ˆCØ�x‰x˜;Ö&ØŸE™E�	Ùñ "ð  ˆØ˜aÔØ—[‘[×&Ñ&×2Ô2�Ø—6‘6˜TŸZ™Z¨�]Ö*Ø&'§k¡k�OÙñ 3ð ":ÐØ—‘×"Ñ"×(Ô(ˆAØ�v‰v˜Ÿ™ A�Ö&Ø)*¯©×);Ñ);×)AÑ)A×)EÑ)EÀaÕ)H×)RÑ)RÐ&Ùñ )ô �$—+‘+Ó Ô"Ø�K‰K×Ñ˜rÖ"à.Ð/¨q°¸µ°
Ð;ˆØ�‰×Ñ˜>Ô*Ø×ÑÜ�K‰K×.Ñ.ØØØ# YÐ0AÐCYÐZð /ó ÷	
ñE *ðT 
‡K�K×Ñ×Ñ×#Ñ# JÔ/Ø	×ÑÔØ€Lr+   c                ó$   € V ^8„  d   QhR\         /# )r8   r¿   r   )r?   s   "r(   r@   r@   ª  s   € ÷ f$ñ f$¤Iñ f$r+   c                 ó¼  aa€ R pRpRp\        \        R V P                  P                  P                  4      4      ^ ,          pV P                  V. RO. RO4      pV P                  VRR.^ ^.4      pVe   TpMVe   TpM\        P                  R	4       R# VR,          pVP                  R8X  Ed“   VR,          p	V P                  V	RR
.^ ^ .4      oSf   \        P                  R4       R# V P                  V	. RO. RO4      p
V
f   \        P                  R4       R# V
^ ,          pSV
R,          8w  d   \        P                  R4       R# \        \        V3R lV P                  P                  P                  4      4      ^ ,          pV P                  P                  P                  P                  V4       V P                  P                  P                  P                  S^ ,          4       V P                  P                  P                  P                  S^,          4       W9P                  ^ &   W;P                  ^ &   EMpV P                  V. R O. R!O4      pVf   \        P                  R4       R# V^,          pV P                  V. R"O. R#O4      oSf   \        P                  R4       R# S^ ,          pVR,          SR,          8w  d   \        P                  R4       R# \        \        V3R lV P                  P                  P                  4      4      ^ ,          pV P                  P                  P                  P                  V4       \        \        V3R lV P                  P                  P                  4      4      ^ ,          pV P                  P                  P                  P                  V4       V P                  P                  P                  P                  S^,          4       V P                  P                  P                  P                  S^,          4       V P                  P                  P                  P                  S^,          4       V P                  P                  P                  P                  S^,          4       W>P                  ^ &   W;P                  ^ &   V P                  P                  P                  P                  \        P                  P                  V\        P                   ^.R7      4       \        P                  P#                  RV.V.V P%                  R4      R7      p\        P                  P                  V\        P                   . R7      p\        P                  P#                  RV.V.V P%                  R4      \        P&                  R7      p\        P                  P                  V\        P&                  . R7      pV P                  P                  P                  P)                  VV.4       V P                  P                  P*                  P)                  VV.4       V P-                  4        W3# )$r‡  Úpast_seq_len_int32Úpast_seq_len_int64c                 ó    € V P                   R 8H  # )ÚLayerNormalizationrÎ  )Úns   &r(   rÐ  Ú*fix_past_sequence_length.<locals>.<lambda>Ð  s   €  §¡Ð.BÒ!Br+   ÚAddrª  ÚRangeÚSliceNzBCannot identify base path for fixing past_sequence_length subgraphr­  zDCannot identify gather path for fixing past_sequence_length subgraphzACannot identify add path for fixing past_sequence_length subgraph:r¯  NNz]Gather path and add path do not share the same nodes for calculating the past_sequence_lengthc                 ó`   <€ V P                   ^ ,          S^ ,          P                  ^,          8H  # ©é    ©r§   rÏ   )rð  Úgather_paths   &€r(   rÐ  rñ    s"   ø€ °1·8±8¸Aµ;À+ÈaÅ.×BVÑBVÐWXÕBYÒ3Yr+   zGCannot identify input_ids path for fixing past_sequence_length subgraphzFCannot identify past_key path for fixing past_sequence_length subgraph:r8   NNziThe input_ids path and past_key path do not share the same nodes for calculating the past_sequence_lengthc                 ó`   <€ V P                   ^ ,          S^,          P                  ^,          8H  # rö  rø  ©rð  Úpast_key_paths   &€r(   rÐ  rñ  '  s$   ø€ °1·8±8¸Aµ;À-ÐPQÕBR×BXÑBXÐYZÕB[Ò3[r+   c                 ó`   <€ V P                   ^ ,          SR,          P                  ^,          8H  # )r÷  éþÿÿÿrø  rû  s   &€r(   rÐ  rñ  )  s$   ø€ °A·H±H¸QµKÀ=ÐQSÕCT×CZÑCZÐ[\ÕC]Ò4]r+   rˆ  ÚSqueeze©ÚinputsÚoutputsrË   ÚCast©r  r  rË   Úto)rò  rª  ÚTileÚExpandÚ	Unsqueezeró  )r÷  r¯  r¯  r÷  r÷  r÷  rs   )rò  rª  r­  ©r¯  r÷  r÷  )r  rò  rª  r­  r®  rÅ   )r8   r÷  r÷  r÷  r÷  r÷  )r  rª  r­  r®  rÅ   )r¯  r÷  r÷  r÷  r÷  )r;   rÕ  r¿   rÊ   r@  Úmatch_parent_pathr–   r—   rÍ   rA  rÏ   r“   rº   rE  rF  r
   r  r‹  Úcreate_node_nameÚINT64r’   rB  rÖ  )r¿   rÈ  rì  rí  r@  Úbase_path_hfÚbase_path_oaiÚ	base_pathÚ	base_nodeÚ
range_nodeÚadd_pathÚadd_nodeÚconstant_in_gatherÚinput_ids_pathÚunsqueeze_nodeÚconstant_in_reshapeÚsqueeze_nodeÚsqueeze_outputÚ	cast_nodeÚcast_outputrù  rü  s   &                   @@r(   Úfix_past_sequence_lengthr  ª  sF  ù€ ðD /ÐØ-ÐØ-Ðä”ÑBÀEÇKÁK×DUÑDU×DZÑDZÓ[Ó\Ð]^Õ_€Dà×*Ñ*ØÚAÚó€Lð
 ×+Ñ+ØØ	�ÐØ	
ˆAˆó€Mð
 ÒØ ‰	Ø	Ò	"Ø!‰	ä�‰ÐXÔYÙØ˜"•€Ià×Ñ˜GÕ#à˜r•]ˆ
à×-Ñ-ØØ�wÐØ�ˆFó
ˆð
 ÒÜ�K‰KÐ^Ô_Ùà×*Ñ*ØÚ&Úó
ˆð
 ÒÜ�K‰KÐ[Ô\ÙØ˜A•;ˆà˜( 2�,Ô&Ü�K‰KÐwÔxÙô "¤&Ô)YÐ[`×[fÑ[f×[lÑ[l×[qÑ[qÓ"rÓsÐtuÕvÐØ�‰×Ñ×Ñ×%Ñ%Ð&8Ô9Ø�‰×Ñ×Ñ×%Ñ% k°!¥nÔ5Ø�‰×Ñ×Ñ×%Ñ% k°!¥nÔ5ð 1×Ñ˜ÑØ.�‰�qÓð ×0Ñ0ØÚKÚó
ˆð
 Ò!Ü�K‰KÐaÔbÙØ! !Õ$ˆà×/Ñ/ØÚDÚó
ˆð
 Ò Ü�K‰KÐ`ÔaÙØ& qÕ)ˆà˜"Õ ¨rÕ!2Ô2Ü�K‰KØ{ôñ ô "¤&Ô)[Ð]b×]hÑ]h×]nÑ]n×]sÑ]sÓ"tÓuÐvwÕxÐØ�‰×Ñ×Ñ×%Ñ%Ð&8Ô9Ü"¤6Ô*]Ð_d×_jÑ_j×_pÑ_p×_uÑ_uÓ#vÓwØõ
Ðð 	�‰×Ñ×Ñ×%Ñ%Ð&9Ô:Ø�‰×Ñ×Ñ×%Ñ% m°AÕ&6Ô7Ø�‰×Ñ×Ñ×%Ñ% m°AÕ&6Ô7Ø�‰×Ñ×Ñ×%Ñ% m°AÕ&6Ô7Ø�‰×Ñ×Ñ×%Ñ% m°AÕ&6Ô7ð #5×Ñ˜QÑØ.�‰�qÑð 
‡K�K×Ñ×Ñ×"Ñ"Ü�‰×*Ñ*Ð+<¼k×>OÑ>OÐXYÐWZÐ*Ó[ôô
 —;‘;×(Ñ(ØØ!Ð"Ø#Ð$Ø×#Ñ# IÓ.ð	 )ó €Lô —[‘[×7Ñ7Ð8JÌK×L]ÑL]ÐegÐ7Óh€NÜ—‘×%Ñ%ØØ"Ð#Ø#Ð$Ø×#Ñ# FÓ+Ü×Ñð &ó €Iô —+‘+×4Ñ4Ð5GÌ×IZÑIZÐbdÐ4Óe€Kð 
‡K�K×Ñ×Ñ×!Ñ! <°Ð";Ô<Ø	‡K�K×Ñ× Ñ ×'Ñ'¨¸Ð(EÔFØ	×ÑÔØÐ#Ð#r+   c                ó0   € V ^8„  d   QhR\         R\        /# rÇ  rÉ  )r?   s   "r(   r@   r@   S  s   € ÷ Dñ D¤)ð DÄñ Dr+   c                 ó4  € R pRpV P                   P                  P                  P                  \        P
                  P                  V\        P                  ^.R7      \        P
                  P                  V\        P                  . ROR7      .4       \        \        R V P                   P                  P                  4      4      p\        V4       EF¯  w  rV^ pVP                   F"  pVP                  R8X  g   K  VP                  p M	  RV^,           2p	\        P
                  P                  V	\        P                   RV^R.R7      p
V^,          ^8X  d0   V P                   P                  P"                  P%                  V
4       \        P
                  P'                  RVP                  ^ ,          VP                  ^,          VP                  ^,          R	R	\)        VP                  4      ^8”  d   VP                  ^,          MR	\)        VP                  4      ^8”  d   VP                  ^,          MR	VVVVP                  ^,          .VP"                  ^ ,          \)        VP                  4      ^8”  d   VP"                  ^,          MR	\)        VP                  4      ^8”  d   VP"                  ^,          MR	V^,          ^8X  d   T	MR	.VP                  P+                  R
R4      RVV^,          ^R7      pV^,          ^ 8X  d   VP"                  P-                  R	4       V P                   P                  P                  P-                  V4       V P                   P                  P                  P                  V.4       EK²  	  V P/                  4        V # )r˜  r™  rˆ  rž  c                 ó    € V P                   R 8H  # rÌ  rÎ  rÏ  s   &r(   rÐ  Ú(replace_mha_with_dmmha.<locals>.<lambda>b  rÒ  r+   r   Úoutput_cross_qk_zencode_sequence_length / 2ÚDecoderMaskedMultiHeadAttentionr   rÍ  úcom.microsoft)r  r  rË   rq  r   Ú	output_qkra   rÓ  )r¿   rÊ   rÏ   r’   rº   rE  rF  r
   r  r;   rÕ  r@  r  rn  rË   r  r  r§   r“   r‹  r”   ÚreplacerA  rÖ  )r¿   rÈ  r˜  r™  rØ  rå  r@  r   ræ  Úqk_output_nameÚ	qk_outputÚ
dmmha_nodes   &&          r(   Úreplace_mha_with_dmmhar)  S  sÐ  € à€JØ+Ðà	‡K�K×Ñ×Ñ×"Ñ"ä�K‰K×.Ñ.¨z¼;×;LÑ;LÐUVÐTWÐ.ÓXÜ�K‰K×.Ñ.Ø!¤;×#4Ñ#4Ò<oð /ó ð	
ôô ”VÑMÈuÏ{É{×O`ÑO`×OeÑOeÓfÓg€IÜ˜y×)‰	ˆàˆ	Ø—>”>ˆCØ�x‰x˜;Ö&ØŸE™E�	Ùñ "ð ,¨C°1­H¨:Ð6ˆÜ—K‘K×6Ñ6ØœK×-Ñ-°lÀIÈqÐRnÐ5oð 7ó 
ˆ	ð ��7�aŒ<Ø�K‰K×Ñ×$Ñ$×+Ñ+¨IÔ6ô —[‘[×*Ñ*Ø-à—
‘
˜1•Ø—
‘
˜1•Ø—
‘
˜1•ØØÜ!$ T§Z¡Z£°1Ô!4�—
‘
˜1–¸"Ü!$ T§Z¡Z£°1Ô!4�—
‘
˜1–¸"Ø!ØØ!Ø—
‘
˜1•ðð —‘˜A•Ü"% d§j¡j£/°AÔ"5�—‘˜A–¸2Ü"% d§j¡j£/°AÔ"5�—‘˜A–¸2Ø"%¨¥'¨Q¤,‘°Bð	ð —‘×"Ñ"Ð#7Ð9ZÓ[Ø"ØØ˜Q•wØ&'ð3 +ó 
ˆ
ð6 ��7�aŒ<à×Ñ×$Ñ$ RÔ(à�‰×Ñ×Ñ×%Ñ% dÔ+Ø�‰×Ñ×Ñ×%Ñ% z l×3ñc *ðf 
×ÑÔØ€Lr+   c          
      óT   € V ^8„  d   QhR\         R\        R\        R\        R\        /# )r8   r¿   Ú	attn_maskÚkv_num_headsÚ
world_sizeÚwindow_size)r   r<   r}   )r?   s   "r(   r@   r@   š  sA   € ÷ kñ kÜðkäðkô ðkô ð	kô
 ñkr+   c                 ón  € V P                  \        P                  P                  R \        P
                  ^.^.R7      4       \        P                  P                  RVR .VR,           .V P                  R4      R7      p\        P                  P                  RVR,           R .R.V P                  R4      R7      p\        P                  P                  RR.R.V P                  R4      \        P                  R	7      p\        P                  P                  R
V.VR,           .V P                  R
4      R7      p\        P                  P                  RVR,           R .R.V P                  R4      ^ R7      p	\        P                  P                  RR.R.V P                  R4      \        P                  R	7      p
V P                  P                  P                  P                  VVVVV	V
.4       \        \        R V P                  P                  P                  4      4      p\        V4       EFƒ  w  rÍV P!                  V. R"O. R#O4      pV P!                  VRR.^ ^ .4      pRRRpppVe   Vw  pppM	Ve   Vw  ppV P!                  V. R"O. R$O4      pV P!                  VRR.^^ .4      pRRRpppVe   Vw  pppM	Ve   Vw  ppV P!                  VRR.^^ .4      pV P!                  VR.^.4      pRRppVe   Vw  ppMVe
   V^ ,          p^ pVe7   Ve3   VP"                   F"  pVP$                  R8X  g   K  VP&                  pK$  	  ^ pVP"                   F"  pVP$                  R8X  g   K  VP&                  pK$  	  VP(                  ^ ,          VP(                  ^ ,          8H  ;'       d(    VP(                  ^ ,          VP(                  ^ ,          8H  pVRJ;'       d    VRJ;'       d    VRJp VRJ ;'       d    VRJ ;'       d    VRJ p!RRRp$p#p"V'       Ed�   V '       g
   V!'       EdŒ   \*        P,                  ! V P/                  VP(                  ^,          4      4      p%\*        P,                  ! V P/                  VP(                  ^,          4      4      p&\*        P,                  ! V P/                  VP(                  ^,          4      4      p'V%P0                  R%,          p(\2        P4                  ! V%V&V'3^R7      P7                  V(^V(,          4      p)\        P8                  P;                  V)RV 2R7      p)V P                  V)4       \        P                  P                  RVP(                  ^ ,          V)P$                  .V)P$                   R2.V P                  R4      R7      p*V P                  P                  P                  P                  V*.4       V P                  P                  P                  P=                  V4       V P                  P                  P                  P=                  V4       V P                  P                  P                  P=                  V4       V*P>                  ^ ,          p"V '       Ed9   \*        P,                  ! V P/                  VP(                  ^,          4      4      p+\*        P,                  ! V P/                  VP(                  ^,          4      4      p,\*        P,                  ! V P/                  VP(                  ^,          4      4      p-V+P0                  R%,          p(\2        P4                  ! V+V,V-3^ R7      P7                  ^V(,          4      p.\        P8                  P;                  V.RV 2R7      p.V P                  V.4       \        P                  P                  RV*P>                  ^ ,          V.P$                  .V.P$                   R2.R7      p/V P                  P                  P                  P                  V/.4       V P                  P                  P                  P=                  V4       V P                  P                  P                  P=                  V4       V P                  P                  P                  P=                  V4       V/P>                  ^ ,          p"M9VP>                  ^ ,          p"VP>                  ^ ,          p#VP>                  ^ ,          p$\        P                  P                  RT"T#T$VP(                  ^,          VP(                  ^,          VP>                  ^ ,          V
P>                  ^ ,          Ve   VP(                  ^,          MRVe   VP(                  ^,          MR.	VP>                  VP$                  PA                  RR4      R VV,          V^ 8X  d
   VV,          MW#,          T\C        VRJ;'       d    VRJ4      VR!7
      p0V P                  P                  P                  P=                  V4       V P                  P                  P                  P                  V0.4       Ve0   V P                  P                  P                  P=                  V4       Vf   EKT  V P                  P                  P                  P=                  V4       EK†  	  V # )&Úone©rË   rÑ   rÓ   ÚvalsÚ	ReduceSumÚ	_row_sumsr   ÚSubÚseqlens_k_int64r  Ú	seqlens_kr  r­  Ú_shaperª  Útotal_seq_len_int64)r  r  rË   rÉ   Útotal_seq_lenc                 ó    € V P                   R 8H  # rÌ  rÎ  rÏ  s   &r(   rÐ  Ú&replace_mha_with_gqa.<locals>.<lambda>   rÒ  r+   ÚRotaryEmbeddingrò  rÄ   NÚinterleavedr   r   rÈ   ÚQKV_Weight_©rË   Ú_outputÚ	QKV_Bias_)r  r  ÚGroupQueryAttentionrÍ  r#  )	r  r  rË   rq  r   r,  Úlocal_window_sizeÚ	do_rotaryÚrotary_interleaved)r=  rò  rÄ   )r÷  r÷  r÷  r	  rs   )"Úadd_initializerrº   rE  Úmake_tensorr
   r  r‹  r  r  r¿   rÊ   r@  r’   r;   rÕ  r  r
  rn  rË   r  rÏ   r   rÛ   rÎ   rD  r×   ÚstackÚreshaperC  Ú
from_arrayrA  r§   r%  r}   )1r¿   r+  r,  r-  r.  Úreduce_sum_nodeÚsub_nodeÚseqlen_k_cast_noderÂ  Úgather_nodeÚtotal_seqlen_cast_noderØ  rå  r@  Úq_path_1Úq_path_2Úq_rotaryÚq_addÚq_matmulÚk_path_1Úk_path_2Úk_rotaryÚk_addÚk_matmulÚv_path_1Úv_path_2Úv_addÚv_matmulr>  ræ  r   Úroot_input_is_sameÚall_paths_have_biasÚall_paths_have_no_biasÚq_input_to_attentionÚk_input_to_attentionÚv_input_to_attentionÚqwÚkwÚvwry  Ú
qkv_weightÚpacked_matmul_nodeÚqbÚkbÚvbÚqkv_biasÚpacked_add_nodeÚgqa_nodes1   &&&&&                                            r(   Úreplace_mha_with_gqarp  š  sf	  € ð& 
×ÑÜ�‰×ÑØÜ!×'Ñ'Ø�Ø�ð	 	 ó 	
ôô —k‘k×+Ñ+ØØ˜5Ð!Ø˜[Õ(Ð)Ø×#Ñ# KÓ0ð	 ,ó €Oô �{‰{×$Ñ$ØØ˜KÕ'¨Ð/Ø"Ð#Ø×#Ñ# EÓ*ð	 %ó €Hô Ÿ™×.Ñ.ØØ!Ð"Ø�Ø×#Ñ# FÓ+Ü×Ñð /ó Ðô —‘×&Ñ&ØØˆ{Ø˜XÕ%Ð&Ø×#Ñ# GÓ,ð	 'ó €Jô —+‘+×'Ñ'ØØ˜HÕ$ eÐ,Ø&Ð'Ø×#Ñ# HÓ-Øð (ó €Kô "Ÿ[™[×2Ñ2ØØ%Ð&Ø Ð!Ø×#Ñ# FÓ+Ü×Ñð 3ó Ðð 
‡K�K×Ñ×Ñ×!Ñ!àØØØØØ"ð	
ô	ôH ”VÑMÈuÏ{É{×O`ÑO`×OeÑOeÓfÓg€IÜ˜y×)‰	ˆà×*Ñ*¨4Ò1UÒW`ÓaˆØ×*Ñ*¨4Ð2CÀXÐ1NÐQRÐTUÐPVÓWˆà$(¨$°˜�%ˆØÒØ(0Ñ%ˆH�e™XØÒ!Ø!)ÑˆH�hð ×*Ñ*¨4Ò1UÒW`ÓaˆØ×*Ñ*¨4Ð2CÀXÐ1NÐQRÐTUÐPVÓWˆà$(¨$°˜�%ˆØÒØ(0Ñ%ˆH�e™XØÒ!Ø!)ÑˆH�hð ×*Ñ*¨4°%¸Ð1BÀQÈÀFÓKˆØ×*Ñ*¨4°(°¸a¸SÓAˆà ˆxˆØÒØ&‰OˆE‘8ØÒ!Ø •{ˆHð ˆØÒ HÒ$8Ø×)Ô)�Ø—8‘8˜}Ö,Ø"%§%¡%’Kñ *ð
 ˆ	Ø—>”>ˆCØ�x‰x˜;Ö&ØŸE™E’	ñ "ð
 &Ÿ^™^¨AÕ.°(·.±.ÀÕ2CÑC×nÐnÈÏÉÐWXÕHYÐ]e×]kÑ]kÐlmÕ]nÑHnÐð $¨4Ð/×[Ð[°EÀÐ4E×[Ð[È%ÐW[ÐJ[ÐØ!&¨$ ×!RÐ!R°5¸D°=×!RÐ!RÀUÈdÀ]Ðð LNÈrÐSUÐ4HÐ2ÐßÐ×#6×:PÐ:PÜ×%Ò% e×&;Ñ&;¸H¿N¹NÈ1Õ<MÓ&NÓOˆBÜ×%Ò% e×&;Ñ&;¸H¿N¹NÈ1Õ<MÓ&NÓOˆBÜ×%Ò% e×&;Ñ&;¸H¿N¹NÈ1Õ<MÓ&NÓOˆBà—(‘(˜2•,ˆCÜŸš 2 r¨2 ,°QÔ7×?Ñ?ÀÀQÈÅWÓMˆJÜ×*Ñ*×5Ñ5°jÈÐUXÐTYÐGZÐ5Ó[ˆJØ×!Ñ! *Ô-ä!%§¡×!6Ñ!6ØØ Ÿ™ qÕ)¨:¯?©?Ð;Ø&ŸO™OÐ,¨GÐ4Ð5Ø×+Ñ+¨HÓ5ð	 "7ó "Ðð �K‰K×Ñ×"Ñ"×)Ñ)Ð+=Ð*>Ô?Ø�K‰K×Ñ×"Ñ"×)Ñ)¨(Ô3Ø�K‰K×Ñ×"Ñ"×)Ñ)¨(Ô3Ø�K‰K×Ñ×"Ñ"×)Ñ)¨(Ô3Ø#5×#<Ñ#<¸QÕ#?Ð ÷ #Ð"Ü ×)Ò)¨%×*?Ñ*?ÀÇÁÈAÅÓ*OÓP�Ü ×)Ò)¨%×*?Ñ*?ÀÇÁÈAÅÓ*OÓP�Ü ×)Ò)¨%×*?Ñ*?ÀÇÁÈAÅÓ*OÓP�à—h‘h˜r•l�ÜŸ8š8 R¨¨R L°qÔ9×AÑAÀ!ÀcÅ'ÓJ�Ü×,Ñ,×7Ñ7¸ÈÐSVÐRWÐGXÐ7ÓY�Ø×%Ñ% hÔ/Ü"&§+¡+×"7Ñ"7ØØ.×5Ñ5°aÕ8¸(¿-¹-ÐHØ (§¡˜¨gÐ6Ð7ð #8ó #�ð
 —‘×!Ñ!×&Ñ&×-Ñ-¨Ð.?Ô@Ø—‘×!Ñ!×&Ñ&×-Ñ-¨eÔ4Ø—‘×!Ñ!×&Ñ&×-Ñ-¨eÔ4Ø—‘×!Ñ!×&Ñ&×-Ñ-¨eÔ4Ø'6×'=Ñ'=¸aÕ'@Ð$øð $,§?¡?°1Õ#5Ð Ø#+§?¡?°1Õ#5Ð Ø#+§?¡?°1Õ#5Ð ô —;‘;×(Ñ(Ø!à$Ø$Ø$Ø—
‘
˜1•Ø—
‘
˜1•Ø"×)Ñ)¨!Õ,Ø&×-Ñ-¨aÕ0Ø&.Ò&:�—‘ Ö"ÀØ&.Ò&:�—‘ Ö"Àð
ð —K‘KØ—‘×"Ñ"Ð#7Ð9NÓOØ"Ø :Õ-Ø5AÀQÔ5F˜) zÖ1ÈLÕLfØ)Ü˜(¨$Ð.×GÐG°8À4Ð3GÓHØ*ð) )ó 
ˆð, 	�‰×Ñ×Ñ×%Ñ% dÔ+Ø�‰×Ñ×Ñ×%Ñ% x jÔ1àÒØ�K‰K×Ñ×"Ñ"×)Ñ)¨(Ô3ØÕØ�K‰K×Ñ×"Ñ"×)Ñ)¨(×3ñE *ðH €Lr+   c                ó$   € V ^8„  d   QhR\         /# r€  r‚  )r?   s   "r(   r@   r@   ˆ  s   € ÷ &fñ &f¼ñ &fr+   c           	      óX  € ^pV P                    Uu. uF  q"P                  NK  	  ppV^8  d)   W1,          P                  R4      '       g   V^,          pK/  ^p\        V P                  4      V,
          ^,          p^V,          V,           p\        V4       Uu/ uF.  qpP                   V^,          V,           ,          P                  VbK0  	  pp\        RV 24       \        V P                   V,          4      p	\        RV	 24       V	^ ,          p
V	^,          pV	^,          p^ pV P                   EFU  pVP                  R8X  g   K  VP                   ^,          V9   g   K1  \        RVP                   RVP                   24       V^,          pWŽP                   ^,          ,          pRV 2pR.^\        VP                  4      ,
          ,          pVP                  V4       VP                  P                  V4       VP                  P                  \        P                  P                  R	^4      .4       \        P                  P!                  V\"        P$                  W«^V.4      pV P                  P                  V.4       EKX  	  WÕ8w  d   \'        R
V RV 24      hR# u upi u upi )r¯  Úpastz    -- past_key_cross_inputs = zpast_key_cross_0_shape is r"  z'    -- add cross QK output from: node: z with output: r!  r   r$  z#Did not add cross QK for all layersz vs N)rÏ   rË   r³  r”   r§   r  Úprintr~  r@  rÍ   r“   r’   rn  rº   rE  Úmake_attributerF  r
   r  r  )r�  Úinput_self_past_0Úgir¹  Úoutput_self_present_0Ú
num_layersÚinput_cross_past_0ÚlayerÚpast_key_cross_inputsÚinput_past_key_cross_0_shapeÚbatch_size_dimÚnum_heads_dimÚcross_seq_len_dimÚnum_layer_output_qkr@  Úcross_attention_out_nameÚappended_namesÚcross_attentions   &                 r(   Ú.update_decoder_subgraph_output_cross_attentionr…  ˆ  s^  € ØÐà+/¯:ª:Ó6©: RŸœ©:ÐÐ6Ø
˜aÔ
Ð(9Õ(L×(WÑ(WÐX^×(_Ò(_Ø˜QÕÒØÐä�d—k‘kÓ"Ð%:Õ:¸qÕ@€JØ˜Z�Ð*;Õ;ÐÜafÐgqÔarÓsÑarÐX]ŸZ™Z¨°­	Ð4FÕ(FÕG×LÑLÈeÒSÑarÐÐsÜ	Ð+Ð,AÐ+BÐ
CÔDä#+¨D¯J©JÐ7IÕ,JÓ#KÐ Ü	Ð&Ð'CÐ&DÐ
EÔFØ1°!Õ4€NØ0°Õ3€MØ4°QÕ7ÐàÐØ—	•	ˆØ�L‰LÐ=Ö=ÀDÇJÁJÈqÅMÐUjÖDjÜÐ;¸D¿I¹I¸;ÀnÐUY×U`ÑU`ÐTaÐbÔcØ 1Õ$ÐØ)¯*©*°Q­-Õ8ˆEØ)9¸%¸Ð'AÐ$Ø ˜T Q¬¨T¯[©[Ó)9Õ%9Õ:ˆNØ×!Ñ!Ð":Ô;Ø�K‰K×Ñ˜~Ô.Ø�N‰N×!Ñ!¤4§;¡;×#=Ñ#=¸kÈ1Ó#MÐ"NÔOä"Ÿk™k×@Ñ@Ø(Ü×!Ñ!Ø°Ð3DÐEóˆOð
 �K‰K×Ñ Ð0×1ñ! ð" Ô(ÜÐ>¸z¸lÈ$ÐObÐNcÐdÓeÐeñ )ùòE 7ùò ts   ‘J"Â4J'c                ó$   € V ^8„  d   QhR\         /# r€  rT  )r?   s   "r(   r@   r@   ±  s   € ÷ \ñ \Ì*ñ \r+   c           
      óB  € ^pV P                    Uu. uF  q"P                  NK  	  ppV^8  d)   W1,          P                  R4      '       g   V^,          pK/  ^p\        \	        V P                   4      V,
          ^,          4      p^V,          V,           p. p. pV P
                   F(  p	V	P                  R8X  g   K  VP                  V	.4       K*  	  \	        V4      V8  d   R# Rp
V P
                   F  p	V	P                  R8X  g   K  T	p
 M	  . R#OpRp\        V 4      w  rÞ\	        V4      ^ 8”  d¶   V F  p\        RV R	V R
24       K  	  V F(  p\        RVP                   RVP                   24       K*  	  \        P                  P                  RR.R.RR7      p\        P                  P                  RR.V.R\        P                  R7      pVP                  VV.4       V P
                   EF)  p	\	        V	P                  4      ^ 8”  d”   V
e�   V	P                  ^ ,          V
P                   ^,          8X  dg   \        P                  P                  RR.R.R\        P                  R7      pVP                  ^ ,          V	P                   ^&   VP                  V.4       V	P                  R8X  Ed   \!        V	4      pVP#                  4        F  pVV9  g   K  VV K  	  V	P                   ^ ,          V	P                   ^,          V	P                   ^,          .pTP                  \	        V	P                   4      ^8”  d   V	P                   ^,          MR.4       TP                  \	        V	P                   4      ^8”  d   V	P                   ^,          MR.4       TP                  \	        V	P                   4      ^8”  d   V	P                   ^,          MR.4       TP                  \	        V	P                   4      ^8”  d   V	P                   ^,          MR.4       VP                  R.4       VP                  R.4       VP                  R.4       TP                  \	        V	P                   4      ^8”  d   V	P                   ^,          MR.4       ^VR&   \        P                  P                  ! RVV	P                  3RV	P                  /VB p	Wž9  g   EKá  \%        V	P                   4       F  w  ppVV9   g   K  WÉP                   V&   K  	  VP                  V	.4       EK,  	  V P'                  R4       V P
                  P                  V4       V P                    Uu. uF  pVP                  NK  	  pp. p\%        V P                   4       F”  w  ppVV8¼  dv   VV8  do   \)        V4      p\        P                  P+                  VP                  VP,                  P.                  P0                  V^ ,          V^,          RV^,          .R7      pVP                  V.4       K–  	  RV9  dL   VP                  \        P                  P+                  R\        P                  P2                  ^.R7      .4       RV9  dL   VP                  \        P                  P+                  R\        P                  P2                  ^.R7      .4       RV9  dM   VP                  \        P                  P+                  R\        P                  P2                  . R$OR7      .4       V P'                  R 4       V P                   P                  V4       . p\%        V P                  4       F�  w  ppVV8¼  do   \)        V4      p\        P                  P+                  VP                  VP,                  P.                  P0                  V^ ,          V^,          RV^,          .R7      pVP                  V.4       K�  	  V P'                  R!4       V P                  P                  V4       R"# u upi u upi )%r¯  rs  rÍ  FNÚRelativePositionBiasra   Ú#past_sequence_length_squeezed_int64zFound tensor name `z` to be renamed to `Ú`zFound node to remove: type = z	, name = rÿ  r‡  Úpast_sequence_length_squeezedÚ!node_past_sequence_length_squeezer@  r  Ú&node_past_sequence_length_squeeze_cast)rË   r  Úpast_sequence_length_int64Úpast_sequence_length_castr   r˜  r™  r"  rË   r@  r…  r†  rˆ  rÏ   r§   TrŸ  r�  )rÏ   rË   r³  r}   r”   r@  rÍ   r’   rÅ  rt  rº   rE  r‹  r
   r  r§   rv  r£  r  rŠ  r~  rF  rE   r  r  r  )r�  rv  rw  r¹  Úoutput_self_past_0ry  rz  r�  Ú	old_nodesr@  Úrel_pos_bias_noder¥  Útarget_squeezed_past_seq_namerµ  r¶  Úname_to_renameÚnrr  r  rs  r¦  r’  r·  rË   r¸  Úorig_input_namesrŽ  r  r|  rD  r�  s   &                              r(   Ú?update_decoder_subgraph_share_buffer_and_use_decoder_masked_mhar—  ±  sâ  € ØÐà+/¯:ª:Ó6©: RŸœ©:ÐÐ6Ø
˜aÔ
Ð(9Õ(L×(WÑ(WÐX^×(_Ò(_Ø˜QÕÒØÐä”c˜$Ÿ*™*“oÐ(9Õ9¸QÕ>Ó?€JØ˜Z�Ð*;Õ;Ðà€IØ€IØ—	”	ˆØ�<‰<Ð/Ö/Ø×Ñ˜d˜VÖ$ñ ô
 ˆ9ƒ~˜
Ô"Ùð ÐØ—	”	ˆØ�<‰<Ð1Ö1Ø $ÐÙñ ò
/Ð+ð %JÐ!Ü.EÀdÓ.KÑ+ÐÜ
Ð!Ó" QÔ&Û4ˆNÜÐ'¨Ð'7Ð7KÐLiÐKjÐjkÐlÖmñ 5ã!ˆBÜÐ1°"·*±*°¸YÀrÇwÁwÀiÐPÖQñ "ô —{‘{×,Ñ,ØØ#Ð$Ø,Ð-Ø4ð	 -ó 
ˆô —K‘K×)Ñ)ØØ,Ð-Ø*Ð+Ø9Ü× Ñ ð *ó 
ˆ	ð 	×Ñ˜,¨	Ð2Ô3à—	•	ˆÜˆt�{‰{Ó˜aÔÐ$5Ò$AÀdÇkÁkÐRSÅnÐXi×XoÑXoÐpqÕXrÔFrÜŸ™×-Ñ-ØØ'Ð(Ø-Ð.Ø0Ü×$Ñ$ð .ó ˆIð &×,Ñ,¨QÕ/ˆD�J‰J�q‰MØ×Ñ˜i˜[Ô)à�<‰<Ð/Õ/Ü˜t“_ˆFØ—[‘[–]�ØÐCÖCØ˜qš	ñ #ð —
‘
˜1•Ø—
‘
˜1•Ø—
‘
˜1•ðˆCð �J‰J¬¨T¯Z©Z«¸1Ô)<˜Ÿ
™
 1žÀ"ÐEÔFØ�J‰J¬¨T¯Z©Z«¸1Ô)<˜Ÿ
™
 1žÀ"ÐEÔFØ�J‰J¬¨T¯Z©Z«¸1Ô)<˜Ÿ
™
 1žÀ"ÐEÔFØ�J‰J¬¨T¯Z©Z«¸1Ô)<˜Ÿ
™
 1žÀ"ÐEÔFØ�J‰JÐ.Ð/Ô0Ø�J‰J˜�~Ô&Ø�J‰JÐ+Ð,Ô-Ø�J‰J¬¨T¯Z©Z«¸1Ô)<˜Ÿ
™
 1žÀ"ÐEÔFà23ˆFÐ.Ñ/ä—;‘;×(Ò(Ø1ØØ—‘ñð —Y‘Yð	ð
 ñˆDð ×&Ü(¨¯©Ö4‘��tØÐ1Ö1Ø(E—J‘J˜uÓ%ñ  5ð ×Ñ˜d˜V×$ña ðd 	‡O�O�FÔØ‡I�I×Ñ�YÔØ,0¯JªJÓ7©J S˜Ÿœ©JÐÐ7à€JÜ˜4Ÿ:™:Ö&‰ˆˆ2ØÐ!Ô! aÐ*<Ô&<Ü˜R“LˆEÜ—‘×3Ñ3Ø—‘ØŸ'™'×-Ñ-×7Ñ7Ø˜Q•x  q¥¨=¸%À½(ÐCð 4ó ˆBð
 	×Ñ˜2˜$Öñ 'ð Ð%5Ô5Ø×ÑÜ�[‰[×/Ñ/Ð0FÌ×HXÑHX×H^ÑH^ÐghÐfiÐ/ÓjÐkô	
ð Ð+Ô+Ø×Ñœ4Ÿ;™;×=Ñ=¸lÌD×L\ÑL\×LbÑLbÐklÐjmÐ=ÓnÐoÔpØÐ"2Ô2Ø×Ñä—‘×2Ñ2Ø'Ü×$Ñ$×*Ñ*ÚEð 3ó ðô	
ð 	‡O�O�GÔØ‡J�J×Ñ�jÔ!à€KÜ˜4Ÿ;™;Ö'‰ˆˆ2ØÐ"Ô"Ü˜R“LˆEÜ—‘×3Ñ3Ø—‘ØŸ'™'×-Ñ-×7Ñ7Ø˜Q•x  q¥¨=¸%À½(ÐCð 4ó ˆBð
 	×Ñ˜B˜4Ö ñ (ð 	‡O�O�HÔØ‡K�K×Ñ�{Ô#áùòs 7ùòZ 8s   ‘`Õ`c                ó$   € V ^8„  d   QhR\         /# )r8   Úmodel_protorT  )r?   s   "r(   r@   r@   P  s   € ÷ 8ñ 8´ñ 8r+   c                 óÈ  € \        V 4      pVP                  4       p. p. pVP                  4        EFë  pVP                  R 8X  g   K  RVP                  ^,          9   d   RVP                  ^,          9   d   KI  W%P                  ^ ,          ,          pW%P                  ^,          ,          pW%P                  ^,          ,          pVP                  VP                  ^,          4      p	VP                  VP                  ^,          4      p
VP                  VP                  ^,          4      pV	'       d   V
'       d	   V'       g    R# \        P                  ! V	4      p\        P                  ! V
4      p\        P                  ! V4      p\        P                  ! WÍV.^R7      pVP                  RRR7      p\        P                  P                  VR,           V	P                  ^8X  d   \        P                   M\        P"                  VP$                  ^ ,          VP$                  ^,          .VP'                  4       P)                  4       R	7      pV P*                  P,                  P/                  V.4       \        P                  P1                  RVP                  ^ ,          VR,           .VR
,           .VR7      pVP2                  ^ ,          VP                  ^ &   RVP                  ^&   RVP                  ^&   VP/                  V.4       VP/                  WgV.4       EKî  	  VP5                  V4       VP7                  V4       VP9                  4        VP;                  4        R# )r"  Úpast_key_crossÚpast_value_crossFrÈ   rÄ   Ú
MatMul_QKV)Úname_prefixÚ_weightr1  Ú_outr   r   T)r   rÌ   ÚnodesrÍ   rÏ   rÎ   r   rÛ   r×   rÚ   r  rº   rE  rH  rÑ   r
   r  r|   rD  ÚflattenÚtolistrÊ   r=  r’   r‹  r§   Ú	add_nodesÚremove_nodesÚupdate_graphrÖ  )r™  Ú
onnx_modelrÌ   Únodes_to_addr¶  r@  rU  rZ  r^  Úq_weightÚk_weightÚv_weightre  rf  rg  rh  Úmatmul_node_nameÚweightrà   s   &                  r(   Úpack_qkv_for_decoder_masked_mhar®  P  s¬  € Ü˜;Ó'€JØ$×8Ñ8Ó:Ðà€LØ€OØ× Ñ ×"ˆØ�<‰<Ð<Ö<Ø 4§:¡:¨a¥=Ô0Ð5GÈ4Ï:É:ÐVWÍ=Ô5XÙØ*¯:©:°a­=Õ9ˆHØ*¯:©:°a­=Õ9ˆHØ*¯:©:°a­=Õ9ˆHà!×1Ñ1°(·.±.ÀÕ2CÓDˆHØ!×1Ñ1°(·.±.ÀÕ2CÓDˆHØ!×1Ñ1°(·.±.ÀÕ2CÓDˆHß§¯hÚä×%Ò% hÓ/ˆBÜ×%Ò% hÓ/ˆBÜ×%Ò% hÓ/ˆBäŸš¨°¨¸1Ô=ˆJà)×:Ñ:¸8ÐQ]Ð:Ó^ÐÜ—[‘[×,Ñ,Ø%¨	Õ1Ø08×0BÑ0BÀaÔ0Gœ;×,Ò,Ì[×M`ÑM`Ø ×&Ñ& qÕ)¨:×+;Ñ+;¸AÕ+>Ð?Ø×'Ñ'Ó)×0Ñ0Ó2ð	 -ó ˆFð ×Ñ×)Ñ)×0Ñ0°&°Ô:äŸ+™+×/Ñ/ØØ Ÿ™ qÕ)Ð+;¸iÕ+GÐHØ)¨FÕ2Ð3Ø%ð	 0ó ˆKð (×.Ñ.¨qÕ1ˆD�J‰J�q‰MØˆD�J‰J�q‰MØˆD�J‰J�q‰Mà×Ñ  Ô.Ø×"Ñ" H¸Ð#A×BñU #ðX ×Ñ˜Ô&Ø×Ñ˜OÔ,Ø×ÑÔà×ÑÔ!ár+   c                ó0   € V ^8„  d   QhR\         R\        /# )r8   Údecoder_onnx_pathrW   r¯   )r?   s   "r(   r@   r@   ‹  s   € ÷ ñ Ä#ð Ôaeñ r+   c                ón  € \         P                  ! V RR7      p\        \        VP                  P
                  4      4       F×  pVP                  P
                  V,          P                  R8X  g/   VP                  P
                  V,          P                  R8X  g   K]  VP                  P
                  V,          P                  P                  P                  P                  ^,          pVP                  R4      '       d   VP                  4        ^Vn        KÙ  	  \        P                  ! VV VR7       R# )a=  Update the input shapes for the inputs "input_ids" and "position_ids" and make the sequence length dim value 1 for each of them.
   The decoder model will be over-written.

Args:
    decoder_onnx_path (str): Path of GPT-2 decoder onnx model
    use_external_data_format(bool): output tensors to external data or not.
Tr³   r  r	  rz  r·   )rº   r»   r  r”   rÊ   rÏ   rË   rE   r  rD  ry  ÚHasFieldÚClearr{  r   r½   )r°  rW   rÝ   r  Úshape_dim_protos   &&   r(   Ú*update_input_shapes_for_gpt2_decoder_modelrµ  ‹  sï   € ô Ÿ/š/Ð*;ÐPTÔUÐÜ”3Ð*×0Ñ0×6Ñ6Ó7Ö8ˆà×%Ñ%×+Ñ+¨AÕ.×3Ñ3°{ÔBØ"×(Ñ(×.Ñ.¨qÕ1×6Ñ6¸.ÖHà1×7Ñ7×=Ñ=¸aÕ@×EÑE×QÑQ×WÑW×[Ñ[Ð\]Õ^ˆOð ×'Ñ'¨×4Ò4Ø×%Ñ%Ô'ð )*ˆOÖ%ñ 9ô ‡N‚NØØØ6õñ
 r+   c                óH   € V ^8„  d   QhR\         R\         R\        R\        /# )r8   r°  Úinit_decoder_onnx_pathrW   r:   r¯   )r?   s   "r(   r@   r@   «  s6   € ÷ hñ hÜðhäðhô #ðhô 
ñ	hr+   c           	     óŒ	  € \         P                  ! V RR7      pVP                  P                  ^ ,          P                  p\        V4      pVP                  4       pWF9   g   Q hWd,          pVP                  R8w  d   R# VP                  V. RO. RO4      pVf   VP                  V. RO. RO4      pVf3   VP                  V. RO. RO4      pVf   VP                  V. RO. RO4      pVf   R# VR,          p	V	P                  R8H  p
V
'       gx   ^ pVP                  V	. ROV^ ^ ^ .4      pVf   ^pVP                  V	. ROV^ ^ ^ .4      pVf   ^ pVP                  V	. ROV^ ^ .4      pVf   ^pVP                  V	. ROV^ ^ .4      pMr^ pVP                  V	. ROV^ ^ .4      pVf   ^pVP                  V	. ROV^ ^ .4      pVf   ^ pVP                  V	RR.V^ .4      pVf   ^pVP                  V	RR.V^ .4      pVf   R# V^8X  d   ^ M^pV
'       g   VP                  V	RV4      pMVP                  V	RV4      pVf   R# VR,          pVR,          p\         P                  P                  R\        P                  ^.R.R	7      p\         P                  P                  R
\        P                  ^.R.R	7      p\         P                  P                  R\        P                  ^.^.R	7      p\         P                  P                  R\        P                  ^.R.R	7      pVP                  V4       VP                  V4       VP                  V4       VP                  V4       RVP                  ^ ,          ,           p\         P                  P                  RVP                  ^ ,          RR
RR.V.VP!                  RR4      R7      pV
'       g   VP                  ^ ,          MVP                  ^,          pRVP                  ^ ,          ,           p\         P                  P                  RVRR
RR.V.VP!                  RR4      R7      pVP#                  V4       VP#                  V4       VP%                  VVP                  ^ ,          V4       VP%                  V	VV4       VP'                  4        \
        P(                  ! VVVR7       R# ) al  Generates the initial decoder GPT2 subgraph and saves it for downstream use.
   The initial decoder model will be saved to init_decoder_onnx_path.

Args:
    decoder_onnx_path (str): Path of GPT-2 decoder onnx model
    init_decoder_onnx_path (str): Path of GPT-2 init decoder onnx model
    use_external_data_format(bool): output tensors to external data or not.
Tr³   rÄ   Frò  ÚSkipLayerNormalizationr‰  ÚSliceLastTokenStartsr1  ÚSliceLastTokenEndsÚSliceLastTokenAxesÚSliceLastTokenStepsÚedge_modified_rô  ÚGatherLastToken_0_r   ÚGatherLastToken_1_r·   )r  rï  rò  rò  r  rÄ   r  ÚFastGelur  rÄ   r  rï  rò  )r÷  r÷  r÷  r¯  r÷  r÷  r÷  r÷  r÷  r÷  r÷  r÷  r÷  )
r  r¹  r  rÄ   r  rÁ  r  rÄ   r  r¹  )
r÷  r÷  r¯  r÷  r÷  r÷  r÷  r÷  r÷  r÷  )rï  rò  rò  rÄ   rÁ  rÄ   rï  rò  )r÷  r÷  r¯  r÷  r÷  r÷  r÷  r÷  )r¹  rÄ   rÁ  rÄ   r¹  )r÷  r¯  r÷  r÷  r÷  rs   )rò  r  rÄ   r‰  )rò  rÄ   r‰  )r  rÄ   r‰  rþ  )rº   r»   rÊ   r§   rË   r   rÌ   rÍ   r
  rÐ   rE  rH  r
   r  rG  r‹  r  r  Úreplace_node_inputrÖ  r½   )r°  r·  rW   Úinit_decoder_model_protorÞ   Úgpt2_init_decoder_modelrÌ   Úlogits_matmul_nodeÚ"logits_matmul_to_residual_add_pathÚresidual_add_nodeÚis_skiplayernorm_pathÚ&residual_add_to_attention_parent_indexÚresidual_add_to_attention_pathÚ residual_add_to_add_parent_indexÚadd_before_residual_addÚ	attentionÚmatmul_after_attentionÚslice_startsÚ
slice_endsÚ
slice_axesÚslice_stepsÚslice_0_output_nameÚslice_node_0Úadd_before_residual_add_outputÚslice_1_output_nameÚslice_node_1s   &&&                       r(   Úgenerate_gpt2_init_decoderrØ  «  s•  € ô  $ŸšÐ/@ÐUYÔZÐà1×7Ñ7×>Ñ>¸qÕA×FÑFÐä'Ð(@ÓAÐà1×EÑEÓGÐØÔ4Ð4Ð4à,Õ@Ðð ×!Ñ! XÔ-Ùð *A×)RÑ)RØò	
ò 	0ó#*Ð&ð* *Ò1Ø-D×-VÑ-VØòò +ó.
Ð*ð$ *Ò1à-D×-VÑ-VØò	ò %ó.
Ð*ð  .Ò5Ø1H×1ZÑ1ZØ"òò  ó
2Ð.ð *Ò1Ùà:¸2Õ>Ðð .×5Ñ5Ð9QÑQÐ÷ !Ø12Ð.Ø)@×)RÑ)RØÚ2Ø3°Q¸¸1Ð=ó*
Ð&ð *Ò1Ø56Ð2Ø-D×-VÑ-VØ!Ú6Ø7¸¸A¸qÐAó.Ð*ð *Ò1Ø56Ð2Ø-D×-VÑ-VØ!Ú.Ø7¸¸AÐ>ó.Ð*ð *Ò1Ø56Ð2Ø-D×-VÑ-VØ!Ú.Ø7¸¸AÐ>ó.Ð*øð 23Ð.Ø)@×)RÑ)RØÚ+Ø3°Q¸Ð:ó*
Ð&ð *Ò1Ø56Ð2Ø-D×-VÑ-VØ!Ú/Ø7¸¸AÐ>ó.Ð*ð *Ò1Ø56Ð2Ø-D×-VÑ-VØ!Ø˜;Ð'Ø7¸Ð;ó.Ð*ð *Ò1Ø56Ð2Ø-D×-VÑ-VØ!Ø˜;Ð'Ø7¸Ð;ó.Ð*ð &Ò-Ùà,RÐVWÔ,W¡qÐ]^Ð$÷ !Ø"9×"FÑ"FØ˜uÐ&Fó#
Ñð #:×"FÑ"FØØ$Ø,ó#
Ðð Ò&Ùà.¨rÕ2€IØ;¸BÕ?Ðä—;‘;×*Ñ*Ø#Ü×#Ñ#ØˆSØˆTð	 +ó €Lô —‘×(Ñ(Ø!Ü×#Ñ#ØˆSØˆTð	 )ó €Jô —‘×(Ñ(Ø!Ü×#Ñ#ØˆSØˆSð	 )ó €Jô —+‘+×)Ñ)Ø"Ü×#Ñ#ØˆSØˆTð	 *ó €Kð ×+Ñ+¨LÔ9Ø×+Ñ+¨JÔ7Ø×+Ñ+¨JÔ7Ø×+Ñ+¨KÔ8ð +¨Y×-=Ñ-=¸aÕ-@Õ@ÐÜ—;‘;×(Ñ(Øà×Ñ˜QÕØ"Ø Ø Ø!ð
ð %Ð%Ø$×5Ñ5°gÐ?SÓTð )ó €L÷" 2GÐ×&Ñ& qÖ)ÐLc×LjÑLjÐklÕLmð #ð +Ð-D×-KÑ-KÈAÕ-NÕNÐÜ—;‘;×(Ñ(Øà*Ø"Ø Ø Ø!ð
ð %Ð%Ø$×5Ñ5°gÐ?SÓTð )ó €Lð ×$Ñ$ \Ô2Ø×$Ñ$ \Ô2ð ×.Ñ.Ð/EÀy×GWÑGWÐXYÕGZÐ\oÔpØ×.Ñ.Ð/@ÐB`ÐbuÔvð ×,Ñ,Ô.ô ‡N‚NØ ØØ6õñ
 r+   c                ó>  € \        ^4      p\        VP                  4      p\        VP                  4      p\        VP                  4      pV P                  P
                   F�  pVP                  P                  P                  P                   F^  pVP                  R4      '       g   K  VP                  VVVV39   g   K3  \        VP                  4      pVP                  4        W‡n        K`  	  K‘  	  V P                  P                   F�  pVP                  P                  P                  P                   F^  pVP                  R4      '       g   K  VP                  VVVV39   g   K3  \        VP                  4      pVP                  4        W‡n        K`  	  K‘  	  R# )z_Make dim_proto numeric.

Args:
    model: T5 encoder and decoder model.
    config: T5 config.
rz  N)r<   r   Úd_modelÚd_kvrÊ   r§   rE   r  rD  ry  r²  rz  r}   r³  r{  rÏ   )	r¿   Úconfigrâ  r   Úhidden_sizeÚ	head_sizerb  Ú	dim_protor{  s	   &&       r(   Úmake_dim_proto_numeric_t5rà  Ö	  sG  € ô ˜!“f€OÜ�F×$Ñ$Ó%€IÜ�f—n‘nÓ%€KÜ�F—K‘KÓ €Ià—+‘+×$Ô$ˆØŸ™×0Ñ0×6Ñ6×:Ô:ˆIØ×!Ñ! +×.Ô.°9×3FÑ3FØØØØð	Kö 4ô   	× 3Ñ 3Ó4�	Ø—‘Ô!Ø&/Ö#ó ;ñ %ð —+‘+×#Ô#ˆØŸ™×0Ñ0×6Ñ6×:Ô:ˆIØ×!Ñ! +×.Ô.°9×3FÑ3FØØØØð	Kö 4ô   	× 3Ñ 3Ó4�	Ø—‘Ô!Ø&/Ö#ó ;ó $r+   c                óD   € V ^8„  d   QhR\         P                  R\        /# )r8   r†   Úgeneration_type)r=   r>   r    )r?   s   "r(   r@   r@   û	  s&   € ÷ y0ñ y0Ü
×
Ñ
ðy0ä#ñy0r+   c                óâ0  € V P                   R8H  pV\        P                  8H  pV\        P                  8H  pV\        P                  8H  pV P
                  p\        P                  RV 24       \        V P                  4      ^8X  d’   V P                  ^ ,          R8X  dz   V'       dk   V P                  \        P                  P                  8X  dB   . RoOV n	        \        P                  RV P                   24       \        P                  R4       M. V n	        V'       g	   V'       dN   V'       g   \        R4      hV P                  '       d   \        R4      hV P                   '       d   \        R4      hV'       d&   V'       d   V P"                  '       g   \%        R	4      hV P"                  '       d   V'       g   \%        R
4      hV P"                  '       d   V P&                  '       g   \%        R4      hV'       Ed   V P(                  '       dT   \*        P,                  P/                  V P(                  4      '       d%   \        P                  RV P(                   24       EM(V P(                  '       gY   V P0                   RV P                   R2p\3        \3        V P4                  4      P6                  V4      P9                  4       V n        \        P                  RV P0                   RV P(                   R24       \;        V 4       M‚V P(                  '       dC   V P<                  '       d1   \        P                  RV P(                   RV P<                   24       M.\        P                  RV P0                   R24       \?        V 4       RpV P@                  '       gª   V P                  \        P                  P                  8X  d�   V'       dy   V'       g   V'       g	   V'       da   \        P                  RV P(                   R24       \C        V P(                  V PD                  4      pV'       g   \        PG                  R4       Rp	Rp
V PH                  '       Eg   V'       dú   V'       g   V'       g	   V'       dâ   \        P                  RV P(                   R24       RV P                   R2p\3        \3        V P4                  4      P6                  V4      P9                  4       p
\K        V P(                  V
V PD                  4      p	V	'       g   \        PG                  R4       V	'       d2   \M        V P(                  V PD                  4      '       g   \%        R4      hV'       g   V PN                  '       g	   V	'       dz   \        P                  R V P(                   R24       \Q        V P(                  V PD                  4       V	'       d/   \        P                  R V
 R24       \Q        W PD                  4       V'       d.   \R        PT                  ! V P0                  V PV                  R!7      pMjV P                   R"8X  d.   \X        PT                  ! V P0                  V PV                  R!7      pM,\Z        PT                  ! V P0                  V PV                  R!7      pV P\                  '       d   \        P                  R#V 24       VP^                  pV'       d   VP^                  MVP`                  pVPb                  pV Pb                  Rp8w  d   V Pb                  pV P^                  Rp8w  d   V P^                  pV P`                  Rp8w  d   V P`                  p\d        Pf                  ! V P(                  R$R%7      pV P                    R&2VPh                  n5        RpV P                   R8X  d€   \m        VPh                  V P                  4       V	'       dW   \d        Pf                  ! V
R$R%7      pV P                    R'2VPh                  n5        \m        VPh                  V P                  4       M \o        VPh                  V P                  4       RpV'       d   . RqOpMV'       g	   V'       d   . RrOpV Pp                  '       d   VPs                  R/4       MVPs                  R04       V Pt                  '       d   VPs                  R14       MVPs                  R04       V Pv                  '       d   VPs                  R24       MVPs                  R04       V'       dk   V Px                  '       d%   V Pz                  '       d   VPs                  R34       MVPs                  R04       V P|                  '       d   VPs                  R44       R5.pV P                  '       d   VPs                  R64       V P                   '       d+   V P                  '       g   Q R74       hVPs                  R84       RpV'       d2   \d        P~                  P�                  R9VVR:V P                    2R;7      pMqV'       d2   \d        P~                  P�                  R<VVR=V P                    2R;7      pM8V'       d1   \d        P~                  P�                  R>VVR?V P                    2R;7      pR@VnA        RpV'       dÑ   \d        P~                  P…                  RAV4      \d        P~                  P…                  RBV4      \d        P~                  P…                  RCV P†                  4      \d        P~                  P…                  RDV Pˆ                  '       d   ^M^ 4      \d        P~                  P…                  REV P                   R8X  d   ^ M^4      .pEM?V'       dž   \d        P~                  P…                  RAV4      \d        P~                  P…                  RBV4      \d        P~                  P…                  REV P                   R8X  d   ^ M^4      \d        P~                  P…                  RCV P†                  4      .pEMšV'       Ed’   \d        P~                  P…                  RAV4      \d        P~                  P…                  RBV4      \d        P~                  P…                  REV P                   R8X  d   ^ M^4      \d        P~                  P…                  RCV P†                  4      \d        P~                  P…                  RFV PŠ                  4      \d        P~                  P…                  RGV PŒ                  4      \d        P~                  P…                  RHV PŽ                  4      \d        P~                  P…                  RIV P�                  4      \d        P~                  P…                  RJV Px                  4      \d        P~                  P…                  RKV P’                  4      .
pV'       d1   VP•                  \d        P~                  P…                  RLV4      .4       VP–                  P•                  V4       . pV P                   Rs9   EdÀ   V PN                  '       dD   \        P                  RMV P<                   R24       \Q        V P<                  V PD                  4       \d        Pf                  ! V P<                  R$R%7      p\        VPh                  P˜                  4      ^8X  d   RNMROpV P                    RPV 2VPh                  n5        \›        VPh                  V P                  4       \�        VV4       \�        VV4       V'       dµ   V P"                  '       g   \%        RQ4      h\        P                  RR4       \Ÿ        VPh                  4      '       d   \        P                  RS4       M\        P                  RT4       \¡        V4      '       d   \        P                  RU4       M\        P                  RV4       V P¢                  '       gK   \¥        VV4      p\        P                  \        V4       RWV Uu. uF  pVPj                  NK  	  up RX24       VP¦                  ^ 8¼  g   Q RY4       hVP–                  P•                  \d        P~                  P…                  RNVPh                  4      \d        P~                  P…                  RZVPh                  4      \d        P~                  P…                  R[VP¦                  4      .4       EMúV	'       Ed   V P¢                  '       gK   \¥        VV4      p\        P                  \        V4       RWV Uu. uF  pVPj                  NK  	  up R\24       V'       d+   \        P                  R]4       \©        VPh                  4       V P"                  '       d)   \«        VPh                  VR4      '       g   \%        R^4      hVP–                  Ps                  \d        P~                  P…                  R_VPh                  4      4       M6\­        VPh                  4      p\        P                  \        V4       R`24       V'       d+   \        P                  Ra4       \©        VPh                  4       V P"                  '       d)   \«        VPh                  VR$4      '       g   \%        Rb4      hVP–                  Ps                  \d        P~                  P…                  RZVPh                  4      4       \d        P~                  P¯                  R(\°        P²                  RcRd.4      p\d        P~                  P¯                  R)\°        P²                  ^.4      p\d        P~                  P¯                  R*\°        P²                  ^.4      p\d        P~                  P¯                  R+\°        P²                  ^.4      p\d        P~                  P¯                  R,\°        P²                  ^.4      p\d        P~                  P¯                  R-\°        P´                  ^.4      p\d        P~                  P¯                  R.\°        P´                  ^.4      p Rp!V'       d   VVVVVVV .p!MV'       g	   V'       d   VVVV .p!V Pp                  '       dB   \d        P~                  P¯                  R/\°        P²                  V.4      p"V!Ps                  V"4       V Pt                  '       dC   \d        P~                  P¯                  R1\°        P²                  RcV.4      p#V!Ps                  V#4       V Pv                  '       dC   \d        P~                  P¯                  R2\°        P²                  RcRd.4      p$V!Ps                  V$4       V Px                  '       dU   V Pz                  '       dC   \d        P~                  P¯                  R3\°        P²                  RcV.4      p%V!Ps                  V%4       V'       dT   V P|                  '       dB   \d        P~                  P¯                  R4\°        P²                  ^.4      p&V!Ps                  V&4       Rp'V'       d3   \d        P~                  P¯                  R5\°        P²                  . RtO4      p'MAV'       g	   V'       d2   \d        P~                  P¯                  R5\°        P²                  RcR).4      p'V'.p(V P                  '       dC   \d        P~                  P¯                  R6\°        P´                  RcR,.4      p)V(Ps                  V)4       V P                   '       dE   \d        P~                  P¯                  R8\°        P´                  ReRcR+V.4      p*V(Ps                  V*4       \d        P~                  P·                  V.V'       g   V P                    Rf2MV P                    Rg2V!V(V4      p+\d        P~                  P¹                  V+RhVPº                  Ri7      p,V PD                  '       dt   ^ RjI^H_p- V-PÁ                  \d        PÂ                  4      V-PÁ                  Rk4      8  d   \        PG                  Rl4       \Ä        PÆ                  ! V,V P4                  R$R$Rm7       M!\d        PÆ                  ! V,V P4                  4       \        P                  RnV P4                   24       R# u upi u upi )uz|Convert model according to command line arguments.

Args:
    args (argparse.Namespace): arguments parsed from command line
rG   z**** past_present_share_buffer=rT   z**** Setting op_block_list to zI**** use --op_block_list if you want to override the block operator list.z<Currently only gpt2 with greedy search/sampling is supportedzLoutput_sequences_scores currently is not supported in greedy search/samplingzHoutput_token_scores currently is not supported in greedy search/samplingzi`use_decoder_masked_attention` MUST be turned on to use `past_present_share_buffer` in case of BeamSearchzS`past_present_share_buffer` MUST be turned on to use `use_decoder_masked_attention`z?`use_decoder_masked_attention` option is only supported on GPUsz)skip convert_to_onnx since path existed: Ú_past_z.onnxzConvert GPT model z	 to onnx z ...z,skip convert_to_onnx since paths specified: z and zConvert model z to onnx ...Fz=Pad logits MatMul weights for optimal MatMul perf in fp16 on z. The file will be overwritten.z]Tried and failed to pad logits MatMul weights. Performance may be sub-optimal for this MatMulNz*Creating an initial run GPT2 decoder from z. Úgpt2_init_past_zuTried and failed to generate the init decoder GPT2 model. Performance may be sub-optimal for the initial decoding runzGCould not update the input shapes for the non-initial decoder subgraph.z Run symbolic shape inference on ©r‘   rq   zConfig=Tr³   z decoderz init decoderr  Ú
max_lengthÚ
min_lengthÚ	num_beamsÚnum_return_sequencesÚlength_penaltyÚrepetition_penaltyr`   r   rc   r
  re   rf   Ú	sequencesÚsequences_scoresz8--output_token_scores requires --output_sequences_scoresÚscoresÚ
BeamSearchÚBeamSearch_r   ÚGreedySearchÚGreedySearch_ÚSamplingÚ	Sampling_r#  Úeos_token_idÚpad_token_idÚno_repeat_ngram_sizer_   r¤   ÚtemperatureÚtop_pÚfilter_valueÚmin_tokens_to_keepÚcustomÚpresence_penaltyÚ
vocab_sizezSymbolic shape inference on r[  zencoder and decoder initÚ zMpast_present_share_buffer is only supported with use_decoder_masked_attentionzl*****update t5 decoder subgraph to share past/present buffer and use decoder_masked_multihead_attention*****z4*****update t5 decoder subgraph successfully!!!*****zF*****DecoderMaskedMultiHeadAttention is not applied to T5 decoder*****z9*****pack qkv for decoder masked mha successfully!!!*****z3*****pack qkv for decoder masked mha failed!!!*****z shared initializers (z>) in encoder and decoder subgraphs are moved to the main graphz%decoder_start_token_id should be >= 0r\  Údecoder_start_token_idzC) in decoder and init decoder subgraphs are moved to the main graphzY*****update init decoder subgraph to make past and present share buffer******************zLCould not update the init decoder subgraph to use DecoderMaskedSelfAttentionÚinit_decoderz: initializers from the decoder are moved to the main graphzT*****update decoder subgraph to make past and present share buffer******************zGCould not update the decoder subgraph to use DecoderMaskedSelfAttentionrž  râ  zmax_length - sequence_lengthz beam searchz greedy searchzonnxruntime.transformers)Úproducer_nameÚopset_imports)Úversionz1.12.0z0Require onnx >= 1.12 to save large (>2GB) model!)r¸   Úall_tensors_to_one_filezmodel save to )rò  rï  r¹  rÁ  rs   )r  rç  rè  ré  rê  rë  rì  )r  rç  rè  rì  ©rq   rr   )rž  rê  rç  )dr¤   r    r1   r2   r3   ra   r–   r—   r”   r•   r�   r   r|   r&   ÚNotImplementedErrorr]   r^   rb   r  rl   r�   rx   ry   ÚexistsrŽ   r   r§   r¨   Úas_posixr›   rª   r¬   rY   ré   rW   r¾   rZ   rØ  rµ  rX   rÁ   r   Úfrom_pretrainedr‘   r   r   rO   rö  r÷  rÿ  rº   r»   rÊ   rË   r  r)  r`   r“   rc   rd   rý  re   rf   rE  r‹  rq  ru  rø  r_   rù  rú  rû  rü  rþ  r’   rn  rÏ   r3  rà  r—  r®  r[   r^  r  r“  r§  rc  rF  r
   r  r  Ú
make_graphÚ
make_modelÚopset_importÚ	packagingr  ÚparseÚ__version__r   r½   ).r†   râ  Úis_gpt2Úis_beamsearchÚis_greedysearchÚis_samplingra   Úonnx_filenameÚlogits_matmul_weight_paddedÚgpt2_init_decoder_generatedÚgpt2_init_decoder_onnx_pathÚgpt2_init_decoder_onnx_filenamerÜ  rö  r÷  rÿ  rß   rÄ  r  r  r@  Úattr_to_extendr]  rS  Úsuffixr  r  rç  rè  ré  rê  rë  rì  Úgraph_inputsr`   rc   r
  re   rf   rí  Úgraph_outputsrî  rï  Ú	new_graphÚ	new_modelr  s.   &&                                            r(   Úconvert_generation_modelr!  û	  s~  € ð —O‘O vÑ-€GØ)¬^×-FÑ-FÑF€MØ+¬~×/JÑ/JÑJ€OØ'¬>×+BÑ+BÑB€KØ&*×&DÑ&DÐä
‡K�KÐ1Ð2KÐ1LÐMÔNÜ
ˆ4×ÑÓ !Ô#¨×(:Ñ(:¸1Õ(=ÀÔ(Gß�t—~‘~¬×):Ñ):×)@Ñ)@Ô@ò"ˆDÔô �K‰KÐ8¸×9KÑ9KÐ8LÐMÔNÜ�K‰KÐcÕdà!#ˆDÔçŸ+ßÜ%Ð&dÓeÐeØ×'×'Ð'Ü%Ð&tÓuÐuØ×#×#Ð#Ü%Ð&pÓqÐq÷ !§]¸4×;\×;\Ð;\ÜØwó
ð 	
ð ×(×(Ð(×1JÜÐnÓoÐoð ×(×(Ð(°··°ÜÐZÓ[Ð[ç€wØ××Ð¤§¡§¡°×0AÑ0A×!BÒ!BÜ�K‰KÐCÀD×DUÑDUÐCVÐWÖXà×$×$Ð$Ø#'×#:Ñ#:Ð";¸6À$Ç.Á.ÐAQÐQVÐ W�Ü$(¬¨d¯k©kÓ):×)AÑ)AÀ=Ó$Q×$ZÑ$ZÓ$\�Ô!ä�K‰KÐ,¨T×-DÑ-DÐ,EÀYÈt×O`ÑO`ÐNaÐaeÐfÔgÜ˜Õà××Ð ×!?×!?Ð!?Ü�K‰KØ>¸t×?PÑ?PÐ>QÐQVÐW[×WuÑWuÐVvÐwõô �K‰K˜.¨×)@Ñ)@Ð(AÀÐNÔOÜ�tÔð #(Ðà×'×'Ð'Ø�N‰Nœi×/Ñ/×5Ñ5Ô5ßßŸo·ä�‰ØKÈD×L]ÑL]ÐK^ð _,ð ,ô	
ô 'CÀ4×CTÑCTÐVZ×VsÑVsÓ&tÐ#ß*Ü�N‰NØoôð #(ÐØ"&Ðà×;×;Ñ;ßßŸo·ä�‰Ð@À×ARÑARÐ@SÐSUÐVÔWà,;¸D¿N¹NÐ;KÈ5Ð*QÐ'ä&*¬4°·±Ó+<×+CÑ+CÐEdÓ&e×&nÑ&nÓ&pÐ#ä&@Ø×ÑØ'Ø×)Ñ)ó'
Ð#÷ +Ü�N‰NðNô÷ 'Ô/YØ×Ñ˜t×<Ñ<÷0
ò 0
ô ÐfÓgÐg÷
 # d×&>×&>Ð&>×B]Ü�‰Ð6°t×7HÑ7HÐ6IÐIhÐiÔjÜ˜×)Ñ)¨4×+HÑ+HÔIß&Ü�K‰KÐ:Ð;VÐ:WÐWvÐwÔxÜÐ7×9VÑ9VÔWçÜ×+Ò+¨D×,CÑ,CÈtÏ~É~Ô^‰Ø	�‰˜DÔ	 Ü×)Ò)¨$×*AÑ*AÈTÏ^É^Ô\‰ä×*Ò*¨4×+BÑ+BÈdÏnÉnÔ]ˆà‡|‡|€|Ü�‰�g˜f˜XÐ&Ô'à×&Ñ&€Lß*1�6×&Ò&°v×7JÑ7J€LØ×"Ñ"€Jð ‡�˜"ÔØ—_‘_ˆ
à×Ñ˜BÔØ×(Ñ(ˆØ×Ñ˜BÔØ×(Ñ(ˆä—O’O D×$5Ñ$5È$ÔO€MØ"&§/¡/Ð!2°(Ð;€M×ÑÔà"ÐØ‡�˜&Ô Ü˜]×0Ñ0°$·.±.ÔA÷ 'Ü&*§o¢oÐ6QÐfjÔ&kÐ#Ø48·O±OÐ3DÀMÐ1RÐ#×)Ñ)Ô.Ü Ð!8×!>Ñ!>ÀÇÁÔOøä" =×#6Ñ#6¸¿¹ÔGà€Fßò
‰÷ 
ŸKò
ˆð ‡‡€Ø�‰�lÕ#à�‰�bÔà××ÐØ�‰Ð)Õ*à�‰�bÔà×!×!Ð!Ø�‰Ð&Õ'à�‰�bÔçØ�;�;ˆ;˜4×-×-Ð-Ø�M‰M˜/Õ*à�M‰M˜"Ôà�9�9ˆ9Ø�M‰M˜&Ô!àˆm€GØ×#×#Ð#Ø�‰Ð)Ô*à××ÐØ×+×+Ð+ÐgÐ-gÓgÐ+Ø�‰�xÔ à€DßÜ�{‰{×$Ñ$ØØØØ˜tŸ™Ð/Ð0ð	 %ó 
‰÷ 
Ü�{‰{×$Ñ$ØØØØ  §¡Ð 1Ð2ð	 %ó 
‰÷ 
Ü�{‰{×$Ñ$ØØØØ˜TŸ_™_Ð-Ð.ð	 %ó 
ˆð "€D„Kà€Nßä�K‰K×&Ñ& ~°|ÓDÜ�K‰K×&Ñ& ~°|ÓDÜ�K‰K×&Ñ&Ð'=¸t×?XÑ?XÓYÜ�K‰K×&Ñ&Ð'7¸d×>Q×>QÐ>Q¹ÐWXÓYÜ�K‰K×&Ñ& |¸$¿/¹/ÈVÔ:S±QÐYZÓ[ð
Š÷ 
ä�K‰K×&Ñ& ~°|ÓDÜ�K‰K×&Ñ& ~°|ÓDÜ�K‰K×&Ñ& |¸$¿/¹/ÈVÔ:S±QÐYZÓ[Ü�K‰K×&Ñ&Ð'=¸t×?XÑ?XÓYð	
Š÷ 
ˆä�K‰K×&Ñ& ~°|ÓDÜ�K‰K×&Ñ& ~°|ÓDÜ�K‰K×&Ñ& |¸$¿/¹/ÈVÔ:S±QÐYZÓ[Ü�K‰K×&Ñ&Ð'=¸t×?XÑ?XÓYÜ�K‰K×&Ñ& }°d×6FÑ6FÓGÜ�K‰K×&Ñ& w°·
±
Ó;Ü�K‰K×&Ñ& ~°t×7HÑ7HÓIÜ�K‰K×&Ñ&Ð';¸T×=TÑ=TÓUÜ�K‰K×&Ñ& x°·±Ó=Ü�K‰K×&Ñ&Ð'9¸4×;PÑ;PÓQð
ˆ÷ #Ø×ÑœtŸ{™{×9Ñ9¸,È
ÓSÐTÔUà‡N�N×Ñ˜.Ô)à€Là‡�˜-Õ'Ø×#×#Ð#Ü�K‰KÐ6°t×7UÑ7UÐ6VÐVuÐvÔwÜ˜D×:Ñ:¸D×<YÑ<YÔZÜŸš¨×(FÑ(FÐ[_Ô`ˆÜ! -×"5Ñ"5×";Ñ";Ó<ÀÔA‘ÐGaˆØ&*§o¡oÐ%6°a¸°xÐ#@ˆ×ÑÔ Ü/°×0CÑ0CÀTÇ^Á^ÔTä! -°Ô8Ü! -°Ô8÷ %Ø×4×4Ð4Ü Ð!pÓqÐqä�K‰KØ~ôô OÈ}×ObÑOb×cÒcÜ—‘ÐRÕSä—‘ÐdÔeä.¨}×=Ò=Ü—‘ÐWÕXä—‘ÐQÔRà×/×/Ð/ä2°=À-ÓPˆLÜ�K‰KÜ�|Ó$Ð%Ð%;É\Ó<ZÉ\È¸Q¿V¼VÉ\Ñ<ZÐ;[ð  \Zð  [ôð ×,Ñ,°Ô1ÐZÐ3ZÓZÐ1à�‰×Ñä—‘×*Ñ*¨9°m×6IÑ6IÓJÜ—‘×*Ñ*¨9°m×6IÑ6IÓJÜ—‘×*Ñ*Ð+CÀV×EbÑEbÓcðö	
÷ 'Ð&ð ×3×3Ð3ä6Ð7NÐP]Ó^�Ü—‘Ü˜<Ó(Ð)Ð)?ÑQ]Ó@^ÑQ]ÈAÀÇÄÑQ]Ñ@^Ð?_ð  `cð  dô÷
 )Ü—‘ÐwÔxÜAÐBY×B_ÑB_Ô`ð ×0×0Ð0Ô9mØ'×-Ñ-¨}¸e÷:ò :ô !Ð!oÓpÐpà�N‰N×!Ñ!¤$§+¡+×"<Ñ"<¸^ÐMd×MjÑMjÓ"kÕlô -¨]×-@Ñ-@ÓAˆLÜ�K‰Kœ3˜|Ó,Ð-Ð-gÐhÔi÷ %Ü�K‰KÐnÔoÜ=¸m×>QÑ>QÔRð ×,×,Ð,Ô5iØ×Ñ °÷6
ò 6
ô ÐfÓgÐgà�‰×ÑœdŸk™k×8Ñ8¸ÀM×DWÑDWÓXÔYô —‘×2Ñ2°;Ä×@QÑ@QÐT`ÐbsÐStÓu€IÜ—‘×3Ñ3°LÄ+×BSÑBSÐVWÐUXÓY€JÜ—‘×3Ñ3°LÄ+×BSÑBSÐVWÐUXÓY€JÜ—‘×2Ñ2°;Ä×@QÑ@QÐTUÐSVÓW€IÜŸ;™;×=Ñ=Ð>TÔVa×VgÑVgÐjkÐilÓmÐÜ—[‘[×7Ñ7Ð8HÌ+×J[ÑJ[Ð^_Ð]`Óa€NÜŸ™×;Ñ;Ð<PÔR]×RcÑRcÐfgÐehÓiÐà€LßàØØØØ ØØð
‰÷ 
ŸKàØØØð	
ˆð ‡‡€Ü—[‘[×7Ñ7¸Äk×FWÑFWÐZdÐYeÓfˆ
Ø×Ñ˜JÔ'à××ÐÜ ŸK™K×>Ñ>Ø¤×!2Ñ!2°\À:Ð4Nó
Ðð 	×ÑÐ-Ô.à×!×!Ð!ÜŸ™×;Ñ;Øœk×/Ñ/°,Ð@QÐ1Ró
ˆð 	×Ñ˜NÔ+à‡{‡{€{�t×)×)Ð)ÜŸ™×:Ñ:Øœ[×.Ñ.°¸zÐ0Jó
ˆð 	×Ñ˜MÔ*ç�t—y—y�yÜ�{‰{×1Ñ1°&¼+×:KÑ:KÈaÈSÓQˆØ×Ñ˜DÔ!ð €IßÜ—K‘K×6Ñ6ØÜ×ÑÚ@ó
‰	÷
 
ŸKÜ—K‘K×6Ñ6ØÜ×ÑØ˜<Ð(ó
ˆ	ð �K€Mà×#×#Ð#ÜŸ;™;×=Ñ=ØÜ×ÑØÐ1Ð2ó
Ðð
 	×ÑÐ-Ô.à××ÐÜ—‘×3Ñ3ØÜ×ÑØ+¨\¸;È
ÐSó
ˆð
 	×Ñ˜VÔ$ä—‘×&Ñ&Ø	ˆß1@ˆD�O‰OÐ˜LÑ	)ÈÏÉÐHYÐYgÐFhØØØó€Iô —‘×&Ñ&ØØ0Ø#×0Ñ0ð 'ó €Ið ×$×$Ð$Ý%à�=‰=œ×)Ñ)Ó*¨W¯]©]¸8Ó-DÔDÜ�N‰NÐMÔNä�ŠØØ�K‰KØ"&Ø$(ö		
ô 	�	Š	�)˜TŸ[™[Ô)Ü
‡K�K�. §¡ Ð.Ö/ùòk =[ùò8 A_s   ÿ?Aa'ÁDAa,c                ó  € V ^8„  d   QhR\         P                  R\        \        ,          R\        P
                  R\        P
                  R\        R\        R\        \        \        ,          ,          R\        \        \        3,          /# )	r8   r†   r¿   r  r
  rö  r÷  Úbad_words_idsr:   )r=   r>   r   r   ÚtorchÚTensorr}   r;   r;  r<   r   )r?   s   "r(   r@   r@   ÷  s|   € ÷ ?9ñ ?9Ü
×
Ñ
ð?9äÔ7Õ7ð?9ô �|‰|ð?9ô —L‘Lð	?9ô
 ð?9ô ð?9ô œœS�	•?ð?9ô 
Œ#Œsˆ(…^ñ?9r+   c                ó
  € V P                   '       d0   \        P                  P                  4       '       g   \	        R4      hV P
                  \        P                  P                  8X  d   VP                  4        \        P                  ! V P                   '       d   RMR4      pVP                  V4       \        P                  ! R4       VP                  V4      pVP                  V4      p. p\        V P                  4       FÛ  p	\        P                  ! 4       p
TP!                  TTV P"                  V P$                  V P&                  V P(                  V P*                  TTV P,                  V P.                  V P0                  V'       d   TMRRV P2                  ;'       g    V P4                  R7      p	VP7                  \        P                  ! 4       V
,
          4       KÝ  	  VP8                  ^ ,          p^ RIHp V! W‹4      # )	ah  Test PyTorch performance of text generation.

Args:
    args (argparse.Namespace): arguments parsed from command line
    model (Union[GPT2LMHeadModel, T5ForConditionalGeneration]): PyTorch model
    input_ids (torch.Tensor): input_ids
    attention_mask (torch.Tensor): Attention mask
    eos_token_id (int): EOS token ID
    pad_token_id (int): Padding token ID
    bad_words_ids (List[List[int]]): Words shall not be generated.

Raises:
    RuntimeError: PyTorch with CUDA is not available for --use_gpu

Returns:
    Dict[str, Any]: A dictionary with string with metric name, and value can be integer or string.
z=Please install PyTorch with Cuda for testing gpu performance.zcuda:0ÚcpuFNT©r  r
  rç  rè  ré  r_   rø  rö  r÷  rê  rë  rì  r#  Úreturn_dict_in_generateÚoutput_scores©Úget_latency_result)rl   r$  ÚcudaÚis_availableró   r�   r   r|   r&   ÚhalfÚdevicer  Úset_grad_enabledr  Ú
total_runsÚtimeÚgeneraterç  rè  ré  r_   rø  rê  rë  rì  r]   r^   r“   rD  Úbenchmark_helperr,  )r†   r¿   r  r
  rö  r÷  r#  r0  Útorch_latencyrá  Ústartrž  r,  s   &&&&&&&      r(   Útest_torch_performancer8  ÷  sz  € ð4 ‡|‡|€|œEŸJ™J×3Ñ3×5Ò5ÜÐZÓ[Ð[à‡~�~œ×*Ñ*×0Ñ0Ô0Ø�
‰
Œä�\Š\ d§l§l l™(¸Ó>€FØ	‡H�HˆVÔä	×Ò˜5Ô!Ø—‘˜VÓ$€IØ#×&Ñ& vÓ.€Nà€MÜ�4—?‘?Ö#ˆÜ—	’	“ˆØ�N‰NØØ)Ø—‘Ø—‘Ø—n‘nØ×.Ñ.Ø!%×!:Ñ!:Ø%Ø%Ø!%×!:Ñ!:Ø×.Ñ.Ø#×6Ñ6ß+8™-¸dØ$(Ø×6Ñ6×RÐR¸$×:RÑ:Rð ó 
ˆð" 	×ÑœTŸYšY›[¨5Õ0Ö1ñ' $ð( —‘ Õ#€JÝ3á˜mÓ8Ð8r+   c                 óX  € \         P                  ! V P                  \         P                  R 7      p\	        V P                  ^ ,          4       FY  p^ p\	        V P                  ^,          4       F4  pW,          V,          V8X  d   V^ 8X  d   ^ W#,          V&   K+  V^,          pK6  	  K[  	  V# )rÆ   )r×   ÚonesrD  Úint32r  )r  r÷  r
  r  Úabs_posrO  s   &&    r(   Úcreate_attention_maskr=  9  s�   € Ü—W’W˜YŸ_™_´B·H±HÔ=€NÜ�9—?‘? 1Õ%Ö&ˆØˆÜ�y—‘ qÕ)Ö*ˆAØ�|˜A� ,Ô.°7¸a´<Ø'(�Õ! !Ó$à˜1•’ó	 +ñ 'ð Ðr+   c                ót   € V ^8„  d   QhR\         P                  R\        \        ,          R,          R\        /# )r8   r†   Ú	sentencesNÚ	is_greedy)r=   r>   r;   r<   r°   )r?   s   "r(   r@   r@   E  s:   € ÷ Tñ TÜ
×
Ñ
ðTä”C�y˜4ÕðTô ñTr+   c                ó‚  € V P                   R8X  g   Q h\        P                  ! V P                  V P                  R7      pRVn        VP                  Vn        \        P                  ! V P                  V P                  VP                  R7      pVf   . R6OpV! VRRR7      pVR	,          pVR
,          pRpVP                  VRR7      p	V	 U
u. uF  qª.NK  	  p	p
V P                  '       d   \        P                  RV	4       M. p	VP                  pVP                  pVP                  pVP                  p. pRpV P                   '       Eg   \#        R74       \#        R4       TP%                  TTV P&                  V P(                  V P*                  V P,                  V P.                  TTV P0                  V P2                  V P4                  V	'       d   T	MRRV P6                  ;'       g    V P8                  R7      p\#        R	V4       \#        R4       \#        RVP:                  4       V P6                  '       d   \#        RVP<                  4       V P8                  '       d   \#        RVP>                  4       \A        VP:                  4       F:  w  ppVPC                  VRR7      pVPE                  V4       \#        V RV 24       K<  	  \#        R74       \#        R4       V'       dÓ   R	VPG                  4       PI                  4       PK                  \L        PN                  4      R\L        PP                  ! V P&                  .\L        PN                  R7      R\L        PP                  ! V P(                  .\L        PN                  R7      R\L        PP                  ! V P4                  .\L        PR                  R7      /pEMcR	VPG                  4       PI                  4       PK                  \L        PN                  4      R\L        PP                  ! V P&                  .\L        PN                  R7      R\L        PP                  ! V P(                  .\L        PN                  R7      R\L        PP                  ! V P*                  .\L        PN                  R7      R\L        PP                  ! V P0                  .\L        PN                  R7      R\L        PP                  ! V P2                  .\L        PR                  R7      R\L        PP                  ! V P4                  .\L        PR                  R7      /pV P                  '       dL   \L        PT                  ! V\L        PN                  R7      pV P                  '       d   V	 F  p^ VV&   K
  	  VVR&   V PV                  '       d   \Y        Wm4      VR
&   VPZ                  ^ ,          pV P\                  '       dC   \        P_                  R4       \L        PT                  ! VV3\L        PN                  R7      pVVR &   V P`                  '       d´   \c        V Pd                  4      Pf                  Pi                  4       p\        P                  R!V4       ^ R"I5H6p \        P_                  R#V R$24       V.p\A        V4       F?  w  pp\n        Pp                  Ps                  VR%\u        V4      ,           4      pV! VV4       KA  	  \        P                  R&V4       V Pv                  '       d   R# \        P                  R'4       \y        V Pd                  V Pz                  V P|                  4      p\        P                  R(4       VP                  RV4      p. p\�        V P‚                  4       FU  p\„        P„                  ! 4       p VP                  RV4      pVPE                  \„        P„                  ! 4       V ,
          4       KW  	  ^ R)ICHDp! VPZ                  ^ ,          pV!! VV4      p"\#        R*4       V^ ,          p#\#        RV#4       V P6                  '       d   \#        RV^,          4       V P8                  '       d   \#        RV^,          4       V'       da   V#PZ                  w  pp$. p%\�        V4       F?  pVPC                  V#V,          RR7      pV%PE                  V4       \#        R+V R,V 24       KA  	  M|V#PZ                  w  pp&p$. p%\�        V4       F[  p\�        V&4       FI  p'VPC                  V#V,          V',          RR7      pV%PE                  V4       \#        R+V R-V' RV 24       KK  	  K]  	  V'       dÁ   VP:                  P‹                  VV P0                  R84      p(\Œ        PŽ                  ! V#4      p)\#        R74       \#        R.4       \#        V(4       \#        V4       \#        R74       \#        R/4       \#        V)4       \#        V%4       \#        R74       VV%8H  p*\#        R0V*'       d   R1MR24       V*V"R3&   V P�                  '       d   \“        V VVVVVV	4      p+\#        R4V+4       \#        R5V"4       V"# u up
i )9a!  Test GPT-2 model

Args:
    args (argparse.Namespace): arguments parsed from command line
    sentences (Optional[List[str]], optional): input text. Defaults to None.

Returns:
    Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
rG   ræ  Úleft)r‘   r÷  NÚptT©Úreturn_tensorsræ   r  r
  úwalk in park)Úadd_prefix_spacer#  úCTest PyTorch model and beam search with huggingface transformers...r(  ú!huggingface transformers outputs:rí  rî  rï  ©Úskip_special_tokensú: ú'Testing beam search with onnxruntime...rç  rÆ   rè  rì  ré  rê  rë  r`   zYUse prefix vocab mask with all ones in ORT, but no corresponding setting for Torch model.rc   Útest_data_dir©Úoutput_test_datazSaving test_data to z/test_data_set_* ...Útest_data_set_ú
ORT inputszCreating ort session......zRun ort session......r+  úORT outputs:úbatch z sequence: ú
 sequence úTorch Sequences:úORT Sequences:zTorch and ORT result isÚsameÚ	differentÚparityúTorch LatencyÚORT)zThe product is releasedzI enjoy walking in the parkzTest best way to investú2--------------------------------------------------rs   )Jr¤   r   r  rŽ   r‘   Úpadding_sideÚ	eos_tokenÚ	pad_tokenr   rö  Úencoder`   r–   r©   rÜ  rÿ  rm   rt  r4  rç  rè  ré  r_   rø  rê  rë  rì  r]   r^   rí  rî  rï  r  Údecoder“   r'  ÚnumpyÚastyper×   r;  ÚarrayÚfloat32r:  rd   r=  rD  rc   r—   rp   r   r§   r¨   r
  Úbert_test_datarP  rx   ry   rw   r<   rn   rù   rl   rj   Úrunr  r2  r3  r5  r,  rJ  r$  Ú
LongTensorro   r8  ),r†   r?  r@  Ú	tokenizerr¿   r  r  r
  Ú	bad_wordsr#  Úword_idrÜ  rö  r÷  rÿ  Útorch_decoded_sequencesÚbeam_outputsr  ÚsequenceÚdecoded_sequencer`   Úbad_word_idrž  rc   rN  rP  Ú
all_inputsÚdirrø   ÚresultÚlatencyrá  r7  r,  r§   rí  rç  Úort_decoded_sequencesÚnum_sequencesrO  Útorch_sequencesÚort_sequencesÚis_sameÚtorch_latency_outputs,   &&&                                         r(   Útest_gpt_modelr|  E  s  € ð �?‰?˜fÔ$Ð$Ð$ä×-Ò-¨d×.EÑ.EÐQU×Q_ÑQ_Ô`€IØ#€IÔØ#×-Ñ-€IÔä×+Ò+Ø×ÑØ—.‘.Ø×+Ñ+ô€Eð Òò
ˆ	ñ �y°¸tÔD€FØ�{Õ#€IØÐ,Õ-€Nà€IØ×$Ñ$ YÀÐ$ÓF€MÙ.;Ó<©m 7“Y©m€MÐ<Ø‡‡€Ü�‰�_ mÕ4àˆà�\‰\€FØ×&Ñ&€LØ×&Ñ&€LØ×"Ñ"€Jà ÐØ€LØ××ÑÜˆhŒÜÐSÔTØ—~‘~ØØ)Ø—‘Ø—‘Ø—n‘nØ×.Ñ.Ø!%×!:Ñ!:Ø%Ø%Ø!%×!:Ñ!:Ø×.Ñ.Ø#×6Ñ6ß+8™-¸dØ$(Ø×6Ñ6×RÐR¸$×:RÑ:Rð &ó 
ˆô" 	ˆk˜9Ô%ÜÐ1Ô2Üˆk˜<×1Ñ1Ô2Ø×'×'Ð'ÜÐ$ l×&CÑ&CÔDØ×#×#Ð#Ü�(˜L×/Ñ/Ô0Ü$ \×%;Ñ%;Ö<‰KˆAˆxØ(×/Ñ/°ÈdÐ/ÓSÐØ#×*Ñ*Ð+;Ô<Ü�Q�C�rÐ*Ð+Ð,Ö-ñ =ô
 
ˆ(„OÜ	Ð
3Ô4çà˜Ÿ™›×.Ñ.Ó0×7Ñ7¼¿¹ÓAØœ"Ÿ(š( D§O¡OÐ#4¼B¿H¹HÔEØœ"Ÿ(š( D§O¡OÐ#4¼B¿H¹HÔEØ ¤"§(¢(¨D×,CÑ,CÐ+DÌBÏJÉJÔ"Wð	
Šð ˜Ÿ™›×.Ñ.Ó0×7Ñ7¼¿¹ÓAØœ"Ÿ(š( D§O¡OÐ#4¼B¿H¹HÔEØœ"Ÿ(š( D§O¡OÐ#4¼B¿H¹HÔEØœŸš 4§>¡>Ð"2¼"¿(¹(ÔCØ"¤B§H¢H¨d×.GÑ.GÐ-HÔPR×PXÑPXÔ$YØœbŸhšh¨×(;Ñ(;Ð'<ÄBÇJÁJÔOØ ¤"§(¢(¨D×,CÑ,CÐ+DÌBÏJÉJÔ"Wð
ˆð ‡‡€Ü—W’W˜j´·±Ô:ˆ
Ø�?�?ˆ?Û,�Ø*+�
˜;Ó'ñ  -à)ˆˆ|Ñà×!×!Ð!Ü#8¸Ó#QˆÐÑ à—‘ Õ#€JØ××ÐÜ�‰ÐoÔpÜŸGšG Z°Ð$<ÄBÇHÁHÔMÐØ&7ˆÐ"Ñ#à××ÐÜ˜TŸ[™[Ó)×0Ñ0×9Ñ9Ó;ˆÜ�‰�_ mÔ4Ý3ä�‰Ð*¨=¨/Ð9MÐNÔOà�Xˆ
Ü" :Ö.‰IˆAˆvÜ—'‘'—,‘,˜}Ð.>ÄÀQÃÕ.GÓHˆCÙ˜S &Ö)ñ /ô ‡L�L�˜vÔ&à××ÐÙä
‡L�LÐ-Ô.Ü$ T§[¡[°$·,±,À×@XÑ@XÓY€Kä
‡L�LÐ(Ô)Ø�_‰_˜T 6Ó*€Fð €GÜ�4—?‘?Ö#ˆÜ—	’	“ˆØ�O‰O˜D &Ó)ˆØ�‰”t—y’y“{ UÕ*Ö+ñ $õ
 4à—‘ Õ#€JÙ ¨Ó4€Fä	ˆ.ÔØ�q•	€IÜ	ˆ+�yÔ!Ø×#×#Ð#ÜÐ  &¨¥)Ô,Ø××ÐÜˆh˜˜q�	Ô"çØ#,§?¡?Ñ ˆ�ZØ "ÐÜ�zÖ"ˆAØ(×/Ñ/°	¸!µÐRVÐ/ÓWÐØ!×(Ñ(Ð)9Ô:Ü�F˜1˜#˜[Ð)9Ð(:Ð;Ö<ò #ð
 3<·/±/Ñ/ˆ�] JØ "ÐÜ�zÖ"ˆAÜ˜=Ö)�Ø#,×#3Ñ#3°I¸aµLÀµOÐY]Ð#3Ó#^Ð Ø%×,Ñ,Ð-=Ô>Ü˜˜q˜c ¨A¨3¨bÐ1AÐ0BÐCÖDó *ñ #÷ Ø&×0Ñ0×8Ñ8¸ÀT×E^ÑE^Ð`bÓcˆÜ×(Ò(¨Ó3ˆÜˆhŒÜÐ Ô!ÜˆoÔÜÐ%Ô&ÜˆhŒÜÐÔÜˆmÔÜÐ#Ô$ÜˆhŒà)Ð-BÑBˆÜÐ'·7©ÀÔLØ"ˆˆxÑà××ÐÜ5ØØØØØØØó 
Ðô 	ˆoÐ3Ô4ä	ˆ%�Ôà€MùòY =s   Ãh<c                óh   € V ^8„  d   QhR\         P                  R\        \        ,          R,          /# )r8   r†   r?  N)r=   r>   r;   r<   )r?   s   "r(   r@   r@     s-   € ÷ zñ zœ×*Ñ*ð z´t¼CµyÀ4Õ7Gñ zr+   c                óB  € V P                   R39   g   Q hV P                  '       d   \        P                  R4       R# \        P
                  ! V P                  V P                  R7      pRVn        V P                   R8X  d.   \        P
                  ! V P                  V P                  R7      pM,\        P
                  ! V P                  V P                  R7      pVf   RR.pV! VRR	R
7      pVR,          pVR,          pRpVP                  V4      RR4 pV U	u. uF  q™.NK  	  pp	V P                  '       d   \        P                  RV4       M. pVP                  p
V
P                  pV
P                  pV
P                   p\        P                  RV RV RV 24       . pV P"                  '       Eg   \%        R54       \%        R4       TP'                  TTV P(                  V P*                  V P,                  V P.                  V P0                  TTV P2                  V P4                  V P6                  V'       d   TMRR	V P8                  ;'       g    V P:                  R7      p\%        RV4       \%        R4       \%        RVP<                  4       V P8                  '       d   \%        RVP>                  4       V P:                  '       d   \%        RVP@                  4       \C        VP<                  4       F:  w  ppVPE                  VR	R7      pVPG                  V4       \%        V RV 24       K<  	  \%        R54       \%        R4       \H        PJ                  ! V\H        PL                  R7      pV P                  '       d   V F  p^ VV&   K
  	  RVPO                  4       PQ                  4       PS                  \H        PL                  4      R\H        PT                  ! V P(                  .\H        PL                  R7      R\H        PT                  ! V P*                  .\H        PL                  R7      R\H        PT                  ! V P,                  .\H        PL                  R7      R\H        PT                  ! V P2                  .\H        PL                  R7      R \H        PT                  ! V P4                  .\H        PV                  R7      R!\H        PT                  ! V P6                  .\H        PV                  R7      /pV P                  '       d   VVR"&   V PX                  '       d   \[        W\4      VR&   V P\                  '       d›   \_        V P`                  4      Pb                  Pe                  4       p\        P                  R#V4       ^ R$I3H4p V.p\C        V4       F?  w  pp\j        Pl                  Po                  VR%\q        V4      ,           4      pV! VV4       KA  	  \        P                  R&V4       \s        V P`                  V Pt                  V Pv                  4      p. p\y        V Pz                  4       FU  p\|        P|                  ! 4       pVP                  RV4      pVPG                  \|        P|                  ! 4       V,
          4       KW  	  VP€                  ^ ,          p^ R'IAHBp V! VV4      p \%        R(4       X^ ,          p!\%        RV!4       V P8                  '       d   \%        RV^,          4       V P:                  '       d   \%        RV^,          4       V!P€                  w  pp"p#. p$\y        V4       F[  p\y        V"4       FI  p%VPE                  V!V,          V%,          R	R7      pV$PG                  V4       \%        R)V R*V% RV 24       KK  	  K]  	  V P"                  '       gÁ   XP<                  P‡                  VV P2                  R44      p&\ˆ        PŠ                  ! V!4      p'\%        R54       \%        R+4       \%        V&4       \%        V4       \%        R54       \%        R,4       \%        V'4       \%        V$4       \%        R54       VV$8H  p(\%        R-V('       d   R.MR/4       V(V R0&   V PŒ                  '       d   \�        V VVVVVV4      p)\%        R1V)4       \%        R2V 4       V # u up	i )6a%  Test T5 or MT5 model

Args:
    args (argparse.Namespace): arguments parsed from command line
    sentences (Optional[List[str]], optional): input text. Defaults to None.

Returns:
    Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
rq   zLSkipping parity test as prefix vocab mask is not implemented by Hugging FaceNræ  rB  z4translate English to French: The product is releasedz¬summarize: research continues to show that pets bring real health benefits to their owners. Having a dog around can lead to lower levels of stress for both adults and kids.rC  TrD  r  r
  rF  r#  zeos_token_id:z, pad_token_id:z, vocab_size:rH  r(  rI  rí  rî  rï  rJ  rL  rM  rÆ   rç  rè  ré  rê  rë  rì  r`   rN  rO  rQ  rR  r+  rS  rT  rU  rV  rW  zTorch and ORT result is rX  rY  rZ  r[  r\  r  rs   r]  )Hr¤   rc   r–   r©   r   r  rŽ   r‘   r^  r   r   ra  r`   rÜ  rö  r÷  rÿ  rm   rt  r4  rç  rè  ré  r_   rø  rê  rë  rì  r]   r^   rí  rî  rï  r  rb  r“   r×   r:  r;  r'  rc  rd  re  rf  rd   r=  rp   r   r§   r¨   r
  rg  rP  rx   ry   rw   r<   rù   rl   rj   r  r2  r3  rh  rD  r5  r,  rJ  r$  ri  ro   r8  )*r†   r?  rj  r¿   r  r  r
  rk  r#  rl  rÜ  rö  r÷  rÿ  rm  rn  r  ro  rp  r`   rq  rN  rP  rr  rs  rø   ru  rá  r7  rt  rž  r,  r§   rí  rw  rç  rv  rO  rx  ry  rz  r{  s*   &&                                        r(   Útest_t5_modelr    sñ  € ð �?‰?˜mÔ+Ð+Ð+à××ÐÜ�‰ÐcÔdÙä×+Ò+¨D×,CÑ,CÈtÏ~É~Ô^€IØ#€IÔà‡�˜$ÔÜ*×:Ò:Ø×#Ñ#Ø—n‘nô
‰ô
 ,×;Ò;Ø×#Ñ#Ø—n‘nô
ˆð ÒàBð {ð
ˆ	ñ �y°¸tÔD€FØ�{Õ#€IØÐ,Õ-€Nà€IØ×$Ñ$ YÓ/°°Ð4€MÙ.;Ó<©m 7“Y©m€MÐ<Ø‡‡€Ü�‰�_ mÕ4àˆà�\‰\€FØ×&Ñ&€LØ×&Ñ&€LØ×"Ñ"€JÜ
‡L�L�=  ¨o¸l¸^È=ÐYcÐXdÐeÔfà ÐØ××ÑÜˆhŒÜÐSÔTØ—~‘~ØØ)Ø—‘Ø—‘Ø—n‘nØ×.Ñ.Ø!%×!:Ñ!:Ø%Ø%Ø!%×!:Ñ!:Ø×.Ñ.Ø#×6Ñ6ß+8™-¸dØ$(Ø×6Ñ6×RÐR¸$×:RÑ:Rð &ó 
ˆô$ 	ˆk˜9Ô%ÜÐ1Ô2Üˆk˜<×1Ñ1Ô2Ø×'×'Ð'ÜÐ$ l×&CÑ&CÔDØ×#×#Ð#Ü�(˜L×/Ñ/Ô0Ü$ \×%;Ñ%;Ö<‰KˆAˆxØ(×/Ñ/°ÈdÐ/ÓSÐØ#×*Ñ*Ð+;Ô<Ü�Q�C�rÐ*Ð+Ð,Ö-ñ =ô
 
ˆ(„OÜ	Ð
3Ô4ä—’˜*¬R¯X©XÔ6€JØ‡‡€Û(ˆKØ&'ˆJ�{Ó#ñ )ð 	�Y—]‘]“_×*Ñ*Ó,×3Ñ3´B·H±HÓ=Ø”b—h’h §¡Ð0¼¿¹ÔAØ”b—h’h §¡Ð0¼¿¹ÔAØ”R—X’X˜tŸ~™~Ð.´b·h±hÔ?Ø¤§¢¨$×*CÑ*CÐ)DÌBÏHÉHÔ UØœ"Ÿ(š( D×$7Ñ$7Ð#8ÄÇ
Á
ÔKØœbŸhšh¨×(?Ñ(?Ð'@ÌÏ
É
ÔSð€Fð ‡‡€Ø)ˆˆ|Ñà×!×!Ð!Ü#8¸Ó#QˆÐÑ à××ÐÜ˜TŸ[™[Ó)×0Ñ0×9Ñ9Ó;ˆÜ�‰�_ mÔ4Ý3à�Xˆ
Ü" :Ö.‰IˆAˆvÜ—'‘'—,‘,˜}Ð.>ÄÀQÃÕ.GÓHˆCÙ˜S &Ö)ñ /ô ‡L�L�˜vÔ&ä$ T§[¡[°$·,±,À×@XÑ@XÓY€Kð €GÜ�4—?‘?Ö#ˆÜ—	’	“ˆØ—‘  vÓ.ˆØ�‰”t—y’y“{ UÕ*Ö+ñ $ð —‘ Õ#€JÝ3á ¨Ó4€Fä	ˆ.ÔØ�q•	€IÜ	ˆ+�yÔ!Ø×#×#Ð#ÜÐ  &¨¥)Ô,Ø××ÐÜˆh˜˜q�	Ô"à.7¯o©oÑ+€Z� 
ØÐÜ�:ÖˆÜ�}Ö%ˆAØ(×/Ñ/°	¸!µ¸QµÐUYÐ/ÓZÐØ!×(Ñ(Ð)9Ô:Ü�F˜1˜#˜Z¨ s¨"Ð-=Ð,>Ð?Ö@ó &ñ ð ××ÐØ&×0Ñ0×8Ñ8¸ÀT×E^ÑE^Ð`bÓcˆÜ×(Ò(¨Ó3ˆÜˆhŒÜÐ Ô!ÜˆoÔÜÐ%Ô&ÜˆhŒÜÐÔÜˆmÔÜÐ#Ô$ÜˆhŒà)Ð-BÑBˆÜÐ(·G©&ÀÔMØ"ˆˆxÑà××ÐÜ5ØØØØØØØó 
Ðô 	ˆoÐ3Ô4ä	ˆ%�ÔØ€Mùò[ =s   Äbc                óx   € V ^8„  d   QhR\         \        ,          R,          R\         \        ,          R,          /# )r8   r9   Nr?  )r;   r<   )r?   s   "r(   r@   r@   Ù  s,   € ÷ 8ñ 8Œt”C�y˜4Õð 8´4¼µ9¸tÕ3Cñ 8r+   c                ó¬  € \        V 4      p\        VP                  4       VP                  R9   Ed   VP                  '       dH   \
        P                  P                  VP                  4      '       g   \        RVP                   24      hVP                  '       dH   \
        P                  P                  VP                  4      '       g   \        RVP                   24      hVP                  '       d   VP                  '       d%   VP                  '       d   VP                  '       g   \        R4      hVP                  ^8H  ;'       d    VP                  ^8H  pVP                  R8X  d­   V'       d¥   VP                  R8”  dy   VP                  R8  dh   \        V\        P                  4       \         P#                  R4       VP                  R8”  g%   VP$                  '       g   VP&                  '       d   R	# M&\        V\        P(                  4       M\        V4       \         P#                  R
4       VP                  R9   d   \+        W!R7      pM\-        W!VR7      pV'       dg   VP.                  '       d3   \         P#                  RVP0                   RVP0                   R24       V# \         P#                  RVP0                   24       V# )a  Main entry function

Args:
    argv (Optional[List[str]], optional): _description_. Defaults to None.
    sentences (Optional[List[str]], optional): input text. Defaults to None.

Raises:
    ValueError: Path does not exist: --encoder_decoder_init_onnx
    ValueError: Path does not exist: --decoder_onnx
    ValueError: --decoder_onnx and --encoder_decoder_init_onnx are not used together for T5

Returns:
    Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
z1Path does not exist: --encoder_decoder_init_onnx z$Path does not exist: --decoder_onnx zB--decoder_onnx shall use together with --encoder_decoder_init_onnxrG   ri   rg   z�The test for gpt2_sampling onnx model is limited to non-custom model with small top_p(e.g <=0.01) value. The result should be the same as gpt2 greedy search.g{®Gáz„?Nzstart testing model...)r?  )r?  r@  zOutput files: rC   z.datazOutput file: r  )r‡   r   rO   r¤   rª   rx   ry   r	  r  r�   ré  rê  rú  r!  r    r3   r–   r—   rý  rf   r2   r  r|  rW   r§   )r9   r?  r†   r@  rt  s   &&   r(   r   r   Ù  s  € ô  ˜4Ó €DÜ�—‘Ôà‡�˜-Õ'Ø×)×)Ð)´"·'±'·.±.À×A_ÑA_×2`Ò2`ÜÐPÐQU×QoÑQoÐPpÐqÓrÐrØ××Ð¤R§W¡W§^¡^°D×4EÑ4E×%FÒ%FÜÐCÀD×DUÑDUÐCVÐWÓXÐXØ×*×*Ð*°4×3D×3DÐ3DØ××Ð d×&D×&DÐ&DäÐaÓbÐbà—‘ !Ñ#×FÐF¨×(AÑ(AÀQÑ(F€Ià‡�˜&Ô §YØ�:‰:˜Ô §
¡
¨SÔ 0Ü$ T¬>×+BÑ+BÔCÜ�K‰Kð pôð �z‰z˜DÔ  D§K§K K°4·9·9°9Ùð 4=ô % T¬>×+FÑ+FÕGä  Ô&ä
‡K�KÐ(Ô)Ø‡�˜-Ô'Ü˜tÔ9‰ä ÀYÔOˆçØ×(×(Ð(Ü�K‰K˜.¨¯©¨°R¸¿¹°}ÀEÐJÔKð €Mô �K‰K˜-¨¯© }Ð5Ô6à€Mr+   Ú__main__r%   )T)Úshared_é   NN)r„  )r÷  r¯  rs   )NF)NN)ZÚ__doc__r=   ÚloggingrÔ   rx   r3  Úenumr   Úpathlibr   Útypingr   rc  r×   rº   r$  r5  r   r   Úfusion_utilsr   r   r	   r
   r§  r   Útransformersr   r   r   r   r   r   r   r   Úonnxruntimer   r   r   r   Ú4onnxruntime.transformers.models.gpt2.convert_to_onnxr   r˜   Ú0onnxruntime.transformers.models.gpt2.gpt2_helperr   Ú2onnxruntime.transformers.models.t5.convert_to_onnxr   r¦   Ú,onnxruntime.transformers.models.t5.t5_helperr   r   Ú	getLoggerr–   r    r‡   r›   r¬   rÁ   ré   rù   r  r)  r3  rQ  r^  rc  ro  rv  r~  r“  r§  rÅ  rÙ  ré  r  r)  rp  r…  r—  r®  rµ  rØ  rà  r1   r!  r8  r=  r|  r  r-   r,   r+   r(   Ú<module>r’     sŽ  ðñ
#óJ Û Û Û 	Û Ý Ý Ý ã Û Û ß 4Ý $ß 4Ñ 4Ý  ÷	÷ 	ó 	÷ó õõ Tõ÷ð
 
×	Ò	˜2Ó	€ô�Tô ÷Qõh*)õZ!÷<O÷$Kõ\õB5õpIoõXYa÷xg!õT÷$ò>"#òJòiõ.õbSõlf3õRð* 9:ÐWY÷ 1õhf$õRD÷Nkõ\&fõR\õ~8÷v÷@hòV	"0ðN '5×&?Ñ&?÷y0õx?9òD	÷T÷nz÷z8ðv ˆzÔÙ†Fñ r+   