+
    G-jD  ã                   óÊ  € R t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RIt^ RI	H
t
HtHtHtHtHtHtHtHtHtHt ^ RIHt ^ RIHtHt ^ RIHtHtHtHt ^ RIH t  ^ RI!H"t" ]PF                  ! R	4      t$]PJ                  ! R
R7      t%R]PL                  9  d   ]'! ]%4      ]PL                  R&   ^ RI(t(^ RI)H*t*H+t+H,t, R t-R t.R R lt/R t0R t1R t2]3R8X  d
   ]2! 4        R# R# )a>  Benchmarking the inference of pretrained transformer models.
PyTorch/TorchScript benchmark is based on https://github.com/huggingface/transformers/blob/master/examples/benchmarks.py.
One difference is that random input_ids is generated in this benchmark.

For onnxruntime, this script will convert a pretrained model to ONNX, and optimize it when -o parameter is used.

Example commands:
    Export all models to ONNX, optimize and validate them:
        python benchmark.py -b 0 -o -v -i 1 2 3
    Run OnnxRuntime on GPU for all models:
        python benchmark.py -g
    Run OnnxRuntime on GPU for all models with fp32 optimization:
        python benchmark.py -g -o
    Run OnnxRuntime on GPU with fp16 optimization:
        python benchmark.py -g -o -p "fp16"
    Run TorchScript on GPU for all models:
        python benchmark.py -e torchscript -g
    Run TorchScript on GPU for all models with fp16:
        python benchmark.py -e torchscript -g -p "fp16"
    Run ONNXRuntime and TorchScript on CPU for all models with quantization:
        python benchmark.py -e torchscript onnxruntime -p "int8" -o
    Run OnnxRuntime with bfloat16 fastmath mode kernels on aarch64 platforms with bfloat16 support:
        python benchmark.py --enable_arm64_bfloat16_fastmath_mlas_gemm

It is recommended to use run_benchmark.sh to launch benchmark.
N)Údatetime)ÚConfigModifierÚOptimizerInfoÚ	PrecisionÚcreate_onnxruntime_sessionÚget_latency_resultÚinference_ortÚinference_ort_with_io_bindingÚoutput_detailsÚoutput_fusion_statisticsÚoutput_summaryÚsetup_logger)ÚFusionOptions)ÚMODEL_CLASSESÚMODELS)Úcreate_onnxruntime_inputÚexport_onnx_model_from_ptÚexport_onnx_model_from_tfÚload_pretrained_model)Úversion)ÚQuantizeHelperÚ F)ÚlogicalÚOMP_NUM_THREADS)Ú
AutoConfigÚAutoTokenizerÚLxmertConfigc           #      ó^
  € ^ RI p. pV '       dW   RVP                  4       9  dB   RVP                  4       9  d-   RVP                  4       9  d   \        P                  R4       V# ^ pVR8X  d?   \        P
                  p^pRVP                  4       9  d   \        P                  R4       V# V\        P
                  8X  d   \        P                  R	V R
24       V EF3  p\        V,          ^ ,          pV
 EF  pV\        V4      8”  d    K1  VRV p\        V,          ^,          Vn	        \        P                  ! V4      pRV9   d}   \        P                  ! 4       ;_uu_ 4        \        V\        V,          ^,          \        V,          ^,          \        V,          ^,          VVVVVV VVVVVVV4      w  pp p!p"RRR4       RV9   dW   \        V\        V,          ^,          \        V,          ^,          \        V,          ^,          VVVVVV VVVVVVV4      w  pp p!p"X '       g   EK6  \!        XV VRVVVR7      p#V#f   EKO  V#P#                  4        U$u. uF  p$V$P$                  NK  	  p%p$. p&V '       d   RMRp'\&        P(                  ! VVR7      p(\*        P,                  ! \/        V4      \/        V4      \/        X!V(P0                  4      .4      p)\*        P,                  ! \/        V4      V(P0                  .4      p*V EF  p+V+^ 8:  d   K  V EFñ  p,X"e
   V,V"8”  d   K  RV9   d   \*        P2                  M\*        P4                  p-\7        V!V+V,VV(V-4      p.RRRVP8                  RVRV'RVRVRV'       * RVRVRVRV+RV,RVP;                  4       R \=        \>        P@                  ! 4       4      /p/V(P                  R$9   d5   \        PC                  R!V R"V+^V(PD                  V(PD                  . 24       M\        PC                  R!V R"V+V,. 24       V'       d   \G        V#V.V/V	V+V4      p0M±V#PI                  V%V.4      p1V).p2\K        \        V14      4       FG  p3V3^8X  d-   \        V,          ^,          R#8X  d   V2PM                  V*4       K6  V2PM                  V)4       KI  	  RV9   d   \*        PN                  M\*        PP                  p4\S        V#V.V/V	V%V1V&V2V+V'V4V4      p0\        PC                  V04       VPM                  V04       EKô  	  EK  	  EK  	  EK6  	  V#   + '       g   i     ELc; iu up$i )%é    NÚCUDAExecutionProviderÚMIGraphXExecutionProviderÚDmlExecutionProviderzŽPlease install onnxruntime-gpu or onnxruntime-directml package instead of onnxruntime, and use a machine with GPU for testing gpu performance.ÚtensorrtÚTensorrtExecutionProviderzhPlease install onnxruntime-gpu-tensorrt package, and use a machine with GPU for testing gpu performance.zOptimizerInfo is set to zA, graph optimizations specified in FusionOptions are not applied.ÚptÚtfT)Úenable_all_optimizationÚnum_threadsÚverboseÚ(enable_mlas_gemm_fastmath_arm64_bfloat16ÚcudaÚcpu©Ú	cache_dirÚengineÚonnxruntimer   Ú	providersÚdeviceÚ	optimizerÚ	precisionÚ
io_bindingÚ
model_nameÚinputsÚthreadsÚ
batch_sizeÚsequence_lengthÚcustom_layer_numr   zRun onnxruntime on ú with input shape Úgpt©ÚvitÚswin)*r/   Úget_available_providersÚloggerÚerrorr   ÚNOOPTÚwarningr   ÚlenÚ
model_typer   ÚparseÚtorchÚno_gradr   r   r   Úget_outputsÚnamer   Úfrom_pretrainedÚnumpyÚprodÚmaxÚhidden_sizeÚint64Úint32r   Ú__version__Úget_layer_numÚstrr   ÚnowÚinfoÚ
image_sizer   ÚrunÚrangeÚappendÚlonglongÚintcr	   )5Úuse_gpuÚproviderÚmodel_namesÚmodel_classÚconfig_modifierr3   r'   Úbatch_sizesÚsequence_lengthsÚrepeat_timesÚinput_countsÚoptimizer_infoÚvalidate_onnxr-   Úonnx_dirr(   Ú	overwriteÚdisable_ort_io_bindingÚuse_raw_attention_maskÚmodel_fusion_statisticsÚmodel_sourceÚ(enable_arm64_bfloat16_fastmath_mlas_gemmÚargsr/   ÚresultsÚwarm_up_repeatr5   Úall_input_namesÚ
num_inputsÚinput_namesÚfusion_optionsÚonnx_model_fileÚis_valid_onnx_modelÚ
vocab_sizeÚmax_sequence_lengthÚort_sessionÚnode_argÚort_output_namesÚoutput_buffersr1   ÚconfigÚmax_last_state_sizeÚmax_pooler_sizer8   r9   Úinput_value_typeÚ
ort_inputsÚresult_templateÚresultÚort_outputsÚoutput_buffer_max_sizesÚiÚ	data_types5   &&&&&&&&&&&&&&&&&&&&&&&                              Ús/Volumes/fast/ai/experiments/nudenet-smoke/.venv/lib/python3.14/site-packages/onnxruntime/transformers/benchmark.pyÚrun_onnxruntimer‹   X   sA  € ó2 à€GçØ$¨K×,OÑ,OÓ,QÔQØ(°×0SÑ0SÓ0UÔUØ#¨;×+NÑ+NÓ+PÔPä�‰ð ]ô	
ð ˆà€NØ�:ÔÜ&×,Ñ,ˆØˆØ&¨k×.QÑ.QÓ.SÔSÜ�L‰LØzôð ˆNàœ×,Ñ,Ô,Ü�‰Ø& ~Ð&6Ð6wÐxô	
ô "ˆ
Ü  Õ,¨QÕ/ˆÜ&ˆJØœC Ó0Ô0Úà)¨+¨:Ð6ˆKÜ$ ZÕ0°Õ3ˆDŒOÜ*×0Ò0°Ó6ˆNà�|Ô#Ü—]’]—_•_ô 2Ø"Ü˜zÕ*¨1Õ-Ü˜zÕ*¨1Õ-Ü˜zÕ*¨1Õ-Ø#Ø'Ø!Ø Ø#ØØ!Ø&Ø%Ø.Ø!Ø/Ø&ó#ñØ'Ø+Ø"Ø+÷ %ð2 �|Ô#ô .ØÜ˜:Õ& qÕ)Ü˜:Õ& qÕ)Ü˜:Õ& qÕ)ØØ#ØØØØØØ"Ø!Ø*ØØ+Ø"ó#ñØ#Ø'ØØ'÷* 'Úä4ØØØØ(,Ø'ØØ9aôˆKð Ò"Úà>I×>UÑ>UÔ>WÓXÑ>W°( §¤Ñ>WÐÐXØˆNß&‘V¨EˆFÜ×/Ò/°
ÀiÔPˆFÜ"'§*¢*ä˜Ó$ÜÐ(Ó)Ü˜
 F×$6Ñ$6Ó7ðó#Ðô $Ÿjšj¬#¨kÓ*:¸F×<NÑ<NÐ)OÓPˆOÜ)�
Ø ”?ÙÜ'7�OØ*Ò6¸?ÐM`Ô;`Ù à6:¸lÔ6J¤u§{¢{ÔPU×P[ÑP[Ð$Ü!9Ø"Ø"Ø'Ø#ØØ(ó"�Jð ! -Ø! ;×#:Ñ#:Ø# XØ  &Ø# ^Ø# YØ$Ð*@Ô&@Ø$ jØ  *Ø! ;Ø$ jØ)¨?Ø*¨O×,IÑ,IÓ,KØ"¤C¬¯ª«Ó$7ð'�Oð" ×(Ñ(¨OÔ;ÜŸ™Ø1°*°Ð=OÐQ[Ð]^Ð`f×`qÑ`qÐsy÷  tEñ  tEð  QFð  PGð  Hõô Ÿ™Ð&9¸*¸ÐEWÐYcÐetÐXuÐWvÐ$wÔxç-Ü!.Ø'Ø&Ø+Ø(Ø&Ø*ó"™ð '2§o¡oÐ6FÈ
Ó&S˜Ø3FÐ2GÐ/Ü!&¤s¨;Ó'7Ö!8˜AØ  Aœv¬&°Õ*<¸QÕ*?À5Ô*Hà 7× >Ñ >¸Ö Oà 7× >Ñ >Ð?RÖ Sñ "9ð 7;¸lÔ6J¤E§N¢NÔPU×PZÑPZ˜	Ü!>Ø'Ø&Ø+Ø(Ø,Ø'Ø*Ø3Ø&Ø"Ø%Ø*ó"˜ô —K‘K Ô'Ø—N‘N 6×*ôS (8ô *ôo 'ñ "ðN €N÷y %—_�_üòD  Ys   Å%AT	ÉT*ÔT'c           "      ó€  aa€ . pV '       d<   \         P                  P                  4       '       g   \        P	                  R 4       V# \         P
                  ! R4       V EFŒ  p\        P                  ! WéVR7      pVP                  V4       \        VVVVR7      pVP                  R$9   d   V^ ,          .pM#\        P                  ! WëR7      pVP                  p\        P                  RV 24       \        P                  RVP                  4        24       V\        P                   8X  d   VP#                  4        \         P$                  ! V '       d   RMR4      pVP'                  V4       V\        P(                  8X  d   \*        P,                  ! V4      pV EFK  pV^ 8:  d   K  V EF7  pVP                  R$9   d™   \        P/                  R	V R
V^VP0                  VP0                  . 24       \         P2                  ! V^VP0                  VP0                  3V\        P                   8X  d   \         P4                  M\         P6                  VR7      oMfXe
   VV8”  d   Kº  \        P/                  R	V R
VV. 24       \         P8                  ! ^ VP:                  ^,
          VV3\         P<                  VR7      o V	'       d!   \         P>                  PA                  VS4      MV
'       d   \         PB                  ! V4      MToS! S4       \D        PF                  ! VV3R lV^R7      pRV	'       d   RMV
'       d   RMRR\         PH                  RRRV '       d   RMRRRRVRRRVR^RVR VR!VR"VPK                  4       R#\M        \N        PP                  ! 4       4      /pVPS                  \U        VV4      4       \        P/                  V4       VPW                  V4       EK:  	  EKN  	  EK�  	  V#   \X         d@   p\        P[                  T4       \         P                  P]                  4         Rp?EK�  Rp?ii ; i)%zYPlease install PyTorch with Cuda, and use a machine with GPU for testing gpu performance.F)Útorchscriptr-   )r   r-   Úcustom_model_classr,   zModel zNumber of parameters zcuda:0r+   zRun PyTorch on r;   )ÚsizeÚdtyper1   N)ÚlowÚhighr�   r�   r1   c                  ó   <€ S ! S4      # ©N© ©Ú	inferenceÚ	input_idss   €€rŠ   Ú<lambda>Úrun_pytorch.<locals>.<lambda>Œ  ó
   ø€ ±Y¸yÔ5Ió    ©ÚrepeatÚnumberr.   r�   Útorch2rH   r   r0   ÚNAr1   r*   r2   r   r3   r4   r5   r6   r7   r8   r9   r:   r   r=   )/rH   r*   Úis_availablerA   rB   Úset_grad_enabledr   rL   Úmodifyr   rF   r   Úmodel_max_lengthÚdebugÚnum_parametersr   ÚFLOAT16Úhalfr1   ÚtoÚINT8r   Úquantize_torch_modelrW   rX   ÚrandnÚfloat16Úfloat32Úrandintry   ÚlongÚjitÚtraceÚcompileÚtimeitrž   rS   rT   rU   r   rV   Úupdater   r[   ÚRuntimeErrorÚ	exceptionÚempty_cache)r^   r`   ra   rb   r3   r'   rc   rd   re   r�   r    r-   r(   rq   r5   r   ÚmodelÚ	tokenizerÚmax_input_sizer1   r8   r9   Úruntimesr…   Úer—   r˜   s   &&&&&&&&&&&&&            @@rŠ   Úrun_pytorchr¿   8  sz  ù€ ð €Gß”u—z‘z×.Ñ.×0Ò0Ü�‰ÐpÔqØˆä	×Ò˜5Ô!ä!ˆ
Ü×+Ò+¨JÐ[dÔeˆØ×Ñ˜vÔ&Ü%ØØØØ*ô	
ˆð ×Ñ Ô/à 0°Õ 3Ð4Ñä%×5Ò5°jÔVˆIà&×7Ñ7ˆNä�‰�v˜e˜WÐ%Ô&Ü�‰Ð,¨U×-AÑ-AÓ-CÐ,DÐEÔFàœ	×)Ñ)Ô)Ø�J‰JŒLä—’¯'™h°uÓ=ˆØ�‰�Ôàœ	Ÿ™Ô&Ü"×7Ò7¸Ó>ˆEä%ˆJØ˜QŒÙä#3�Ø×$Ñ$¨Ô7Ü—K‘KØ)¨*¨Ð5GÈÐUVÐX^×XiÑXiÐkq×k|Ñk|ÐH}ÐG~Ðôô !&§¢Ø(¨!¨V×->Ñ->À×@QÑ@QÐRØ/8¼I×<MÑ<MÔ/MœeŸmšmÔSX×S`ÑS`Ø%ô!‘Ið &Ò1°oÈÔ6VÙ ä—K‘K /°*°Ð=OÐQ[Ð]lÐPmÐOnÐ oÔpÜ %§¢ØØ#×.Ñ.°Õ2Ø(¨/Ð:Ü#Ÿj™jØ%ô!�Ið-ç=HœŸ	™	Ÿ™¨¨yÔ9×flÌeÏmÊmÐ\aÔNbÐrwð ñ ˜iÔ(ä%Ÿ}š}Õ-IÐR^ÐghÔi�Hð !·;¡-×PVÁHÐ\cØ!¤5×#4Ñ#4Ø# TØ ¯G¡&¸Ø# RØ# YØ$ bØ$ jØ  !Ø! ;Ø$ jØ)¨?Ø*¨O×,IÑ,IÓ,KØ"¤C¬¯ª«Ó$7ð�Fð  —M‘MÔ"4°X¸zÓ"JÔKÜ—K‘K Ô'Ø—N‘N 6×*ôa $4ô	 &ñ= "ðn €Nøô	 $ô -Ü×$Ñ$ QÔ'Ü—J‘J×*Ñ*×,Ó,ûð-ús8   Ê?O3Ë'O3Ë/AO3Ì6	O3Í O3ÍBO3Ï3P=	Ï>3P8	Ð8P=	c                ó0   € V ^8„  d   QhR\         R\         /# )é   Údo_eager_modeÚuse_xla)Úbool)Úformats   "rŠ   Ú__annotate__rÆ   ¨  s   € ÷ ñ ¬Tð ¼Dñ rœ   c                 ó4   a aaa€ ^ RI Ho ^ RIoV VVV3R lpV# )r   )ÚwrapsNc                 ó¢   <a € S! S 4      V 3R  l4       pS! S 4      SP                  SR7      V 3R l4       4       pSRJ d   SRJ g   Q R4       hV# V# )c                  ó   <€ S! V / VB # r”   r•   ©rp   ÚkwargsÚfuncs   *,€rŠ   Úrun_in_eager_modeÚFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_eager_mode®  s   ø€ á˜Ð( Ñ(Ð(rœ   )Újit_compilec                  ó   <€ S! V / VB # r”   r•   rË   s   *,€rŠ   Úrun_in_graph_modeÚFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_graph_mode²  s   ø€ ñ ˜Ð( Ñ(Ð(rœ   TFzcCannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.)Úfunction)rÍ   rÎ   rÒ   rÂ   r%   rÃ   rÈ   s   f  €€€€rŠ   Úrun_funcÚ+run_with_tf_optimizations.<locals>.run_func­  st   ù€ Ù	ˆt‹ô	)ó 
ð	)ñ 
ˆt‹Ø	�‰ ˆÓ	)ô	)ó 
*ó 
ð	)ð ˜DÓ Ø˜eÓ#ð ØuóÐ#ð %Ð$à$Ð$rœ   )Ú	functoolsrÈ   Ú
tensorflow)rÂ   rÃ   rÕ   r%   rÈ   s   ff @@rŠ   Úrun_with_tf_optimizationsrÙ   ¨  s   û€ Ýã÷%ð %ð$ €Orœ   c           "      ó‚  aaaa a!€ . p^ RI o!S!P                  P                  P                  V4       V '       g   S!P                  P	                  . R4       V '       d8   S!P
                  P                  4       '       g   \        P                  R4       V# V '       d‰   S!P                  P                  R4      p S!P                  P	                  V^ ,          R4       S!P                  P                  P                  V^ ,          R4       S!P                  P                  RR7       V\         P"                  8X  g   V\         P$                  8X  d   \'        R4      hV EFW  p\(        P*                  ! WéR7      oVP-                  S4       \/        VSV	VRR	7      o \0        P*                  ! WéR7      pVP2                  p\5        R
R
R7      V 3R l4       p\5        R
R
R7      V 3R l4       p\5        R
R
R7      VV V!3R l4       pSP6                  '       d   VoM\9        S\:        4      '       d   VoMVoV EF}  pV^ 8:  d   K  V EFi  pVe
   VV8”  d   K  \        P=                  RV RVV. 24       \>        P@                  ! 4       p\C        VV,          4       Uu. uF&  pVPE                  ^ SPF                  ^,
          4      NK(  	  ppS!PI                  VVV3S!PJ                  R7      o S! S4       \L        PN                  ! VV3R lV^R7      pRRRS!PP                  RRRV '       d   RMRRRRVRRR VR!^R"VR#VR$VR%VPS                  4       R&\U        \V        PX                  ! 4       4      /pVP[                  \]        VV4      4       \        P=                  V4       VP_                  V4       EKl  	  EK€  	  EKZ  	  V#   \         d!   p\        P                  T4        Rp?EL»Rp?ii ; iu upi   \         dH   p\        P                  T4       ^ R'I0H1p TPe                  4       pTPg                  4         Rp?EKú  Rp?ii ; i)(r   NÚGPUzVPlease install Tensorflow-gpu, and use a machine with GPU for testing gpu performance.Tz/gpu:0)r1   z+Mixed precision is currently not supported.r,   )r   r-   rŽ   Úis_tf_modelF)rÂ   rÃ   c                 ó   <€ S! V R R7      # )F)Útrainingr•   ©r˜   rº   s   &€rŠ   Úencoder_forwardÚ'run_tensorflow.<locals>.encoder_forwardü  s   ø€ á˜¨UÔ3Ð3rœ   c                 ó   <€ S! W R R7      # )F)Údecoder_input_idsrÞ   r•   rß   s   &€rŠ   Úencoder_decoder_forwardÚ/run_tensorflow.<locals>.encoder_decoder_forward   s   ø€ á˜È%ÔPÐPrœ   c                 ó¼   <€ SP                   P                  ^^SP                  .4      pSP                   P                  ^^SP                  .4      pS! V VVRR7      # )é   F)Úvisual_featsÚ
visual_posrÞ   )ÚrandomÚnormalÚvisual_feat_dimÚvisual_pos_dim)r˜   ÚfeatsÚposr   rº   r%   s   &  €€€rŠ   Úlxmert_forwardÚ&run_tensorflow.<locals>.lxmert_forward  s^   ø€ à—I‘I×$Ñ$ a¨¨F×,BÑ,BÐ%CÓDˆEØ—)‘)×"Ñ" A q¨&×*?Ñ*?Ð#@ÓAˆCÙØØ"ØØô	ð rœ   zRun Tensorflow on r;   )Úshaper�   c                  ó   <€ S ! S4      # r”   r•   r–   s   €€rŠ   r™   Ú run_tensorflow.<locals>.<lambda>'  r›   rœ   r�   r.   rØ   r   r0   r¡   r1   r*   r+   r2   r   r3   r4   r5   r6   r7   r8   r9   r:   r   )r*   )4rØ   r   Ú	threadingÚ set_intra_op_parallelism_threadsÚset_visible_devicesÚtestÚis_built_with_cudarA   rB   Úlist_physical_devicesÚexperimentalÚset_memory_growthÚ
distributeÚOneDeviceStrategyr·   r¸   r   r¨   r«   ÚNotImplementedErrorr   rL   r¤   r   r   r¥   rÙ   Úis_encoder_decoderÚ
isinstancer   rW   rê   ÚRandomrZ   r°   ry   ÚconstantrR   rµ   rž   rS   rT   rU   r   rV   r¶   r   r[   Únumbar*   Úget_current_deviceÚreset)"r^   r`   ra   rb   r3   r'   rc   rd   re   r-   r(   rq   Úphysical_devicesr¾   r5   r»   r¼   rà   rä   rð   r8   r9   Úrngrˆ   Úvaluesr½   r…   r*   r1   r   r—   r˜   rº   r%   s"   &&&&&&&&&&&                  @@@@@rŠ   Úrun_tensorflowr
  Â  s²  ü€ ð €Gãà‡I�I×Ñ×8Ñ8¸ÔEçØ
�	‰	×%Ñ% b¨%Ô0ç�r—w‘w×1Ñ1×3Ò3Ü�‰ÐmÔnØˆçØŸ9™9×:Ñ:¸5ÓAÐð	 Ø�I‰I×)Ñ)Ð*:¸1Õ*=¸uÔEØ�I‰I×"Ñ"×4Ñ4Ð5EÀaÕ5HÈ$ÔOØ�M‰M×+Ñ+°8Ð+Ô<ð ”I×%Ñ%Ô%¨´i·n±nÔ)DÜ!Ð"OÓPÐPä!ˆ
Ü×+Ò+¨JÔLˆØ×Ñ˜vÔ&ä%ØØØØ*Øô
ˆô "×1Ò1°*ÔRˆ	à"×3Ñ3ˆô 
#°ÀÔ	Fô	4ó 
Gð	4ô 
#°ÀÔ	Fô	Qó 
Gð	Qô 
#°ÀÔ	Fö	ó 
Gð	ð ×$×$Ð$Ø/‰IÜ˜¤×-Ò-Ø&‰Ià'ˆIä%ˆJØ˜QŒÙä#3�Ø!Ò-°/ÀNÔ2RÙä—‘Ð0°°Ð<NÐPZÐ\kÐOlÐNmÐnÔoä—m’m“o�ÜINÈzÐ\kÕOkÔIlÓmÑIlÀA˜#Ÿ+™+ a¨×):Ñ):¸QÕ)>Ö?ÑIl�ÐmØŸK™K¨°zÀ?Ð6SÐ[]×[cÑ[c˜KÓd�	ð#Ù˜iÔ(ä%Ÿ}š}Õ-IÐR^ÐghÔi�Hð ! ,Ø! 2§>¡>Ø# TØ ¯G¡&¸Ø# RØ# YØ$ bØ$ jØ  !Ø! ;Ø$ jØ)¨?Ø*¨O×,IÑ,IÓ,KØ"¤C¬¯ª«Ó$7ð�Fð  —M‘MÔ"4°X¸zÓ"JÔKÜ—K‘K Ô'Ø—N‘N 6×*ôC $4ô	 &ñ] "ðv €NøôC ô 	 Ü×Ñ˜Q×Òûð	 üò~ nøô6 $ô #Ü×$Ñ$ QÔ'Ý*à!×4Ñ4Ó6�FØ—L‘L—N“Nûð#ús8   Â8A,N9 Ê,O'
Ë%CO,Î9O$ÏOÏO$Ï,P>	Ï7;P9	Ð9P>	c                  óP  € \         P                  ! 4       p V P                  R RRR\        . RTO\	        \
        P                  ! 4       4      RRP                  \
        P                  ! 4       4      ,           R7       V P                  RR^\        RRR	.R
R7       V P                  RR\        R\	        \        4      RRP                  \        4      ,           R7       V P                  RRRR\        R.. RUORR7       V P                  RRR\        \        P                  P                  RR4      RR7       V P                  RR\        \        P                  P                  RR4      RR7       V P                  RRRRRR 7       V P                  R!R\        RR"R7       V P                  R#R$\        \        P                  \	        \        4      R%R&7       V P                  R'RRR(R 7       V P                  R)RRR*R 7       V P                  R+R,\        \        P                  \	        \        4      R-R&7       V P                  R.R/RRR0R 7       V P                  R1R2RRR3R47       V P                  R5R6RRR7R47       V P                  R8R9RRR:R47       V P                  R;R<RR^.\        . RVOR=R>7       V P                  R?R@R^d\        RARB7       V P                  RCRDR\        ^.RE7       V P                  RFRGR\        . RWORE7       V P                  RHRRRIR 7       V P!                  RRJ7       V P                  RKRLRR\        ^ .RMRN7       V P                  ROR\        RRPR7       V P                  RQRRRRR 7       V P!                  RRS7       \"        P$                  ! V 4       V P'                  4       pV# )Xz-mz--modelsFÚ+z Pre-trained models in the list: z, )ÚrequiredÚnargsÚtypeÚdefaultÚchoicesÚhelpz--model_sourcer$   r%   zExport onnx from pt or tfz--model_classNz!Model type selected in the list: )r  r  r  r  r  z-ez	--enginesr/   zEngines to benchmarkz-cz--cache_dirÚ.Úcache_modelsz%Directory to cache pre-trained models)r  r  r  r  z
--onnx_dirÚonnx_modelszDirectory to store onnx modelsz-gz	--use_gpuÚ
store_truezRun on gpu device)r  Úactionr  z
--providerzExecution provider to usez-pz--precisionzfPrecision of model to run. fp32 for full precision, fp16 for half precision, and int8 for quantization)r  r  r  r  z	--verbosezPrint more informationz--overwritezOverwrite existing modelsz-oz--optimizer_infozjOptimizer info: Use optimizer.py to optimize onnx model as default. Can also choose from by_ort and no_optz-vz--validate_onnxzValidate ONNX modelz-fz--fusion_csvz:CSV file for saving summary results of graph optimization.)r  r  r  z-dz--detail_csvz#CSV file for saving detail results.z-rz--result_csvz$CSV file for saving summary results.z-iz--input_countszXNumber of ONNX model inputs. Please use 1 for fair comparison with Torch or TorchScript.)r  r  r  r  r  r  z-tz--test_timesz8Number of repeat times to get average inference latency.)r  r  r  r  z-bz--batch_sizes)r  r  r  z-sz--sequence_lengthsz--disable_ort_io_bindingz=Disable running ONNX Runtime with binded inputs and outputs. )rk   z-nz--num_threadszThreads to use)r  r  r  r  r  z--force_num_layersz%Manually set the model's layer numberz*--enable_arm64_bfloat16_fastmath_mlas_gemmzHEnable bfloat16 mlas gemm kernels on aarch64. Supported only for CPU EP )ro   )zbert-base-casedzroberta-baseÚgpt2)r/   rH   r    r�   rØ   )rç   rÁ   é   )é   é   é   é    é@   é€   é   )ÚargparseÚArgumentParserÚadd_argumentrU   Úlistr   ÚkeysÚjoinr   ÚosÚpathr   ÚFLOAT32r   ÚBYSCRIPTÚintÚset_defaultsr   Úadd_argumentsÚ
parse_args)Úparserrp   s     rŠ   Úparse_argumentsr0  F  s0  € Ü×$Ò$Ó&€Fà
×ÑØØØØÜÚ;Ü”V—[’[“]Ó#Ø/°$·)±)¼F¿KºK»MÓ2JÕJð ô 	ð ×ÑØØØÜØØ�t�Ø(ð ô ð ×ÑØØÜØÜ”]Ó#Ø0°4·9±9¼]Ó3KÕKð ô ð ×ÑØØØØÜØ�ÚOØ#ð ô 	ð ×ÑØØØÜÜ—‘—‘˜S .Ó1Ø4ð ô ð ×ÑØØÜÜ—‘—‘˜S -Ó0Ø-ð ô ð ×Ñ˜˜k°EÀ,ÐUhÐÔià
×ÑØØÜØØ(ð ô ð ×ÑØØÜÜ×!Ñ!Ü”Y“Øuð ô ð ×Ñ˜¨e¸LÐOgÐÔhà
×ÑØØØØ(ð	 ô ð ×ÑØØÜÜ×&Ñ&Ü”]Ó#Øyð ô ð ×ÑØØØØØ"ð ô ð ×ÑØØØØØIð ô ð ×ÑØØØØØ2ð ô ð ×ÑØØØØØ3ð ô ð ×ÑØØØØØ�ÜÚØgð ô 	ð ×ÑØØØØÜØGð ô ð ×Ñ˜˜o°S¼sÈQÈCÐÔPà
×ÑØØØÜÚ,ð ô ð ×ÑØ"ØØØLð	 ô ð ×Ñ¨uÐÔ5à
×ÑØØØØÜØ�Øð ô ð ×ÑØØÜØØ4ð ô ð ×ÑØ4ØØØWð	 ô ð ×ÑÀÐÔGä×Ò Ô'à×ÑÓ€DØ€Krœ   c                  óŠ  € \        4       p \        V P                  4       V P                  \        P
                  8X  d*   V P                  '       g   \        P                  R 4       R# V P                  \        P                  8X  d;   V P                  '       d)   V P                  R9  d   \        P                  R4       R# \        V P                  4      ^8X  d3   \        V P                  ^ ,          ,          ^,          R9   d	   R.V n        \        V P                    Uu0 uF  q^ 8:  d   \"        MTkK  	  up4      V n        \        P%                  RV  24       \&        P(                  P+                  V P,                  4      '       g"    \&        P.                  ! V P,                  4       RV P2                  9   pRV P2                  9   pRV P2                  9   pR	V P2                  9   pR
V P2                  9   pV'       df   \4        P6                  ! \8        P:                  4      \4        P6                  ! R4      8  d)   \        P                  R\8        P:                   24       R# \=        V P>                  4      p. pV P                    EF„  p	\8        P@                  ! V	4       \        PC                  \8        PD                  PG                  4       4       V'       g   V'       g
   V'       Ed¨   V PH                  ^.8w  d   \        PK                  R4       V'       dy   V\M        V P                  V P                  V PN                  VV P                  V	V PP                  V P                  V PR                  RRV P,                  V P                  4      ,          pV'       dy   V\M        V P                  V P                  V PN                  VV P                  V	V PP                  V P                  V PR                  RRV P,                  V P                  4      ,          pV'       dy   V\M        V P                  V P                  V PN                  VV P                  V	V PP                  V P                  V PR                  RRV P,                  V P                  4      ,          pV'       dw   V\U        V P                  V P                  V PN                  VV P                  V	V PP                  V P                  V PR                  V P,                  V P                  4      ,          p/ p
V'       g   EK–   V PV                  '       * pV\Y        V P                  V P                  V P                  V PN                  VV P                  V	V PP                  V P                  V PR                  V PH                  V PZ                  V P\                  V P,                  V P^                  V P                  V P`                  V Pb                  VV
V Pd                  V Pf                  V 4      ,          pEK‡  	  \l        Pn                  ! 4       Pq                  R4      pX
'       d&   V Pr                  ;'       g    RV R2p\u        W­4       \        V4      ^ 8X  d*   V PP                  ^ .8w  d   \        PK                  R4       R# V Pv                  ;'       g    RV R2p\y        W�4       V Pz                  ;'       g    RV R2p\}        W�V 4       R# u upi   \0         d%    \        P                  RT P,                  4        ELni ; i  \h         d    \        Pk                  R4        EK»  i ; i)zfp16 is for GPU onlyNzint8 is for CPU onlyr   zArguments: z#Creation of the directory %s failedrH   r    r�   r/   rØ   z2.0.0z2PyTorch version must be >=2.0.0 and you are using zB--input_counts is not implemented for torch or torchscript engine.TFÚ	Exceptionz%Y%m%d-%H%M%SÚbenchmark_fusion_z.csvzNo any result available.Úbenchmark_detail_Úbenchmark_summary_)Úmigraphx)r>   Úswim)?r0  r   r(   r3   r   r¨   r^   rA   rB   r«   r_   rE   Úmodelsr   rd   Úsortedr'   Ú	cpu_countrW   r'  r(  Úexistsr-   ÚmkdirÚOSErrorÚenginesr   rG   rH   rS   r   Úforce_num_layersÚset_num_threadsr¦   Ú
__config__Úparallel_inforf   rD   r¿   ra   rc   Ú
test_timesr
  Úuse_mask_indexr‹   rg   rh   ri   rj   rk   rn   ro   r2  r¸   r   rV   ÚstrftimeÚ
fusion_csvr   Ú
detail_csvr
   Ú
result_csvr   )rp   ÚxÚenable_torchÚenable_torch2Úenable_torchscriptÚenable_onnxruntimeÚenable_tensorflowrb   rq   r'   rm   rl   Ú
time_stampÚcsv_filenames                 rŠ   ÚmainrQ    s€  € ÜÓ€Dä�—‘Ôà‡~�~œ×*Ñ*Ô*°4·<·<°<Ü�‰Ð+Ô,Ùà‡~�~œŸ™Ô'¨D¯L¯L¨L¸T¿]¹]ÐR^Ô=^Ü�‰Ð+Ô,Ùä
ˆ4�;‰;Ó˜1Ô¤¨¯©°A­Õ!7¸Õ!:¸oÔ!MØ!# ˆÔäÀ4×CSÒCSÓTÑCS¸a°¬F�y¸Ò9ÑCSÑTÓU€DÔä
‡K�K�+˜d˜VÐ$Ô%ä�7‰7�>‰>˜$Ÿ.™.×)Ò)ð	PÜ�HŠH�T—^‘^Ô$ð ˜dŸl™lÑ*€LØ §¡Ñ,€MØ&¨$¯,©,Ñ6ÐØ&¨$¯,©,Ñ6ÐØ$¨¯©Ñ4ÐçœŸš¤u×'8Ñ'8Ó9¼G¿MºMÈ'Ó<RÔRÜ�‰ÐIÌ%×J[ÑJ[ÐI\Ð]Ô^Ùä$ T×%:Ñ%:Ó;€Oà€Gà×'Õ'ˆÜ×Ò˜kÔ*Ü�‰”U×%Ñ%×3Ñ3Ó5Ô6ßŸ=×,>Ð,>Ø× Ñ  Q CÔ'Ü—‘ÐcÔdç!Øœ;Ø—L‘LØ—K‘KØ×$Ñ$Ø#Ø—N‘NØØ×$Ñ$Ø×)Ñ)Ø—O‘OØØØ—N‘NØ—L‘Lóõ �÷  Øœ;Ø—L‘LØ—K‘KØ×$Ñ$Ø#Ø—N‘NØØ×$Ñ$Ø×)Ñ)Ø—O‘OØØØ—N‘NØ—L‘Lóõ �÷  Øœ;Ø—L‘LØ—K‘KØ×$Ñ$Ø#Ø—N‘NØØ×$Ñ$Ø×)Ñ)Ø—O‘OØØØ—N‘NØ—L‘Lóõ �÷  Ø”~Ø—‘Ø—‘Ø× Ñ ØØ—‘ØØ× Ñ Ø×%Ñ%Ø—‘Ø—‘Ø—‘óõ ˆGð #%ÐßÒð.Ø-1×-@Ñ-@Ô)@Ð&Øœ?Ø—L‘LØ—M‘MØ—K‘KØ×$Ñ$Ø#Ø—N‘NØØ×$Ñ$Ø×)Ñ)Ø—O‘OØ×%Ñ%Ø×'Ñ'Ø×&Ñ&Ø—N‘NØ—M‘MØ—L‘LØ—N‘NØ×/Ñ/Ø*Ø+Ø×%Ñ%Ø×AÑAØó/õ “ñ[ (ôR —’“×(Ñ(¨Ó9€JßØ—‘×NÐNÐ,=¸j¸\ÈÐ*NˆÜ Ð!8ÔGä
ˆ7ƒ|�qÔØ×Ñ ˜sÔ"Ü�N‰NÐ5Ô6Ùà—?‘?×JÐJÐ(9¸*¸ÀTÐ&J€LÜ�7Ô)à—?‘?×KÐKÐ(:¸:¸,ÀdÐ&K€LÜ�7¨$Ö/ùò_ Uøô ô 	PÜ�L‰LÐ>ÀÇÁ×Oð	Pûôn ô .Ü× Ñ  ×-Ð-ð.ús+   Ä [&Æ [+ ÔC-\Û++\Ü\Ü ]Ý]Ú__main__)4Ú__doc__r!  Úloggingr'  rê   rµ   r   rM   ÚpsutilÚbenchmark_helperr   r   r   r   r   r   r	   r
   r   r   r   rv   r   Úhuggingface_modelsr   r   Úonnx_exporterr   r   r   r   Ú	packagingr   Úquantize_helperr   Ú	getLoggerrA   r:  ÚenvironrU   rH   Útransformersr   r   r   r‹   r¿   rÙ   r
  r0  rQ  Ú__name__r•   rœ   rŠ   Ú<module>r_     sØ   ðñ ó6 Û Û 	Û Û Ý ã Û ÷÷ ÷ ñ õ )ß 4÷ó õ Ý *à	×	Ò	˜2Ó	€à×Ò UÔ+€	ð ˜BŸJ™JÔ&Ù$'¨	£N€B‡J�JÐ Ñ!ã ß @Ñ @ò]ò@mõ`ò4AòHEòP_0ðD ˆzÔÙ†Fñ rœ   