+
    G-j€W  ã                   ó  € ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHtH	t	 ^ RI
Ht ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIt^ RIt^ RIt^ RIHt ^ RIt]P0                  ! ]4      t ! R	 R
]4      t ! R R]4      t ! R R4      tR]P<                  /tRRR'RRR/ 3R lt R(R lt!R)R lt"R t#R t$R t%R t&R*R lt']PP                  ^ 3R lt)R t*R+R lt+R R lt, ! R R ]4      t- ! R! R"]-4      t. ! R# R$]-4      t/R,R% lt0R& t1R# )-é    N)ÚABCÚabstractmethod)ÚThreadPoolExecutor)Údatetime)ÚEnum)Úsleep)ÚAny)Úversionc                   ó6   a € ] tR t^t o RtRtRtRtR tRt	V t
R# )Ú	PrecisionÚfp32Úfp16Úint8Úint4c                ó   € V P                   # ©N©Úvalue©Úselfs   &Úz/Volumes/fast/ai/experiments/nudenet-smoke/.venv/lib/python3.14/site-packages/onnxruntime/transformers/benchmark_helper.pyÚ__str__ÚPrecision.__str__%   ó   € Ø�z‰zÐó    © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚFLOAT32ÚFLOAT16ÚINT8ÚINT4r   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   r   r      s#   ø‡ € Ø€GØ€GØ€DØ€D÷ð r   r   c                   ó2   a € ] tR t^)t o RtRtRtR tRtV t	R# )ÚOptimizerInfoÚno_optÚby_ortÚ	by_scriptc                ó   € V P                   # r   r   r   s   &r   r   ÚOptimizerInfo.__str__0   r   r   r   N)
r   r   r   r    ÚNOOPTÚBYORTÚBYSCRIPTr   r%   r&   r'   s   @r   r*   r*   )   s    ø‡ € ð €EØ€EØ€H÷ð r   r*   c                   ó2   a € ] tR t^4t o R tR tR tRtV tR# )ÚConfigModifierc                ó   € Wn         R # r   ©Ú
num_layers)r   r7   s   &&r   Ú__init__ÚConfigModifier.__init__5   s   € Ø$Žr   c                óÈ  € V P                   f   R # \        VR4      '       d4   V P                   Vn        \        P	                  RV P                    24       \        VR4      '       d4   V P                   Vn        \        P	                  RV P                    24       \        VR4      '       d6   V P                   Vn        \        P	                  RV P                    24       R # R # )NÚnum_hidden_layersz6Modifying pytorch model's number of hidden layers to: Úencoder_layersz7Modifying pytorch model's number of encoder layers to: zdecoder_layers z7Modifying pytorch model's number of decoder layers to: )r7   Úhasattrr;   ÚloggerÚinfor<   Údecoder_layers)r   Úconfigs   &&r   ÚmodifyÚConfigModifier.modify8   s¶   € Ø�?‰?Ò"ÙÜ�6Ð.×/Ò/Ø'+§¡ˆFÔ$Ü�K‰KÐPÐQU×Q`ÑQ`ÐPaÐbÔcÜ�6Ð+×,Ò,Ø$(§O¡OˆFÔ!Ü�K‰KÐQÐRV×RaÑRaÐQbÐcÔdÜ�6Ð,×-Ò-Ø$(§O¡OˆFÔ!Ü�K‰KÐQÐRV×RaÑRaÐQbÐcÖdñ .r   c                ó   € V P                   # r   r6   r   s   &r   Úget_layer_numÚConfigModifier.get_layer_numE   s   € Ø�‰Ðr   r6   N)	r   r   r   r    r8   rB   rE   r%   r&   r'   s   @r   r4   r4   4   s   ø‡ € ò%òe÷ð r   r4   Úfloat32TFc	                 ó^  € \         P                  ! 4       p	V'       d!   \         P                  P                  V	n        M\         P                  P
                  V	n        V'       d   R V	n        V^ 8”  d)   WIn        \        P                  RV	P                   24       V'       d	   ^ V	n
        M^V	n
        V\         P                  ! 4       9   d   V.p
MMV'       dC   VR8X  d   RR.p
M9VR8X  d   RR.p
M-VR8X  g   Vf   R	R.p
MVR
8X  d   . ROp
M\        RV 24      hR.p
V'       d"   V
 Uu. uF  q»V9   d   W¸V,          3MTNK  	  p
pV'       d   V	P                  RR4       Rp \         P                  ! W	V
R7      pV# u upi   \         d     \        P!                  RT  RT
 24        T# i ; i)Tz%Session option: intra_op_num_threads=ÚdmlÚDmlExecutionProviderÚCPUExecutionProviderÚmigraphxÚMIGraphXExecutionProviderÚcudaNÚCUDAExecutionProviderÚtensorrtz)The execution provider is not supported: z(mlas.enable_gemm_fastmath_arm64_bfloat16Ú1)Ú	providerszFailed to create session for z with providers=)ÚTensorrtExecutionProviderrO   rK   )ÚonnxruntimeÚSessionOptionsÚGraphOptimizationLevelÚORT_ENABLE_ALLÚgraph_optimization_levelÚORT_ENABLE_BASICÚenable_profilingÚintra_op_num_threadsr>   ÚdebugÚlog_severity_levelÚget_available_providersÚRuntimeErrorÚadd_session_config_entryÚInferenceSessionÚ	ExceptionÚ	exception)Úonnx_model_pathÚuse_gpuÚproviderÚenable_all_optimizationÚnum_threadsrZ   ÚverboseÚ(enable_mlas_gemm_fastmath_arm64_bfloat16Úprovider_optionsÚsess_optionsrR   ÚnameÚsessions   &&&&&&&&&    r   Úcreate_onnxruntime_sessionro   O   s²  € ô ×-Ò-Ó/€LçÜ0;×0RÑ0R×0aÑ0aˆÕ-ä0;×0RÑ0R×0cÑ0cˆÔ-çØ(,ˆÔ%à�Q„Ø,7Ô)Ü�‰Ð<¸\×=^Ñ=^Ð<_Ð`ÔaçØ*+ˆÕ'à*+ˆÔ'à”;×6Ò6Ó8Ô8Ø�J‰	ß	Ø�uÔØ/Ð1GÐH‰IØ˜Ô#à+Ø&ð‰Ið ˜Ô 8Ò#3Ø0Ð2HÐI‰IØ˜Ô#ò‰Iô Ð!JÈ8È*ÐUÓVÐVà+Ð,ˆ	çÙfoÓpÑfoÐ^bÐ?OÔ7O�d¨TÕ2Ñ3ÐUYÒYÑfoˆ	Ðpç/Ø×-Ñ-Ð.XÐZ]Ô^à€GðgÜ×.Ò.¨ÐXaÔbˆð €Nùò qøô ô gÜ×ÑÐ8¸Ð8IÐIYÐZcÐYdÐeÕfà€Nðgús   Ä)E=Å#F Æ&F,Æ+F,c                 ó  € V '       d)   \         P                  ! R \         P                  R7       R# \         P                  ! R\         P                  R7       \         P                  ! R4      P                  \         P                  4       R# )z8[%(filename)s:%(lineno)s - %(funcName)20s()] %(message)s)ÚformatÚlevelz%(message)sÚtransformersN)ÚloggingÚbasicConfigÚDEBUGÚINFOÚ	getLoggerÚsetLevelÚWARNING)ri   s   &r   Úsetup_loggerr{   “   sP   € ßÜ×ÒØMÜ—-‘-÷	
ô
 	×Ò =¼¿¹ÕEÜ×Ò˜.Ó)×2Ñ2´7·?±?ÖCr   c                 óJ  € V '       d<   \         P                  P                  V 4      '       g   \         P                  ! V 4       V'       d<   \         P                  P                  V4      '       g   \         P                  ! V4       V'       df   VR 8X  d#   R\        P
                  ! 4       9   g   Q R4       hM<\        \        P
                  ! 4       4      P                  RR.4      '       d   Q R4       h\        P                  R\        P                   24       \        P                  R\        P                   24       \        P                  R\        P                   24       \        P                  ! \        P                  4      \        P                  ! R	4      8¼  g   Q h\        P                  ! \        P                  4      \        P                  ! R
4      8¼  g   Q h\        P                  ! \        P                  4      \        P                  ! R	4      8¼  g   Q hR# )rI   rJ   zBPlease install onnxruntime-directml package to test GPU inference.rO   rM   zSPlease install onnxruntime-gpu package, or install migraphx, to test GPU inference.zPyTorch Version:zTransformers Version:zOnnxRuntime Version:z1.10.0z4.12.0N)ÚosÚpathÚexistsÚmakedirsrT   r^   ÚsetÚ
isdisjointr>   r?   ÚtorchÚ__version__rs   r
   Úparse)Ú	cache_dirÚ
output_dirre   rf   s   &&&&r   Úprepare_environmentrˆ   ž   sw  € ßœŸ™Ÿ™¨	×2Ò2Ü
�Š�IÔçœ"Ÿ'™'Ÿ.™.¨×4Ò4Ü
�Š�JÔçØ�uÔØ)¬[×-PÒ-PÓ-RÔRð ØTóÑRô
 œ;×>Ò>Ó@ÓA×LÑLØ(Ð*EÐF÷ò ð eàdóeð ô ‡K�KÐ"¤5×#4Ñ#4Ð"5Ð6Ô7Ü
‡K�KÐ'¬×(@Ñ(@Ð'AÐBÔCÜ
‡K�KÐ&¤{×'>Ñ'>Ð&?Ð@ÔAô �=Š=œ×*Ñ*Ó+¬w¯}ª}¸XÓ/FÔFÐFÐFÜ�=Š=œ×1Ñ1Ó2´g·m²mÀHÓ6MÔMÐMÐMÜ�=Š=œ×0Ñ0Ó1´W·]²]À8Ó5LÔLÐLÒLr   c                 óÄ  € \        V 4      \        \        V 4      4      ,          R ,          p\        P                  ! V \        P
                  R7      R ,          pVR V,          ,          pR\        V 4      RVR R\        P                  ! V ^Z4      R ,          R R\        P                  ! V ^_4      R ,          R R\        P                  ! V ^c4      R ,          R RVR R	VR /# )
g     @�@)ÚdtypeÚ
test_timesÚlatency_variancez.2fÚlatency_90_percentileÚlatency_95_percentileÚlatency_99_percentileÚaverage_latency_msÚQPS)ÚsumÚfloatÚlenÚnumpyÚvarÚfloat64Ú
percentile)Úlatency_listÚ
batch_sizeÚ
latency_msrŒ   Ú
throughputs   &&   r   Úget_latency_resultr�   º   sÕ   € Ü�\Ó"¤U¬3¨|Ó+<Ó%=Õ=ÀÕF€JÜ—y’y ´U·]±]ÔCÀfÕLÐØ˜v¨
Õ2Õ3€Jð 	”c˜,Ó'ØÐ/°Ð4Ø¤E×$4Ò$4°\À2Ó$FÈÕ$OÐPSÐ#TØ¤E×$4Ò$4°\À2Ó$FÈÕ$OÐPSÐ#TØ¤E×$4Ò$4°\À2Ó$FÈÕ$OÐPSÐ#TØ ¨CÐ 0Ø�*˜SÐ!ðð r   c                 ó&  € \        VR RRR7      ;_uu_ 4       p. ROp\        P                  ! W#R7      pVP                  4        V  F  pVP	                  V4       K  	  RRR4       \
        P                  RV 24       R#   + '       g   i     L*; i)ÚaÚ Úascii©ÚmodeÚnewlineÚencoding©Ú
fieldnamesNz&Detail results are saved to csv file: )Úenginer
   rR   ÚdeviceÚ	precisionÚ	optimizerÚ
io_bindingÚ
model_nameÚinputsÚthreadsrš   Úsequence_lengthÚcustom_layer_numr   r‹   r‘   r�   rŒ   r�   rŽ   r�   )ÚopenÚcsvÚ
DictWriterÚwriteheaderÚwriterowr>   r?   )ÚresultsÚcsv_filenameÚcsv_fileÚcolumn_namesÚ
csv_writerÚresults   &&    r   Úoutput_detailsr½   Ê   sv   € Ü	ˆl ¨b¸7×	CÕ	CÀxò
ˆô0 —^’^ HÔFˆ
Ø×ÑÔ ÛˆFØ×Ñ Ö'ñ ÷7 
Dô< ‡K�KÐ8¸¸ÐGÖH÷= 
D×	Cús   ˜AB Â B	c                 ó  € \        VR RRR7      ;_uu_ 4       p. ROp. pVP                   FU  pVP                  R.8X  d   VP                  R	V 24       K+  VP                   F  pVP                  R	V R
V 24       K  	  KW  	  \        P
                  ! W4V,           R7      pVP                  4        VP                   EFˆ  p	R EF}  p
VP                   EFh  pR EF]  pVP                   EFH  p/ pV  EF  pVR,          V	8X  g   K  VR,          V
8X  g   K$  VR,          V8X  g   K4  VR,          V8X  g   KD  VR,          V8X  g   KT  VP                  4        UUu/ uF  w  ppVV9   g   K  VVbK  	  pppV'       g8   VP                  V4       VP                  \        P                  VR4      4       M V F  pVV,          VV,          8X  d   K  Q h	  VR,          pVR,          pV'       d   VR,          VR	V R
V 2&   EK  VR,          VR	V 2&   EK"  	  V'       g   EK7  VP                  V4       EKK  	  EK`  	  EKk  	  EK€  	  EK‹  	  RRR4       \        P!                  RV 24       R# u uppi   + '       g   i     L0; i)rŸ   r    r¡   r¢   r­   r®   r¨   r¬   r¯   ÚbÚ_sr¦   rš   r°   r�   Nz'Summary results are saved to csv file: )r­   r®   r±   r¨   r
   rR   r©   rª   r«   r¬   r¯   )é   é   é   )TFr    )r²   Úbatch_sizesÚsequence_lengthsÚappendr³   r´   rµ   ÚmodelsÚenginesrh   ÚitemsÚupdateÚdictÚfromkeysr¶   r>   r?   )r·   r¸   Úargsr¹   Úheader_namesÚ
data_namesrš   r°   r»   r­   Úinput_countÚengine_namer¬   r¯   Úrowr¼   ÚkÚvÚheadersr¿   Úss   &&&                  r   Úoutput_summaryr×   ì   s4  € Ü	ˆl ¨b¸7×	CÕ	CÀxò
ˆð ˆ
Ø×*Ô*ˆJØ×$Ñ$¨¨Ô,Ø×!Ñ! A j \Ð"2Ö3à'+×'<Ô'<�OØ×%Ñ%¨¨*¨°R¸Ð7HÐ&IÖJó (=ñ	 +ô —^’^ HÈ
Õ9RÔSˆ
Ø×ÑÔ ØŸ+�+ˆJÜ(�Ø#'§<¥<�KÜ&7˜
Ø'+×'7Õ'7˜GØ"$˜CÜ*1 à$*¨<Õ$8¸JÖ$FØ(.¨xÕ(8¸KÖ(GØ(.¨xÕ(8¸KÖ(GØ(.¨|Õ(<À
Ö(JØ(.¨yÕ(9¸WÖ(Dà@FÇÁÄÔ.dÁ¹¸¸1ÐRSÐWcÑRc¬t¨q°!ªtÁ GÑ.dß+.Ø(+¯
©
°7Ô(;Ø(+¯
©
´4·=±=ÀÈRÓ3PÕ(Qã1=¨AØ36°qµ6¸WÀQ½ZÖ3GÐ,GÐ3Gñ 2>à(.¨|Õ(< AØ(.Ð/@Õ(A Aß'(Ø<BÐCWÕ<X¨¨a°¨s°"°Q°C¨LÔ(9à7=Ð>RÕ7S¨¨a°¨s¨G¬ñ) +2÷*  #šsØ *× 3Ñ 3°C× 8ô1 (8ô '8ô $0ô  )ñ &÷1 
Dôl ‡K�KÐ9¸,¸ÐHÖIùó! /e÷M 
D×	CúsU   ˜C)I3ÄI3ÄI3Ä&I3Ä6I3ÅI3ÅI-Å*I-Å1AI3ÇI3Ç-.I3È!*I3É-I3É3J	c                 ól  € \        VR RRR7      ;_uu_ 4       p. RO\        \        \        V P	                  4       4      4      P                  4       4      Op\        P                  ! W#R7      pVP                  4        V  F~  p\        \        P                  ! 4       4      W,          R&   \        P                  W,          R&   \        P                  W,          R&   WPV,          R&   VP                  W,          4       K€  	  R	R	R	4       \         P#                  R
V 24       R	#   + '       g   i     L*; i)rŸ   r    r¡   r¢   Úmodel_filenamer   rs   rƒ   r¦   Nz(Fusion statistics is saved to csv file: )rÙ   r   rs   rƒ   )r²   ÚlistÚnextÚiterÚvaluesÚkeysr³   r´   rµ   Ústrr   Únowrs   r„   rƒ   r¶   r>   r?   )Úmodel_fusion_statisticsr¸   r¹   rº   r»   Úkeys   &&    r   Úoutput_fusion_statisticsrã   &  sõ   € Ü	ˆl ¨b¸7×	CÕ	CÀxò
ô
 ”$”tÐ3×:Ñ:Ó<Ó=Ó>×CÑCÓEÓFð
ˆô —^’^ HÔFˆ
Ø×ÑÔ Û*ˆCÜ7:¼8¿<º<»>Ó7JÐ#Õ(¨Ñ4Ü;G×;SÑ;SÐ#Õ(¨Ñ8Ü49×4EÑ4EÐ#Õ(¨Ñ1Ø=@ CÕ(Ð)9Ñ:Ø×ÑÐ 7Õ <Ö=ñ +÷ 
Dô  ‡K�KÐ:¸<¸.ÐIÖJ÷! 
D×	Cús   ˜C)D#Ä#D3	c                 ó  a a€ / p\         P                  ! VV 3R  l^VR7       \         P                  ! VV 3R l^VR7      pVP                  V4       VP                  RR/4       VP                  \        Wt4      4       V# )c                  ó(   <€ SP                  R S 4      # r   ©Úrun©Ú
ort_inputsÚort_sessions   €€r   Ú<lambda>Úinference_ort.<locals>.<lambda><  s   ø€ ˜+Ÿ/™/¨$°
Ô;r   ©ÚnumberÚrepeatc                  ó(   <€ SP                  R S 4      # r   ræ   rè   s   €€r   rë   rì   =  s   ø€ ¨¯©¸¸zÔ)Jr   r¬   F)Útimeitrï   rÊ   r�   )rê   ré   Úresult_templateÚrepeat_timesrš   Úwarm_up_repeatr¼   r™   s   ff&&&&  r   Úinference_ortrõ   :  sb   ù€ Ø€FÜ
‡M‚MÕ;ÀAÈnÕ]Ü—=’=Õ!JÐSTÐ]iÔj€LØ
‡M�M�/Ô"Ø
‡M�M�< Ð'Ô(Ø
‡M�MÔ$ \Ó>Ô?Ø€Mr   c           
      óš  a a€ / pS P                  4       oV FŸ  p\        P                  ! W,          4      P                  V	4      p\        P                  \        W,          P                  4      V
4      pSP                  VVP                  P                  ^ VVP                  VP                  4       4       K¡  	  \        V4      ^ 8X  d   \        WgV	4       \        V4       Fk  w  ppSP!                  VVV,          P                  P                  ^ \"        P$                  VV,          P                  VV,          P                  4       4       Km  	  \&        P(                  ! VV 3R l^VR7       \&        P(                  ! VV 3R l^VR7      pVP+                  V4       VP+                  RR/4       VP+                  \-        VV4      4       V# )r   c                  ó&   <€ SP                  S 4      # r   ©Úrun_with_iobinding©r¬   rê   s   €€r   rë   Ú/inference_ort_with_io_binding.<locals>.<lambda>q  ó   ø€ �×.Ñ.¨zÔ:r   rí   c                  ó&   <€ SP                  S 4      # r   rø   rú   s   €€r   rë   rû   w  rü   r   r¬   T)r¬   rƒ   Ú
from_numpyÚtoÚIO_BINDING_DATA_TYPE_MAPÚgetrß   rŠ   Ú
bind_inputr©   ÚtypeÚshapeÚdata_ptrr”   ÚallocateOutputBuffersÚ	enumerateÚbind_outputr•   rG   rñ   rï   rÊ   r�   )rê   ré   rò   ró   Úort_output_namesÚort_outputsÚoutput_buffersÚoutput_buffer_max_sizesrš   r©   Ú	data_typerô   r¼   rm   Únp_inputÚ
input_typeÚiÚort_output_namer™   r¬   s   f&&&&&&&&&&&       @r   Úinference_ort_with_io_bindingr  D  s|  ù€ ð €Fð ×'Ñ'Ó)€JãˆÜ×#Ò# JÕ$4Ó5×8Ñ8¸Ó@ˆÜ-×1Ñ1´#°jÕ6F×6LÑ6LÓ2MÈyÓYˆ
Ø×ÑØØ�O‰O× Ñ ØØØ�N‰NØ×ÑÓö	
ñ ô ˆ>Ó˜aÔÜ˜nÀvÔNä'Ð(8Ö9Ñˆˆ?Ø×ÑØØ˜1Õ×$Ñ$×)Ñ)ØÜ�M‰MØ˜�N× Ñ Ø˜1Õ×&Ñ&Ó(ö	
ñ :ô ‡M‚MÝ:ØØõô —=’=Ý:ØØô€Lð
 ‡M�M�/Ô"Ø
‡M�M�< Ð&Ô'Ø
‡M�MÔ$ \°:Ó>Ô?Ø€Mr   c           	      ó„   € V F9  pV P                  \        P                  ! V\        P                  VR 7      4       K;  	  R# ))rŠ   r©   N)rÆ   rƒ   ÚemptyrG   )r  r  r©   r  s   &&& r   r  r  �  s-   € ó %ˆØ×ÑœeŸkšk¨!´5·=±=ÈÔPÖQó %r   c                ó  € \         P                  ! V 4       \        P                   P                  V 4       \        P                  ! V 4       \        P
                  P	                  V 4       \        P
                  P                  V 4       R# )z5Set random seed manually to get deterministic resultsN)ÚrandomÚseedr•   rƒ   Úmanual_seedrN   Úmanual_seed_all)r  s   &r   Úset_random_seedr  ‰  sR   € ä
‡K‚K�ÔÜ	‡L�L×Ñ�dÔÜ	×Ò�dÔÜ	‡J�J×Ñ˜4Ô Ü	‡J�J×Ñ˜tÖ$r   c                ój   € V ^8„  d   QhR\         \        \        \        3,          ,          R,          /# ©rÂ   ÚreturnN©rÚ   rË   rß   r	   )rq   s   "r   Ú__annotate__r  •  s%   € ÷ #ñ #”dœ4¤¤S �>Õ*¨TÕ1ñ #r   c                  óÈ  € ^ RI Hp HpHpHpHpHpHp  V! 4        . pV! 4       p\        V\        4      '       g   R# \        V4       Fo  p	V! V! V	4      4      p
\        V
\        4      '       d    R# VP                  RV	RV! V! V	4      4      RV
P                  RV
P                  RV
P                  /4       Kq  	  V! 4        V#   T  d   p\!        RT4        Rp?R# Rp?ii ; i)	r   ©Ú	NVMLErrorÚnvmlDeviceGetCountÚnvmlDeviceGetHandleByIndexÚnvmlDeviceGetMemoryInfoÚnvmlDeviceGetNameÚnvmlInitÚnvmlShutdownNÚidrm   ÚtotalÚfreeÚusedú-Error fetching GPU information using nvml: %s)Úpy3nvml.py3nvmlr"  r#  r$  r%  r&  r'  r(  Ú
isinstanceÚintÚrangerß   rÆ   r*  r+  r,  Úprint)r"  r#  r$  r%  r&  r'  r(  r¼   Údevice_countr  r?   Úerrors               r   Úget_gpu_infor5  •  sÓ   € ÷÷ ñ ðÙŒ
ØˆÙ)Ó+ˆÜ˜,¬×,Ò,Ùä�|Ö$ˆAÙ*Ñ+EÀaÓ+HÓIˆDÜ˜$¤×$Ò$ÚØ�M‰Mà˜!ØÑ-Ñ.HÈÓ.KÓLØ˜TŸZ™ZØ˜DŸI™IØ˜DŸI™Iðöñ	 %ñ 	ŒØˆøØô ÜÐ=¸uÔEÝûðús#   ”&C ¼3C Á1AC ÃC!Ã
CÃC!c                   óL   a € ] tR tRt o RR ltR t]V 3R lR l4       tRtV t	R# )	ÚMemoryMonitori»  c                ó   € Wn         R # r   ©Úkeep_measuring)r   r:  s   &&r   r8   ÚMemoryMonitor.__init__¼  s   € Ø,Ör   c                óì   € ^ RI p^ p \        W!P                  \        P                  ! 4       4      P                  4       P                  R,          4      p\        R4       V P                  '       d   Kl   V# )r   Nç{®Gázt?é   )	ÚpsutilÚmaxÚProcessr}   ÚgetpidÚmemory_infoÚrssr   r:  )r   r?  Ú	max_usages   &  r   Úmeasure_cpu_usageÚMemoryMonitor.measure_cpu_usage¿  sW   € Ûàˆ	ØÜ˜I§~¡~´b·i²i³kÓ'B×'NÑ'NÓ'P×'TÑ'TÐW^Õ'^Ó_ˆIÜ�%ŒLØ×&×&Ò&ØØÐr   c                óT   <€ V ^8„  d   QhRS[ S[S[S[3,          ,          R,          /# r  r  )rq   r(   s   "€r   r  ÚMemoryMonitor.__annotate__Ë  s&   ø€ ÷ $ñ $¡4©©S±#¨X­Õ#7¸$Õ#>ñ $r   c                ó   € \        4       hr   )ÚNotImplementedErrorr   s   &r   Úmeasure_gpu_usageÚMemoryMonitor.measure_gpu_usageÊ  s   € ä!Ó#Ð#r   r9  N©T)
r   r   r   r    r8   rF  r   rL  r%   r&   r'   s   @r   r7  r7  »  s#   ø‡ € ô-ò	ð ÷$ó ö$r   r7  c                   óH   a a€ ] tR tRt oRV 3R lltV3R lR ltRtVtV ;t# )ÚCudaMemoryMonitoriÏ  c                ó&   <€ \         SV `  V4       R # r   )Úsuperr8   )r   r:  Ú	__class__s   &&€r   r8   ÚCudaMemoryMonitor.__init__Ð  s   ø€ Ü‰Ñ˜Ö(r   c                óT   <€ V ^8„  d   QhRS[ S[S[S[3,          ,          R,          /# r  r  )rq   r(   s   "€r   r  ÚCudaMemoryMonitor.__annotate__Ó  s&   ø€ ÷ +ñ +¡4©©S±#¨X­Õ#7¸$Õ#>ñ +r   c           
     ó4  € ^ RI HpHpHpHpHpHpHp . p. p	 V! 4        V! 4       p
\        V
\        4      '       g   \        P                  RV
 24       R# \        V
4       Uu. uF  p^ NK  	  pp\        V
4       Uu. uF  qµ! V! V4      4      NK  	  p	p \        V
4       Fg  pV! V! V4      4      p\        V\        4      '       d   \        P                  RV 24        R# \        W‹,          VP                  R
,          4      W‹&   Ki  	  \!        R4       V P"                  '       d   K–   T! 4        \        T
4       Uu. uF  pRTRY›,          RY‹,          /NK  	  up# u upi u upi u upi   T d"   p\        P                  R	T4        Rp?R# Rp?ii ; i)r   r!  z*nvmlDeviceGetCount result is not integer: Nz%nvmlDeviceGetMemoryInfo returns str: r=  Ú	device_idrm   Úmax_used_MBr-  r>  )r.  r"  r#  r$  r%  r&  r'  r(  r/  r0  r>   r4  r1  rß   r@  r,  r   r:  )r   r"  r#  r$  r%  r&  r'  r(  Úmax_gpu_usageÚgpu_namer3  r  r?   r4  s   &             r   rL  Ú#CudaMemoryMonitor.measure_gpu_usageÓ  s†  € ÷	
÷ 	
ñ 	
ð ˆØˆð	ÙŒJÙ-Ó/ˆLÜ˜l¬C×0Ò0Ü—‘ÐIÈ,ÈÐXÔYÙä(-¨lÔ(;Ó<Ñ(; 1›QÑ(;ˆMÐ<ÜRWÐXdÔReÓfÑReÈQÐ)Ñ*DÀQÓ*GÖHÑReˆHÐfØÜ˜|Ö,�AÙ2Ñ3MÈaÓ3PÓQ�DÜ! $¬×,Ò,ÜŸ™Ð'LÈTÈFÐ%SÔTÚ#Ü'*¨=Õ+;¸T¿Y¹YÈÕ=PÓ'Q�MÓ$ñ -ô �e”Ø×*×*Ò*ØÙŒNô ˜|Ô,óñ -�Að	   Ø˜H�KØ! =Õ#3óñ
 -ñð ùò =ùÚfùòøð ô 	Ü�L‰LÐHÈ%ÔPÝûð	úsU   ˜<E/ ÁE/ Á#E Á.E/ Á=E%ÂAE/ Ã#AE/ Ä+E/ Å E*ÅE/ Å E/ Å/FÅ6FÆFr   rN  )	r   r   r   r    r8   rL  r%   r&   Ú__classcell__©rS  r(   s   @@r   rP  rP  Ï  s   ù‡ € ÷)÷+÷ +ð +r   rP  c                   óB   a a€ ] tR tRt oRV 3R lltR tR tRtVtV ;t	# )ÚRocmMemoryMonitori  c                óX  <€ \         SV `  V4       R p\        P                  P	                  V4      '       d5   V\
        P                  9  d    \
        P                  P                  V4        ^ RIpW0n        V P                  P                  4        R#   \         d    RT n         R# i ; i)z/opt/rocm/libexec/rocm_smiN)
rR  r8   r}   r~   r   ÚsysrÆ   Úrocm_smiÚinitializeRsmiÚImportError)r   r:  Úrocm_smi_pathrc  rS  s   &&  €r   r8   ÚRocmMemoryMonitor.__init__  sw   ø€ Ü‰Ñ˜Ô(Ø4ˆÜ�7‰7�>‰>˜-×(Ò(Ø¤C§H¡HÔ,Ü—‘—‘ Ô.ð	!Ûà$ŒMØ�M‰M×(Ñ(Ö*øÜô 	!Ø ˆD�Mð	!ús   Á-$B ÂB)Â(B)c                ó„   € V P                   f   R# V P                   P                  VR4      ^ ,          R,          R,          # )NÚVRAMi   éÿÿÿÿ)rc  Ú
getMemInfo)r   Údevs   &&r   Úget_used_memoryÚ!RocmMemoryMonitor.get_used_memory  s7   € Ø�=‰=Ò ØˆIØ�}‰}×'Ñ'¨¨VÓ4°QÕ7¸$Õ>ÀÕEÐEr   c           
     ó0  € V P                   f   R # V P                   e$   \        V P                   P                  4       4      M^ p\        V4       Uu. uF  p^ NK  	  pp\        V4       Uu. uF  pRV 2NK
  	  pp \        V4       F&  p\	        W2,          V P                  V4      4      W2&   K(  	  \        P                  ! R4       V P                  '       d   K`   \        T4       Uu. uF  pRTRYB,          RY2,          /NK  	  up# u upi u upi u upi )NÚGPUr=  rX  rm   rY  )	rc  r”   ÚlistDevicesr1  r@  rm  Útimer   r:  )r   r3  r  rZ  r[  s   &    r   rL  Ú#RocmMemoryMonitor.measure_gpu_usage  s  € Ø�=‰=Ò Ùà;?¿=¹=Ò;T”s˜4Ÿ=™=×4Ñ4Ó6Ô7ÐZ[ˆÜ$)¨,Ô$7Ó8Ñ$7˜q›Ñ$7ˆÐ8Ü',¨\Ô':Ó;Ñ': !�c˜!˜“IÑ':ˆÐ;ØÜ˜<Ö(�Ü#& }Õ'7¸×9MÑ9MÈaÓ9PÓ#Q�Ó ñ )ä�JŠJ�uÔØ×&×&Ò&Øô ˜<Ô(ó
ñ )�ð	 ˜QØ˜�Ø˜}Õ/óñ
 )ñ
ð 	
ùò 9ùÚ;ùò
s   ÁD	Á+DÃ)D)rc  rN  )
r   r   r   r    r8   rm  rL  r%   r&   r]  r^  s   @@r   r`  r`    s   ù‡ € ÷!òF÷

ò 
r   r`  c           	      ó   € R pVR8X  d   \         pM\        pV! R4      pV '       EdN   Ve   TpMVP                  4       pVf   R # Vf   V# \        4       ;_uu_ 4       pV! 4       pVP	                  VP                  4      p VP	                  V4      p	V	P                  4       p
RVn        VP                  4       pVf    R R R 4       R # \        P                  RV RV 24       \        V4      ^8¼  dy   \        V4      ^8¼  di   \        V4      \        V4      8X  dP   ^ p\        V4       F2  w  rÞVR,          pW½,          R,          pVV,
          p\        VV4      pK4  	  VuuR R R 4       # R R R 4       R # Ve   TpMVP                  4       pVf   V# \        4       ;_uu_ 4       pV! 4       pVP	                  VP                  4      p VP	                  V4      p	V	P                  4       p
RVn        VP                  4       p\        P                  RVR RVR R	24       W¶,
          uuR R R 4       #   RTn        TP                  4       pi ; i  + '       g   i     R # ; i  RTn        TP                  4       pi ; i  + '       g   i     R # ; i)
NÚrocmFzGPU memory usage: before=z  peak=rY  zCPU memory usage: before=z.1fz
 MB, peak=z MB)r`  rP  rL  r   Úsubmitr¼   r:  r>   r?   r”   r  r@  rF  )Úis_gpuÚfuncÚmonitor_typeÚstart_memoryÚmemory_monitor_typeÚmonitorÚmemory_before_testÚexecutorÚ
mem_threadÚ	fn_threadÚ_rE  Úmax_usedr  Úmemory_beforeÚbeforeÚafterr,  s   &&&&              r   Úmeasure_memoryr†  ,  st  € ØÐØ�vÔÜ/Ñä/Ðá! %Ó(€Gç€vØÒ#Ø!-Ñà!(×!:Ñ!:Ó!<ÐØÒ%ÙàŠ<Ø%Ð%ä×!Ô! XÙ)Ó+ˆGØ!Ÿ™¨×)BÑ)BÓCˆJð0Ø$ŸO™O¨DÓ1�	Ø×$Ñ$Ó&�à).�Ô&Ø&×-Ñ-Ó/�	àÒ Ø÷ "Ñ!ô �K‰KÐ3Ð4FÐ3GÀwÈyÈkÐZÔ[ÜÐ%Ó&¨!Ô+´°I³À!Ô0CÌÐL^ÓH_ÔcfÐgpÓcqÔHqà�Ü(1Ð2DÖ(EÑ$�AØ*¨=Õ9�FØ%�L¨Õ7�EØ  6�>�DÜ" 8¨TÓ2’Hñ	 )Fð
  ÷- "×!Ò!ñ. ð ÒØ)Ñà$×6Ñ6Ó8Ðà‚|Ø!Ð!ä	×	Ô	 Ù%Ó'ˆØ—_‘_ W×%>Ñ%>Ó?ˆ
ð	,Ø Ÿ™¨Ó-ˆIØ× Ñ Ó"ˆAà%*ˆGÔ"Ø"×)Ñ)Ó+ˆIä�‰Ð/Ð0BÀ3Ð/GÀzÐR[Ð\_ÐQ`Ð`cÐdÔeØÕ-÷ 
Ò	øð7 */�Ô&Ø&×-Ñ-Ó/‘	ú÷ "Ö!ñ. ûð$ &+ˆGÔ"Ø"×)Ñ)Ó+‰Iú÷ 
×	Ð	úsO   Á#IÂ!H0Â"IÃBIÆ$#I<Ç!I Ç)<I<È0I	É	IÉI	É I9É9I<É<J	c                  ó˜   € . ROp RpV  F=  p\         P                  ! V4      pVf   K  V'       d
   VR,          pW RV 2,          pK?  	  V# )ÚORT_DISABLE_FUSED_ATTENTIONr    Ú,Ú=)rˆ  Ú!ORT_ENABLE_FUSED_CAUSAL_ATTENTIONÚ!ORT_DISABLE_FUSED_CROSS_ATTENTIONÚORT_DISABLE_TRT_FLASH_ATTENTIONÚ&ORT_DISABLE_MEMORY_EFFICIENT_ATTENTIONÚORT_TRANSFORMER_OPTIONSÚORT_CUDA_GEMM_OPTIONS)r}   Úgetenv)Ú	env_namesÚenvrm   r   s       r   Úget_ort_environment_variablesr”  p  sU   € ò€Ið €CÛˆÜ—	’	˜$“ˆØŠ=ÙßØ�3�JˆCØ��q˜˜Ð Õ Šñ ð €Jr   rj  rN  r   )r   )é{   )rN   N)2r³   rt   r}   r  rb  rr  rñ   Úabcr   r   Úconcurrent.futuresr   r   Úenumr   r   Útypingr	   r•   rƒ   rs   Ú	packagingr
   rT   rx   r   r>   r   r*   r4   rG   r   ro   r{   rˆ   r�   r½   r×   rã   rõ   Úlonglongr  r  r  r5  r7  rP  r`  r†  r”  r   r   r   Ú<module>rœ     s  ðó Û Û 	Û Û 
Û Û ß #Ý 1Ý Ý Ý Ý ã Û Û Ý ã à	×	Ò	˜8Ó	$€ô�ô ô�Dô ÷ñ ð, ˆu�}‰}ðÐ ð Ø ØØØØ-2ØôAôHDôMò8ò IòD7JòtKô(ð* �n‰nØô:òzRô%õ#ôL$�Cô $ô(/˜ô /ôd(
˜ô (
ôVA.ôHr   