Ë
    (täiqß ã                  óJ  — U d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZmZ d dlZd dlmZ d dlmc mZ ej                   j-                  «       r	d dlmc mZ ddlmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- dd	l.m/Z/ d
dl0m1Z1 erd
dl0m2Z2 dZ3dZ4dZ5dZ6dZ7dZ8 ee9«      Z: e«       xr	  e de3«      Z; e«       Z< e«       xr	  ede4«      Z= e#«       xr	  e$de5«      Z> e&de6«      Z? e%«       Z@ e'«       xr	  e(de7«      ZA e)«       xr	  e*de8«      ZBe;r	 d dlCmDZDmEZE d dlFmGZGmHZH ndZDdZEdZGdZHe<r	 d dlNmDZO d dlNmEZP ndZOdZPe=r	 d dlQmDZR ndZRe>r	 d dlSmTZTmUZUmVZVmWZWmXZXmYZY ndZTdZWdZXdZUdZVdZYe?r	 d dlZmc m[c m\Z\ ndZ\e@r	 d d!l]m^Z^ ndZ^eAr	 d d#l_m`Za ndZaeBr	 d dlbmcZd ndZdejÊ                  d&k\  r-ejÌ                  jÎ                  ZhejÌ                  jÒ                  Zjnd¢ddd'œd(„Zkd¢dd
d)œd*„ZlekZhelZj G d+„ d,eme	«      Zn G d-„ d.«      Zoe G d/„ d0«      «       Zpenjâ                   epd1d2d3d4d
¬5«      enjä                   epd1d6d
¬7«      enjæ                   epd8d2d9d:d
¬5«      enjè                   epd8d6d
¬7«      enjê                   epd;d<d
¬7«      iZvd=ewd><   ejð                  enjò                  fd£d?„«       Zz	 	 	 	 	 	 d¤dddAœ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d¥dB„Z{d¦dC„Z|d§dD„Z}d§dE„Z~d¨dF„Zd§dG„Z€d§dH„Z�	 d¢	 	 	 	 	 	 	 	 	 d©dI„Z‚dªdJ„Zƒ e�j                  dK¬L«      	 d¢	 	 	 	 	 	 	 d«dM„«       Z…	 d¢	 	 	 	 	 	 	 d¬dN„Z†	 	 d­	 	 	 	 	 	 	 	 	 	 	 d®dO„Z‡d¯dP„ZˆdQ„ Z‰d°dR„ZŠdªdS„Z‹ ehdTdUdV¬W«      	 	 	 	 	 	 	 	 	 	 	 	 d±	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d²dX„«       ZŒ ejdT«      	 	 	 	 	 	 	 	 	 	 	 	 d±	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d²dY„«       Z�	 	 	 	 	 	 	 	 d³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d´d[„ZŽ	 	 	 	 dµd\„Z�	 	 	 	 	 	 	 	 d³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d´d]„Z�	 	 	 	 dµd^„Z‘	 	 	 	 	 	 	 	 d³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d´d_„Z’	 	 	 	 dµd`„Z“	 	 	 	 	 	 	 	 d³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d´da„Z”	 	 	 	 dµdb„Z•	 	 	 	 	 	 	 	 d³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d´dc„Z–	 	 	 	 dµdd„Z—	 	 	 	 	 	 	 	 d³ded@d
ddd dfœ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d¶dg„Z˜	 	 	 	 dµdh„Z™	 	 	 	 	 	 	 	 d³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d´di„Zš	 	 	 	 	 	 	 	 d³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d´dj„Z›	 	 	 	 dµdk„Zœd¢d·dl„Z�	 	 	 	 	 	 	 	 d³	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d´dm„Zž	 	 	 	 dµdn„ZŸdo„ Z d¸dp„Z¡d¹dºdq„Z¢ G dr„ dse�jF                  �jH                  «      Z¥d»dt„Z¦d¼du„Z§d»dv„Z¨d¼dw„Z©d½dx„Zªe/	 	 	 	 	 	 d¾dy„«       Z«e/d¾dz„«       Z¬ G d{„ d|e�jF                  �jH                  «      Z­ G d}„ d~e�jF                  �jH                  «      Z® G d„ d€e�jF                  �jH                  «      Z¯	 	 	 d¿	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÀd�„Z°	 	 	 	 	 	 dÁdd‚œ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÂdƒ„Z±eo�je                  en�jf                  e}e�e‚gdZ¬„«      	 	 	 	 	 	 d¤	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÃd…„«       Z´eo�je                  enjæ                  e}e�e‚gdZ¬„«      	 	 	 	 	 	 d¤	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÃd†„«       Zµeo�je                  enjè                  e}e�e‚gd¬„«      	 	 	 	 	 	 dÄ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÅd‡„«       Z¶eo�je                  en�jn                  e}e�e‚g¬ˆ«      	 	 	 	 	 	 dÄ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÅd‰„«       Z¸eo�je                  en�jr                  e}e�e‚g¬ˆ«      	 	 	 	 	 dÆ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÇdŠ„«       Zºeo�je                  enjâ                  e}e�e‚gdZ¬„«      	 	 	 	 	 	 	 	 dÈ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÉd‹„«       Z»eo�je                  enjä                  e}e�e‚gd¬„«      	 	 	 	 	 dÆ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÇdŒ„«       Z¼eo�je                  en�jz                  e}e�e‚g¬ˆ«      	 	 	 	 	 dÆ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÇd�„«       Z¾eo�je                  en�j~                  e~e�e‚g¬ˆ«      	 	 	 	 	 	 d¤	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÃdŽ„«       ZÀeo�je                  en�j‚                  e|e}e‚g¬ˆ«      	 	 	 	 	 	 dÊ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dËd�„«       ZÂ	 dÌ	 	 	 	 	 	 	 dÍd�„ZÃeo�je                  enjò                  e}e‚gdZ¬„«      	 	 	 	 	 	 	 dÎ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÏd‘„«       ZÄeo�je                  en�jŠ                  e}e�e‚gdZ¬„«      	 	 	 	 	 	 	 dÎ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÏd’„«       ZÆeo�je                  en�jŽ                  e}e‚g¬ˆ«      	 	 	 	 	 	 	 dÎ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÏd“„«       ZÈeo�je                  en�j’                  e}e�e‚gdZ¬„«      	 	 	 	 	 	 	 dÎ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÏd”„«       ZÊeo�je                  en�j–                  e}e‚g¬ˆ«      	 	 	 	 	 	 	 dÎ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÏd•„«       ZÌeo�je                  en�jš                  e}e�e‚gdZ¬„«      	 	 	 	 	 dÐ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÑd–„«       ZÎeo�je                  en�jž                  e}e‚g¬ˆ«      	 	 	 	 dÒ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÓd—„«       ZÐeo�je                  en�j¢                  e~e�e‚gdZ¬„«      	 	 	 	 	 dÔ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÕd˜„«       ZÒeo�je                  enjê                  e~e�e‚gdZ¬„«      	 	 	 	 	 dÔ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÕd™„«       ZÓeo�je                  en�j¨                  e~e�e‚g¬ˆ«      	 	 	 	 	 dÔ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÕdš„«       ZÕeo�je                  en�j¬                   ed›d «      e‚g¬ˆ«      	 	 	 	 	 dÔ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÕdœ„«       Z×eo�je                  en�j°                   ed›d «      e‚g¬ˆ«      	 	 	 	 	 dÔ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÕd�„«       ZÙeo�je                  en�j´                   edžd «      e‚g¬ˆ«      	 	 	 	 	 dÔ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÕdŸ„«       ZÛeo�je                  en�j¸                   edžd «      e‚g¬ˆ«      	 	 	 	 	 dÔ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÕd „«       ZÝeo�je                  en�j¼                  e|e}e‚g¬ˆ«      	 	 	 	 	 	 	 dÎ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dÏd¡„«       Zßy# eIeJeKf$ r*ZLe:j›                  deL› d�«       dZ;dZDdZEdZGdZHY dZL[L�ŒdZL[Lww xY w# eIeJeKf$ r&ZLe:j›                  deL› d�«       dZ<dZOdZPY dZL[L�Œ dZL[Lww xY w# eIeJeKf$ r$ZLe:j›                  deL› d�«       dZ=dZRY dZL[L�ŒCdZL[Lww xY w# eIeJeKf$ r.ZLe:j›                  deL› d�«       dZ>dZTdZUdZVdZWdZXdZYY dZL[L�Œ\dZL[Lww xY w# eIeJeKf$ r$ZLe:j›                  d eL› d�«       dZ?dZ\Y dZL[L�ŒydZL[Lww xY w# eIeJeKf$ r$ZLe:j›                  d"eL› d�«       dZ@dZ^Y dZL[L�ŒœdZL[Lww xY w# eIeJeKf$ r$ZLe:j›                  d$eL› d�«       dZAdZaY dZL[L�Œ¿dZL[Lww xY w# eIeJeKf$ r$ZLe:j›                  d%eL› d�«       dZBdZdY dZL[L�ŒâdZL[Lww xY w)Öé    )ÚannotationsN)Ú	dataclass)ÚEnum)ÚTYPE_CHECKINGÚAnyÚCallableé   )Ú
get_loggerÚis_aiter_availableÚis_aiter_versionÚis_flash_attn_3_availableÚis_flash_attn_availableÚis_flash_attn_versionÚis_kernels_availableÚis_kernels_versionÚis_sageattention_availableÚis_sageattention_versionÚis_torch_npu_availableÚis_torch_versionÚis_torch_xla_availableÚis_torch_xla_versionÚis_xformers_availableÚis_xformers_version)ÚDIFFUSERS_ATTN_BACKENDÚDIFFUSERS_ATTN_CHECKS)Úmaybe_allow_in_graphé   )Úgather_size_by_comm)ÚParallelConfigz2.6.3z0.1.5z2.1.1ú2.5.0z2.2z0.0.29ú>=)Úflash_attn_funcÚflash_attn_varlen_func)Ú_wrapped_flash_attn_backwardÚ_wrapped_flash_attn_forwardz.flash_attn is installed but failed to import: z+. Falling back to native PyTorch attention.F)r"   )r#   zflash_attn_3 failed to import: z#. Falling back to native attention.zaiter failed to import: )ÚsageattnÚsageattn_qk_int8_pv_fp8_cudaÚ!sageattn_qk_int8_pv_fp8_cuda_sm90Úsageattn_qk_int8_pv_fp16_cudaÚsageattn_qk_int8_pv_fp16_tritonÚsageattn_varlenz sageattention failed to import: z!flex_attention failed to import: )Únpu_fusion_attentionztorch_npu failed to import: )Úflash_attentionztorch_xla failed to import: zxformers failed to import: z2.4.0)Údevice_typesÚschemac              ó   — d„ }|€|S |S )Nc                ó   — | S ©N© ©Úfuncs    úr/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/diffusers/models/attention_dispatch.pyÚwrapzcustom_op_no_op.<locals>.wrapÊ   ó   € ØˆKó    r3   )ÚnameÚfnÚmutates_argsr.   r/   r7   s         r6   Úcustom_op_no_opr=   É   ó   € ò	ð �zˆtÐ) rÐ)r9   )ÚlibÚ_stacklevelc              ó   — d„ }|€|S |S )Nc                ó   — | S r2   r3   r4   s    r6   r7   z!register_fake_no_op.<locals>.wrapÐ   r8   r9   r3   )Úopr;   r?   r@   r7   s        r6   Úregister_fake_no_oprD   Ï   r>   r9   c                  óp   — e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZy)ÚAttentionBackendNameÚflashÚ	flash_hubÚflash_varlenÚflash_varlen_hubÚ_flash_3Ú_flash_varlen_3Ú_flash_3_hubÚ_flash_3_varlen_hubÚaiterÚflexÚnativeÚ_native_cudnnÚ_native_efficientÚ_native_flashÚ_native_mathÚ_native_npuÚ_native_xlaÚsageÚsage_hubÚsage_varlenÚ_sage_qk_int8_pv_fp8_cudaÚ_sage_qk_int8_pv_fp8_cuda_sm90Ú_sage_qk_int8_pv_fp16_cudaÚ_sage_qk_int8_pv_fp16_tritonÚxformersN)Ú__name__Ú
__module__Ú__qualname__ÚFLASHÚ	FLASH_HUBÚFLASH_VARLENÚFLASH_VARLEN_HUBÚ_FLASH_3Ú_FLASH_VARLEN_3Ú_FLASH_3_HUBÚ_FLASH_3_VARLEN_HUBÚAITERÚFLEXÚNATIVEÚ_NATIVE_CUDNNÚ_NATIVE_EFFICIENTÚ_NATIVE_FLASHÚ_NATIVE_MATHÚ_NATIVE_NPUÚ_NATIVE_XLAÚSAGEÚSAGE_HUBÚSAGE_VARLENÚ_SAGE_QK_INT8_PV_FP8_CUDAÚ_SAGE_QK_INT8_PV_FP8_CUDA_SM90Ú_SAGE_QK_INT8_PV_FP16_CUDAÚ_SAGE_QK_INT8_PV_FP16_TRITONÚXFORMERSr3   r9   r6   rF   rF   à   s�   „ ð €EØ€IØ!€LØ)ÐØ€HØ'€OØ!€LØ/Ðð €Eð €DØ€FØ#€MØ+ÐØ#€MØ!€LØ€KØ€Kð €DØ€HØ€KØ ;ÐØ%EÐ"Ø!=ÐØ#AÐ ð �Hr9   rF   c                  ó¨   — e Zd Zi Zi Zi Z e«       Z ee	«      Z
eZe	 	 d	 	 	 	 	 dd„«       Zed„ «       Zed	d„«       Zed„ «       Ze	 	 	 	 d
d„«       Zy)Ú_AttentionBackendRegistryNc                óT   ‡ ‡‡‡— t         j                  d‰› d‰› �«       ˆˆ ˆˆfd„}|S )NzRegistering attention backend: z with constraints: c                ó   •— | ‰j                   ‰<   ‰xs g ‰j                  ‰<   t        t        j                  | «      j
                  j                  «       «      ‰j                  ‰<   ‰r%‰j                  j                  ‰j                  «       | S r2   )Ú	_backendsÚ_constraintsÚsetÚinspectÚ	signatureÚ
parametersÚkeysÚ_supported_arg_namesÚ_supports_context_parallelÚaddÚvalue)r5   ÚbackendÚclsÚconstraintsÚsupports_context_parallels    €€€€r6   Ú	decoratorz5_AttentionBackendRegistry.register.<locals>.decorator  su   ø€ Ø%)ˆC�M‰M˜'Ñ"Ø(3Ò(9°rˆC×Ñ˜WÑ%Ü03´G×4EÑ4EÀdÓ4K×4VÑ4V×4[Ñ4[Ó4]Ó0^ˆC×$Ñ$ WÑ-Ù(Ø×.Ñ.×2Ñ2°7·=±=ÔAàˆKr9   )ÚloggerÚdebug)rŒ   r‹   r�   rŽ   r�   s   ```` r6   Úregisterz"_AttentionBackendRegistry.register  s/   û€ ô 	�‰Ð6°w°iÐ?RÐS^ÐR_Ð`Ôa÷	ð Ðr9   c                óL   — | j                   | j                  | j                      fS r2   )Ú_active_backendr€   ©rŒ   s    r6   Úget_active_backendz,_AttentionBackendRegistry.get_active_backend&  s"   € à×"Ñ" C§M¡M°#×2EÑ2EÑ$FÐFÐFr9   c                ó   — || _         y r2   )r”   )rŒ   r‹   s     r6   Úset_active_backendz,_AttentionBackendRegistry.set_active_backend*  s
   € à%ˆÕr9   c                óH   — t        | j                  j                  «       «      S r2   )Úlistr€   r†   r•   s    r6   Úlist_backendsz'_AttentionBackendRegistry.list_backends.  s   € ä�C—M‘M×&Ñ&Ó(Ó)Ð)r9   c                ó6   — |j                   | j                  v }|S r2   )rŠ   rˆ   )rŒ   r‹   rŽ   s      r6   Ú_is_context_parallel_availablez8_AttentionBackendRegistry._is_context_parallel_available2  s   € ð
 %,§M¡M°S×5SÑ5SÐ$SÐ!Ø(Ð(r9   )NF)r‹   rF   r�   zlist[Callable] | NonerŽ   Úbool)r‹   Ústr)r‹   rF   Úreturnrž   )r`   ra   rb   r€   r�   r‡   r‚   rˆ   rF   r   r”   r   Ú_checks_enabledÚclassmethodr’   r–   r˜   r›   r�   r3   r9   r6   r}   r}   
  sÄ   „ Ø€IØ€LØÐÙ!$£ÐÙ*Ð+AÓB€OØ+€Oàð .2Ø*/ð	à%ðð +ðð $(ò	ó ðð& ñGó ðGð ò&ó ð&ð ñ*ó ð*ð ð)à%ð)ð 
ò)ó ñ)r9   r}   c                  óˆ   — e Zd ZU dZded<   ded<   dZded<   dZded	<   dZd
ed<   dZded<   dZ	ded<   dZ
d
ed<   dZd
ed<   y)Ú_HubKernelConfigzEConfiguration for downloading and using a hub-based attention kernel.rŸ   Úrepo_idÚfunction_attrNz
str | NoneÚrevisionz
int | NoneÚversionzCallable | NoneÚ	kernel_fnÚwrapped_forward_attrÚwrapped_backward_attrÚwrapped_forward_fnÚwrapped_backward_fn)r`   ra   rb   Ú__doc__Ú__annotations__r§   r¨   r©   rª   r«   r¬   r­   r3   r9   r6   r¤   r¤   ;  sZ   … áOàƒLØÓØ€HˆjÓØ€GˆZÓØ!%€IˆÓ%Ø'+Ð˜*Ó+Ø(,Ð˜:Ó,Ø*.Ð˜Ó.Ø+/Ð˜Ô/r9   r¤   zkernels-community/flash-attn3r"   z(flash_attn_interface._flash_attn_forwardz)flash_attn_interface._flash_attn_backward)r¥   r¦   rª   r«   r¨   r#   )r¥   r¦   r¨   zkernels-community/flash-attn2z0flash_attn_interface._wrapped_flash_attn_forwardz1flash_attn_interface._wrapped_flash_attn_backwardz kernels-community/sage-attentionr&   z.dict['AttentionBackendName', _HubKernelConfig]Ú_HUB_KERNELS_REGISTRYc              #  óB  K  — | t         j                  vrt        d| › d�«      ‚t        | «      } t	        | «       t        | «       t         j                  }t         j                  | «       	 d–— t         j                  |«       y# t         j                  |«       w xY w­w)z>
    Context manager to set the active attention backend.
    zBackend z is not registered.N)r}   r€   Ú
ValueErrorrF   Ú%_check_attention_backend_requirementsÚ"_maybe_download_kernel_for_backendr”   r˜   )r‹   Úold_backends     r6   Úattention_backendr¶   l  s…   è ø€ ð
 Ô/×9Ñ9Ñ9Ü˜8 G 9Ð,?Ð@ÓAÐAä" 7Ó+€GÜ)¨'Ô2Ü& wÔ/ä+×;Ñ;€KÜ×0Ñ0°Ô9ðBÛä!×4Ñ4°[ÕAøÔ!×4Ñ4°[ÕAüs   ‚A(BÁ+B Á/BÂBÂBç        )r‹   Úparallel_configc	               ój  — |xs i }|	€t         j                  «       \  }}n*t        |	«      }t         j                  j	                  |«      }| ||||||dœ|¥d|
i¥}t        dd«      r||d<   t         j                  rrt        |«      t        t         j                  |   «      z
  }|rt        j                  d|› d|› d�«       t         j                  j	                  |«      D ]
  } |d	i |¤Ž Œ |j                  «       D ��ci c]  \  }}|t         j                  |   v sŒ||“Œ  }}} |d	i |¤ŽS c c}}w )
N)ÚqueryÚkeyrŠ   Ú	attn_maskÚ	dropout_pÚ	is_causalÚscaleÚ_parallel_configr!   r    Ú
enable_gqaz5Removing unsupported arguments for attention backend z: Ú.r3   )r}   r–   rF   r€   Úgetr   r¡   r‚   r‡   r�   Úwarningr�   Úitems)rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   Úattention_kwargsr‹   r¸   Úbackend_nameÚ
backend_fnÚkwargsÚremoved_kwargsÚcheckÚkÚvs                     r6   Údispatch_attention_fnrÎ   �  sR  € ð (Ò-¨2Ðà€ô $=×#OÑ#OÓ#QÑ ˆ‘jä+¨GÓ4ˆÜ.×8Ñ8×<Ñ<¸\ÓJˆ
ð ØØØØØØñ
ð ð
ð 	˜Oñ
€Fô ˜˜gÔ&Ø)ˆˆ|Ñä ×0Ò0Ü˜V›¤sÔ+D×+YÑ+YÐZfÑ+gÓ'hÑhˆÙÜ�N‰NÐRÐS_ÐR`Ð`bÐcqÐbrÐrsÐtÔuÜ.×;Ñ;×?Ñ?ÀÖMˆEÙ‰O�F‹Oð Nð  &Ÿ|™|œ~Ôs™~‘t�q˜!°Ô6O×6dÑ6dÐeqÑ6rÒ1rˆa�‰d˜~€FÑsáÑ˜ÑÐùó ts   ÄD/ÄD/c                ó$   — | �|rt        d«      ‚y y )Nz8`is_causal` cannot be True when `attn_mask` is not None.)r²   )r¼   r¾   rÉ   s      r6   Ú_check_attn_mask_or_causalrÐ   ·  s   € ØÐ¡ÜÐSÓTÐTð "+Ðr9   c                óø   — | j                   |j                   k7  s| j                   |j                   k7  rt        d«      ‚| j                  |j                  k7  s| j                  |j                  k7  rt        d«      ‚y )Nz1Query, key, and value must be on the same device.z/Query, key, and value must have the same dtype.)Údevicer²   Údtype©rº   r»   rŠ   rÉ   s       r6   Ú_check_devicerÕ   ¼  s]   € Ø‡|�|�s—z‘zÒ! U§\¡\°U·\±\Ò%AÜÐLÓMÐMØ‡{�{�c—i‘iÒ 5§;¡;°%·+±+Ò#=ÜÐJÓKÐKð $>r9   c                óf   — t        | ||«       | j                  j                  dk7  rt        d«      ‚y )NÚcudaz/Query, key, and value must be on a CUDA device.)rÕ   rÒ   Útyper²   rÔ   s       r6   Ú_check_device_cudarÙ   Ã  s1   € Ü�%˜˜eÔ$Ø‡|�|×Ñ˜FÒ"ÜÐJÓKÐKð #r9   c                ó   ‡ ‡— dˆ ˆfd„}|S )Nc                ó    •— t        | ||«       t        j                  j                  | j                  «      ‰‰fk  rt        d‰› d‰› d�«      ‚y )NzJQuery, key, and value must be on a CUDA device with compute capability >= rÂ   )rÙ   Útorchr×   Úget_device_capabilityrÒ   r²   )rº   r»   rŠ   rÉ   ÚmajorÚminors       €€r6   Úcheck_device_cudaz:_check_device_cuda_atleast_smXY.<locals>.check_device_cudaÊ  sY   ø€ Ü˜5 # uÔ-Ü�:‰:×+Ñ+¨E¯L©LÓ9¸UÀE¸NÒJÜØ\Ð]bÐ\cÐcdÐejÐdkÐklÐmóð ð Kr9   ©rº   útorch.Tensorr»   râ   rŠ   râ   r    ÚNoner3   )rÞ   rß   rà   s   `` r6   Ú_check_device_cuda_atleast_smXYrä   É  s   ù€ öð Ðr9   c                ó”   — | j                   |j                   k7  rt        d«      ‚| j                   |j                   k7  rt        d«      ‚y )Nz'Query and key must have the same dtype.z)Query and value must have the same dtype.)rÓ   r²   rÔ   s       r6   Ú_check_qkv_dtype_matchræ   Ô  s?   € Ø‡{�{�c—i‘iÒÜÐBÓCÐCØ‡{�{�e—k‘kÒ!ÜÐDÓEÐEð "r9   c                óŒ   — t        | ||«       | j                  t        j                  t        j                  fvrt        d«      ‚y )Nz9Query, key, and value must be either bfloat16 or float16.)ræ   rÓ   rÜ   Úbfloat16Úfloat16r²   rÔ   s       r6   Ú_check_qkv_dtype_bf16_or_fp16rê   Û  s8   € Ü˜5 # uÔ-Ø‡{�{œ5Ÿ>™>¬5¯=©=Ð9Ñ9ÜÐTÓUÐUð :r9   c                ó  — | j                   d   |j                   d   k7  rt        d«      ‚|j                   d   |j                   d   k7  rt        d«      ‚|�+|j                   d   |j                   d   k7  rt        d«      ‚y y )Néÿÿÿÿz0Query and key must have the same head dimension.éýÿÿÿz1Key and value must have the same sequence length.z4Attention mask must match the key's sequence length.)Úshaper²   )rº   r»   rŠ   r¼   rÉ   s        r6   Ú_check_shaperï   á  s   € ð ‡{�{�2�˜#Ÿ)™) B™-Ò'ÜÐKÓLÐLØ
‡y�y��}˜Ÿ™ B™Ò'ÜÐLÓMÐMØÐ §¡°Ñ!4¸¿	¹	À"¹Ò!EÜÐOÓPÐPð "FÐr9   c                ó|  — | t         j                  t         j                  fv r't        s t	        d| j
                  › dt        › d�«      ‚y | t         j                  t         j                  fv r t        st	        d| j
                  › d�«      ‚y | t         j                  t         j                  t         j                  t         j                  t         j                  fv rIt        «       st	        d| j
                  › d�«      ‚t!        dd	«      st	        d| j
                  › d
�«      ‚y | t         j"                  k(  r't$        s t	        d| j
                  › dt&        › d�«      ‚y | t         j(                  t         j*                  t         j,                  t         j.                  t         j0                  t         j2                  fv r't4        s t	        d| j
                  › dt6        › d�«      ‚y | t         j8                  k(  r t:        st	        d| j
                  › d�«      ‚y | t         j<                  k(  r t>        st	        d| j
                  › d�«      ‚y | t         j@                  k(  r'tB        s t	        d| j
                  › dtD        › d�«      ‚y | t         jF                  k(  r'tH        s t	        d| j
                  › dtJ        › d�«      ‚y y )NzFlash Attention backend 'zb' is not usable because of missing package or the version is too old. Please install `flash-attn>=z`.zFlash Attention 3 backend 'zp' is not usable because of missing package or the version is too old. Please build FA3 beta release from source.z	Backend 'zl' is not usable because the `kernels` package isn't available. Please install it with `pip install kernels`.r!   z0.12zj' needs to be used with a `kernels` version of at least 0.12. Please update with `pip install -U kernels`.zAiter Attention backend 'z]' is not usable because of missing package or the version is too old. Please install `aiter>=zSage Attention backend 'ze' is not usable because of missing package or the version is too old. Please install `sageattention>=zFlex Attention backend 'zd' is not usable because of missing package or the version is too old. Please install `torch>=2.5.0`.zNPU Attention backend 'za' is not usable because of missing package or the version is too old. Please install `torch_npu`.zXLA Attention backend 'za' is not usable because of missing package or the version is too old. Please install `torch_xla>=zXformers Attention backend 'z`' is not usable because of missing package or the version is too old. Please install `xformers>=)&rF   rc   re   Ú_CAN_USE_FLASH_ATTNÚRuntimeErrorrŠ   Ú_REQUIRED_FLASH_VERSIONrg   rh   Ú_CAN_USE_FLASH_ATTN_3rd   rf   ri   rj   ru   r   r   rk   Ú_CAN_USE_AITER_ATTNÚ_REQUIRED_AITER_VERSIONrt   rv   rw   rx   ry   rz   Ú_CAN_USE_SAGE_ATTNÚ_REQUIRED_SAGE_VERSIONrl   Ú_CAN_USE_FLEX_ATTNrr   Ú_CAN_USE_NPU_ATTNrs   Ú_CAN_USE_XLA_ATTNÚ_REQUIRED_XLA_VERSIONr{   Ú_CAN_USE_XFORMERS_ATTNÚ_REQUIRED_XFORMERS_VERSION)r‹   s    r6   r³   r³   ù  sA  € ØÔ'×-Ñ-Ô/C×/PÑ/PÐQÑQÝ"ÜØ+¨G¯M©M¨?ð  ;]ô  ^uð  ]vð  vxð  yóð ð #ð
 
Ô)×2Ñ2Ô4H×4XÑ4XÐYÑ	YÝ$ÜØ-¨g¯m©m¨_ð  =mð  nóð ð %ð
 
Ü×&Ñ&Ü×-Ñ-Ü×)Ñ)Ü×0Ñ0Ü×%Ñ%ðñ 
ô $Ô%ÜØ˜GŸM™M˜?ð  +Wð  Xóð ô " $¨Ô/ÜØ˜GŸM™M˜?ð  +Uð  Vóð ð 0ð
 
Ô(×.Ñ.Ò	.Ý"ÜØ+¨G¯M©M¨?ð  ;Xô  Ypð  Xqð  qsð  tóð ð #ð
 
Ü×!Ñ!Ü×(Ñ(Ü×6Ñ6Ü×;Ñ;Ü×7Ñ7Ü×9Ñ9ðñ 
õ "ÜØ*¨7¯=©=¨/ð  :_ô  `vð  _wð  wyð  zóð ð "ð
 
Ô(×-Ñ-Ò	-Ý!ÜØ*¨7¯=©=¨/ð  :^ð  _óð ð "ð
 
Ô(×4Ñ4Ò	4Ý ÜØ)¨'¯-©-¨ð  9Zð  [óð ð !ð
 
Ô(×4Ñ4Ò	4Ý ÜØ)¨'¯-©-¨ð  9Zô  [pð  Zqð  qsð  tóð ð !ð
 
Ô(×1Ñ1Ò	1Ý%ÜØ.¨w¯}©}¨oð  >^ô  _yð  ^zð  z|ð  }óð ð &ð 
2r9   é€   )Úmaxsizec                ó<  — t        j                  | f|t         j                  |¬«      }t        j                  | f|t         j                  |¬«      }t        j                  | dz   t         j                  |¬«      }t        j                  | dz   t         j                  |¬«      }t        j                  |d¬«      |dd  t        j                  |d¬«      |dd  |j                  «       j                  «       }|j                  «       j                  «       }	||f||f||	ffS )N©rÓ   rÒ   r   r   ©Údim)rÜ   ÚfullÚint32ÚzerosÚcumsumÚmaxÚitem)
Ú
batch_sizeÚ	seq_len_qÚ
seq_len_kvrÒ   Ú	seqlens_qÚ	seqlens_kÚcu_seqlens_qÚcu_seqlens_kÚmax_seqlen_qÚmax_seqlen_ks
             r6   Ú3_prepare_for_flash_attn_or_sage_varlen_without_maskr  B  sà   € ô —
‘
˜J˜=¨)¼5¿;¹;ÈvÔV€IÜ—
‘
˜J˜=¨*¼E¿K¹KÐPVÔW€IÜ—;‘;˜z¨A™~´U·[±[ÈÔP€LÜ—;‘;˜z¨A™~´U·[±[ÈÔP€LÜ—|‘| I°1Ô5€L��ÐÜ—|‘| I°1Ô5€L��ÐØ—=‘=“?×'Ñ'Ó)€LØ—=‘=“?×'Ñ'Ó)€LØ�yÐ! L°,Ð#?À,ÐP\ÐA]Ð]Ð]r9   c                ó.  — t        j                  | f|t         j                  |¬«      }|j                  dt         j                  ¬«      }t        j                  | dz   t         j                  |¬«      }t        j                  | dz   t         j                  |¬«      }t        j
                  |d¬«      |dd  t        j
                  |d¬«      |dd  |j                  «       j                  «       }|j                  «       j                  «       }	||f||f||	ffS )Nr  r   )r  rÓ   r   r  )rÜ   r  r  Úsumr  r  r	  r
  )
r  r  r¼   rÒ   r  r  r  r  r  r  s
             r6   Ú0_prepare_for_flash_attn_or_sage_varlen_with_maskr  T  sÛ   € ô —
‘
˜J˜=¨)¼5¿;¹;ÈvÔV€IØ—‘ !¬5¯;©;�Ó7€IÜ—;‘;˜z¨A™~´U·[±[ÈÔP€LÜ—;‘;˜z¨A™~´U·[±[ÈÔP€LÜ—|‘| I°1Ô5€L��ÐÜ—|‘| I°1Ô5€L��ÐØ—=‘=“?×'Ñ'Ó)€LØ—=‘=“?×'Ñ'Ó)€LØ�yÐ! L°,Ð#?À,ÐP\ÐA]Ð]Ð]r9   c                ó>   — |€t        | |||«      S t        | |||«      S r2   )r  r  )r  r  r  r¼   rÒ   s        r6   Ú&_prepare_for_flash_attn_or_sage_varlenr  e  s0   € ð ÐÜBÀ:ÈyÐZdÐflÓmÐmÜ;¸JÈ	ÐS\Ð^dÓeÐer9   c           	     ó¶  — | j                   t        j                  k7  rt        d| j                   › d�«      ‚| j                  dk(  r#| j                  d«      j                  ||«      } �n@| j                  dk(  rG| j                  d«      d|fvrt        d| j                  d   › d|› d�«      ‚| j                  ||«      } nê| j                  d	k(  rY| j                  d«      d|fvrt        d| j                  d   › d|› d
�«      ‚| j                  d¬«      } | j                  ||«      } n‚| j                  dk(  r[| j                  d«      d|fvrt        d| j                  d   › d|› d�«      ‚| j                  |dd|«      } | j                  d¬«      } nt        d| j                  › �«      ‚| j                  ||fk7  rt        d| j                  › d|› d|› d�«      ‚| S )zÕ
    Normalize an attention mask to shape [batch_size, seq_len_k] (bool) suitable for inferring seqlens_[q|k] in
    FlashAttention/Sage varlen.

    Supports 1D to 4D shapes and common broadcasting patterns.
    z)Attention mask must be of type bool, got rÂ   r   r   r	   zattn_mask.shape[0] (z) must be 1 or z for 2D attention mask.é   z for 3D attention mask.r  é   z for 4D attention mask.rì   )r   r	   z"Unsupported attention mask shape: z.Normalized attention mask shape mismatch: got z, expected (z, Ú))
rÓ   rÜ   rž   r²   ÚndimÚ	unsqueezeÚexpandÚsizerî   Úany)r¼   r  Ú	seq_len_ks      r6   Ú_normalize_attn_maskr$  q  sþ  € ð ‡�œ%Ÿ*™*Ò$ÜÐDÀYÇ_Á_ÐDUÐUVÐWÓXÐXà‡~�~˜Òà×'Ñ'¨Ó*×1Ñ1°*¸iÓHŠ	à	�‰˜1Ò	à�>‰>˜!Ó Q¨
 OÑ3ÜØ& y§¡°qÑ'9Ð&:¸/È*ÈÐUlÐmóð ð ×$Ñ$ Z°Ó;‰	à	�‰˜1Ò	ð �>‰>˜!Ó Q¨
 OÑ3ÜØ& y§¡°qÑ'9Ð&:¸/È*ÈÐUlÐmóð ð —M‘M a�MÓ(ˆ	Ø×$Ñ$ Z°Ó;‰	à	�‰˜1Ò	à�>‰>˜!Ó Q¨
 OÑ3ÜØ& y§¡°qÑ'9Ð&:¸/È*ÈÐUlÐmóð ð ×$Ñ$ Z°°R¸ÓCˆ	Ø—M‘M f�MÓ-‰	ô Ð=¸i¿o¹oÐ=NÐOÓPÐPà‡�˜: yÐ1Ò1ÜØ<¸Y¿_¹_Ð<MÈ\ÐZdÐYeÐegÐhqÐgrÐrsÐtó
ð 	
ð Ðr9   c                ó   — ||k\  S r2   r3   )Ú	batch_idxÚhead_idxÚq_idxÚkv_idxs       r6   Ú_flex_attention_causal_mask_modr*  ¥  s   € Ø�F‰?Ðr9   c                óž   — | }|j                  d«      D ]6  }t        ||«      st        d| j                  › d|› d�«      ‚t	        ||«      }Œ8 |S )NrÂ   zKernel module 'z"' does not define attribute path 'z'.)ÚsplitÚhasattrÚAttributeErrorr`   Úgetattr)ÚmoduleÚ	attr_pathÚtargetÚattrs       r6   Ú_resolve_kernel_attrr4  ª  sZ   € Ø€FØ—‘ Ö$ˆÜ�v˜tÔ$Ü  ?°6·?±?Ð2CÐCeÐfoÐepÐprÐ!sÓtÐtÜ˜ Ó&‰ð %ð €Mr9   c                óH  — | t         vry t         |    }|j                  d u }|j                  d uxr |j                  d u }|j                  d uxr |j
                  d u }|s|s|sy 	 ddlm}  ||j                  |j                  |j                  ¬«      }|rt        ||j                  «      |_        |rt        ||j                  «      |_        |rt        ||j                  «      |_        y y # t        $ r+}t        j                  d|j                  › d|› �«       ‚ d }~ww xY w)Nr   )Ú
get_kernel)r§   r¨   z)An error occurred while fetching kernel 'z' from the Hub: )r°   r©   rª   r¬   r«   r­   Úkernelsr6  r¥   r§   r¨   r4  r¦   Ú	Exceptionr�   Úerror)r‹   ÚconfigÚneeds_kernelÚneeds_wrapped_forwardÚneeds_wrapped_backwardr6  Úkernel_moduleÚes           r6   r´   r´   ³  s!  € ØÔ+Ñ+ØÜ" 7Ñ+€Fà×#Ñ# tÐ+€LØ"×7Ñ7¸tÐCÒiÈ×HaÑHaÐeiÐHiÐØ#×9Ñ9ÀÐEÒlÈ&×JdÑJdÐhlÐJlÐáÑ1Ñ5KØðÝ&á" 6§>¡>¸F¿O¹OÐU[×UcÑUcÔdˆÙÜ3°MÀ6×CWÑCWÓXˆFÔá Ü(<¸]ÈF×LgÑLgÓ(hˆFÔ%á!Ü)=¸mÈV×MiÑMiÓ)jˆFÕ&ð "øô ò Ü�‰Ð@ÀÇÁÐ@PÐP`ÐabÐ`cÐdÔeØûðús   Á%BC- Ã-	D!Ã6&DÄD!z,_diffusers_flash_attn_3::_flash_attn_forwardr3   r×   )r<   r.   c                ó¶   — d}t        di d| “d|“d|“d|“d|“d|“d|“d	|“d
|“d|“d|	“d|
“d|“d|“d|“d|“dd“Ž}|^}}}|j                  ddd«      }||fS )N©rì   rì   ÚqrÌ   rÍ   Úsoftmax_scaleÚcausalÚqvÚ	q_descaleÚ	k_descaleÚ	v_descaleÚwindow_sizeÚattention_chunkÚsoftcapÚ
num_splitsÚpack_gqaÚdeterministicÚ	sm_marginÚreturn_attn_probsTr   r	   r   r3   )Úflash_attn_3_funcÚpermute)rB  rÌ   rÍ   rC  rD  rE  rF  rG  rH  rJ  rK  rL  rM  rN  rO  rI  ÚresultÚoutÚlseÚ_s                       r6   Ú_wrapped_flash_attn_3rW  Õ  sá   € ð& €KÜò Ù
ðá
ðñ ðñ $ð	ñ
 ðñ ðñ ðñ ðñ ðñ  ðñ (ðñ ðñ ðñ ðñ $ðñ  ð!ñ" ð#€Fð& €L€CˆˆqØ
�+‰+�a˜˜AÓ
€CØ�ˆ8€Or9   c                ó~   — d}| j                   \  }}}}|||f}t        j                  | «      | j                  |«      fS )NrA  )rî   rÜ   Ú
empty_likeÚ	new_empty)rB  rÌ   rÍ   rC  rD  rE  rF  rG  rH  rJ  rK  rL  rM  rN  rO  rI  r  Úseq_lenÚ	num_headsÚhead_dimÚ	lse_shapes                        r6   rV  rV    sH   € ð$ €Kð 01¯w©wÑ,€J�˜ HØ˜W iÐ0€IÜ×Ñ˜AÓ §¡¨IÓ 6Ð6Ð6r9   Tc           
     ó:  — |	rt        d«      ‚|
r6| j                  |||«       || _        || _        || _        || _        || _        d„ |||fD «       \  }}}t        j                  j                  j                  ||||||||¬«      }|j                  dddd«      }|S )Nz1Native attention does not support return_lse=Truec              3  óD   K  — | ]  }|j                  d ddd«      –— Œ y­w©r   r	   r   r  N©rR  ©Ú.0Úxs     r6   Ú	<genexpr>z/_native_attention_forward_op.<locals>.<genexpr>9  ó"   è ø€ ÐLÑ8K°1˜Ÿ™ 1 a¨¨A×.Ñ8Kùó   ‚ ©rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   r   r	   r   r  )r²   Úsave_for_backwardr¼   r½   r¾   r¿   rÁ   rÜ   ÚnnÚ
functionalÚscaled_dot_product_attentionrR  ©Úctxrº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   Ú
return_lseÚ	_save_ctxrÀ   rT  s                r6   Ú_native_attention_forward_oprr    s·   € ñ ÜÐLÓMÐMñ Ø×Ñ˜e S¨%Ô0Ø!ˆŒØ!ˆŒØ!ˆŒØˆŒ	Ø#ˆŒáL¸ÀÀUÑ8KÓLÑ€Eˆ3�Ü
�(‰(×
Ñ
×
:Ñ
:ØØØØØØØØð ;ó 	€Cð �+‰+�a˜˜A˜qÓ
!€Cà€Jr9   c           
     ó”  — | j                   \  }}}|j                  d«       |j                  d«       |j                  d«       d„ |||fD «       \  }}}	t        j                  j                  j                  |||	| j                  | j                  | j                  | j                  | j                  ¬«      }
|
j                  dddd«      }
|j                  dddd«      }t        j                  j                  |
|||	g|d¬	«      \  }}}|j                  dddd«      }|j                  dddd«      }|j                  dddd«      }|||fS )
NTc              3  óD   K  — | ]  }|j                  d ddd«      –— Œ y­wra  rb  rc  s     r6   rf  z0_native_attention_backward_op.<locals>.<genexpr>U  s"   è ø€ ÐRÑ>Q¸˜qŸy™y¨¨A¨q°!×4Ñ>Qùrh  ri  r   r	   r   r  F)ÚoutputsÚinputsÚgrad_outputsÚretain_graph)Úsaved_tensorsÚrequires_grad_rÜ   rk  rl  rm  r¼   r½   r¾   r¿   rÁ   rR  ÚautogradÚgrad)ro  Úgrad_outÚargsrÉ   rº   r»   rŠ   Úquery_tÚkey_tÚvalue_trT  Ú
grad_out_tÚgrad_query_tÚ
grad_key_tÚgrad_value_tÚ
grad_queryÚgrad_keyÚ
grad_values                     r6   Ú_native_attention_backward_opr‰  I  sR  € ð ×)Ñ)Ñ€Eˆ3�à	×Ñ˜ÔØ×Ñ�tÔØ	×Ñ˜ÔáR¸uÀcÈ5Ñ>QÓRÑ€GˆU�GÜ
�(‰(×
Ñ
×
:Ñ
:ØØØØ—-‘-Ø—-‘-Ø—-‘-Ø�i‰iØ—>‘>ð ;ó 	€Cð �+‰+�a˜˜A˜qÓ
!€Cà×!Ñ! ! Q¨¨1Ó-€JÜ-2¯^©^×-@Ñ-@Ø˜W e¨WÐ5ÀJÐ]bð .Aó .Ñ*€L�*˜lð ×%Ñ% a¨¨A¨qÓ1€JØ×!Ñ! ! Q¨¨1Ó-€HØ×%Ñ% a¨¨A¨qÓ1€Jà�x Ð+Ð+r9   c                óˆ  — |rt        d«      ‚d}|j                  dd«      j                  «       }|j                  dd«      j                  «       }|j                  dd«      j                  «       }||||fz  }t        j                  j
                  j                  |||||	||d|¬«	      \	  }}}}}}}}}|||||||fz  }|
r9 | j                  |Ž  || _        || _	        || _
        || _        || _        || _        |j                  dd«      j                  «       }|� |j                  dd«      j                  «       }|	r||fS |S )Nz6`enable_gqa` is not yet supported for cuDNN attention.r3   r   r	   F)	rº   r»   rŠ   Ú	attn_biasÚcompute_log_sumexpr½   r¾   Úreturn_debug_maskr¿   )r²   Ú	transposeÚ
contiguousrÜ   ÚopsÚatenÚ#_scaled_dot_product_cudnn_attentionrj  r½   r¾   r¿   r¼   Úmax_qÚmax_k©ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rq  rÀ   Útensors_to_saverT  rU  Ú	cum_seq_qÚ	cum_seq_kr“  r”  Úphilox_seedÚphilox_offsetÚdebug_attn_masks                         r6   Ú_cudnn_attention_forward_oprœ  q  sd  € ñ ÜÐQÓRÐRà€Oð �O‰O˜A˜qÓ!×,Ñ,Ó.€EØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ�O‰O˜A˜qÓ!×,Ñ,Ó.€EØ˜˜s EÐ*Ñ*€Oô 	�	‰	�‰×:Ñ:ØØØØØ)ØØØ#Øð 	;ó 
	
ñ ^€Cˆˆi˜ E¨5°+¸}Èoð ˜˜S )¨Y¸À]ÐSÑS€OÙØˆ×Ñ˜Ñ/Ø!ˆŒØ!ˆŒØˆŒ	Ø!ˆŒØˆŒ	ØˆŒ	à
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ
€Ø�m‰m˜A˜qÓ!×,Ñ,Ó.ˆÙ#ˆC�ˆ:Ð,¨Ð,r9   c                ó
  — | j                   \	  }}}}}}	}
}}|j                  dd«      j                  «       }|j                  dd«      j                  «       }|j                  dd«      j                  «       }t        j                  j
                  j                  ||||||||| j                  |	|
| j                  | j                  | j                  | j                  | j                  ¬«      \  }}}d„ |||fD «       \  }}}|||fS )Nr   r	   )Ú	logsumexpr™  rš  r‹  r—  r˜  r“  r”  r½   r¾   r¿   c              3  ó\   K  — | ]$  }|j                  d d«      j                  «       –— Œ& y­w©r   r	   N©rŽ  r�  rc  s     r6   rf  z/_cudnn_attention_backward_op.<locals>.<genexpr>Ê  ó'   è ø€ Ð'sÑPrÈ1¨¯©°A°qÓ(9×(DÑ(D×(FÑPrùó   ‚*,)ry  rŽ  r�  rÜ   r�  r‘  Ú,_scaled_dot_product_cudnn_attention_backwardr¼   r“  r”  r½   r¾   r¿   ©ro  r}  r~  rÉ   rº   r»   rŠ   rT  rU  r—  r˜  r™  rš  r†  r‡  rˆ  s                   r6   Ú_cudnn_attention_backward_opr¦  «  s  € ð UX×TeÑTeÑQ€Eˆ3��s˜C ¨I°{ÀMà×!Ñ! ! QÓ'×2Ñ2Ó4€HØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ�O‰O˜A˜qÓ!×,Ñ,Ó.€Eô (-§y¡y§~¡~×'bÑ'bØØØØØØØØ#Ø—-‘-ØØØ�i‰iØ�i‰iØ—-‘-Ø—-‘-Ø�i‰ið! (có (Ñ$€J�˜*ñ$ (tÐQ[Ð]eÐgqÑPrÓ'sÑ$€J�˜*à�x Ð+Ð+r9   c           	     óv  — |rt        d«      ‚d}|j                  dd«      j                  «       }|j                  dd«      j                  «       }|j                  dd«      j                  «       }||||fz  }t        j                  j
                  j                  |||||d|¬«      \	  }}}}}}}}}|||||||fz  }|
r2 | j                  |Ž  || _        || _	        || _
        || _        || _        |j                  dd«      j                  «       }|� |j                  dd«      j                  «       }|	r||fS |S )Nz=`enable_gqa` is not yet supported for native flash attention.r3   r   r	   F)rº   r»   rŠ   r½   r¾   r�  r¿   )r²   rŽ  r�  rÜ   r�  r‘  Ú#_scaled_dot_product_flash_attentionrj  r½   r¾   r¿   r“  r”  r•  s                         r6   Ú"_native_flash_attention_forward_opr©  Ò  sU  € ñ ÜÐXÓYÐYà€Oà�O‰O˜A˜qÓ!×,Ñ,Ó.€EØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ�O‰O˜A˜qÓ!×,Ñ,Ó.€EØ˜˜s EÐ*Ñ*€Oô 	�	‰	�‰×:Ñ:ØØØØØØ#Øð 	;ó 	
ñ ^€Cˆˆi˜ E¨5°+¸}Èoð ˜˜S )¨Y¸À]ÐSÑS€OÙØˆ×Ñ˜Ñ/Ø!ˆŒØ!ˆŒØˆŒ	ØˆŒ	ØˆŒ	à
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ
€Ø�m‰m˜A˜qÓ!×,Ñ,Ó.ˆÙ#ˆC�ˆ:Ð,¨Ð,r9   c                óô  — | j                   \	  }}}}}}	}
}}|j                  dd«      j                  «       }|j                  dd«      j                  «       }|j                  dd«      j                  «       }t        j                  j
                  j                  |||||||||	|
| j                  | j                  | j                  | j                  | j                  ¬«      \  }}}d„ |||fD «       \  }}}|||fS )Nr   r	   )
rž  r™  rš  r—  r˜  r“  r”  r½   r¾   r¿   c              3  ó\   K  — | ]$  }|j                  d d«      j                  «       –— Œ& y­wr   r¡  rc  s     r6   rf  z6_native_flash_attention_backward_op.<locals>.<genexpr>%  r¢  r£  )ry  rŽ  r�  rÜ   r�  r‘  Ú,_scaled_dot_product_flash_attention_backwardr“  r”  r½   r¾   r¿   r¥  s                   r6   Ú#_native_flash_attention_backward_opr­    s  € ð UX×TeÑTeÑQ€Eˆ3��s˜C ¨I°{ÀMà×!Ñ! ! QÓ'×2Ñ2Ó4€HØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ�O‰O˜A˜qÓ!×,Ñ,Ó.€Eä',§y¡y§~¡~×'bÑ'bØØØØØØØØ#ØØØ�i‰iØ�i‰iØ—-‘-Ø—-‘-Ø�i‰ið (có (Ñ$€J�˜*ñ" (tÐQ[Ð]eÐgqÑPrÓ'sÑ$€J�˜*à�x Ð+Ð+r9   c                ó@  — |�t        d«      ‚|rt        d«      ‚d}d}d }d}t        d„ |||fD «       «      }|€|j                  d   dz  }|s|�"|j                  j                  d	kD  r	|d
kD  r|nd}t        j                  |«      5  t        |||||||d
   |d	   |||	«      \  }}}}|j                  d
dd	«      }d d d «       |
rG| j                  |||«       || _
        || _        || _        || _        || _        || _        || _        |	rfS S # 1 sw Y   ŒZxY w)Nz2`attn_mask` is not yet supported for flash-attn 2.z3`enable_gqa` is not yet supported for flash-attn 2.rA  r·   Fc              3  ó4   K  — | ]  }|j                   –— Œ y ­wr2   ©Úrequires_gradrc  s     r6   rf  z._flash_attention_forward_op.<locals>.<genexpr>C  ó   è ø€ ÐDÑ0C¨1�q—•Ñ0Cùó   ‚rì   ç      à¿r   r   ç ÂëþKH´9r	   )r²   r"  rî   Úcontext_parallel_configÚ_world_sizerÜ   Úset_grad_enabledr%   rR  rj  r½   r¿   r¾   rI  rK  Úalibi_slopesrN  )ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rq  rÀ   rI  rK  r¹  rN  Úgrad_enabledrT  rU  ÚS_dmaskÚ	rng_states                        r6   Ú_flash_attention_forward_opr½  +  sc  € ð ÐÜÐMÓNÐNÙÜÐNÓOÐOð €KØ€GØ€LØ€MÜÑD°¸¸UÑ0CÓDÓD€Là€}Ø—‘˜B‘ DÑ)ˆñ Ð(Ð4Ð9I×9aÑ9a×9mÑ9mÐpqÒ9qØ!*¨Q¢‘I°Eˆ	ä	×	Ñ	 Õ	-Ü'BØØØØØØØ˜‰NØ˜‰NØØØó(
Ñ$ˆˆS�'˜9ð �k‰k˜!˜Q Ó"ˆ÷ 
.ñ  Ø×Ñ˜e S¨%°°c¸9ÔEØ!ˆŒØˆŒ	Ø!ˆŒØ%ˆŒØˆŒØ'ˆÔØ)ˆÔá#ˆC�ˆ:Ð,¨Ð,÷5 
.Ð	-ús   Â4DÄDc                ó  — | j                   \  }}}}}}	t        j                  |«      t        j                  |«      t        j                  |«      }}}
t        |||||||
||| j                  | j
                  | j                  | j                  d   | j                  d   | j                  | j                  | j                  |	«      }|
dd |j                  d   …f   }
|dd |j                  d   …f   }|dd |j                  d   …f   }|
||fS )Nr   r   .rì   )ry  rÜ   rY  r$   r½   r¿   r¾   rI  rK  r¹  rN  rî   )ro  r}  r~  rÉ   rº   r»   rŠ   rT  rU  r¼  r†  r‡  rˆ  Úlse_ds                 r6   Ú_flash_attention_backward_oprÀ  i  s$  € ð .1×->Ñ->Ñ*€Eˆ3��s˜C Ü',×'7Ñ'7¸Ó'>Ä×@PÑ@PÐQTÓ@UÔW\×WgÑWgÐhmÓWn˜*�€Jä(ØØØØØØØØØØ�‰Ø�	‰	Ø�‰Ø�‰˜ÑØ�‰˜ÑØ�‰Ø×ÑØ×ÑØó%€Eð, ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€JØ˜Ð1˜xŸ~™~¨bÑ1Ð1Ð1Ñ2€HØ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€Jà�x Ð+Ð+r9   c                óÒ  — |�t        d«      ‚|rt        d«      ‚t        t        j                     }|j                  }|j
                  }|�|€t        d«      ‚|€|j                  d   dz  }d}d}d }d}t        d	„ |||fD «       «      }|s|�"|j                  j                  d
kD  r	|dkD  r|nd}t        j                  |«      5   ||||||||d   |d
   |||	«      \  }}}}|j                  ddd
«      j                  «       }d d d «       |
rG| j                  |||«       || _        || _        || _        || _        || _        || _        || _        |	rfS S # 1 sw Y   ŒZxY w)Nz<`attn_mask` is not yet supported for flash-attn hub kernels.z=`enable_gqa` is not yet supported for flash-attn hub kernels.zˆFlash attention hub kernels must expose `_wrapped_flash_attn_forward` and `_wrapped_flash_attn_backward` for context parallel execution.rì   r´  rA  r·   Fc              3  ó4   K  — | ]  }|j                   –— Œ y ­wr2   r°  rc  s     r6   rf  z2_flash_attention_hub_forward_op.<locals>.<genexpr>²  r²  r³  r   r   rµ  r	   )r²   r°   rF   rd   r¬   r­   rò   rî   r"  r¶  r·  rÜ   r¸  rR  r�  rj  r½   r¿   r¾   rI  rK  r¹  rN  )ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rq  rÀ   r:  r¬   r­   rI  rK  r¹  rN  rº  rT  rU  r»  r¼  s                           r6   Ú_flash_attention_hub_forward_oprÃ  �  s±  € ð ÐÜÐWÓXÐXÙÜÐXÓYÐYä"Ô#7×#AÑ#AÑB€FØ×2Ñ2ÐØ ×4Ñ4ÐØÐ!Ð%8Ð%@Üð.ó
ð 	
ð
 €}Ø—‘˜B‘ DÑ)ˆà€KØ€GØ€LØ€MÜÑD°¸¸UÑ0CÓDÓD€LáÐ(Ð4Ð9I×9aÑ9a×9mÑ9mÐpqÒ9qØ!*¨Q¢‘I°Eˆ	ä	×	Ñ	 Õ	-Ù'9ØØØØØØØ˜‰NØ˜‰NØØØó(
Ñ$ˆˆS�'˜9ð �k‰k˜!˜Q Ó"×-Ñ-Ó/ˆ÷ 
.ñ  Ø×Ñ˜e S¨%°°c¸9ÔEØ!ˆŒØˆŒ	Ø!ˆŒØ%ˆŒØˆŒØ'ˆÔØ)ˆÔá#ˆC�ˆ:Ð,¨Ð,÷5 
.Ð	-ús   Ã?EÅE&c                ór  — t         t        j                     }|j                  }|€t	        d«      ‚| j
                  \  }}}}	}
}t        j                  |«      t        j                  |«      t        j                  |«      }}} ||||||	|
|||| j                  | j                  | j                  | j                  d   | j                  d   | j                  | j                  | j                  |«      }|dd |j                  d   …f   }|dd |j                  d   …f   }|dd |j                  d   …f   }|||fS )NzfFlash attention hub kernels must expose `_wrapped_flash_attn_backward` for context parallel execution.r   r   .rì   )r°   rF   rd   r­   rò   ry  rÜ   rY  r½   r¿   r¾   rI  rK  r¹  rN  rî   )ro  r}  r~  rÉ   r:  r­   rº   r»   rŠ   rT  rU  r¼  r†  r‡  rˆ  rV  s                   r6   Ú _flash_attention_hub_backward_oprÅ  Ô  sY  € ô #Ô#7×#AÑ#AÑB€FØ ×4Ñ4ÐØÐ"ÜØtó
ð 	
ð .1×->Ñ->Ñ*€Eˆ3��s˜C Ü',×'7Ñ'7¸Ó'>Ä×@PÑ@PÐQTÓ@UÔW\×WgÑWgÐhmÓWn˜*�€JáØØØØØØØØØØ�‰Ø�	‰	Ø�‰Ø�‰˜ÑØ�‰˜ÑØ�‰Ø×ÑØ×ÑØó%	€Að* ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€JØ˜Ð1˜xŸ~™~¨bÑ1Ð1Ð1Ñ2€HØ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€Jà�x Ð+Ð+r9   rA  ©rI  rK  rL  rM  rN  rO  c               ó  — |�t        d«      ‚|dk7  rt        d«      ‚|rt        d«      ‚t        t        j                     }|j                  }|€t        d«      ‚|€|j                  d   dz  } ||||d d d d d d d d d d d d d d d d d d d d |f||d   |d	   d||||d
œŽ^}}}|	r!|j                  ddd	«      j                  «       nd }|
r?| j                  |||||«       || _
        || _        || _        || _        || _        || _        |	r||fS |S )Nz>`attn_mask` is not yet supported for flash-attn 3 hub kernels.r·   z>`dropout_p` is not yet supported for flash-attn 3 hub kernels.z?`enable_gqa` is not yet supported for flash-attn 3 hub kernels.ztFlash attention 3 hub kernels must expose `flash_attn_interface._flash_attn_forward` for context parallel execution.rì   r´  r   r   )rD  Úwindow_size_leftÚwindow_size_rightrJ  rK  rL  rM  rO  r	   )r²   r°   rF   ri   r¬   rò   rî   rR  r�  rj  r¿   r¾   rI  rK  rN  rO  )ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rq  rÀ   rI  rK  rL  rM  rN  rO  r:  r¬   rT  Úsoftmax_lserV  rU  s                           r6   Ú!_flash_attention_3_hub_forward_oprË     s‚  € ð* ÐÜÐYÓZÐZØ�CÒÜÐYÓZÐZÙÜÐZÓ[Ð[ä"Ô#7×#DÑ#DÑE€FØ×2Ñ2ÐØÐ!Üð.ó
ð 	
ð
 €}Ø—‘˜B‘ DÑ)ˆá-ØØØØØØØØØØØØØØØØØØØØØØØØð1!ð2 Ø$ Q™Ø% a™.ØØØØØòA!Ð€Cˆ�qñF 8Bˆ+×
Ñ
˜a  AÓ
&×
1Ñ
1Ô
3Àt€CáØ×Ñ˜e S¨%°°kÔBØˆŒ	Ø!ˆŒØ%ˆŒØˆŒØ)ˆÔØ!ˆŒá#ˆC�ˆ:Ð,¨Ð,r9   c                ód  — t         t        j                     }|j                  }|€t	        d«      ‚| j
                  \  }}}}	}
t        j                  |«      }t        j                  |«      }t        j                  |«      } ||||||	|
d d d d d d |||| j                  | j                  | j                  d   | j                  d   | j                  | j                  | j                  «       |dd |j                  d   …f   }|dd |j                  d   …f   }|dd |j                  d   …f   }|||fS )NzuFlash attention 3 hub kernels must expose `flash_attn_interface._flash_attn_backward` for context parallel execution.r   r   .rì   )r°   rF   ri   r­   rò   ry  rÜ   rY  r¿   r¾   rI  rK  rN  rO  rî   )ro  r}  r~  rÉ   r:  r­   rº   r»   rŠ   rT  rÊ  r†  r‡  rˆ  s                 r6   Ú"_flash_attention_3_hub_backward_oprÍ  X  s\  € ô #Ô#7×#DÑ#DÑE€FØ ×4Ñ4ÐØÐ"Üð.ó
ð 	
ð
 +.×*;Ñ*;Ñ'€Eˆ3��s˜KÜ×!Ñ! %Ó(€JÜ×Ñ Ó$€HÜ×!Ñ! %Ó(€JáØØØØØØØØØØØØØØØØ�	‰	Ø�‰Ø�‰˜ÑØ�‰˜ÑØ�‰Ø×ÑØ�‰ô-ð2 ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€JØ˜Ð1˜xŸ~™~¨bÑ1Ð1Ð1Ñ2€HØ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€Jà�x Ð+Ð+r9   c           	     óÂ   — |�t        d«      ‚|dkD  rt        d«      ‚|rt        d«      ‚t        |||d|||	¬«      }d }|	r|^}}}|j                  ddd	«      }|	r||fS |S ©
Nz4`attn_mask` is not yet supported for Sage attention.r·   z4`dropout_p` is not yet supported for Sage attention.z5`enable_gqa` is not yet supported for Sage attention.ÚNHD©rB  rÌ   rÍ   Útensor_layoutr¾   Úsm_scalerp  r   r	   r   )r²   r&   rR  )ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rq  rÀ   rT  rU  rV  s                  r6   Ú_sage_attention_forward_oprÔ  ‹  s�   € ð ÐÜÐOÓPÐPØ�3‚ÜÐOÓPÐPÙÜÐPÓQÐQä
Ø
Ø
Ø
ØØØØô€Cð €CÙØˆˆˆS�1Ø�k‰k˜!˜Q Ó"ˆá#ˆC�ˆ:Ð,¨Ð,r9   c           	     ó  — |�t        d«      ‚|dkD  rt        d«      ‚|rt        d«      ‚t        t        j                     j                  } ||||d|||	¬«      }d }|	r&|^}}}|j                  ddd	«      j                  «       }|	r||fS |S rÏ  )r²   r°   rF   ru   r©   rR  r�  )ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rq  rÀ   r5   rT  rU  rV  s                   r6   Ú_sage_attention_hub_forward_oprÖ  ±  s¯   € ð ÐÜÐOÓPÐPØ�3‚ÜÐOÓPÐPÙÜÐPÓQÐQä Ô!5×!>Ñ!>Ñ?×IÑI€DÙ
Ø
Ø
Ø
ØØØØô€Cð €CÙØˆˆˆS�1Ø�k‰k˜!˜Q Ó"×-Ñ-Ó/ˆá#ˆC�ˆ:Ð,¨Ð,r9   c                ó   — t        d«      ‚)Nz4Backward pass is not implemented for Sage attention.©ÚNotImplementedError©ro  r}  r~  s      r6   Ú_sage_attention_backward_oprÛ  Ù  s   € ô
 ÐTÓ
UÐUr9   c                óô  — |�t        j                  |dk7  «      rd }|�Ù|j                  dk(  rÊ|j                  d   | j                  d   k(  r«|j                  d   |j                  d   k(  rŒ|j                  d   | j                  d   |j                  d   }}}|j	                  t         j
                  «       }|j                  d«      j                  |||«      j                  d«      j                  «       }|S )Nr   r	   r   )	rÜ   Úallr  rî   Útorž   r  r   r�  )rº   r»   r¼   ÚBÚSqÚSkvs         r6   Ú_maybe_modify_attn_mask_npurâ  á  sÖ   € àÐ¤§¡¨9¸©>Ô!:Øˆ	ð
 	ÐØ�N‰N˜aÒØ�O‰O˜AÑ %§+¡+¨a¡.Ò0Ø�O‰O˜AÑ #§)¡)¨A¡,Ò.à—_‘_ QÑ'¨¯©°Q©¸¿¹À1¹ˆsˆ2ˆØ—\‘\¤%§*¡*Ó-Ð-ˆ	Ø×'Ñ'¨Ó*×1Ñ1°!°R¸Ó=×GÑGÈÓJ×UÑUÓWˆ	àÐr9   c                óâ   — |	rt        d«      ‚t        |||«      }t        ||||j                  d«      |dd |€%dt	        j
                  |j                  d   «      z  n|ddd|z
  dd¬	«      d   }|S )
NúANPU attention backend does not support setting `return_lse=True`.r	   ÚBSNDç      ð?rì   é   Fr   ©	Ú
atten_maskÚinput_layoutÚpser¿   Úpre_tockensÚnext_tockensÚ	keep_probÚsyncÚinner_precise)r²   râ  r,   r!  ÚmathÚsqrtrî   rn  s                r6   Ú_npu_attention_forward_opró  õ  s‹   € ñ ÜÐ\Ó]Ð]ä+¨E°3¸	ÓB€Iä
ØØØØ�
‰
�1‹ØØØØ27°-ˆc”D—I‘I˜eŸk™k¨"™oÓ.Ò.ÀUØØØ˜	‘/ØØôð ñ	€Cð  €Jr9   c                ó   — t        d«      ‚)Nz:Backward pass is not implemented for Npu Fusion Attention.rØ  )ro  r}  r~  rÉ   s       r6   Ú_npu_attention_backward_oprõ    s   € ô ÐZÓ
[Ð[r9   c                óZ   — t        | t        j                  «      r| j                  «       } | S r2   )Ú
isinstanceÚfuncolÚAsyncCollectiveTensorÚwait)Útensors    r6   Ú_wait_tensorrü  ,  s"   € Ü�&œ&×6Ñ6Ô7Ø—‘“ˆØ€Mr9   c                ó¦   — | j                   }| j                  «       } t        j                  | d d |«      } | j	                  |«      } t        | «      } | S r2   )rî   Úflattenrø  Úall_to_all_singleÚreshaperü  )re  Úgrouprî   s      r6   Ú_all_to_all_singler  2  sJ   € Ø�G‰G€Eð
 	
�	‰	‹€AÜ× Ñ   D¨$°Ó6€AØ	�	‰	�%Ó€AÜ�Q‹€AØ€Hr9   c                ó<  — t         j                  j                  |«      }|dk(  r²|dk(  r­| j                  \  }}}}||z  }	||z  }
| j	                  ||||
|«      j                  dd«      j                  «       }|dkD  rt        ||¬«      }n|}|j	                  |	||
|«      j                  dddd«      j                  «       }|j	                  ||	|
|«      }|S |dk(  r·|dk(  r²| j                  \  }}	}
}|
|z  }|	|z  }| j	                  ||||
|«      j                  ddddd«      j	                  ||
|||«      }|dkD  rt        ||«      }n|}|j	                  ||||«      j                  dd«      j                  «       }|j	                  ||||«      }|S t        d«      ‚)ua  
    Perform dimension sharding / reassembly across processes using _all_to_all_single.

    This utility reshapes and redistributes tensor `x` across the given process group, across sequence dimension or
    head dimension flexibly by accepting scatter_idx and gather_idx.

    Args:
        x (torch.Tensor):
            Input tensor. Expected shapes:
            - When scatter_idx=2, gather_idx=1: (batch_size, seq_len_local, num_heads, head_dim)
            - When scatter_idx=1, gather_idx=2: (batch_size, seq_len, num_heads_local, head_dim)
        scatter_idx (int) :
            Dimension along which the tensor is partitioned before all-to-all.
        gather_idx (int):
            Dimension along which the output is reassembled after all-to-all.
        group :
            Distributed process group for the Ulysses group.

    Returns:
        torch.Tensor: Tensor with globally exchanged dimensions.
            - For (scatter_idx=2 â†’ gather_idx=1): (batch_size, seq_len, num_heads_local, head_dim)
            - For (scatter_idx=1 â†’ gather_idx=2): (batch_size, seq_len_local, num_heads, head_dim)
    r	   r   r   ©r  r  r  z<Invalid scatter/gather indices for _all_to_all_dim_exchange.)
rÜ   ÚdistributedÚget_world_sizerî   r   rŽ  r�  r  rR  r²   )re  Úscatter_idxÚ
gather_idxr  Úgroup_world_sizer  Úseq_len_localr\  r]  r[  Únum_heads_localÚx_temprT  Úoutputs                 r6   Ú_all_to_all_dim_exchanger  ?  sÏ  € ô0 ×(Ñ(×7Ñ7¸Ó>Ðà�aÒ˜J¨!šOð :;¿¹Ñ6ˆ
�M 9¨hØÐ"2Ñ2ˆØ#Ð'7Ñ7ˆð �I‰I�j -Ð1AÀ?ÐT\Ó]ß‰Y�q˜!‹_ß‰Z‹\ð 	ð ˜aÒÜ$ V°5Ô9‰CàˆCà�k‰k˜' :¨ÀÓI×QÑQÐRSÐUVÐXYÐ[\Ó]×hÑhÓjˆØ�k‰k˜* g¨ÀÓIˆØˆ
Ø	˜Ò	˜j¨Ašoð :;¿¹Ñ6ˆ
�G˜_¨hØ#Ð&6Ñ6ˆ	ØÐ#3Ñ3ˆð �I‰I�jÐ"2°MÀ?ÐT\Ó]ß‰W�Q˜˜1˜a Ó#ß‰WÐ% ¸ÀzÐS[Ó\ð 	ð ˜aÒÜ'¨°Ó6‰FàˆFØ—‘ 	¨=¸*ÀhÓO×YÑYÐZ[Ð]^Ó_×jÑjÓlˆØ—‘ 
¨M¸9ÀhÓOˆØˆäÐWÓXÐXr9   c                  ó2   — e Zd ZdZedd„«       Zed„ «       Zy)ÚSeqAllToAllDimz�
    all_to_all operation for unified sequence parallelism. uses _all_to_all_dim_exchange, see _all_to_all_dim_exchange
    for more info.
    c                óH   — || _         || _        || _        t        ||||«      S r2   )r  Ú
scatter_idÚ	gather_idr  )ro  r  Úinputr  r  s        r6   ÚforwardzSeqAllToAllDim.forwardŽ  s(   € àˆŒ	Ø#ˆŒØ!ˆŒÜ'¨¨z¸9ÀeÓLÐLr9   c                óz   — t         j                  | j                  || j                  | j                  «      }d |d d fS r2   )r  Úapplyr  r  r  )ro  rw  Ú
grad_inputs      r6   ÚbackwardzSeqAllToAllDim.backward•  s;   € ä#×)Ñ)Ø�I‰IØØ�M‰MØ�N‰Nó	
ˆ
ð �j $¨Ð-Ð-r9   N)r	   r   )r`   ra   rb   r®   Ústaticmethodr  r  r3   r9   r6   r  r  ˆ  s1   „ ñð
 òMó ðMð ñ.ó ñ.r9   r  c                óÞ   — t        j                  |¬«      }d}||z  dk7  rJ|||z  z
  }||z   |z  }||k  sJ d|› d|› �«       ‚t        j                  | ddd|f«      j	                  «       } | |fS )zèMaybe pad the head dimension to be divisible by world_size.
    x: torch.Tensor, shape (B, S_LOCAL, H, D) H: int, original global head num return: tuple[torch.Tensor, int], padded
    tensor (B, S_LOCAL, H + H_PAD, D) and H_PAD
    r  r   úPadding head num ú( should be less than new local head num )Údistr  ÚFÚpadr�  )re  ÚHr  Ú
world_sizeÚH_PADÚNEW_H_LOCALs         r6   Ú_maybe_pad_qkv_headr%  ¡  s’   € ô
 ×$Ñ$¨5Ô1€JØ€EØˆ:�~˜ÒØ˜a *™nÑ-ˆØ˜5‘y ZÑ/ˆð �{Ò"ÐtÐ&7¸°wÐ>fÐgrÐfsÐ$tÓtÐ"Ü�E‰E�!�a˜˜A˜uÐ%Ó&×1Ñ1Ó3ˆØˆeˆ8€Or9   c                ó¸   — t        j                  |¬«      }t        j                  |¬«      }|dkD  r||dz
  k(  r| dd…dd…d| …dd…f   } | j                  «       S )zÂMaybe unpad the head dimension.
    x: torch.Tensor, shape (B, S_GLOBAL, H_LOCAL + H_PAD, D) H_PAD: int, head padding num return: torch.Tensor,
    unpadded tensor (B, S_GLOBAL, H_LOCAL, D)
    r  r   r   N)r  Úget_rankr  r�  )re  r#  r  Úrankr"  s        r6   Ú_maybe_unpad_qkv_headr)  ²  sY   € ô
 �=‰=˜uÔ%€DÜ×$Ñ$¨5Ô1€Jàˆq‚y�T˜Z¨!™^Ò+ØŠa’�G�e�V�GšQÐÑˆØ�<‰<‹>Ðr9   c                ó&  — |€| dfS t        j                  |¬«      }t        j                  |¬«      }d}||z  dk7  rR|||z  z
  }||z   |z  }||k  sJ d|› d|› �«       ‚||dz
  k(  r(t        j                  | ddd|f«      j                  «       } | |fS )zöMaybe pad the head dimension to be divisible by world_size.
    x: torch.Tensor, shape (B, S_GLOBAL, H_LOCAL, D) H: int, original global head num return: tuple[torch.Tensor, int],
    padded tensor (B, S_GLOBAL, H_LOCAL + H_PAD, D) and H_PAD
    r   r  r  r  r   )r  r'  r  r  r   r�  )re  r!  r  r(  r"  r#  r$  s          r6   Ú_maybe_pad_o_headr+  ¿  sº   € ð
 	€yØ�!ˆtˆä�=‰=˜uÔ%€DÜ×$Ñ$¨5Ô1€JØ€Eàˆ:�~˜Òð ˜a *™nÑ-ˆØ˜5‘y ZÑ/ˆØ�{Ò"ÐtÐ&7¸°wÐ>fÐgrÐfsÐ$tÓtÐ"Ø�: ‘>Ò!Ü—‘�a˜!˜Q  5Ð)Ó*×5Ñ5Ó7ˆAØˆeˆ8€Or9   c                óP   — |dkD  r| dd…dd…d| …dd…f   } | j                  «       S )zÂMaybe unpad the head dimension.
    x: torch.Tensor, shape (B, S_LOCAL, H_GLOBAL + H_PAD, D) H_PAD: int, head padding num return: torch.Tensor,
    unpadded tensor (B, S_LOCAL, H_GLOBAL, D)
    r   N)r�  )re  r#  r  s      r6   Ú_maybe_unpad_o_headr-  Ö  s0   € ð
 ˆq‚yØŠa’�G�e�V�GšQÐÑˆØ�<‰<‹>Ðr9   c                ó�   — t        | j                  «      dk(  sJ d«       ‚i }| j                  d   |d<   | j                  d   |d<   |S )Nr  zEQuery tensor must be 4-dimensional of shape (B, S_LOCAL, H_GLOBAL, D)r	   ÚNUM_QO_HEADr   Ú	Q_S_LOCAL)Úlenrî   )rº   rÉ   Úextra_kwargss      r6   Úulysses_anything_metadatar3  à  sN   € äˆu�{‰{Ó˜qÒ ÐiÐ"iÓiÐ Ø€LØ"'§+¡+¨a¡.€L�ÑØ %§¡¨A¡€L�ÑàÐr9   c                ó~  ‡ ‡‡— t        j                  ‰¬«      }‰ j                  \  }}}}t        ‰ |‰«      \  Š Š|‰z   |z  }‰ j	                  |||||«      j                  ddddd«      j                  «       Š |g|z  }	t        |‰«      }
‰ j                  dd«      Š t        j                  ‰ |
|	‰«      Š dˆˆˆ fd„}|S )	zk
    x: torch.Tensor, shape (B, S_LOCAL, H, D) return: Callable that returns (B, S_GLOBAL, H_LOCAL, D)
    r  r	   r   r   r  r  c                 ó|   •— t        ‰«      Š‰j                  dddd«      j                  «       Št        ‰‰ ‰«      Š‰S )Nr   r   r	   r  )rü  rR  r�  r)  )r#  r  re  s   €€€r6   rú  z-all_to_all_single_any_qkv_async.<locals>.wait   s>   ø€ ä˜‹Oˆð �I‰I�a˜˜A˜qÓ!×,Ñ,Ó.ˆÜ! ! U¨EÓ2ˆØˆr9   ©r    râ   )r  r  rî   r%  r   rR  r�  r   rþ  rø  rÿ  )re  r  rÉ   r"  rß  ÚS_LOCALr!  ÚDÚH_LOCALÚinput_split_sizesÚoutput_split_sizesrú  r#  s   ``          @r6   Úall_to_all_single_any_qkv_asyncr<  ê  sÈ   ú€ ô ×$Ñ$¨5Ô1€JØ—w‘wÑ€A€w��1Ü" 1 a¨Ó/�H€A€uØ�5‰y˜ZÑ'€Gà	�	‰	�!�W˜j¨'°1Ó5×=Ñ=¸aÀÀAÀqÈ!ÓL×WÑWÓY€Aà ˜	 JÑ.Ðô -¨W°eÓ<ÐØ	�	‰	�!�Q‹€AÜ× Ñ  Ð$6Ð8IÈ5ÓQ€A÷ð €Kr9   c           	     ó€  ‡ ‡‡	‡
‡‡‡‡— |j                  dd«      }t        j                  ‰¬«      Št        ‰ |‰«      \  Š Š‰ j                  }|\  Š	}ŠŠ
|j                  d«      Št        ‰‰«      }‰ j                  dddd«      j                  «       Š ‰g‰z  }t        j                  ‰ ||‰«      Š d
ˆ	ˆ
ˆˆˆˆˆˆ fd	„}|S )zr
    x: torch.Tensor, shape (B, S_GLOBAL, H_LOCAL, D) return: Callable that returns (B, S_LOCAL, H_GLOBAL, D)
    r/  Nr  r0  r   r   r	   r  c                 óÖ   •— t        ‰«      Š‰j                  ‰‰‰ ‰‰«      Š‰j                  ddddd«      j                  «       Š‰j                  ‰ ‰‰‰z  ‰«      Št	        ‰‰‰«      Š‰S )Nr	   r   r   r  r  )rü  r   rR  r�  r-  )rß  r8  r9  r#  r7  r  r"  re  s   €€€€€€€€r6   rú  z+all_to_all_single_any_o_async.<locals>.wait(  sm   ø€ ä˜‹OˆØ�I‰I�j '¨1¨g°qÓ9ˆØ�I‰I�a˜˜A˜q !Ó$×/Ñ/Ó1ˆØ�I‰I�a˜ *¨wÑ"6¸Ó:ˆÜ  5¨%Ó0ˆØˆr9   r6  )
rÃ   r  r  r+  rî   r   rR  r�  rø  rÿ  )re  r  rÉ   r!  rî   ÚS_GLOBALr:  r;  rú  rß  r8  r9  r#  r7  r"  s   ``       @@@@@@r6   Úall_to_all_single_any_o_asyncr@    s¾   ÿ€ ð 	�
‰
�= $Ó'€AÜ×$Ñ$¨5Ô1€Jä   A uÓ-�H€A€uØ�G‰G€EØ %Ñ€Qˆ�'˜1ð �j‰j˜Ó%€GÜ+¨G°UÓ;ÐØ	�	‰	�!�Q˜˜1Ó×(Ñ(Ó*€AØ!˜ ZÑ/ÐÜ× Ñ  Ð$6Ð8IÈ5ÓQ€A÷ô ð €Kr9   c                  óf   — e Zd Ze	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       Ze	 	 	 	 dd„«       Zy)ÚTemplatedRingAttentionNc                ó  — |j                   j                  }|j                   j                  }|j                   j                  }|dz   |z  }d x}}|
| _        || _        |j                  | _        |j                  | _        || _	        t        j                  |j                  «       |j                  «       g«      j                  «       }t        j                  |d|j!                  «       ¬«      }|j#                  |«      }t%        |«      D �]B  }|dkD  rE||   }|j'                  «       }|d | j)                  |«      }||d  j)                  |«      }|dz   |z  } |
| ||||||||d|dk(  |¬«      \  }}|j                   j*                  r>|j-                  t        j.                  «      }|j-                  t        j.                  «      }t1        dd«      r|j3                  d«      }|�d|t        j4                  j6                  j9                  ||z
  «      ||z
  z  z
  }|t        j4                  j6                  j;                  ||z
  «      z
  }|}|}�ŒE j-                  |j<                  «      }j?                  d«      }|	r||fS |S )	Nr   r   ©Ú
gather_dimr  T©rq  rÀ   Ú<ú2.9.0rì   ) r¶  Ú
_ring_meshÚ_ring_local_rankÚring_degreeÚ
forward_opÚbackward_oprî   Úq_shapeÚkv_shaperÀ   rÜ   Úcatrþ  r�  rø  Úall_gather_tensorÚ	get_groupÚchunkÚrangeÚnumelÚ
reshape_asÚconvert_to_fp32rÞ  Úfloat32r   r  rk  rl  ÚsigmoidÚ
logsigmoidrÓ   Úsqueeze)ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rL  rM  rÀ   Ú	ring_meshr(  r"  Ú	next_rankÚprev_outÚprev_lseÚ	kv_bufferÚiÚkvÚ	key_numelrT  rU  s                            r6   r  zTemplatedRingAttention.forward5  sN  € ð  %×<Ñ<×GÑGˆ	Ø×7Ñ7×HÑHˆØ%×=Ñ=×IÑIˆ
Ø˜A‘X Ñ+ˆ	Ø"Ð"ˆ�8à#ˆŒØ%ˆŒØ—k‘kˆŒØ—y‘yˆŒØ/ˆÔä—I‘I˜sŸ{™{›}¨e¯m©m«oÐ>Ó?×JÑJÓLˆ	Ü×,Ñ,¨YÀ1ÈI×L_ÑL_ÓLaÔbˆ	Ø—O‘O JÓ/ˆ	ä�z×"ˆAØ�1ŠuØ˜yÑ)�ØŸI™I›K�	Ø˜˜)�n×/Ñ/°Ó4�Ø˜9˜:˜×1Ñ1°%Ó8�Ø&¨™]¨jÑ8�	á!ØØØØØØØØØØØ˜q™&Ø!1ô‰HˆC�ð  ×7Ñ7×GÒGØ—f‘fœUŸ]™]Ó+�Ø—f‘fœUŸ]™]Ó+�ô    WÔ-Ø—m‘m BÓ'�ØÐ#Ø¤§¡×!4Ñ!4×!<Ñ!<¸SÀ8¹^Ó!LÐPXÐ[^ÑP^Ñ!_Ñ_�Ø¤§¡×!4Ñ!4×!?Ñ!?ÀÈ3ÁÓ!OÑO�ØˆHØŠHðG #ðJ �f‰f�U—[‘[Ó!ˆØ�k‰k˜"‹oˆá'��SˆzÐ0¨SÐ0r9   c                óL  ‡— | j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }|dz   |z  }t        t        d|«      «      dgz   }| j                   j                  j                  rt        j                  n‰j                  }t        j                  | j                  |‰j                  ¬«      }	t        j                  | j                  |‰j                  ¬«      }
t        j                  | j                  |‰j                  ¬«      }d }| j                  ^}}}}t        j                   |j#                  «       |j#                  «       g«      j%                  «       }t'        j(                  |d|j+                  «       ¬«      }|j-                  |«      }t        |«      D �]*  }|dkD  rE||   }|j/                  «       }|d | j1                  |«      }||d  j1                  |«      }|dz   |z  }| j3                  | ‰«      ^}}}}|dkD  rCt5        |«      }|
j/                  «       }|d | j1                  |
«      }
||d  j1                  |«      }|	|z  }	|
|z  }
||z  }||dz
  k  sŒÄt        j                   |
j#                  «       |j#                  «       g«      j%                  «       }t'        j6                  |||j+                  «       ¬«      }�Œ- ˆfd„|	|
|fD «       \  }	}
}|	|
|d d d d d d d d d fS )Nr   r   r  rD  r  c              3  óT   •K  — | ]  }|j                  ‰j                  «      –— Œ! y ­wr2   )rÞ  rÓ   )rd  re  r}  s     €r6   rf  z2TemplatedRingAttention.backward.<locals>.<genexpr>®  s!   øè ø€ Ð+mÑJlÀQ¨A¯D©D°·±×,@ÑJlùs   ƒ%()rÀ   r¶  rI  rJ  rK  rš   rT  rW  rÜ   rX  rÓ   r  rN  rÒ   rO  ry  rP  rþ  r�  rø  rQ  rR  rS  rU  rV  rM  rü  Úpermute_tensor)ro  r}  r~  r\  r(  r"  r]  Ú
next_ranksÚaccum_dtyper†  r‡  rˆ  Únext_grad_kvrº   r»   rŠ   rV  r`  ra  rb  rc  Úgrad_query_opÚgrad_key_opÚgrad_value_opÚgrad_kv_bufferÚgrad_key_numels    `                        r6   r  zTemplatedRingAttention.backward  sÏ  ø€ ð ×(Ñ(×@Ñ@×KÑKˆ	Ø×#Ñ#×;Ñ;×LÑLˆØ×)Ñ)×AÑA×MÑMˆ
Ø˜A‘X Ñ+ˆ	Üœ%  :Ó.Ó/°1°#Ñ5ˆ
à'*×';Ñ';×'SÑ'S×'cÒ'c”e—m’mÐiq×iwÑiwˆÜ—[‘[ §¡°KÈÏÉÔXˆ
Ü—;‘;˜sŸ|™|°;ÀxÇÁÔWˆÜ—[‘[ §¡°[ÈÏÉÔYˆ
Øˆà #× 1Ñ 1Ðˆˆs�E˜AÜ—I‘I˜sŸ{™{›}¨e¯m©m«oÐ>Ó?×JÑJÓLˆ	Ü×,Ñ,¨YÀ1ÈI×L_ÑL_ÓLaÔbˆ	Ø—O‘O JÓ/ˆ	ä�z×"ˆAØ�1ŠuØ˜yÑ)�ØŸI™I›K�	Ø˜˜)�n×/Ñ/°Ó4�Ø˜9˜:˜×1Ñ1°%Ó8�Ø&¨™]¨jÑ8�	à<?¿O¹OÈCÐQYÓ<ZÐ9ˆM˜;¨¸à�1ŠuÜ!-¨lÓ!;�Ø!)§¡Ó!1�Ø)¨/¨>Ð:×EÑEÀhÓO�Ø+¨N¨OÐ<×GÑGÈ
ÓS�
à˜-Ñ'ˆJØ˜Ñ#ˆHØ˜-Ñ'ˆJà�: ‘>Ó!Ü!&§¡¨H×,<Ñ,<Ó,>À
×@RÑ@RÓ@TÐ+UÓ!V×!aÑ!aÓ!c�Ü%×4Ñ4°^ÀZÐW`×WjÑWjÓWlÔm’ð- #ó0 ,nÈ:ÐW_ÐakÑJlÓ+mÑ(ˆ
�H˜jà˜8 Z°°t¸TÀ4ÈÈtÐUYÐ[_ÐaeÐeÐer9   r2   ©ro  ú#torch.autograd.function.FunctionCtxrº   râ   r»   râ   rŠ   râ   r¼   útorch.Tensor | Noner½   Úfloatr¾   rž   r¿   úfloat | NonerÁ   rž   rp  rž   rÀ   ú'ParallelConfig' | None©ro  rp  r}  râ   ©r`   ra   rb   r  r  r  r3   r9   r6   rB  rB  4  s¹   „ Øð 59ðG1Ø0ðG1àðG1ð ðG1ð ð	G1ð
 'ðG1ð ðG1ð ðG1ð ðG1ð ðG1ð ðG1ð 2òG1ó ðG1ðR ð0fØ0ð0fàò0fó ñ0fr9   rB  c                  óf   — e Zd Ze	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       Ze	 	 	 	 dd„«       Zy)ÚTemplatedUlyssesAttentionNc                ó–  ‡— |j                   j                  }|j                   j                  }|j                  «       Š|
| _        || _        || _        |j                  \  }}}}|j                  \  }}}}||z  }|j                  |||||«      j                  ddddd«      j                  «       }|j                  |||||«      j                  ddddd«      j                  «       }|j                  |||||«      j                  ddddd«      j                  «       }ˆfd„|||fD «       \  }}}d„ |||fD «       \  }}} |
| |||||||||	d|¬	«      }|	r|^}}}|j                  |||||«      j                  ddddd«      j                  «       }t        |‰«      }|j                  dd«      j                  dddd«      j                  «       }|	rrj                  ||||«      j                  dddd«      j                  «       }t        |‰«      }|j                  dd«      j                  ddd«      j                  «       }nd }|	r||fS |S )
Nr	   r   r   r  r  c              3  ó6   •K  — | ]  }t        |‰«      –— Œ y ­wr2   ©r  ©rd  re  r  s     €r6   rf  z4TemplatedUlyssesAttention.forward.<locals>.<genexpr>Ò  s   øè ø€ ÐWÑCV¸aÔ/°°5×9ÑCVùó   ƒc              3  ó€   K  — | ]6  }|j                  d d«      j                  dd dd«      j                  «       –— Œ8 y­w©r   r   r	   r  N©rþ  rR  r�  rc  s     r6   rf  z4TemplatedUlyssesAttention.forward.<locals>.<genexpr>Ó  s8   è ø€ ÐkÑWjÐRS˜QŸY™Y q¨!›_×4Ñ4°Q¸¸1¸aÓ@×KÑK×MÑWjùó   ‚<>TrF  )r¶  Ú_ulysses_meshÚulysses_degreerR  rL  rM  rÀ   rî   r   rR  r�  r  rþ  )ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rL  rM  rÀ   Úulysses_meshr"  rß  Ú	S_Q_LOCALr!  r8  rV  Ú
S_KV_LOCALr9  rT  rU  r  s                           @r6   r  z!TemplatedUlyssesAttention.forward´  so  ø€ ð  (×?Ñ?×MÑMˆØ%×=Ñ=×LÑLˆ
Ø×&Ñ&Ó(ˆà#ˆŒØ%ˆŒØ/ˆÔà"Ÿ[™[Ñˆˆ9�a˜Ø!Ÿi™iÑˆˆ:�q˜!Ø�z‘/ˆØ—‘˜a ¨J¸ÀÓC×KÑKÈAÈqÐRSÐUVÐXYÓZ×eÑeÓgˆØ�k‰k˜!˜Z¨°W¸aÓ@×HÑHÈÈAÈqÐRSÐUVÓW×bÑbÓdˆØ—‘˜a ¨Z¸À!ÓD×LÑLÈQÐPQÐSTÐVWÐYZÓ[×fÑfÓhˆÛWÀEÈ3ÐPUÑCVÓWÑˆˆs�EÙkÐX]Ð_bÐdiÑWjÓkÑˆˆs�EáØØØØØØØØØØØØ-ô
ˆñ ØˆLˆC��qà�k‰k˜!˜Z¨°G¸QÓ?×GÑGÈÈ1ÈaÐQRÐTUÓV×aÑaÓcˆÜ   eÓ,ˆØ�k‰k˜!˜QÓ×'Ñ'¨¨1¨a°Ó3×>Ñ>Ó@ˆáØ—+‘+˜a ¨Y¸Ó@×HÑHÈÈAÈqÐRSÓT×_Ñ_ÓaˆCÜ$ S¨%Ó0ˆCØ—+‘+˜a Ó#×+Ñ+¨A¨q°!Ó4×?Ñ?ÓA‰CàˆCá'��SˆzÐ0¨SÐ0r9   c                óz  ‡‡‡‡‡‡— | j                   j                  j                  }| j                   j                  j                  Š|j	                  «       Š|j
                  \  ŠŠ}Š|‰z  Š|j                  ‰‰‰‰‰«      j                  ddddd«      j                  «       }t        |‰«      }|j                  dd«      j                  dddd«      j                  «       }| j                  | |«      ^}}}}ˆˆˆˆˆfd„|||fD «       \  }	}
}ˆfd„|	|
|fD «       \  }	}
}d„ |	|
|fD «       \  }	}
}|	|
|d d d d d d d d d fS )	Nr	   r   r   r  r  c              3  óŠ   •K  — | ]:  }|j                  ‰‰‰‰‰«      j                  d dddd«      j                  «       –— Œ< y­w)r   r  r   r	   r  N)r   rR  r�  )rd  re  rß  r8  r9  r7  r"  s     €€€€€r6   rf  z5TemplatedUlyssesAttention.backward.<locals>.<genexpr>  sF   øè ø€ ð ,
á@�ð �I‰I�a˜ W¨g°qÓ9×AÑAÀ!ÀQÈÈ1ÈaÓP×[Ñ[×]Ù@ùs   ƒA Ac              3  ó6   •K  — | ]  }t        |‰«      –— Œ y ­wr2   r{  r|  s     €r6   rf  z5TemplatedUlyssesAttention.backward.<locals>.<genexpr>
  s   øè ø€ Ð+uÑRtÈQÔ,>¸qÀ%×,HÑRtùr}  c              3  ó€   K  — | ]6  }|j                  d d«      j                  ddd d«      j                  «       –— Œ8 y­wr  r€  rc  s     r6   rf  z5TemplatedUlyssesAttention.backward.<locals>.<genexpr>  s9   è ø€ ð ,
ÙFhÀˆA�I‰I�a˜‹O×#Ñ# A q¨!¨QÓ/×:Ñ:×<ÑFhùr�  )rÀ   r¶  r‚  rƒ  rR  rî   r   rR  r�  r  rþ  rM  )ro  r}  r~  r„  r!  rj  rk  rl  rV  r†  r‡  rˆ  rß  r8  r9  r7  r  r"  s               @@@@@@r6   r  z"TemplatedUlyssesAttention.backwardó  sd  ý€ ð ×+Ñ+×CÑC×QÑQˆØ×)Ñ)×AÑA×PÑPˆ
Ø×&Ñ&Ó(ˆà#Ÿ>™>Ñˆˆ7�A�qØ�z‘/ˆà×#Ñ# A w°
¸GÀQÓG×OÑOÐPQÐSTÐVWÐYZÐ\]Ó^×iÑiÓkˆÜ% h°Ó6ˆØ×#Ñ# A qÓ)×1Ñ1°!°Q¸¸1Ó=×HÑHÓJˆà8;¿¹ÈÈXÓ8VÐ5ˆ�{ M°A÷,
à# [°-Ñ@ó,
Ñ(ˆ
�H˜jó ,vÐS]Ð_gÐisÑRtÓ+uÑ(ˆ
�H˜jñ,
ØGQÐS[Ð]gÑFhó,
Ñ(ˆ
�H˜jð ˜8 Z°°t¸TÀ4ÈÈtÐUYÐ[_ÐaeÐeÐer9   r2   ro  ru  rv  r3   r9   r6   rx  rx  ³  s¬   „ Øð 59ð<1Ø0ð<1àð<1ð ð<1ð ð	<1ð
 'ð<1ð ð<1ð ð<1ð ð<1ð ð<1ð ð<1ð 2ò<1ó ð<1ð| ðfØ0ðfàòfó ñfr9   rx  c                  óf   — e Zd Ze	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       Ze	 	 	 	 dd„«       Zy)Ú!TemplatedUlyssesAnythingAttentionNc                ó  — |j                   j                  }|j                  «       }|
| _        || _        || _        t        |«      }t        ||fi |¤Ž}t        ||fi |¤Ž}t        ||fi |¤Ž} |«       } |«       } |«       } |
| |||||||||	d|¬«      }|	r|^}}}t        ||fi |¤Ž}|	rLj                  d«      }t        ||fi |¤Ž} |«       } |«       }|j                  d«      j                  «       }n	 |«       }d }|	r||fS |S )NFrF  rì   )r¶  r‚  rR  rL  rM  rÀ   r3  r<  r@  r  r[  r�  )ro  rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rL  rM  rÀ   rÉ   r„  r  ÚmetadataÚ
query_waitÚkey_waitÚ
value_waitrT  rU  rV  Úout_waitÚlse_waits                            r6   r  z)TemplatedUlyssesAnythingAttention.forward  s?  € ð" (×?Ñ?×MÑMˆØ×&Ñ&Ó(ˆà#ˆŒØ%ˆŒØ/ˆÔä,¨UÓ3ˆÜ4°U¸EÑNÀXÑNˆ
Ü2°3¸ÑJÀÑJˆÜ4°U¸EÑNÀXÑNˆ
á“ˆÙ‹jˆÙ“ˆáØØØØØØØØØØØØ-ô
ˆñ ØˆLˆC��qô 1°°eÑH¸xÑHˆáà—-‘- Ó#ˆCÜ4°S¸%ÑLÀ8ÑLˆHÙ“*ˆCÙ“*ˆCØ—+‘+˜b“/×,Ñ,Ó.‰Cá“*ˆCØˆCá'��SˆzÐ0¨SÐ0r9   c                ó   — t        d«      ‚)NzQBackward pass for Ulysses Anything Attention in diffusers is not implemented yet.rØ  rÚ  s      r6   r  z*TemplatedUlyssesAnythingAttention.backwardU  s   € ô "Ð"uÓvÐvr9   r2   )ro  rp  rº   râ   r»   râ   rŠ   râ   r¼   râ   r½   rr  r¾   rž   r¿   rr  rÁ   rž   rp  rž   rÀ   rt  ru  rv  r3   r9   r6   rŒ  rŒ    s¬   „ Øð 59ð?1Ø0ð?1àð?1ð ð?1ð ð	?1ð
  ð?1ð ð?1ð ð?1ð ð?1ð ð?1ð ð?1ð 2ò?1ó ð?1ðB ðwØ0ðwàòwó ñwr9   rŒ  c                óþ  — |j                   j                  }|j                  «       }t        j	                  || ||«      } t        j	                  ||||«      }t        j	                  ||||«      }t
        j	                  | |||||||||	|
|«      }|r|^}}}n|}t        j	                  ||||«      }|rJt        dd«      rj                  d«      }t        j	                  |||«      }|j                  d«      }||fS |S )z|
    Unified Sequence Parallelism attention combining Ulysses and ring attention. See: https://arxiv.org/abs/2405.07719
    rG  rH  rì   )	r¶  r‚  rR  r  r  rB  r   r  r[  )rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rL  rM  rÀ   r  r  r„  Úulysses_grouprT  Úcontext_layerrU  rV  r  s                        r6   Ú_templated_unified_attentionr˜  ^  s  € ð& $×;Ñ;×IÑI€LØ ×*Ñ*Ó,€Mä× Ñ  °°{ÀJÓO€EÜ
×
Ñ
˜}¨c°;À
Ó
K€CÜ× Ñ  °°{ÀJÓO€EÜ
 ×
&Ñ
&ØØØØØØØØØØØØó€Cñ Ø!$Ðˆ�s™Qàˆä×!Ñ!ØØØØó	€Fñ ô ˜C Ô)Ø—-‘- Ó#ˆCÜ×"Ñ" =°#°zÀ;ÓOˆØ�k‰k˜"‹oˆØ˜ˆ}ÐØ€Mr9   )rÀ   c	               ó,  — |rt        d«      ‚|rt        d«      ‚|j                  j                  dkD  r/|j                  j                  dkD  rt	        | |||||||||	|
|«      S |j                  j                  dkD  r t
        j                  | |||||||||	|
|«      S |j                  j                  dkD  rV|j                  j                  r t        j                  | |||||||||	|
|«      S t        j                  | |||||||||	|
|«      S t        d«      ‚)Nz>Causal attention is not yet supported for templated attention.z1GQA is not yet supported for templated attention.r   z@Reaching this branch of code is unexpected. Please report a bug.)
r²   r¶  rK  rƒ  r˜  rB  r  Úulysses_anythingrŒ  rx  )rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rL  rM  rÀ   s               r6   Ú%_templated_context_parallel_attentionr›  œ  sm  € ñ ÜÐYÓZÐZÙÜÐLÓMÐMð 	×0Ñ0×<Ñ<¸qÒ@Ø×4Ñ4×CÑCÀaÒGä+ØØØØØØØØØØØØó
ð 	
ð 
×	1Ñ	1×	=Ñ	=ÀÒ	AÜ%×+Ñ+ØØØØØØØØØØØØó
ð 	
ð 
×	1Ñ	1×	@Ñ	@À1Ò	DØ×3Ñ3×DÒDä4×:Ñ:ØØØØØØØØØØØØ óð ô -×2Ñ2ØØØØØØØØØØØØ óð ô Ð[Ó\Ð\r9   )r�   rŽ   c	                ó´   — d }	|�t        d«      ‚|€t        | ||||||¬«      }
|r,|
^}
}	}n&t        | ||d |||d|t        t        |¬«      }
|r|
\  }
}	|r|
|	fS |
S ©Nz.`attn_mask` is not supported for flash-attn 2.)rB  rÌ   rÍ   r½   rC  rD  rP  F©rL  rM  rÀ   )r²   r"   r›  r½  rÀ  )rº   r»   rŠ   r¼   r½   r¾   r¿   rp  rÀ   rU  rT  rV  s               r6   Ú_flash_attentionrŸ  ù  s�   € ð  €CØÐÜÐIÓJÐJàÐÜØØØØØØØ(ô
ˆñ ØˆLˆC�‘qä3ØØØØØØØØØÜ2Ü4Ø-ô
ˆñ Ø‰HˆC�á#ˆC�ˆ:Ð,¨Ð,r9   c	                óð   — d }	|�t        d«      ‚t        t        j                     j                  }
|€ |
| ||||||¬«      }|r,|^}}	}n&t        | ||d |||d|t        t        |¬«      }|r|\  }}	|r||	fS |S r�  )r²   r°   rF   rd   r©   r›  rÃ  rÅ  )rº   r»   rŠ   r¼   r½   r¾   r¿   rp  rÀ   rU  r5   rT  rV  s                r6   Ú_flash_attention_hubr¡  .	  s´   € ð  €CØÐÜÐIÓJÐJä Ô!5×!?Ñ!?Ñ@×JÑJ€DØÐÙØØØØØØØ(ô
ˆñ ØˆLˆC�‘qä3ØØØØØØØØØÜ6Ü8Ø-ô
ˆñ Ø‰HˆC�á#ˆC�ˆ:Ð,¨Ð,r9   c	                óX  — | j                   \  }	}
}}|j                   \  }}}}|�t        ||	|«      }t        |	|
||| j                  ¬«      \  \  }}\  }}\  }}g g }}t	        |	«      D ]7  }||   }|j                  ||d |…f   «       |j                  ||d |…f   «       Œ9 | j                  dd«      }t        j                  |d¬«      }t        j                  |d¬«      }t        t        j                     j                  } ||||||||||||¬«      }|j                  d|	df«      }|S ©N©r¼   rÒ   r   r   r  )rB  rÌ   rÍ   r  r  r  r  r½   rC  rD  rP  rì   )rî   r$  r  rÒ   rT  Úappendrþ  rÜ   rP  r°   rF   rf   r©   Ú	unflatten)rº   r»   rŠ   r¼   r½   r¿   r¾   rp  rÀ   r  r  rV  r  r  r  r  r  r  Ú	key_validÚvalue_validÚbÚ	valid_lenÚquery_packedÚ
key_packedÚvalue_packedr5   rT  s                              r6   Ú_flash_varlen_attention_hubr®  d	  sX  € ð  #(§+¡+Ñ€J�	˜1˜aØŸ)™)Ñ€A€z�1�aàÐÜ(¨°JÀ
ÓKˆ	ô 	/Ø˜	 :¸È5Ï<É<ô	
ñ O�N€Qˆ	Ñ0�\ <Ñ2N°<Àð   ˆ{€IÜ�:ÖˆØ˜a‘Lˆ	Ø×Ñ˜˜Q 
  
˜]Ñ+Ô,Ø×Ñ˜5  J Y J Ñ/Õ0ð ð
 —=‘=  AÓ&€LÜ—‘˜9¨!Ô,€JÜ—9‘9˜[¨aÔ0€Lä Ô!5×!FÑ!FÑG×QÑQ€DÙ
Ø
Ø
Ø
Ø!Ø!Ø!Ø!ØØØØ$ô€Cð �-‰-˜˜J¨Ð+Ó
,€Cà€Jr9   )r�   c	                ó  — | j                   \  }	}
}}|j                   \  }}}}|�t        ||	|«      }t        |	|
||| j                  ¬«      \  \  }}\  }}\  }}g g }}t	        |	«      D ]7  }||   }|j                  ||d |…f   «       |j                  ||d |…f   «       Œ9 | j                  dd«      }t        j                  |d¬«      }t        j                  |d¬«      }t        |||||||||||¬«      }|j                  d|	df«      }|S r£  )rî   r$  r  rÒ   rT  r¥  rþ  rÜ   rP  r#   r¦  )rº   r»   rŠ   r¼   r½   r¿   r¾   rp  rÀ   r  r  rV  r  r  r  r  r  r  r§  r¨  r©  rª  r«  r¬  r­  rT  s                             r6   Ú_flash_varlen_attentionr°  �	  sA  € ð #(§+¡+Ñ€J�	˜1˜aØŸ)™)Ñ€A€z�1�aàÐÜ(¨°JÀ
ÓKˆ	ô 	/Ø˜	 :¸È5Ï<É<ô	
ñ O�N€Qˆ	Ñ0�\ <Ñ2N°<Àð   ˆ{€IÜ�:ÖˆØ˜a‘Lˆ	Ø×Ñ˜˜Q 
  
˜]Ñ+Ô,Ø×Ñ˜5  J Y J Ñ/Õ0ð ð
 —=‘=  AÓ&€LÜ—‘˜9¨!Ô,€JÜ—9‘9˜[¨aÔ0€Lä
 Ø
Ø
Ø
Ø!Ø!Ø!Ø!ØØØØ$ô€Cð �-‰-˜˜J¨Ð+Ó
,€Cà€Jr9   c                óR   — |�t        d«      ‚t        | ||||¬«      \  }}	|r||	fS |S )Nú.`attn_mask` is not supported for flash-attn 3.)rB  rÌ   rÍ   rC  rD  )r²   rW  )
rº   r»   rŠ   r¼   r¿   r¾   rp  rÀ   rT  rU  s
             r6   Ú_flash_attention_3r³  Ô	  sF   € ð ÐÜÐIÓJÐJä$Ø
Ø
Ø
ØØô�H€Cˆñ $ˆC�ˆ:Ð,¨Ð,r9   c                ó²  — |�t        d«      ‚t        t        j                     j                  }|
€D |di d| “d|“d|“d|“d|“dd “dd “d	d “d
d “d|“d|“dd“dd “d|“dd“d|	“Ž}|	r
|d   |d   fS |S t        j                  t        ||dd |d¬«      }t        j                  t        ||dd |d¬«      }t        | ||d d||d|	|||
¬«      }|	r	|\  }}||fS |S )Nr²  rB  rÌ   rÍ   rC  rD  rE  rF  rG  rH  rI  rK  rL  r   rM  rN  rO  r   rP  rÆ  r·   Frž  r3   )
r²   r°   rF   ri   r©   Ú	functoolsÚpartialrË  rÍ  r›  )rº   r»   rŠ   r¼   r¿   r¾   rI  rK  rN  rP  rÀ   r5   rT  rL  rM  rU  s                   r6   Ú_flash_attention_3_hubr·  ï	  s�  € ð$ ÐÜÐIÓJÐJä Ô!5×!BÑ!BÑC×MÑM€DØÐÙò 
Ùð
áð
ñ ð
ñ  ð	
ñ
 ð
ñ ð
ñ ð
ñ ð
ñ ð
ñ $ð
ñ ð
ñ ð
ñ ð
ñ (ð
ñ ð
ñ  0ð!
ˆñ$ $5��A‘˜˜A™ÐÐ=¸#Ð=ä×"Ñ"Ü)ØØØØØ#Øô€Jô ×#Ñ#Ü*ØØØØØ#Øô€Kô 0ØØØØØØØØØØØØ)ô€Cñ Ø‰ˆˆSØ�Cˆxˆà€Jr9   c                óf  — | j                   \  }}	}
}
|j                   \  }
}}
}
|�t        |||«      }t        ||	||| j                  ¬«      \  \  }
}\  }}\  }}g g }}t	        |«      D ]7  }||   }|j                  ||d |…f   «       |j                  ||d |…f   «       Œ9 | j                  dd«      }t        j                  |d¬«      }t        j                  |d¬«      }t        t        j                     j                  } ||||||||||¬«	      ^}}}
|j                  d|df«      }|r||fS |S )Nr¤  r   r   r  )	rB  rÌ   rÍ   r  r  r  r  rC  rD  rì   )rî   r$  r  rÒ   rT  r¥  rþ  rÜ   rP  r°   rF   rj   r©   r¦  )rº   r»   rŠ   r¼   r¿   r¾   rp  rÀ   r  r  rV  r  r  r  r  r  r  r§  r¨  r©  rª  r«  r¬  r­  r5   rT  rU  s                              r6   Ú_flash_attention_3_varlen_hubr¹  A
  sd  € ð #(§+¡+Ñ€J�	˜1˜aØŸ)™)Ñ€A€z�1�aàÐÜ(¨°JÀ
ÓKˆ	ô 	/Ø˜	 :¸È5Ï<É<ô	
ñ O�N€Qˆ	Ñ0�\ <Ñ2N°<Àð   ˆ{€IÜ�:ÖˆØ˜a‘Lˆ	Ø×Ñ˜˜Q 
  
˜]Ñ+Ô,Ø×Ñ˜5  J Y J Ñ/Õ0ð ð
 —=‘=  AÓ&€LÜ—‘˜9¨!Ô,€JÜ—9‘9˜[¨aÔ0€Lä Ô!5×!IÑ!IÑJ×TÑT€DÙØ
Ø
Ø
Ø!Ø!Ø!Ø!ØØô
€L€Cˆˆqð �-‰-˜˜J¨Ð+Ó
,€Cá#ˆC�ˆ:Ð,¨Ð,r9   c                óZ  — | j                   \  }}	}
}
|j                   \  }
}}
}
|�t        |||«      }t        ||	||| j                  ¬«      \  \  }
}\  }}\  }}g g }}t	        |«      D ]7  }||   }|j                  ||d |…f   «       |j                  ||d |…f   «       Œ9 | j                  dd«      }t        j                  |d¬«      }t        j                  |d¬«      }t        ||||||||||¬«
      }t        |t        «      r|^}}}
n|}d }|j                  d|df«      }|r||fS |S )Nr¤  r   r   r  )
rB  rÌ   rÍ   r  r  r  r  rC  rD  rP  rì   )rî   r$  r  rÒ   rT  r¥  rþ  rÜ   rP  Úflash_attn_3_varlen_funcr÷  Útupler¦  )rº   r»   rŠ   r¼   r¿   r¾   rp  rÀ   r  r  rV  r  r  r  r  r  r  r§  r¨  r©  rª  r«  r¬  r­  rS  rT  rU  s                              r6   Ú_flash_varlen_attention_3r½  w
  si  € ð #(§+¡+Ñ€J�	˜1˜aØŸ)™)Ñ€A€z�1�aàÐÜ(¨°JÀ
ÓKˆ	ô 	/Ø˜	 :¸È5Ï<É<ô	
ñ O�N€Qˆ	Ñ0�\ <Ñ2N°<Àð   ˆ{€IÜ�:ÖˆØ˜a‘Lˆ	Ø×Ñ˜˜Q 
  
˜]Ñ+Ô,Ø×Ñ˜5  J Y J Ñ/Õ0ð ð
 —=‘=  AÓ&€LÜ—‘˜9¨!Ô,€JÜ—9‘9˜[¨aÔ0€Lä%Ø
Ø
Ø
Ø!Ø!Ø!Ø!ØØØ$ô€Fô �&œ%Ô ØˆˆˆS‘1àˆØˆØ
�-‰-˜˜J¨Ð+Ó
,€Cá#ˆC�ˆ:Ð,¨Ð,r9   c	           	     ó¶   — |�t        d«      ‚|s*t        j                  «       rt        | |||||d¬«      ^}	}
}nt        | ||||||¬«      }	|r|	^}	}
}|r|	
fS |	S )Nú0`attn_mask` is not supported for aiter attentionT)rB  rÌ   rÍ   r½   rC  rD  rp  )r²   rÜ   Úis_grad_enabledÚaiter_flash_attn_func)rº   r»   rŠ   r¼   r½   r¾   r¿   rp  rÀ   rT  rU  rV  s               r6   Ú_aiter_flash_attentionrÂ  ±
  s‹   € ð ÐÜÐKÓLÐLáœ%×/Ñ/Ô1ä,ØØØØØØØô
ˆˆˆS‘1ô $ØØØØØØØ!ô
ˆñ ØˆLˆC��qá#ˆC�ˆ:Ð,¨Ð,r9   c	           
     óî  ‡— d }	d }
| j                   \  }}}}|j                   \  }}}}‰�t        ‰t        j                  «      r‰}
ní|r)t        j                  t
        ||||| j                  «      }
nÂt        j                  ‰«      r¢‰j                  dk(  r2‰j                  ‰j                  d«      d‰j                  d«      d«      Š‰j                  ||||«      Š‰j                  t        j                  k(  r*ˆfd„}t        j                  ||d ||| j                  «      }
nˆfd„}	nt        d«      ‚d„ | ||fD «       \  } }}t        j                  | |||	|
|||¬«      }|j!                  dddd	«      }|S )
Nr	   r   r   c                ó   •— ‰| |||f   S r2   r3   )r&  r'  r(  r)  r¼   s       €r6   Úmask_modz(_native_flex_attention.<locals>.mask_mod  s   ø€ Ø  ¨H°e¸VÐ!CÑDÐDr9   c                ó   •— | ‰||||f   z   S r2   r3   )Úscorer&  r'  r(  r)  r¼   s        €r6   Ú	score_modz)_native_flex_attention.<locals>.score_mod	  s   ø€ Ø˜y¨°H¸eÀVÐ)KÑLÑLÐLr9   zCAttention mask must be either None, a BlockMask, or a 2D/4D tensor.c              3  óD   K  — | ]  }|j                  d ddd«      –— Œ y­wra  rb  rc  s     r6   rf  z)_native_flex_attention.<locals>.<genexpr>  rg  rh  )rº   r»   rŠ   rÈ  Ú
block_maskr¿   rÁ   rp  r  )rî   r÷  Úflex_attentionÚ	BlockMaskÚcreate_block_maskr*  rÒ   rÜ   Ú	is_tensorr  Úviewr!  r   rÓ   rž   r²   rR  )rº   r»   rŠ   r¼   r¾   r¿   rÁ   rp  rÀ   rÈ  rÊ  r  r  r\  rV  r  rÅ  rT  s      `              r6   Ú_native_flex_attentionrÐ  Þ
  ss  ø€ ð  €IØ€JØ*/¯+©+Ñ'€J�	˜9 aØŸ)™)Ñ€A€z�1�aàÐœJ y´.×2JÑ2JÔKØ‰
Ù	Ü#×5Ñ5Ü+¨Z¸ÀIÈzÐ[`×[gÑ[gó
‰
ô 
�‰˜Ô	#Ø�>‰>˜QÒØ!Ÿ™ y§~¡~°aÓ'8¸!¸Y¿^¹^ÈAÓ=NÐPQÓRˆIà×$Ñ$ Z°¸IÀzÓRˆ	à�?‰?œeŸj™jÒ(ôEô (×9Ñ9Ø˜* d¨I°zÀ5Ç<Á<ó‰Jõ
Mô Ð^Ó_Ð_áL¸ÀÀUÑ8KÓLÑ€Eˆ3�Ü
×
'Ñ
'ØØØØØØØØô	€Cð �+‰+�a˜˜A˜qÓ
!€CØ€Jr9   c                ó  — | j                   t        j                  k(  r3t        j                  | dt	        d«      «      } | j                  |¬«      } n| j                  |¬«      } |r#| j                  \  }}| j                  |dd|«      } | S )a'  
    Convert a 2D attention mask to an additive mask, optionally reshaping to 4D for SDPA.

    This helper is used by both native SDPA and xformers backends to handle both boolean and additive masks.

    Args:
        attn_mask: 2D tensor [batch_size, seq_len_k]
                   - Boolean: True means attend, False means mask out
                   - Additive: 0.0 means attend, -inf means mask out
        target_dtype: The dtype to convert the mask to (usually query.dtype)
        reshape_4d: If True, reshape from [batch_size, seq_len_k] to [batch_size, 1, 1, seq_len_k] for broadcasting

    Returns:
        Additive mask tensor where 0.0 means attend and -inf means mask out. Shape is [batch_size, seq_len_k] if
        reshape_4d=False, or [batch_size, 1, 1, seq_len_k] if reshape_4d=True.
    r·   ú-inf)rÓ   r   )rÓ   rÜ   rž   Úwhererr  rÞ  rî   rÏ  )r¼   Útarget_dtypeÚ
reshape_4dr  r#  s        r6   Ú_prepare_additive_attn_maskrÖ    sy   € ð( ‡�œ%Ÿ*™*Ò$ä—K‘K 	¨3´°f³Ó>ˆ	à—L‘L |�LÓ4‰	ð —L‘L |�LÓ4ˆ	ñ Ø )§¡Ñˆ
�IØ—N‘N :¨q°!°YÓ?ˆ	àÐr9   c
                óî  — |rt        d«      ‚|�m|j                  dk(  r^|j                  d   | j                  d   k(  r?|j                  d   |j                  d   k(  r |j                  d«      j                  d«      }|	€Wd„ | ||fD «       \  } }}t        j
                  j                  j                  | |||||||¬«      }
|
j                  dddd«      }
|
S t        | ||||||||t        t        |	¬«      }
|
S )	NzDNative attention backend does not support setting `return_lse=True`.r	   r   r   c              3  óD   K  — | ]  }|j                  d ddd«      –— Œ y­wra  rb  rc  s     r6   rf  z$_native_attention.<locals>.<genexpr>c  ó"   è ø€ ÐPÑ<O°q˜QŸY™Y q¨!¨Q°×2Ñ<Oùrh  ri  r  rž  )r²   r  rî   r  rÜ   rk  rl  rm  rR  r›  rr  r‰  ©rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rÀ   rT  s              r6   Ú_native_attentionrÛ  B  s  € ñ" ÜÐ_Ó`Ð`ð
 	ÐØ�N‰N˜aÒØ�O‰O˜AÑ %§+¡+¨a¡.Ò0Ø�O‰O˜AÑ #§)¡)¨A¡,Ò.ð ×'Ñ'¨Ó*×4Ñ4°QÓ7ˆ	àÐÙP¸UÀCÈÑ<OÓPÑˆˆs�EÜ�h‰h×!Ñ!×>Ñ>ØØØØØØØØ!ð ?ó 	
ˆð �k‰k˜!˜Q  1Ó%ˆð" €Jô 4ØØØØØØØØØÜ3Ü5Ø-ô
ˆð €Jr9   c
                óê  — d }
|	€¶|s´d„ | ||fD «       \  } }}t         j                  j                  j                  t         j                  j                  j                  j
                  «      5  t         j                  j                  j                  | |||||||¬«      }d d d «       j                  dddd«      }n&t        | ||||||||t        t        |	¬«      }|r|\  }}
|r||
fS |S # 1 sw Y   ŒLxY w)Nc              3  ó`   K  — | ]&  }|j                  d ddd«      j                  «       –— Œ( y­wra  )rR  r�  rc  s     r6   rf  z*_native_cudnn_attention.<locals>.<genexpr>•  s+   è ø€ Ð]ÑI\ÀA˜QŸY™Y q¨!¨Q°Ó2×=Ñ=×?ÑI\ùs   ‚,.ri  r   r	   r   r  rž  )rÜ   rk  Ú	attentionÚsdpa_kernelÚ
SDPBackendÚCUDNN_ATTENTIONrl  rm  rR  r›  rœ  r¦  ©rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rÀ   rU  rT  s               r6   Ú_native_cudnn_attentionrã  ‚  s  € ð" €CØÐ©
Ù]È%ÐQTÐV[ÑI\Ó]Ñˆˆs�EÜ�X‰X×Ñ×+Ñ+¬E¯H©H×,>Ñ,>×,IÑ,I×,YÑ,YÕZÜ—(‘(×%Ñ%×BÑBØØØØ#Ø#Ø#ØØ%ð Có 	ˆC÷ [ð �k‰k˜!˜Q  1Ó%‰ä3ØØØØØØØØØÜ2Ü4Ø-ô
ˆñ Ø‰HˆC�á#ˆC�ˆ:Ð,¨Ð,÷= [ÐZús   Á,2C)Ã)C2c
                óž  — |rt        d«      ‚d„ | ||fD «       \  } }}t        j                  j                  j	                  t        j                  j                  j
                  j                  «      5  t        j                  j                  j                  | |||||||¬«      }
d d d «       
j                  dddd«      }
|
S # 1 sw Y   ŒxY w)NzNNative efficient attention backend does not support setting `return_lse=True`.c              3  óD   K  — | ]  }|j                  d ddd«      –— Œ y­wra  rb  rc  s     r6   rf  z._native_efficient_attention.<locals>.<genexpr>É  rg  rh  ri  r   r	   r   r  )
r²   rÜ   rk  rÞ  rß  rà  ÚEFFICIENT_ATTENTIONrl  rm  rR  rÚ  s              r6   Ú_native_efficient_attentionrç  ·  s»   € ñ  ÜÐiÓjÐjÙL¸ÀÀUÑ8KÓLÑ€Eˆ3�Ü	�‰×	Ñ	×	'Ñ	'¬¯©×(:Ñ(:×(EÑ(E×(YÑ(YÕ	ZÜ�h‰h×!Ñ!×>Ñ>ØØØØØØØØ!ð ?ó 	
ˆ÷ 
[ð �+‰+�a˜˜A˜qÓ
!€CØ€J÷ 
[Ð	Zúó   Á32CÃCc
                ó  — |�t        d«      ‚d }
|	€¶|s´d„ | ||fD «       \  } }}t        j                  j                  j	                  t        j                  j                  j
                  j                  «      5  t        j                  j                  j                  | ||d ||||¬«      }d d d «       j                  dddd«      }n&t        | ||d |||||t        t        |	¬«      }|r|\  }}
|r||
fS |S # 1 sw Y   ŒLxY w)	Nr¿  c              3  óD   K  — | ]  }|j                  d ddd«      –— Œ y­wra  rb  rc  s     r6   rf  z*_native_flash_attention.<locals>.<genexpr>ï  rÙ  rh  ri  r   r	   r   r  rž  )r²   rÜ   rk  rÞ  rß  rà  ÚFLASH_ATTENTIONrl  rm  rR  r›  r©  r­  râ  s               r6   Ú_native_flash_attentionrì  Ù  s  € ð" ÐÜÐKÓLÐLà
€CØÐ©
ÙP¸UÀCÈÑ<OÓPÑˆˆs�EÜ�X‰X×Ñ×+Ñ+¬E¯H©H×,>Ñ,>×,IÑ,I×,YÑ,YÕZÜ—(‘(×%Ñ%×BÑBØØØØØ#Ø#ØØ%ð Có 	ˆC÷ [ð �k‰k˜!˜Q  1Ó%‰ä3ØØØØØØØØØÜ9Ü;Ø-ô
ˆñ Ø‰HˆC�á#ˆC�ˆ:Ð,¨Ð,÷= [ÐZús   Á92C6Ã6C?c
                óž  — |rt        d«      ‚d„ | ||fD «       \  } }}t        j                  j                  j	                  t        j                  j                  j
                  j                  «      5  t        j                  j                  j                  | |||||||¬«      }
d d d «       
j                  dddd«      }
|
S # 1 sw Y   ŒxY w)NzINative math attention backend does not support setting `return_lse=True`.c              3  óD   K  — | ]  }|j                  d ddd«      –— Œ y­wra  rb  rc  s     r6   rf  z)_native_math_attention.<locals>.<genexpr>#  rg  rh  ri  r   r	   r   r  )
r²   rÜ   rk  rÞ  rß  rà  ÚMATHrl  rm  rR  rÚ  s              r6   Ú_native_math_attentionrð    s»   € ñ  ÜÐdÓeÐeÙL¸ÀÀUÑ8KÓLÑ€Eˆ3�Ü	�‰×	Ñ	×	'Ñ	'¬¯©×(:Ñ(:×(EÑ(E×(JÑ(JÕ	KÜ�h‰h×!Ñ!×>Ñ>ØØØØØØØØ!ð ?ó 	
ˆ÷ 
Lð �+‰+�a˜˜A˜qÓ
!€CØ€J÷ 
LÐ	Kúrè  c                ó(  — |rt        d«      ‚|€ct        | ||«      }t        | ||| j                  d«      |dd |€%dt	        j
                  | j                  d   «      z  n|ddd|z
  dd¬	«      d   }|S t        | ||||d |d |t        t        |¬
«      }|S )Nrä  r	   rå  ræ  rì   rç  Fr   rè  rž  )
r²   râ  r,   r!  rñ  rò  rî   r›  ró  rõ  )	rº   r»   rŠ   r¼   r½   r¿   rp  rÀ   rT  s	            r6   Ú_native_npu_attentionrò  3  sÈ   € ñ ÜÐ\Ó]Ð]ØÐÜ/°°s¸IÓFˆ	ä"ØØØØ�J‰J�q‹MØ ØØØ6;°m�#œŸ	™	 %§+¡+¨b¡/Ó2Ò2ÈØØØ˜I‘oØØô
ð ñˆð< €Jô 4ØØØØØØØØØÜ0Ü2Ø-ô
ˆð €Jr9   c                óê   — |�t        d«      ‚|rt        d«      ‚d„ | ||fD «       \  } }}| t        j                  | j                  d   «      z  } t	        | |||¬«      }|j                  dddd	«      }|S )
Nz.`attn_mask` is not supported for XLA attentionzAXLA attention backend does not support setting `return_lse=True`.c              3  óD   K  — | ]  }|j                  d ddd«      –— Œ y­wra  rb  rc  s     r6   rf  z(_native_xla_attention.<locals>.<genexpr>z  rg  rh  rì   )rB  rÌ   rÍ   rD  r   r	   r   r  )r²   rñ  rò  rî   Úxla_flash_attentionrR  )rº   r»   rŠ   r¼   r¾   rp  rÀ   rT  s           r6   Ú_native_xla_attentionrö  i  s‰   € ð ÐÜÐIÓJÐJÙÜÐ\Ó]Ð]ÙL¸ÀÀUÑ8KÓLÑ€Eˆ3�Ø”D—I‘I˜eŸk™k¨"™oÓ.Ñ.€EÜ
Ø
Ø
Ø
Øô	€Cð �+‰+�a˜˜A˜qÓ
!€CØ€Jr9   c                ó´   — |�t        d«      ‚d }|€t        | ||d|||¬«      }	|r,|	^}	}}
n&t        | ||d d||d|t        t        |¬«      }	|r|	\  }	}|r|	|fS |	S ©Nú/`attn_mask` is not supported for sage attentionrÐ  rÑ  r·   Frž  )r²   r&   r›  rÔ  rÛ  )rº   r»   rŠ   r¼   r¾   r¿   rp  rÀ   rU  rT  rV  s              r6   Ú_sage_attentionrú  †  s�   € ð ÐÜÐJÓKÐKØ
€CØÐÜØØØØØØØ!ô
ˆñ ØˆLˆC�‘qä3ØØØØØØØØØÜ1Ü3Ø-ô
ˆñ Ø‰HˆC�á#ˆC�ˆ:Ð,¨Ð,r9   c                óð   — |�t        d«      ‚d }t        t        j                     j                  }	|€ |	| ||d|||¬«      }
|r,|
^}
}}n&t        | ||d d||d|t        t        |¬«      }
|r|
\  }
}|r|
|fS |
S rø  )r²   r°   rF   ru   r©   r›  rÖ  rÛ  )rº   r»   rŠ   r¼   r¾   r¿   rp  rÀ   rU  r5   rT  rV  s               r6   Ú_sage_attention_hubrü  ¹  s´   € ð ÐÜÐJÓKÐKØ
€CÜ Ô!5×!>Ñ!>Ñ?×IÑI€DØÐÙØØØØØØØ!ô
ˆñ ØˆLˆC�‘qä3ØØØØØØØØØÜ5Ü3Ø-ô
ˆñ Ø‰HˆC�á#ˆC�ˆ:Ð,¨Ð,r9   c                ó2  — |rt        d«      ‚| j                  \  }}	}
}
|j                  \  }
}}
}
|�t        |||«      }t        ||	||| j                  ¬«      \  \  }
}\  }}\  }}g g }}t        |«      D ]7  }||   }|j                  ||d |…f   «       |j                  ||d |…f   «       Œ9 | j                  dd«      }t        j                  |d¬«      }t        j                  |d¬«      }t        |||||||||¬«	      }|j                  d|df«      }|S )Nz?Sage varlen backend does not support setting `return_lse=True`.r¤  r   r   r  )	rB  rÌ   rÍ   r  r  r  r  r¾   rÓ  rì   )r²   rî   r$  r  rÒ   rT  r¥  rþ  rÜ   rP  r+   r¦  )rº   r»   rŠ   r¼   r¾   r¿   rp  rÀ   r  r  rV  r  r  r  r  r  r  r§  r¨  r©  rª  r«  r¬  r­  rT  s                            r6   Ú_sage_varlen_attentionrþ  í  sJ  € ñ ÜÐZÓ[Ð[à"'§+¡+Ñ€J�	˜1˜aØŸ)™)Ñ€A€z�1�aàÐÜ(¨°JÀ
ÓKˆ	ô 	/Ø˜	 :¸È5Ï<É<ô	
ñ O�N€Qˆ	Ñ0�\ <Ñ2N°<Àð   ˆ{€IÜ�:ÖˆØ˜a‘Lˆ	Ø×Ñ˜˜Q 
  
˜]Ñ+Ô,Ø×Ñ˜5  J Y J Ñ/Õ0ð ð
 —=‘=  AÓ&€LÜ—‘˜9¨!Ô,€JÜ—9‘9˜[¨aÔ0€Lä
Ø
Ø
Ø
Ø!Ø!Ø!Ø!ØØô
€Cð �-‰-˜˜J¨Ð+Ó
,€Cà€Jr9   é	   c           	     ó@   — |�t        d«      ‚t        | ||d|||¬«      S ©Nrù  rÐ  rÑ  )r²   r'   ©rº   r»   rŠ   r¼   r¾   r¿   rp  rÀ   s           r6   Ú#_sage_qk_int8_pv_fp8_cuda_attentionr  $  s8   € ð ÐÜÐJÓKÐKÜ'Ø
Ø
Ø
ØØØØôð r9   c           	     ó@   — |�t        d«      ‚t        | ||d|||¬«      S r  )r²   r(   r  s           r6   Ú(_sage_qk_int8_pv_fp8_cuda_sm90_attentionr  ?  s8   € ð ÐÜÐJÓKÐKÜ,Ø
Ø
Ø
ØØØØôð r9   é   c           	     ó@   — |�t        d«      ‚t        | ||d|||¬«      S r  )r²   r)   r  s           r6   Ú$_sage_qk_int8_pv_fp16_cuda_attentionr  Z  s8   € ð ÐÜÐJÓKÐKÜ(Ø
Ø
Ø
ØØØØôð r9   c           	     ó@   — |�t        d«      ‚t        | ||d|||¬«      S r  )r²   r*   r  s           r6   Ú&_sage_qk_int8_pv_fp16_triton_attentionr
  u  s8   € ð ÐÜÐJÓKÐKÜ*Ø
Ø
Ø
ØØØØôð r9   c
                óà  — |rt        d«      ‚| j                  \  }
}}}|j                  \  }}}}|rt        j                  «       }ný|�û|j                  dk(  r |j                  d«      }|dz   dz  dz  }t        j                  |
|||f| j                  | j                  ¬«      }t        || j                  ¬«      }||d d …d d …d d …d |…f<   t        d«      |d d …d d …d d …|d …f<   |d d …d d …d d …d |…f   }nL|j                  d	k7  rt        d
«      ‚|j                  d	k(  r#|j                  |
|||«      j                  | «      }|rz||z  dk7  rt        d«      ‚||z  }| j                  d|df«      } |j                  d|df«      j                  ddd|d«      }|j                  d|df«      j                  ddd|d«      }t        j                  | |||||«      }|r|j!                  dd«      }|S )NzFxformers attention backend does not support setting `return_lse=True`.r	   r   é   r  r  )rÔ  rÒ  r  zDOnly 2D and 4D attention masks are supported for xformers attention.r   zKNumber of heads in query must be divisible by number of heads in key/value.rì   r  )r²   rî   ÚxopsÚLowerTriangularMaskr  r!  rÜ   r  rÓ   rÒ   rÖ  rr  r   Útype_asr¦  Úmemory_efficient_attentionrþ  )rº   r»   rŠ   r¼   r½   r¾   r¿   rÁ   rp  rÀ   r  r  Únum_heads_qrV  r  Únum_heads_kvÚoriginal_seq_lenÚaligned_seq_lenÚaligned_maskÚmask_additiveÚnum_heads_per_grouprT  s                         r6   Ú_xformers_attentionr  �  s  € ñ  ÜÐaÓbÐbà,1¯K©KÑ)€J�	˜;¨Ø%(§Y¡YÑ"€A€z�< áÜ×,Ñ,Ó.‰	Ø	Ð	Ø�>‰>˜QÒð
  )Ÿ~™~¨aÓ0ÐØ 0°1Ñ 4¸Ñ:¸aÑ?ˆOô !Ÿ;™;Ø˜[¨)°_ÐEØ—k‘kØ—|‘|ôˆLô 8Ø¨¯©ôˆMð 8EˆLššAšqÐ"3Ð#3Ð"3Ð3Ñ4ä7<¸V³}ˆLššAšqÐ"2Ñ"3Ð3Ñ4ð %¢Qªª1¨k¨z¨kÐ%9Ñ:‰IØ�^‰^˜qÒ ÜÐcÓdÐdØ�^‰^˜qÒ Ø!×(Ñ(¨°[À)ÈZÓX×`Ñ`ÐafÓgˆIáØ˜Ñ%¨Ò*ÜÐjÓkÐkØ)¨\Ñ9ÐØ—‘  L°"Ð#5Ó6ˆØ�m‰m˜A ¨bÐ1Ó2×9Ñ9¸"¸bÀ"ÐFYÐ[]Ó^ˆØ—‘  L°"Ð#5Ó6×=Ñ=¸bÀ"ÀbÐJ]Ð_aÓbˆä
×
)Ñ
)¨%°°e¸YÈ	ÐSXÓ
Y€CáØ�k‰k˜!˜QÓˆà€Jr9   r2   )r‹   zstr | AttentionBackendName)Nr·   FNFN)rº   râ   r»   râ   rŠ   râ   r¼   rq  r½   rr  r¾   rž   r¿   rs  rÁ   rž   rÆ   zdict[str, Any] | Noner‹   zAttentionBackendName | Noner¸   rt  r    râ   )r¼   rq  r¾   rž   r    rã   rá   )rÞ   Úintrß   r  r    r   )
rº   râ   r»   râ   rŠ   râ   r¼   rq  r    rã   )r‹   rF   r    rã   )r  r  r  r  r  r  rÒ   útorch.device | None)r  r  r  r  r¼   râ   rÒ   r  )NN)r  r  r  r  r  r  r¼   rq  rÒ   r  r    rã   )r¼   râ   r  r  r#  r  r    râ   )r1  rŸ   )NFNNNNr   r·   r   NFr   ) rB  râ   rÌ   râ   rÍ   râ   rC  rs  rD  rž   rE  rq  rF  rq  rG  rq  rH  rq  rJ  r  rK  rr  rL  r  rM  úbool | NonerN  rž   rO  r  r    z!tuple[torch.Tensor, torch.Tensor])Nr·   FNFFTN)ro  rp  rº   râ   r»   râ   rŠ   râ   r¼   rq  r½   rr  r¾   rž   r¿   rs  rÁ   rž   rp  rž   rq  rž   rÀ   rt  ru  )$ro  rp  rº   râ   r»   râ   rŠ   râ   r¼   rq  r½   rr  r¾   rž   r¿   rs  rÁ   rž   rp  rž   rq  rž   rÀ   rt  rI  útuple[int, int]rK  rr  rL  r  rM  r  rN  rž   rO  r  )rº   râ   r»   râ   r¼   rq  )re  râ   r    râ   )r	   r   N)re  râ   r  r  r  r  r    râ   )re  râ   r!  r  r  údist.ProcessGroupr    ztuple[torch.Tensor, int])re  râ   r#  r  r  r  r    râ   )rº   râ   r    Údict)re  râ   r  r  r    zCallable[..., torch.Tensor])Nr	   r   )rº   râ   r»   râ   rŠ   râ   r¼   râ   r½   rr  r¾   rž   r¿   rr  rÁ   rž   rp  rž   rÀ   rt  r  r  r  r  )Nr·   FNFF)rº   râ   r»   râ   rŠ   râ   r¼   rq  r½   rr  r¾   rž   r¿   rs  rÁ   rž   rp  rž   rÀ   rt  )rº   râ   r»   râ   rŠ   râ   r¼   rq  r½   rr  r¾   rž   r¿   rs  rp  rž   rÀ   rt  r    râ   )Nr·   NFFN)rº   râ   r»   râ   rŠ   râ   r¼   rq  r½   rr  r¿   rs  r¾   rž   rp  rž   rÀ   rt  r    râ   )NNFFN)rº   râ   r»   râ   rŠ   râ   r¼   rq  r¿   rs  r¾   rž   rp  rž   rÀ   rt  r    râ   )NNFrA  r·   FFN)rº   râ   r»   râ   rŠ   râ   r¼   rq  r¿   rs  r¾   rž   rI  r  rK  rr  rN  rž   rP  rž   rÀ   rt  r    râ   )NFNFFN)rº   râ   r»   râ   rŠ   râ   r¼   z0torch.Tensor | 'flex_attention.BlockMask' | Noner¾   rž   r¿   rs  rÁ   rž   rp  rž   rÀ   rt  r    râ   )T)r¼   râ   rÔ  ztorch.dtyperÕ  rž   r    râ   )Nr·   FNFFN)rº   râ   r»   râ   rŠ   râ   r¼   rq  r½   rr  r¾   rž   r¿   rs  rÁ   rž   rp  rž   rÀ   rt  r    râ   )Nr·   NFN)rº   râ   r»   râ   rŠ   râ   r¼   rq  r½   rr  r¿   rs  rp  rž   rÀ   rt  r    râ   )NFFN)rº   râ   r»   râ   rŠ   râ   r¼   rq  r¾   rž   rp  rž   rÀ   rt  r    râ   )NFNFN)rº   râ   r»   râ   rŠ   râ   r¼   rq  r¾   rž   r¿   rs  rp  rž   rÀ   rt  r    râ   )àÚ
__future__r   Ú
contextlibrµ  rƒ   rñ  Údataclassesr   Úenumr   Útypingr   r   r   rÜ   Útorch.distributedr  r  Útorch.nn.functionalrk  rl  r  Úis_availableÚ)torch.distributed._functional_collectivesÚ_functional_collectivesrø  Úutilsr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Úutils.constantsr   r   Úutils.torch_utilsr   Ú_modeling_parallelr   r   ró   rö   rø   Ú_REQUIRED_FLEX_VERSIONrü   rþ   r`   r�   rñ   rô   rõ   r÷   rù   rú   rû   rý   Ú
flash_attnr"   r#   Úflash_attn.flash_attn_interfacer$   r%   ÚImportErrorÚOSErrorrò   r?  rÄ   Úflash_attn_interfacerQ  r»  rO   rÁ  Úsageattentionr&   r'   r(   r)   r*   r+   Ú!torch.nn.attention.flex_attentionrÞ  rË  Ú	torch_npur,   Ú$torch_xla.experimental.custom_kernelr-   rõ  Úxformers.opsr�  r  Ú__version__ÚlibraryÚ	custom_opÚ
_custom_opÚregister_fakeÚ_register_faker=   rD   rŸ   rF   r}   r¤   ri   rj   rd   rf   ru   r°   r¯   Úcontextmanagerrm   r¶   rÎ   rÐ   rÕ   rÙ   rä   ræ   rê   rï   r³   Ú	lru_cacher  r  r  r$  r*  r4  r´   rW  rV  rr  r‰  rœ  r¦  r©  r­  r½  rÀ  rÃ  rÅ  rË  rÍ  rÔ  rÖ  rÛ  râ  ró  rõ  rü  r  r  r{  ÚFunctionr  r%  r)  r+  r-  r3  r<  r@  rB  rx  rŒ  r˜  r›  r’   rc   rŸ  r¡  r®  re   r°  rg   r³  r·  r¹  rh   r½  rk   rÂ  rl   rÐ  rÖ  rÛ  rn   rã  ro   rç  rp   rì  rq   rð  rr   rò  rs   rö  rt   rú  rü  rv   rþ  rw   r  rx   r  ry   r  rz   r
  r{   r  r3   r9   r6   Ú<module>rA     s—"  ðö #ã Û Û Û Ý !Ý ß /Ñ /ã Ý  ß Ð ð 	×Ñ×!Ñ!Ô#ß>Ð>÷÷ ÷ ÷ ó ÷$ LÝ 4Ý 3ñ Ý2à!Ð Ø!Ð Ø Ð Ø Ð ØÐ Ø%Ð á	�HÓ	€á-Ó/ÒhÑ4IÈ$ÐPgÓ4hÐ Ù1Ó3Ð Ù(Ó*Ò^Ñ/?ÀÐF]Ó/^Ð Ù/Ó1ÒlÑ6NÈtÐUkÓ6lÐ Ù% dÐ,BÓCÐ Ù*Ó,Ð Ù*Ó,ÒbÑ1EÀdÐLaÓ1bÐ Ù.Ó0ÒjÑ5HÈÐOiÓ5jÐ ñ ð+ßFßmÐmð €OØ!ÐØ#'Ð Ø"&Ðñ ð(ÝMÞ[ð ÐØ#Ðáð%ÞBð !Ðáð÷	
÷ 	
ð 	
ð$ €HØ$(Ð!Ø&*Ð#Ø#'Ð Ø(,Ð%Ø€Oñ ð÷ 	CÔBð €Nñ ð$Þ2ð  Ðñ ð#Þ_ð Ðñ ðÞ#ð €Dð 	×Ñ˜ÒØ—‘×(Ñ(€JØ—]‘]×0Ñ0�Nð*ÈÐUYô *ð*°4ÀQô *ð !€JØ(€Nô'˜3 ô '÷T.)ñ .)ðb ÷0ð 0ó ð0ð  ×%Ñ%Ñ'7Ø/Ø'ØGØIØô(ð ×,Ñ,Ñ.>Ø/Ø.Øô/ð
 ×"Ñ"Ñ$4Ø/Ø'ØOØQØô%ð ×)Ñ)Ñ+;Ø/Ø.Øô,ð
 ×!Ñ!Ñ#3Ø2Ø Øô$ð3IÐ ÐEó ðB ×ÑØ<P×<WÑ<Wó Bó ðBð0 &*ØØØØØ.2ð/ ð ,0Ø/3ñ/ Øð/ à	ð/ ð ð/ ð #ð	/ ð
 ð/ ð ð/ ð ð/ ð ð/ ð ,ð/ ð )ð/ ð -ð/ ð ó/ ólUó
LóLóóFóVð &*ð	QØðQà	ðQð ðQð #ð	Qð 
óQó0FðR €×Ò˜SÔ!ð
 #'ð	^Øð^àð^ð ð^ð  ò	^ó "ð^ð* #'ð	^Øð^àð^ð ð^ð  ó	^ð* &*Ø"&ð	fØð	fàð	fð ð	fð #ð		fð
  ð	fð 
ó	fó1òhó
óñD Ð:ÈÐZ`Ôað
 #'ØØ"Ø%)Ø%)Ø%)ØØØØ ØØð(Øð(àð(ð ð(ð  ð	(ð
 ð(ð 	ð(ð #ð(ð #ð(ð #ð(ð ð(ð ð(ð ð(ð ð(ð ð(ð ð(ð  'ò!(ó bð(ñV Ð>Ó?ð
 #'ØØ"Ø%)Ø%)Ø%)ØØØØ ØØð7Øð7àð7ð ð7ð  ð	7ð
 ð7ð 	ð7ð #ð7ð #ð7ð #ð7ð ð7ð ð7ð ð7ð ð7ð ð7ð ð7ð  'ò!7ó @ð7ðB &*ØØØØØØØ04ð(Ø	,ð(àð(ð 
ð(ð ð	(ð
 #ð(ð ð(ð ð(ð ð(ð ð(ð ð(ð ð(ð .ó(ðV",Ø	,ð",àó",ðZ &*ØØØØØØØ04ð5-Ø	,ð5-àð5-ð 
ð5-ð ð	5-ð
 #ð5-ð ð5-ð ð5-ð ð5-ð ð5-ð ð5-ð ð5-ð .ó5-ðt!,Ø	,ð!,àó!,ðX &*ØØØØØØØ04ð0-Ø	,ð0-àð0-ð 
ð0-ð ð	0-ð
 #ð0-ð ð0-ð ð0-ð ð0-ð ð0-ð ð0-ð ð0-ð .ó0-ðl,Ø	,ð,àó,ðP &*ØØØØØØØ04ð;-Ø	,ð;-àð;-ð 
ð;-ð ð	;-ð
 #ð;-ð ð;-ð ð;-ð ð;-ð ð;-ð ð;-ð ð;-ð .ó;-ð|#,Ø	,ð#,àó#,ðV &*ØØØØØØØ04ðB-Ø	,ðB-àðB-ð 
ðB-ð ð	B-ð
 #ðB-ð ðB-ð ðB-ð ðB-ð ðB-ð ðB-ð ðB-ð .óB-ðJ),Ø	,ð),àó),ðb &*ØØØØØØØ04ðU-ð $,ØØØ ØØñ'U-Ø	,ðU-àðU-ð 
ðU-ð ð	U-ð
 #ðU-ð ðU-ð ðU-ð ðU-ð ðU-ð ðU-ð ðU-ð .ðU-ð !ðU-ð ðU-ð  ð!U-ð" ð#U-ð$ ð%U-ð& ó'U-ðp0,Ø	,ð0,àó0,ðp &*ØØØØØØØ04ð#-Ø	,ð#-àð#-ð 
ð#-ð ð	#-ð
 #ð#-ð ð#-ð ð#-ð ð#-ð ð#-ð ð#-ð ð#-ð .ó#-ðV &*ØØØØØØØ04ð%-Ø	,ð%-àð%-ð 
ð%-ð ð	%-ð
 #ð%-ð ð%-ð ð%-ð ð%-ð ð%-ð ð%-ð ð%-ð .ó%-ðPVØ	,ðVàóVôð2 &*ØØØØØØØ04ð#Ø	,ð#àð#ð 
ð#ð ð	#ð
 #ð#ð ð#ð ð#ð ð#ð ð#ð ð#ð ð#ð .ó#ðN\Ø	,ð\àó\ò ó
ôFYôR.�U—^’^×,Ò,ô .ó2ó"
óó.óð ðØðØ-ðà òó ððB ò$ó ð$ôN|f˜UŸ^š^×4Ò4ô |fô~\f §¢× 7Ò 7ô \fô~Iw¨¯ª×(?Ò(?ô Iwðp 15ØØð;Øð;à	ð;ð ð;ð ð	;ð
 ð;ð ð;ð ð;ð ð;ð ð;ð .ð;ð ð;ð ó;ðD &*ØØØØØðW]ð 15ñW]ØðW]à	ðW]ð ðW]ð #ð	W]ð
 ðW]ð ðW]ð ðW]ð ðW]ð ðW]ð .óW]ðz ×#Ò#Ø×ÒØÐ =¸|ÐLØ"ð $ó ð &*ØØØØØ04ð--Øð--à	ð--ð ð--ð #ð	--ð
 ð--ð ð--ð ð--ð ð--ð .ð--ð ò--óð
--ð` ×#Ò#Ø×"Ñ"ØÐ =¸|ÐLØ"ð $ó ð &*ØØØØØ04ð.-Øð.-à	ð.-ð ð.-ð #ð	.-ð
 ð.-ð ð.-ð ð.-ð ð.-ð .ð.-ð ò.-óð
.-ðb ×#Ò#Ø×)Ñ)ØÐ =¸|ÐLØ#ð $ó ð &*ØØØØØ04ð1Øð1à	ð1ð ð1ð #ð	1ð
 ð1ð ð1ð ð1ð ð1ð .ð1ð ò1óð
1ðh ×#Ò#Ø×%Ò%ØÐ =¸|ÐLð $ó ð &*ØØØØØ04ð0Øð0à	ð0ð ð0ð #ð	0ð
 ð0ð ð0ð ð0ð ð0ð .ð0ð ò0ó	ð0ðf ×#Ò#Ø×!Ò!ØÐ =¸|ÐLð $ó ð &*ØØØØ04ð-Øð-à	ð-ð ð-ð #ð	-ð
 ð-ð ð-ð ð-ð .ð-ð ò-ó	ð-ð. ×#Ò#Ø×%Ñ%ØÐ =¸|ÐLØ"ð $ó ð &*ØØØ#+ØØØ#Ø04ðJØðJà	ðJð ðJð #ð	Jð
 ðJð ðJð !ðJð ðJð ðJð ðJð .ðJð òJóð
JðZ ×#Ò#Ø×,Ñ,ØÐ =¸|ÐLØ#ð $ó ð &*ØØØØ04ð.-Øð.-à	ð.-ð ð.-ð #ð	.-ð
 ð.-ð ð.-ð ð.-ð .ð.-ð ò.-óð
.-ðb ×#Ò#Ø×(Ò(ØÐ =¸|ÐLð $ó ð &*ØØØØ04ð3-Øð3-à	ð3-ð ð3-ð #ð	3-ð
 ð3-ð ð3-ð ð3-ð .ð3-ð ò3-ó	ð3-ðl ×#Ò#Ø×ÒØ#Ð%BÀLÐQð $ó ð &*ØØØØØ04ð&-Øð&-à	ð&-ð ð&-ð #ð	&-ð
 ð&-ð ð&-ð ð&-ð ð&-ð .ð&-ð ò&-ó	ð&-ðR ×#Ò#Ø×ÒØ+¨]¸LÐIð $ó ð CGØØØØØ04ð8Øð8à	ð8ð ð8ð @ð	8ð
 ð8ð ð8ð ð8ð ð8ð .ð8ð ò8ó	ð8ðx LPð"Øð"Ø+6ð"ØDHð"àó"ðJ ×#Ò#Ø×ÑØ Ð-Ø"ð $ó ð &*ØØØØØØ04ð8Øð8à	ð8ð ð8ð #ð	8ð
 ð8ð ð8ð ð8ð ð8ð ð8ð .ð8ð ò8óð
8ðv ×#Ò#Ø×&Ò&ØÐ =¸|ÐLØ"ð $ó ð &*ØØØØØØ04ð--Øð--à	ð--ð ð--ð #ð	--ð
 ð--ð ð--ð ð--ð ð--ð ð--ð .ð--ð ò--óð
--ð` ×#Ò#Ø×*Ò*Ø Ð-ð $ó ð &*ØØØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð ðð ðð .ðð òó	ðð< ×#Ò#Ø×&Ò&ØÐ =¸|ÐLØ"ð $ó ð &*ØØØØØØ04ð0-Øð0-à	ð0-ð ð0-ð #ð	0-ð
 ð0-ð ð0-ð ð0-ð ð0-ð ð0-ð .ð0-ð ò0-óð
0-ðf ×#Ò#Ø×%Ò%Ø Ð-ð $ó ð &*ØØØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð ðð ðð .ðð òó	ðð< ×#Ò#Ø×$Ò$ØÐ =¸|ÐLØ"ð $ó ð &*ØØØØ04ð-Øð-à	ð-ð ð-ð #ð	-ð
 ð-ð ð-ð ð-ð .ð-ð ò-óð
-ðb ×#Ò#Ø×$Ò$Ø Ð-ð $ó ð &*ØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð .ðð òó	ðð2 ×#Ò#Ø×ÒØ#Ð%BÀLÐQØ"ð $ó ð &*ØØØØ04ð+-Øð+-à	ð+-ð ð+-ð #ð	+-ð
 ð+-ð ð+-ð ð+-ð .ð+-ð ò+-óð
+-ð\ ×#Ò#Ø×!Ñ!Ø#Ð%BÀLÐQØ"ð $ó ð &*ØØØØ04ð,-Øð,-à	ð,-ð ð,-ð #ð	,-ð
 ð,-ð ð,-ð ð,-ð .ð,-ð ò,-óð
,-ð^ ×#Ò#Ø×$Ò$Ø#Ð%BÀLÐQð $ó ð &*ØØØØ04ð0Øð0à	ð0ð ð0ð #ð	0ð
 ð0ð ð0ð ð0ð .ð0ð ò0ó	ð0ðf ×#Ò#Ø×2Ò2Ù0°°AÓ6¸ÐEð $ó ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð òó	ðð. ×#Ò#Ø×7Ò7Ù0°°AÓ6¸ÐEð $ó ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð òó	ðð. ×#Ò#Ø×3Ò3Ù0°°AÓ6¸ÐEð $ó ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð òó	ðð. ×#Ò#Ø×5Ò5Ù0°°AÓ6¸ÐEð $ó ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð òó	ðð. ×#Ò#Ø×!Ò!Ø+¨]¸LÐIð $ó ð &*ØØØØØØ04ð@Øð@à	ð@ð ð@ð #ð	@ð
 ð@ð ð@ð ð@ð ð@ð ð@ð .ð@ð ò@ó	ñ@øðEj ˜ ,Ð/ò +ð 	�‰ÐGÈÀsÐJuÐvÔwØ#ÐØˆØ!%ÐØ'+Ð$Ø&*Ö#ûð+ûð( ˜ ,Ð/ò (Ø�‰Ð8¸¸Ð;^Ð_Ô`Ø %ÐØ ÐØ#'Ö ûð	(ûð ˜ ,Ð/ò %Ø�‰Ð1°!°Ð4WÐXÔYØ#ÐØ $Öûð%ûð" ˜ ,Ð/ò Ø�‰Ð9¸!¸Ð<_Ð`ÔaØ"ÐØˆØ'+Ð$Ø,0Ð)Ø(,Ð%Ø*.Ð'ØŽûðûð0 ˜ ,Ð/ò Ø�‰Ð:¸1¸#Ð=`ÐaÔbØ"ÐØŽûðûð ˜ ,Ð/ò $Ø�‰Ð5°a°SÐ8[Ð\Ô]Ø!ÐØ#Öûð$ûð ˜ ,Ð/ò #Ø�‰Ð5°a°SÐ8[Ð\Ô]Ø!ÐØ"Öûð#ûð ˜ ,Ð/ò Ø�‰Ð4°Q°CÐ7ZÐ[Ô\Ø!&ÐØŽûðúsÀ   Ä,t Åu Åv Å(v0 Æw) Æx Æ&y Æ2y6 ôuô#uõuõu>õu9õ9u>öv-ö	v(ö(v-ö0w&ö8#w!÷!w&÷)x÷1xøxøyø x?ø?yùy3ùy.ù.y3ù6z"ù>zúz"