+
    QV-jà7  ã                   ó¾  € ^ RI Ht ^RIHt ^RIHt ^RIHtHtH	t	 ^RI
HtHtHtHtHt ]! 4       '       d   ^ RIt^ RIHt ]! 4       '       d   ^ RIHt ]! 4       t]! 4       '       d   ]'       g   ^ RIt]P.                  ! ]4      t ! R	 R
]4      t ! R R]P"                  P6                  4      t ! R R]P:                  4      t]! ^R7      R 4       tRR R llt R# )é    )Ú	lru_cache)ÚACT2FN)ÚConversionOps)Úget_module_from_nameÚ	on_deviceÚshould_convert_module)Úis_accelerate_availableÚis_fbgemm_gpu_availableÚis_torch_availableÚis_torch_xpu_availableÚloggingN)Únn)Úinit_empty_weightsc                   ó<   a € ] tR t^,t o R tRV 3R lR lltRtV tR# )ÚFbgemmFp8Quantizec                ó   € Wn         R # ©N©Úhf_quantizer)Úselfr   s   &&Úu/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/integrations/fbgemm_fp8.pyÚ__init__ÚFbgemmFp8Quantize.__init__-   s   € Ø(Öó    Nc                óê   <€ V ^8„  d   QhRS[ S[S[P                  S[S[P                  ,          ,          3,          RS[P
                  P                  R,          RS[ S[S[P                  3,          /# )é   Ú
input_dictÚmodelNÚreturn)ÚdictÚstrÚtorchÚTensorÚlistr   ÚModule)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚFbgemmFp8Quantize.__annotate__0   se   ø€ ÷ 5`ñ 5`á™™eŸl™l©T±%·,±,Õ-?Õ?Ð?Õ@ð5`ñ �x‰x�‰ Õ%ð5`ñ
 
‰c‘5—<‘<ÐÕ	 ñ5`r   c                óæ  € \        VP                  4       4      ^ ,          w  rEV^ ,          p^RIHp \	        W$4      w  rx\        Wv4      '       EdZ   VR8X  d§   VP                  ^^4      p	V	P                  p
V	P                  RV
R,          4      p\        V4      ;_uu_ 4        \        V4      w  rÍRRR4       XP                  V
4      pVP                  ^^4      pXP                  V
^ ,          ^V
^,          4      pEMVR8X  d¥   VP                  ^^4      p	V	P                  p
V	P                  RV
R,          4      p\        V4      ;_uu_ 4        \        V4      w  rÍRRR4       XP                  V
4      pVP                  ^^4      pXP                  V
^ ,          V
^,          ^4      pMi\        V4      ;_uu_ 4        \        V4      w  rïRRR4       \        P                  P                  XP                  VP                  ^ ,          ^4      4      pV\        P                  P                  X4      V R2X/#   + '       g   i     EL’; i  + '       g   i     Lø; i  + '       g   i     L�; i)r   )ÚFbgemmFp8Llama4TextExpertsÚgate_up_projNÚ	down_projÚ_scaleéÿÿÿÿ)ÚtupleÚitemsÚintegrationsr+   r   Ú
isinstanceÚ	transposeÚshapeÚreshaper   Úquantize_fp8_per_rowr"   r   Ú	ParameterÚview)r   r   r   ÚkwargsÚ
target_keyÚvaluer+   ÚmoduleÚtensor_nameÚtransposed_paramÚoriginal_shapeÚflattened_paramÚnew_value_flatÚweight_scale_flatÚ	new_valueÚweight_scales   &&&,            r   ÚconvertÚFbgemmFp8Quantize.convert0   s  € ô " *×"2Ñ"2Ó"4Ó5°aÕ8Ñˆ
Ø�a•ˆå=ä2°5ÓEÑˆä�f×9Ó9Ø˜nÔ,ð $)§?¡?°1°aÓ#8Ð ð "2×!7Ñ!7�Ø"2×":Ñ":¸2¸~ÈbÕ?QÓ"R�ô ˜×/Õ/Ü8LÈ_Ó8]Ñ5�N÷ 0ð +×2Ñ2°>ÓB�	Ø%×/Ñ/°°1Ó5�	Ø0×8Ñ8¸ÈÕ9JÈAÈ~Ð^_ÕO`Óa’Ø Ô+ð $)§?¡?°1°aÓ#8Ð ð "2×!7Ñ!7�Ø"2×":Ñ":¸2¸~ÈbÕ?QÓ"R�ô ˜×/Õ/Ü8LÈ_Ó8]Ñ5�N÷ 0ð +×2Ñ2°>ÓB�	Ø%×/Ñ/°°1Ó5�	Ø0×8Ñ8¸ÈÕ9JÈNÐ[\ÕL]Ð_`Óa�øä˜5×!Õ!Ü*>¸uÓ*EÑ'�	÷ "ä Ÿ8™8×-Ñ-¨l×.?Ñ.?À×@RÑ@RÐSTÕ@UÐWXÓ.YÓZˆLàœEŸH™H×.Ñ.¨yÓ9¸j¸\ÈÐ;PÐR^Ð_Ð_÷; 0×/Ð/ú÷" 0×/ú÷ "×!ús$   Â"H9ÅIÆ=I È9I
	ÉI	É I0	r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   rF   Ú__static_attributes__Ú__classdictcell__)r'   s   @r   r   r   ,   s   ø‡ € ò)÷5`÷ 5`ð 5`r   r   c                   óR   a a€ ] tR t^ht o]P
                  3V 3R lltR tRtVt	V ;t
# )ÚFbgemmFp8Linearc                ó‚  <€ \         SV `  WV4       Wn        W n        \        P
                  P                  \        P                  ! W!3VR 7      4      V n        \        P
                  P                  \        P                  ! V^3\        P                  R 7      4      V n
        V P                  R\        P                  ! ^.\        P                  R 7      RR7       V'       dU   \        P
                  P                  \        P                  ! V P                  \        P                  R 7      4      V n        R# RV n        R# )©ÚdtypeÚinput_scale_ubF©Ú
persistentN)Úsuperr   Úin_featuresÚout_featuresr"   r   r8   ÚzerosÚweightÚfloat32rE   Úregister_bufferÚfloatÚbias)r   rW   rX   r^   rR   Ú	__class__s   &&&&&€r   r   ÚFbgemmFp8Linear.__init__i   sÑ   ø€ Ü‰Ñ˜°DÔ9Ø&ÔØ(Ôä—h‘h×(Ñ(¬¯ª°lÐ5PÐX]Ô)^Ó_ˆŒÜ!ŸH™H×.Ñ.¬u¯{ª{¸LÈ!Ð;LÔTY×TaÑTaÔ/bÓcˆÔØ×ÑÐ-¬u¯{ª{¸A¸3ÄeÇkÁkÔ/RÐ_dÐÔeçÜŸ™×*Ñ*¬5¯;ª;¸×8IÑ8IÔRW×R_ÑR_Ô+`ÓaˆDŽIàˆDŽIr   c           	     ó.  € . VP                   R R ORN5p\        V4      ;_uu_ 4        \        VP                  RVP                   R,          4      P	                  4       V P
                  R7      w  r4R R R 4       V P                  P                  \        P                  4      p\        '       dg   \        P                  ! XV P                  P                  4       XP                  R4      VP                  4       VP                  V P                   R7      pMZ\        P"                  P$                  P'                  XV P                  XVRR7      pV P                   e   W`P                   ,           MTpVP)                  V4      p??V#   + '       g   i     EL; i)N)Úscale_ub)Úscale_aÚscale_bÚ	out_dtyper^   T©Úuse_fast_accumr/   )r5   r   r7   r9   Ú
contiguousrS   rE   Útor"   r[   Ú_is_torch_xpu_availableÚ
_scaled_mmrZ   ÚtÚ	unsqueezerR   r^   ÚopsÚfbgemmÚf8f8bf16_rowwiser6   )r   ÚxÚoutput_shapeÚx_quantizedÚx_scaleÚweight_scale_float32Úoutputs   &&     r   ÚforwardÚFbgemmFp8Linear.forwardw   s7  € à*˜Ÿ™  "˜Ð* rÑ*ˆô �q�\�\Ü#7Ø—‘�r˜1Ÿ7™7 2�;Ó'×2Ñ2Ó4¸t×?RÑ?Rô$Ñ ˆK÷ ð
  $×0Ñ0×3Ñ3´E·M±MÓBÐß"Ó"Ü×%Ò%ØØ—‘—‘“Ø×)Ñ)¨"Ó-Ø,×.Ñ.Ó0ØŸ'™'Ø—Y‘Yô‰Fô —Y‘Y×%Ñ%×6Ñ6Ø˜TŸ[™[¨'Ð3GÐX\ð 7ó ˆFð ,0¯9©9Ò+@�VŸi™iÖ'ÀfˆFØ—‘ Ó-ˆØ˜Øˆ÷- �\ˆ\ús   ¨A	FÆF	)r^   rW   rX   rZ   rE   )rH   rI   rJ   rK   r"   Úfloat8_e4m3fnr   rw   rL   rM   Ú__classcell__©r_   r'   s   @@r   rO   rO   h   s   ù‡ € Ø>C×>QÑ>Q÷ ÷ò r   rO   c                   óR   a a€ ] tR t^–t o]P
                  3V 3R lltR tRtVt	V ;t
# )r+   c                ó’  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        V P                  V n        \        VP                  ,          V n	        \        P                  P                  \        P                  ! V P                  V P
                  ^V P                  ,          3\        P                  R7      4      V n        \        P                  P                  \        P                  ! V P                  ^V P                  ^,          3\        P                   R7      4      V n        \        P                  P                  \        P                  ! V P                  V P                  V P
                  3\        P                  R7      4      V n        \        P                  P                  \        P                  ! V P                  V P
                  ^3\        P                   R7      4      V n        V P)                  R\        P                  ! ^.\        P*                  R7      RR7       R# )r   rQ   rS   FrT   N)rV   r   Únum_local_expertsÚnum_expertsÚintermediate_sizeÚhidden_sizeÚ
expert_dimr   Ú
hidden_actÚact_fnr"   r   r8   rY   ry   r,   r[   Úgate_up_proj_scaler-   Údown_proj_scaler\   r]   )r   ÚconfigrR   r_   s   &&&€r   r   Ú#FbgemmFp8Llama4TextExperts.__init__—   sŒ  ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ!'×!9Ñ!9ˆÔØ!×-Ñ-ˆÔØ×0Ñ0ˆŒÜ˜V×.Ñ.Õ/ˆŒä!ŸH™H×.Ñ.Ü�KŠK˜×)Ñ)¨4×+;Ñ+;¸QÀÇÁÕ=PÐQÔY^×YlÑYlÔmó
ˆÔô #(§(¡(×"4Ñ"4Ü�KŠK˜×)Ñ)¨1¨d¯o©oÀÕ.AÐBÌ%Ï-É-ÔXó#
ˆÔô Ÿ™×+Ñ+Ü�KŠK˜×)Ñ)¨4¯?©?¸D×<LÑ<LÐMÔUZ×UhÑUhÔió
ˆŒô  %Ÿx™x×1Ñ1Ü�KŠK˜×)Ñ)¨4×+;Ñ+;¸QÐ?ÄuÇ}Á}ÔUó 
ˆÔð 	×ÑÐ-¬u¯{ª{¸A¸3ÄeÇkÁkÔ/RÐ_dÐÖer   c           
     ó  € VP                  V P                  RV P                  4      pRp\        P                  ! V4      p\        V P                  4       EFÑ  pW,          pVP                  RV P                  4      p\        V4      ;_uu_ 4        \        WbV P                  4      w  rxRRR4       V P                  P                  R,          ^,          p	V P                  P                  \        P                  4      p
\        '       Edc   \        P                   ! XV P                  V,          P#                  ^ ^4      RV	 P%                  4       P'                  4       XP)                  R4      W¤,          ^ ,          RV	 P                  R^4      P%                  4       P'                  4       VP*                  R7      p\        P                   ! VV P                  V,          P#                  ^ ^4      V	R P%                  4       P'                  4       VP)                  R4      W¤,          ^ ,          V	R P                  R^4      P%                  4       P'                  4       VP*                  R7      pEM\        P,                  P.                  P1                  XV P                  V,          P#                  ^ ^4      RV	 P%                  4       XW¤,          ^ ,          RV	 P                  R^4      P%                  4       RR7      p\        P,                  P.                  P1                  VV P                  V,          P#                  ^ ^4      V	R P%                  4       VW¤,          ^ ,          V	R P                  R^4      P%                  4       RR7      pWÀP3                  V4      ,          p\        V4      ;_uu_ 4        \        WÒV P                  4      w  rïRRR4       V P4                  P                  \        P                  4      p\        '       d˜   \        P                   ! XV P6                  V,          P#                  ^ ^4      P%                  4       XP)                  R4      VV,          P                  R^4      P%                  4       P'                  4       VP*                  R7      pM‚\        P,                  P.                  P1                  XV P6                  V,          P#                  ^ ^4      P%                  4       XVV,          P                  R^4      P%                  4       RR7      pVW4&   EKÔ  	  VP                  VP8                  4      pVP                  RV P                  4      #   + '       g   i     ELÂ; i  + '       g   i     EL³; i)z�
Args:
    hidden_states (torch.Tensor): (batch_size * token_num, hidden_size)
Returns:
    torch.Tensor: (batch_size * token_num, hidden_size)
N)rc   rd   re   Trf   r/   )r9   r   r�   r"   Ú
empty_likeÚranger6   r   r7   rS   r,   r5   r…   ri   r[   rj   rk   r4   rh   rl   rm   rR   rn   ro   rp   r„   r†   r-   Údevice)r   Úhidden_statesÚ
num_tokensÚnext_statesÚiÚexpert_hiddenÚexpert_hidden_reshapedÚexpert_quantizedÚexpert_scaleÚsharded_expert_dimÚgate_up_proj_scale_float32ÚgateÚupÚ	activatedÚactivated_quantizedÚactivated_scaleÚdown_proj_scale_float32Úexpert_outputs   &&                r   rw   Ú"FbgemmFp8Llama4TextExperts.forward¯   s™  € ð &×*Ñ*¨4×+;Ñ+;¸RÀ×AQÑAQÓRˆØˆ
ô ×&Ò& }Ó5ˆä�t×'Ñ'×(ˆAà)Õ,ˆMØ%2×%:Ñ%:¸2¸t×?OÑ?OÓ%PÐ"äÐ1×2Õ2Ü1EØ*¸×8KÑ8Kó2Ñ.Ð ÷ 3ð "&×!2Ñ!2×!8Ñ!8¸Õ!<ÀÕ!AÐØ)-×)@Ñ)@×)CÑ)CÄEÇMÁMÓ)RÐ&ß&Ô&Ü×'Ò'Ø$Ø×%Ñ% aÕ(×2Ñ2°1°aÓ8Ð9LÐ:LÐM×XÑXÓZ×\Ñ\Ó^Ø(×2Ñ2°2Ó6Ø6Õ9¸!Õ<Ð=PÐ>PÐQ×VÑVÐWYÐ[\Ó]×hÑhÓj×lÑlÓnØ+×1Ñ1ô�ô ×%Ò%Ø$Ø×%Ñ% aÕ(×2Ñ2°1°aÓ8Ð9KÐ9LÐM×XÑXÓZ×\Ñ\Ó^Ø(×2Ñ2°2Ó6Ø6Õ9¸!Õ<Ð=OÐ=PÐQ×VÑVÐWYÐ[\Ó]×hÑhÓj×lÑlÓnØ+×1Ñ1ô’ô —y‘y×'Ñ'×8Ñ8Ø$Ø×%Ñ% aÕ(×2Ñ2°1°aÓ8Ð9LÐ:LÐM×XÑXÓZØ Ø.Õ1°!Õ4Ð5HÐ6HÐI×NÑNÈrÐSTÓU×`Ñ`ÓbØ#'ð 9ó �ô —Y‘Y×%Ñ%×6Ñ6Ø$Ø×%Ñ% aÕ(×2Ñ2°1°aÓ8Ð9KÐ9LÐM×XÑXÓZØ Ø.Õ1°!Õ4Ð5GÐ5HÐI×NÑNÈrÐSTÓU×`Ñ`ÓbØ#'ð 7ó �ð Ÿ[™[¨Ó.Õ.ˆIä˜9×%Õ%Ü7KÈIÐcg×cvÑcvÓ7wÑ4Ð#÷ &ð '+×&:Ñ&:×&=Ñ&=¼e¿m¹mÓ&LÐ#ß&Ó&Ü %× 0Ò 0Ø'Ø—N‘N 1Õ%×/Ñ/°°1Ó5×@Ñ@ÓBØ+×5Ñ5°bÓ9Ø3°AÕ6×;Ñ;¸BÀÓB×MÑMÓO×QÑQÓSØ+×1Ñ1ô!‘ô !&§	¡	× 0Ñ 0× AÑ AØ'Ø—N‘N 1Õ%×/Ñ/°°1Ó5×@Ñ@ÓBØ#Ø+¨AÕ.×3Ñ3°B¸Ó:×EÑEÓGØ#'ð !Bó !�ð +ˆKŒNñE )ðF "—n‘n ]×%9Ñ%9Ó:ˆØ×Ñ  D×$4Ñ$4Ó5Ð5÷ 3×2Ð2ú÷P &×%Ð%ús   ÂU"Î3U6Õ"U3Õ6V)	r„   r-   r†   r‚   r,   r…   r�   r€   r   )rH   rI   rJ   rK   r"   r[   r   rw   rL   rM   rz   r{   s   @@r   r+   r+   –   s   ù‡ € Ø%*§]¡]÷ f÷0R6ò R6r   r+   )Úmaxsizec                  ó’   € \         '       d   ^RIHp  V ! R4      P                  # \        P
                  P                  P                  # )é   ©Ú
get_kernelzkernels-community/fp8-fbgemm)rj   Úhub_kernelsr£   r7   r"   rn   ro   r¢   s    r   Úget_quantize_fp8_per_rowr¥     s3   € çÓÝ+áÐ8Ó9×NÑNÐNÜ�9‰9×Ñ×0Ñ0Ð0r   c                óH   € V ^8„  d   QhR\         \        ,          R,          /# )r   Úmodules_to_not_convertN)r$   r!   )r&   s   "r   r(   r(     s   € ÷ :ñ :Ü#'¬¥9¨tÕ#3ñ:r   c           	     óÜ  € \        4       sRpV'       d   / MRR/pV P                  4        EF  w  rx\        Wq4      '       g   K  Rp	\	        RR7      ;_uu_ 4        VP
                  P                  R8X  dB   \        V P                  RV P                  4      p
\        T
;'       g    V P                  4      p	M_\        V\        P                  4      '       d@   \        VP                  VP                  VP                   RJ3/ VB p	V	P#                  R4       RRR4       V	f   Kù  V P%                  Wy4       RpEK  	  V'       g   \&        P)                  R4       V #   + '       g   i     LM; i)	aß  
A helper function to replace all `torch.nn.Linear` modules by `FbgemmFp8Linear` modules.
This will enable running your models using high performance fp8 kernel from FBGEMM library.

Parameters:
    model (`torch.nn.Module`):
        Input model or `torch.nn.Module` as the function is run recursively.
    modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
        Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons.
    quantization_config (`FbgemmFp8Config`):
        The quantization config object that contains the quantization parameters.
    pre_quantized (`book`, defaults to `False`):
        Whether the model is pre-quantized or not
FrR   NT)Úinclude_buffersÚLlama4TextExpertsÚtext_configzÇYou are loading your model using FP8 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)r¥   r7   Únamed_modulesr   r   r_   rH   Úgetattrr‡   r+   r3   r   ÚLinearrO   rW   rX   r^   Úrequires_grad_Úset_submoduleÚloggerÚwarning)r   r§   Úquantization_configÚpre_quantizedÚtp_planÚhas_been_replacedÚmodule_kwargsÚmodule_namer=   Ú
new_moduler«   s   &&&&&      r   Úreplace_with_fbgemm_fp8_linearrº     s4  € ô$ 4Ó5ÐàÐß'‘B¨g°t¨_€Mà$×2Ñ2×4ÑˆÜ$ [×IÒIÙàˆ
Ü°×5Ö5Ø×Ñ×(Ñ(Ð,?Ô?ô
 & e§l¡l°MÀ5Ç<Á<ÓP�Ü7¸×8SÐ8SÀuÇ|Á|ÓT‘
Ü˜F¤B§I¡I×.Ò.Ü,Ø×&Ñ&Ø×'Ñ'Ø—K‘K tÐ+ñð $ñ	�
ð ×)Ñ)¨%Ô0÷ 6ð" ÒÙà×Ñ˜KÔ4Ø Óñ5  5÷8 Ü�‰ðô	
ð €L÷= 6×5ús   ÁA
EÂ%A1EÅE+)NNFN)!Ú	functoolsr   Úactivationsr   Úcore_model_loadingr   Úquantizers.quantizers_utilsr   r   r   Úutilsr	   r
   r   r   r   r"   r   Ú
accelerater   rj   Úfbgemm_gpu.experimental.gen_aiÚ
fbgemm_gpuÚ
get_loggerrH   r±   r   r®   rO   r%   r+   r¥   rº   © r   r   Ú<module>rÅ      s¿   ðõ  å  Ý .ß `Ñ `÷õ ñ ×ÒÛÝá×ÒÝ-á0Ó2Ð á×Ò×%<Û)à	×	Ò	˜HÓ	%€ô9`˜ô 9`ôx+�e—h‘h—o‘oô +ô\k6 §¡ô k6ñ\ �1Ôñ1ó ð1÷:ñ :r   