+
    QV-jG»  ã                  óŠ  € ^ RI Ht ^ RIt^ RIt^ RIHt ^ RIHt ^ RIt^ RI	H
t
 ^ RI	Ht ^RIHt ^RIHt ^RIHtHt ^R	IHt ^R
IHtHt ^RIHtHtHt ^RIHt ^RIH t H!t! ^RI"H#t# ]PH                  ! ]%4      t&]PN                  t(]PR                  ! ](4      PT                  t+]PR                  ! ](4      PX                  t-]P\                  R R l4       t/R t0]! RR7       ! R R4      4       t1]P\                  R R l4       t2]Pf                  Ph                  R R l4       t5R R lt6R R lt7R8R R llt8R9R  R! llt9R9R" R# llt: ! R$ R%]
Pv                  4      t< ! R& R']<4      t=R( R) lt>R* R+ lt? ! R, R-]
P€                  4      tA ! R. R/] 4      tB]B! 4       tCR:R0 R1 lltD ! R2 R3]4      tE ! R4 R5]4      tF ! R6 R7]4      tGR# );é    )ÚannotationsN)ÚCallable)Ú	dataclass)Ú
functional)ÚACT2FN)ÚConversionOps)Úget_module_from_nameÚshould_convert_module)Úlogging)Úis_kernels_availableÚis_torchdynamo_compiling)Ú deepgemm_fp8_fp4_experts_forwardÚdeepgemm_fp8_fp4_linearÚ(deepgemm_fp8_fp4_megamoe_experts_forward)Úlazy_load_kernel)ÚExpertsInterfaceÚuse_experts_implementation)Úto_localc               ó   € V ^8„  d   QhRR/# )é   Úreturnútorch.dtype© )Úformats   "Úz/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/integrations/finegrained_fp8.pyÚ__annotate__r   1   s   € ÷  ñ  ˜+ñ  ó    c                 óˆ   € \        \        R4      '       g   \        R\        P                   R24      h\        P                  # )zTReturn ``torch.float8_e8m0fnu`` or raise a clear error on torch without FP8 support.Úfloat8_e8m0fnuzbscale_fmt='ue8m0' requires torch.float8_e8m0fnu, which is only available in PyTorch >= 2.7 (found z.). Upgrade torch to use UE8M0 FP8 checkpoints.)ÚhasattrÚtorchÚRuntimeErrorÚ__version__r   r   r   r   Ú_get_ue8m0_dtyper$   0   sG   € ô ”5Ð*×+Ò+Üð%Ü%*×%6Ñ%6Ð$7Ð7eðgó
ð 	
ô ×ÑÐr   c                ó–   € V F!  p\        W4      '       g   K  \        W4      u # 	  \        \        V 4      P                   R V 24      h)z has none of: )r    ÚgetattrÚAttributeErrorÚtypeÚ__name__)ÚobjÚnamesÚnames   &* r   Ú_first_attrr-   ;   sE   € ÛˆÜ�3×ÔÜ˜3Ó%Ò%ñ ô œD ›I×.Ñ.Ð/¨~¸e¸WÐEÓ
FÐFr   T)Úfrozenc                  ó:   € ] tR t^Bt$ RtR]R&   R]R&   R]R&   RtR# )ÚFineGrainedFP8zNEntry points exposed by the `kernels-community/finegrained-fp8` Triton kernel.r   ÚmatmulÚbatched_matmulÚgrouped_matmulr   N)r)   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__r   r   r   r0   r0   B   s   ‡ áXàÓØÓØ×r   r0   c               ó   € V ^8„  d   QhRR/# ©r   r   r0   r   )r   s   "r   r   r   L   s   € ÷ +ñ + nñ +r   c                 ó–  € \        4       '       g   \        4       '       g   \        R4      h\        R4      p V f   \        R4      h\	        V RR4      p\	        V RR4      p\	        V RR4      pRV3RV3RV33 UUu. uF  w  rEVe   K  VNK  	  pppV'       d   \        RR	P                  V4       R
24      h\        VVVR7      # u uppi )z¸
Load the finegrained-fp8 Triton kernel once and return its entry points.

Raises `ImportError` if the `kernels` package is missing, or the kernel or required
symbols cannot be found.
z`finegrained-fp8 kernel requires the `kernels` package. Install it with `pip install -U kernels`.zfinegrained-fp8Nu‰   Failed to load the finegrained-fp8 kernel â€” check that `kernels-community/finegrained-fp8` has a build matching the current torch/CUDA.r1   Úmatmul_batchedÚmatmul_groupedz4finegrained-fp8 kernel is missing required symbols: ú, zA. Please update the `kernels` package (`pip install -U kernels`).)r1   r2   r3   )r   r   ÚImportErrorr   r&   Újoinr0   )Úkernelr1   r2   r3   r,   ÚattrÚmissings          r   Ú_load_finegrained_fp8_kernelrE   K   s  € ô $×%Ò%Ü#×%Ò%ÜØróð ô Ð/Ó0€FØ‚~Üð;ó
ð 	
ô
 �V˜X tÓ,€FÜ˜VÐ%5°tÓ<€NÜ˜VÐ%5°tÓ<€Nð
 �vÐØ˜~Ð.Ø˜~Ð.ñ
ôñ
‰JˆDð
 ÷ 	ˆñ
ð ñ ÷ ÜØBÀ4Ç9Á9ÈWÓCUÐBVð WNð Nó
ð 	
ô
 ØØ%Ø%ôð ùós   Á;CÂCc               ó   € V ^8„  d   QhRR/# )r   r   ÚNoner   )r   s   "r   r   r   {   s   € ÷ ñ ¨$ñ r   c                 ó   € \        4       p R # ©N)rE   )Ú_s    r   Ú _populate_finegrained_fp8_kernelrK   z   s   € ä$Ó&€AÙr   c               ó   € V ^8„  d   QhRR/# r;   r   )r   s   "r   r   r   €   s   € ÷ *ñ * ^ñ *r   c                 óJ   € \        4       '       d   \        4        \        4       # rI   )r   rK   rE   r   r   r   Úload_finegrained_fp8_kernelrN   €   s   € Ü×!Ò!Ü(Ô*Ü'Ó)Ð)r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   ÚaÚintÚbr   r   )r   s   "r   r   r   †   s!   € ÷ ñ ˆSð �Sð ˜Sñ r   c                ó.   € W,           ^,
          V,          # )zCeiling division.r   )rP   rR   s   &&r   Ú_cdivrT   †   s   € à�E�A�I˜!ÕÐr   c               ó@   € V ^8„  d   QhRRRRRRRRRRRRR	R
RR
RRRR/
# )r   Únum_expertsrQ   Úproj_outÚproj_inÚweight_dtyper   Úsf_dtypeÚweight_k_divÚ	sf_gran_nz
int | NoneÚ	sf_gran_kÚ
min_sf_outr   z!tuple[nn.Parameter, nn.Parameter]r   )r   s   "r   r   r   ‹   sl   € ÷ ñ Øðàðð ðð ð	ð
 ðð ðð ðð ðð ðð 'ñr   c	                ób  € \         P                  ! WW%,          VR7      p	\        P                  ! W™P	                  4       R7      p
\        Ve   \        W4      M^V4      pVe   \        W'4      M^p\         P                  ! WWÄR7      p\        P                  ! WÝP	                  4       R7      pW®3# )u¿  Allocate `(weight, weight_scale_inv)` parameters for one expert projection.

`weight_k_div` halves the K dim for FP4-packed storage (2 e2m1 values per byte).
`sf_gran_n` / `sf_gran_k` set per-block (None â†’ per-row/per-tensor) SF granularity.
`min_sf_out` floors the SF tensor's output dim â€” used by the fused gate_up
projection to keep room for both halves (pass `2`) even when `proj_out < sf_gran_n`
would otherwise collapse the SF dim to 1.
©Údtype)Úrequires_grad)r!   ÚemptyÚnnÚ	ParameterÚis_floating_pointÚmaxrT   )rV   rW   rX   rY   rZ   r[   r\   r]   r^   Úweight_tÚweightÚsf_outÚsf_inÚsf_tÚsfs   &&&&&&&&&      r   Ú_alloc_expert_projrn   ‹   s‰   € ô& �{Š{˜;°'Õ2IÐQ]Ô^€HÜ�\Š\˜(×2LÑ2LÓ2NÔO€FÜ¨yÒ/D”�xÔ+È!ÈZÓX€FØ)2Ò)>ŒE�'Ô%ÀA€EÜ�;Š;�{¨EÔB€DÜ	�Š�d×*@Ñ*@Ó*BÔ	C€BØˆ:Ðr   c               ó8   € V ^8„  d   QhRRRRRRRRRRR	RR
RRR/# ©r   Úinputútorch.Tensorri   Úweight_scale_invÚ
block_sizezlist[int] | NoneÚbiasútorch.Tensor | NoneÚactivation_scaleÚoutput_dtypeútorch.dtype | Noner   r   )r   s   "r   r   r   §   sX   € ÷ ñ Øðàðð #ðð !ð	ð
 ðð *ðð %ðð ñr   c           	     ór   € \        4       pVP                  V VVVVVR7      pVe   VP                  V4       V# )u   Triton FP8/FP4 linear: fused act-quant + matmul, then optional bias add.

``activation_scale=None`` â†’ dynamic per-K-block scales (inline); set it for
static per-tensor quant. ``weight_scale_inv`` accepts fp32 or UE8M0; the
dispatcher routes FP4 (``int8``-packed) weights automatically.
©rw   )rN   r1   Úadd_)	rq   ri   rs   rt   ru   rw   rx   Úfinegrained_fp8Úoutputs	   &&&&&&&  r   Úfinegrained_fp8_linearr   §   sL   € ô 2Ó3€OØ×#Ñ#ØØØØØØ)ð $ó €Fð ÒØ�‰�DÔØ€Mr   c               ó8   € V ^8„  d   QhRRRRRRRRRRR	RR
RRR/# rp   r   )r   s   "r   r   r   Ä   sl   € ÷ Buñ BuØðBuàðBuð #ðBuð !ð	Buð
 ðBuð *ðBuð %ðBuð ñBur   c           
     ó¢  € VRJ ;'       dì    VP                   P                  R8H  ;'       dË    \        P                  P	                  4       P
                  ^	8¬  ;'       d˜    VP                  \        P                  8H  ;'       g9    VRJ;'       dg    V^ ,          V^,          u;8H  ;'       d    ^€8H  Mu ;'       d;    \        P                  P                  RR4      R8g  ;'       d    \        4       '       * pV'       d    \        V VVVVVVR7      # \!        WW#WEV4      #   \         d#   p\        P                  RT 24        Rp?L6Rp?ii ; i)u¨  End-to-end FP8/FP4 linear used by `FP8Linear` and the eager `FP8Experts` loop.

Dispatch order â€” both backends handle FP8 and FP4 weights with fp32 or UE8M0 scales:
  1. DeepGEMM (`deepgemm_fp8_fp4_linear`) â€” 3-6Ã— faster on the shapes it supports.
     Preferred for FP4, UE8M0 SFs, and 128Ã—128 block FP8.
  2. Triton finegrained-fp8 fallback â€” used when DeepGEMM is unavailable, when the
     caller passes ``activation_scale`` (DeepGEMM is dynamic-only), or for any
     shape DeepGEMM declined.

Args:
    input: (..., K) bf16/fp16 activations.
    weight: (N, K) `float8_e4m3fn` or (N, K // 2) `int8` (FP4-packed).
    weight_scale_inv: per-block weight scales â€” `float32` (V3-style) or `float8_e8m0fnu`
        (V4-style; reinterpreted as int32 at the DeepGEMM kernel boundary).
    block_size: [block_n, block_k] for FP8 block-wise quant, or None/[N, K] for per-tensor.
        Ignored for FP4 weights (the kernel infers SF granularity from the dtype).
    bias: optional bias added to the matmul output.
    activation_scale: pass a per-tensor scalar to use static activation quant; leave `None`
        for dynamic (per-token) quant.
    output_dtype: desired output dtype.
NÚcudaÚ$TRANSFORMERS_DISABLE_DEEPGEMM_LINEARÚ0Ú1)rt   rx   rw   ru   zDDeepGEMM unavailable for this call, falling back to Triton. Reason: )Údevicer(   r!   r‚   Úget_device_propertiesÚmajorra   Úint8ÚosÚenvironÚgetr   r   r@   ÚloggerÚwarning_oncer   )	rq   ri   rs   rt   ru   rw   rx   Údeepgemm_preferredÚes	   &&&&&&&  r   Ú
fp8_linearr‘   Ä   sO  € ðT 	˜DÐ ÷ 	+ð 	+Ø�M‰M×Ñ &Ñ(÷	+ð 	+ä�J‰J×,Ñ,Ó.×4Ñ4¸Ñ9÷	+ð 	+ð �\‰\œUŸZ™ZÑ'×mÐm¨J¸dÐ,B÷	+ð -mÀzÐRSÅ}ÐXbÐcdÕXe×GlÔGlÐilÔGl÷	+ð 	+ô �J‰J�N‰NÐAÀ3ÓGÈ3ÑN÷		+ð 	+ô
 )Ó*Ô*ð ÷ ð	lÜ*ØØØ Ø%Ø)Ø!1Øôð ô " %Ð1AÈtÐgsÓtÐtøô ô 	lô ×ÑÐ"fÐghÐfiÐ j×kÑkûð	lús   ÄD! Ä!EÄ,E	Å	Ec                  ó>   a € ] tR tRtRR V 3R llltR R ltRtV ;t# )Ú	FP8Lineari	  c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   Úin_featuresrQ   Úout_featuresrt   útuple[int, int] | NoneÚactivation_schemeÚstrÚ	scale_fmtÚhas_biasÚboolr   )r   s   "r   r   ÚFP8Linear.__annotate__
  sF   € ÷ !2ñ !2àð!2ð ð!2ð +ð	!2ð
 ð!2ð ð!2ð ñ!2r   c                	ój  <€ \         S
V `  W4       W`n        W0n        W@n        \
        P                  P                  \
        P                  ! W!\        R 7      4      V n
        V P                  fA   \        P                  ! \
        P                  ! R\
        P                  R 7      4      V n        MÂVR8X  d   \        4       M\
        P                  pW P                  ^ ,          ,           ^,
          V P                  ^ ,          ,          pWP                  ^,          ,           ^,
          V P                  ^,          ,          p	\        P                  ! \
        P                  ! W‰VR 7      4      V n        V P                  R8X  dA   \        P                  ! \
        P                  ! R\
        P                  R 7      4      V n        MV P!                  RR4       V P                  '       d<   \        P                  ! \
        P                  ! V P"                  4      4      V n        R# V P!                  RR4       R# )r`   Nç      ð?Úue8m0Ústaticrw   ru   )ÚsuperÚ__init__r›   rt   r˜   r!   rd   re   rc   Ú
_FP8_DTYPEri   ÚtensorÚfloat32rs   r$   rw   Úregister_parameterr–   ru   )Úselfr•   r–   rt   r˜   rš   r›   rZ   Úscale_out_featuresÚscale_in_featuresÚ	__class__s   &&&&&&&   €r   r£   ÚFP8Linear.__init__
  se  ø€ ô 	‰Ñ˜Ô3à ŒØ$ŒØ!2ÔÜ—h‘h×(Ñ(¬¯ª°\ÔV`Ô)aÓbˆŒà�?‰?Ò"ä$&§L¢L´·²¸cÌÏÉÔ1WÓ$XˆDÕ!à-6¸'Ô-AÔ'Ô)ÄuÇ}Á}ˆHØ".·±ÀÕ1CÕ"CÀaÕ"GÈDÏOÉOÐ\]ÕL^Õ!^ÐØ!,¯©¸qÕ/AÕ!AÀAÕ!EÈ$Ï/É/ÐZ[ÕJ\Õ \ÐÜ$&§L¢L´·²Ð=OÐjrÔ1sÓ$tˆDÔ!à×!Ñ! XÔ-Ü$&§L¢L´·²¸cÌÏÉÔ1WÓ$XˆDÕ!à×#Ñ#Ð$6¸Ô=à�=�=ˆ=ÜŸš¤U§[¢[°×1BÑ1BÓ%CÓDˆDŽIà×#Ñ# F¨DÖ1r   c               ó    € V ^8„  d   QhRRRR/# )r   rq   rr   r   r   )r   s   "r   r   r�   -  s   € ÷ 
ñ 
˜\ð 
¨lñ 
r   c           
     	ó^  € V P                   P                  4       ^8”  d,   \        P                  ! WP                   V P                  4      # \        V P                   4      p\        V P                  4      p\        VVVV P                  V P                  VP                  V P                  R7      # )é   )rt   rw   rx   ru   )ri   Úelement_sizeÚFÚlinearru   r   rs   r‘   rt   rw   ra   )r¨   rq   ri   Ú	scale_invs   &&  r   ÚforwardÚFP8Linear.forward-  s‚   € Ø�;‰;×#Ñ#Ó%¨Ô)Ü—8’8˜E§;¡;°·	±	Ó:Ð:ä˜$Ÿ+™+Ó&ˆÜ˜T×2Ñ2Ó3ˆ	äØØØØ—‘Ø!×2Ñ2ØŸ™Ø—‘ô
ð 	
r   )rw   r˜   ru   rt   r›   ri   rs   ©NÚdynamicÚfloatF)r)   r4   r5   r6   r£   r´   r9   Ú__classcell__©r«   s   @r   r“   r“   	  s   ø† ÷!2ò !2÷F
ó 
r   r“   c                  óB   a € ] tR tRtRtRR V 3R llltR R ltRtV ;t# )	ÚFP8GroupedLineari?  uÓ  FP8 drop-in for block-diagonal grouped linears.

The underlying nn.Linear stores a single `(n_groups * out_per_group, in_per_group)`
weight; logically that's `n_groups` independent `(out_per_group, in_per_group)`
sub-matrices, each consuming a disjoint slice of the input's last-but-one dim.
Forward expects input of shape `(..., n_groups, in_per_group)` and returns
`(..., n_groups, out_per_group)` â€” same contract as the vanilla bf16 grouped
linear it replaces.

c               ó4   € V ^8„  d   QhRRRRRRRRRRR	RR
R/# )r   Úin_features_per_grouprQ   r–   Ún_groupsrt   r—   r˜   r™   rš   r›   rœ   r   )r   s   "r   r   ÚFP8GroupedLinear.__annotate__K  sP   € ÷ !ñ !à"ð!ð ð!ð ð	!ð
 +ð!ð ð!ð ð!ð ñ!r   c           	     	ó>   <€ \         SV `  VVVVVVR 7       W0n        R# )©r•   r–   rt   r˜   rš   r›   N)r¢   r£   r¿   )	r¨   r¾   r–   r¿   rt   r˜   rš   r›   r«   s	   &&&&&&&&€r   r£   ÚFP8GroupedLinear.__init__K  s0   ø€ ô 	‰ÑØ-Ø%Ø!Ø/ØØð 	ô 	
ð !Žr   c               ó    € V ^8„  d   QhRRRR/# )r   Úxrr   r   r   )r   s   "r   r   rÀ   _  s   € ÷ $ñ $˜ð $¨,ñ $r   c           	     	ón  € VP                   R R pVP                   R,          pV P                  P                  4       ^8”  dõ   V P                  P                  V P                  RV4      P                  ^^4      pVP                  RV P                  V4      P                  ^ ^4      p\        P                  ! W4      P                  ^ ^4      pVP                  ! . VOV P                  NRN5!  pV P                  '       d6   VP                  V P                  P                  V P                  R4      4       V# \        V P                  4      p\        V P                  4      pVP                  V P                  RV4      pVP                  R^ 4      P                  RV4      pVP                  V P                  VP                  ^ 4      V P                  ,          VP                  ^4      4      pVP                  ^ 4      V P                  ,          p\        P                   ! V P                  3WqP"                  \        P$                  R7      p\        P&                  ! ^V P                  ^,           VP"                  \        P$                  R7      V,          p	\)        4       p
V
P+                  VVVV	VV P,                  R7      pVP                  ! V P                  .VORN5!  P                  ^ R4      pV P                  '       d6   VP                  V P                  P                  V P                  R4      4       V# )N)r†   ra   ©ÚoffsetsÚtokens_per_expertrt   éþÿÿÿéÿÿÿÿ)Úshaperi   r°   Úviewr¿   Ú	transposeÚreshaper!   Úbmmr›   r|   ru   r   rs   ÚmovedimÚsizeÚfullr†   Úint32ÚarangerN   r3   rt   )r¨   rÅ   Úinput_shapeÚ
hidden_dimÚwÚyr³   Útokens_per_grouprÉ   rÈ   r}   s   &&         r   r´   ÚFP8GroupedLinear.forward_  s`  € Ø—g‘g˜c˜r�lˆØ—W‘W˜R•[ˆ
à�;‰;×#Ñ#Ó%¨Ô)Ø—‘× Ñ  §¡°°JÓ?×IÑIÈ!ÈQÓOˆAØ—	‘	˜"˜dŸm™m¨ZÓ8×BÑBÀ1ÀaÓHˆAÜ—	’	˜!“×)Ñ)¨!¨QÓ/ˆAØ—	’	Ð:˜;Ð:¨¯©Ð:°rÓ:ˆAØ�}�}ˆ}Ø—‘�t—y‘y—~‘~ d§m¡m°RÓ8Ô9ØˆHä�T—[‘[Ó!ˆÜ˜T×2Ñ2Ó3ˆ	à�F‰F�4—=‘= " jÓ1ˆØ�I‰I�b˜!Ó×$Ñ$ R¨Ó4ˆØ—N‘N 4§=¡=°)·.±.ÀÓ2CÀtÇ}Á}Õ2TÐV_×VdÑVdÐefÓVgÓhˆ	àŸ6™6 !›9¨¯©Õ5ÐÜ!ŸJšJ¨¯©Ð'7Ð9I×RZÑRZÔbg×bmÑbmÔnÐÜ—,’,˜q $§-¡-°!Õ"3¸A¿H¹HÌEÏKÉKÔXÐ[kÕkˆä5Ó7ˆØ×*Ñ*ØØØØØ/Ø—‘ð +ó 
ˆð �IŠI�d—m‘mÐ6 kÐ6°2Ó6×>Ñ>¸qÀ"ÓEˆØ�=�=ˆ=Ø�F‰F�4—9‘9—>‘> $§-¡-°Ó4Ô5Øˆr   )r¿   r¶   )	r)   r4   r5   r6   r7   r£   r´   r9   r¹   rº   s   @r   r¼   r¼   ?  s   ø† ñ	÷!ò !÷($ó $r   r¼   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# ©r   r¨   ztorch.nn.ModuleÚhidden_statesrr   Útop_k_indexÚtop_k_weightsr   r   )r   s   "r   r   r   †  sA   € ÷ G7ñ G7Ø
ðG7àðG7ð ðG7ð  ð	G7ð
 ñG7r   c                ób  € V P                   R 8X  d   \        R4      h\        4       pVP                  R4      pVP                  ^ 4      pVP                  R4      pVP	                  V^ R7      pVP                  R4      p	VP                  R4      p
W P                  8¬  P                  R4      p\        V P                  '       d   V P                  MV P                  4      p\        V P                  '       d   V P                  MV P                  4      p\        V P                  4      p\        V P                  4      pVP!                  VVVV P"                  V
R7      pV P                  '       d   V P%                  V4      pMV P'                  V4      pVP!                  VVVV P"                  V
R7      pVV	P)                  VP*                  4      P                  R4      ,          pVP-                  VR4       VP/                  WeV4      P1                  ^R7      pVP)                  VP*                  4      # )r¡   z‘batched_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.©Údim)rt   Ú
expert_idsç        rË   )r˜   ÚNotImplementedErrorrN   rÒ   Úrepeat_interleaverÏ   rV   Ú	unsqueezer   Úhas_gateÚgate_up_projÚup_projÚgate_up_proj_scale_invÚup_proj_scale_invÚ	down_projÚdown_proj_scale_invr2   rt   Ú_apply_gateÚact_fnÚtora   Úmasked_fill_rÍ   Úsum)r¨   rÞ   rß   rà   r}   Ú	num_top_kÚ
num_tokensr×   Úselected_hidden_statesÚsample_weightsrä   Úsentinel_maskÚ	weight_upÚweight_scale_upÚweight_downÚweight_scale_downrW   Úweighted_outÚfinal_hidden_statess   &&&&               r   Úfp8_batched_mm_experts_forwardr   †  s  € ð ×Ñ Ô)Ü!ðWó
ð 	
ô
 2Ó3€Oà× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jð +×<Ñ<¸YÈAÐ<ÓNÐØ"×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jð
  ×#3Ñ#3Ñ3×>Ñ>¸rÓB€Mä¨d¯m¯m¨m˜×*Ò*ÀÇÁÓN€IÜ¸d¿m¿m¸m˜t×:Ò:ÐQU×QgÑQgÓh€OÜ˜4Ÿ>™>Ó*€KÜ  ×!9Ñ!9Ó:Ðð ×-Ñ-ØØØØ—?‘?Øð .ó €Hð ‡}‡}€}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆð ×-Ñ-ØØØØ—?‘?Øð .ó €Hð ˜n×/Ñ/°·±Ó?×IÑIÈ"ÓMÕM€Lð ×Ñ˜m¨SÔ1ð '×+Ñ+¨JÀ:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# rÝ   r   )r   s   "r   r   r   Ð  sA   € ÷ [7ñ [7Ø
ð[7àð[7ð ð[7ð  ð	[7ð
 ñ[7r   c           	     ó`  € V P                   R 8X  d   \        R4      h\        4       pVP                  pVP	                  R	4      pVP	                  ^ 4      pVP	                  R	4      pVP                  R	4      p	VP                  R	4      p
\        P                  ! V
4      w  r¼WV,          ,          pWœ,          pVP                  R8X  d   VP                  4       MVP                  4       p\        P                  ! WðP                  ^ V P                  ^,
          R7      p\        P                  ! V^ \        P                  R7      pW°P                  8¬  P                  R	4      p\!        V P"                  '       d   V P$                  MV P&                  4      p\!        V P"                  '       d   V P(                  MV P*                  4      p\!        V P,                  4      p\!        V P.                  4      pVP1                  VVVVVV P2                  R7      pV P"                  '       d   V P5                  V4      pMV P7                  V4      pVP1                  VVVVVV P2                  R7      pVVP9                  VP:                  4      P                  R	4      ,          pVP=                  VR4       \        P>                  ! V4      p\        P@                  ! VP	                  ^ 4      VR7      VV&   VV,          pVPC                  WvV4      PE                  ^R7      pVP9                  VP:                  4      # )
r¡   z‘grouped_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.Úcpu)ÚbinsÚminrg   )rã   ra   rÇ   rå   )r†   râ   rË   )#r˜   ræ   rN   r†   rÒ   rÏ   r!   Úsortr(   r¸   rQ   ÚhistcrV   ÚcumsumrÔ   rè   r   ré   rê   rë   rì   rí   rî   rï   r3   rt   rð   rñ   rò   ra   ró   Ú
empty_likerÕ   rÍ   rô   )r¨   rÞ   rß   rà   r}   r†   rõ   rö   r×   rø   rä   Úexpert_ids_gÚpermÚselected_hidden_states_gÚsample_weights_gÚhistc_inputrÉ   rÈ   rù   rú   rû   rü   rý   rW   rþ   Úinv_permrÿ   s   &&&&                       r   Úfp8_grouped_mm_experts_forwardr  Ð  sÃ  € ð ×Ñ Ô)Ü!ðWó
ð 	
ô
 2Ó3€Oà×!Ñ!€FØ× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jð #×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jô Ÿš JÓ/Ñ€LØ,°YÕ->Õ?ÐØ%Õ+Ðð
 +1¯+©+¸Ô*>�,×$Ñ$Ô&ÀL×DTÑDTÓDV€KÜŸš K×6FÑ6FÈAÐSW×ScÑScÐfgÕSgÔhÐÜ�lŠlÐ,°!¼5¿;¹;ÔG€Gð "×%5Ñ%5Ñ5×@Ñ@ÀÓD€Mä¨d¯m¯m¨m˜×*Ò*ÀÇÁÓN€IÜ¸d¿m¿m¸m˜t×:Ò:ÐQU×QgÑQgÓh€OÜ˜4Ÿ>™>Ó*€KÜ  ×!9Ñ!9Ó:Ðð ×-Ñ-Ø ØØØØ+Ø—?‘?ð .ó €Hð ‡}‡}€}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆð ×-Ñ-ØØØØØ+Ø—?‘?ð .ó €Hð Ð.×1Ñ1°(·.±.ÓA×KÑKÈBÓOÕO€Lð ×Ñ˜m¨SÔ1ô ×Ò Ó%€HÜ—\’\ $§)¡)¨A£,°vÔ>€HˆT�NØ Õ)€Lð '×+Ñ+¨JÀ:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   c                  óv   a € ] tR tRt$ RRRRR//tR]R&   RR	 V 3R
 llltR R ltR R ltRR R llt	Rt
V ;t# )Ú
FP8Expertsi.  Údeepgemm_megamoeÚmoe_tp_expertsÚmegamoe_expertsÚ	ep_routerÚmegamoe_routerzdict[str, dict[str, str]]Ú_impl_tp_layer_overridesc          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )	r   rt   r—   r˜   r™   rš   r›   rœ   ré   r   )r   s   "r   r   ÚFP8Experts.__annotate__=  sL   € ÷ Dnñ Dnð +ðDnð ð	Dnð
 ðDnð ðDnð ñDnr   c           
     	óÆ  <€ \         S
V `  4        VR J g   Q R4       hWn        WPn        W`n        W n        VP                  V n        W0n        \        VRR4      V n
        \        VRR4      V n        \        VRR4      V n        \        VRR4      V n        \        \        VR	R
4      ,          V n        \        VRR4      V n        \        VRR4      R8H  pVR8X  d   \%        4       M\&        P(                  pV'       d   R\&        P*                  RVR^R^R^ /p	M(R\,        RTRVe
   V^ ,          MRRVe
   V^,          MR/p	V P                  '       dV   \/        V P                  ^V P                  ,          V P                  3R^/V	B w  V n        V n        V P5                  RR4       MK\/        V P                  V P                  V P                  3/ V	B w  V n        V n        V P5                  RR4       \/        V P                  V P                  V P                  3/ V	B w  V n        V n        V P5                  RR4       V P                  R8X  d•   \>        P@                  ! \&        PB                  ! V P                  \&        P(                  R7      4      V n"        \>        P@                  ! \&        PB                  ! V P                  \&        P(                  R7      4      V n#        R# R# )FzWFP8Experts does not support bias for now, please open an issue if you want this featureÚnum_local_expertsrV   Úmoe_intermediate_sizeÚintermediate_sizeÚswiglu_alphaNÚswiglu_limitÚhidden_activationÚ
hidden_actÚexpert_dtypeÚfp8Úfp4r    rY   rZ   r[   r\   r]   r^   Úgate_up_proj_biasÚup_proj_biasÚdown_proj_biasr¡   r`   )$r¢   r£   Úconfigr›   ré   rt   Úhidden_sizer×   r˜   r-   rV   Úintermediate_dimr&   r  r   r   rñ   Úlimitr$   r!   r¦   r‰   r¤   rn   rê   rì   r§   rë   rí   rî   rï   rd   re   ÚonesÚgate_up_proj_activation_scaleÚdown_proj_activation_scale)r¨   r)  rt   r˜   rš   r›   ré   Úis_fp4rZ   Úalloc_kwargsr«   s   &&&&&&&   €r   r£   ÚFP8Experts.__init__=  s’  ø€ ô 	‰ÑÔà˜5Ó ð 	
Øeó	
Ð ð ŒØ ŒØ ŒØ$ŒØ ×,Ñ,ˆŒØ!2ÔÜ& vÐ/BÀMÓRˆÔÜ +¨FÐ4KÐM`Ó aˆÔÜ# F¨N¸DÓAˆÔÜ# F¨N¸DÓAˆÔÜœ[¨Ð1DÀlÓSÕTˆŒÜ˜V ^°TÓ:ˆŒ
ô ˜ °Ó7¸5Ñ@ˆØ)2°gÔ)=Ô#Ô%Ä5Ç=Á=ˆßà¤§
¡
Ø˜HØ Ø˜QØ˜Rð‰Lð ¤
Ø˜HØ¨jÒ.D˜Z¨ž]È$Ø¨jÒ.D˜Z¨ž]È$ð	ˆLð �=�=ˆ=Ü=OØ× Ñ  ! d×&;Ñ&;Õ";¸T¿_¹_ñ>ØYZð>Ø^jñ>Ñ:ˆDÔ˜tÔ:ð ×#Ñ#Ð$7¸Õ>ä3EØ× Ñ  $×"7Ñ"7¸¿¹ñ4ØLXñ4Ñ0ˆDŒL˜$Ô0ð ×#Ñ# N°DÔ9ä3EØ×Ñ˜dŸo™o¨t×/DÑ/Dñ4
ØHTñ4
Ñ0ˆŒ˜Ô0ð 	×ÑÐ 0°$Ô7à×!Ñ! XÔ-Ü13·²¼e¿jºjÈ×IYÑIYÔaf×anÑanÔ>oÓ1pˆDÔ.Ü.0¯lªl¼5¿:º:Àd×FVÑFVÔ^c×^kÑ^kÔ;lÓ.mˆDÖ+ñ .r   c               ó    € V ^8„  d   QhRRRR/# )r   Úgate_uprr   r   r   )r   s   "r   r   r  ƒ  s   € ÷ &ñ & <ð &°Lñ &r   c                	ó  € VP                  ^RR7      w  r#V P                  e‚   VP                  V P                  R7      pVP                  V P                  ) V P                  R7      pV\        P
                  ! W P                  ,          4      ,          pVR,           V,          # V P                  eE   VP                  V P                  R7      pVP                  V P                  ) V P                  R7      pV P                  V4      V,          # )r   râ   )rg   ©r  rg   rŸ   rË   )Úchunkr  Úclampr   r!   Úsigmoidr,  rñ   )r¨   r4  ÚgateÚupÚglus   &&   r   rð   ÚFP8Experts._apply_gateƒ  sÒ   € Ø—=‘= ¨�=Ó+‰ˆØ×ÑÒ(à—:‘: $×"3Ñ"3�:Ó4ˆDØ—‘˜t×0Ñ0Ð0°d×6GÑ6G�ÓHˆBØœŸš t×.?Ñ.?Õ'?Ó@Õ@ˆCØ˜•H Õ#Ð#Ø�Z‰ZÒ#Ø—:‘: $§*¡*�:Ó-ˆDØ—‘˜tŸz™z˜k¨t¯z©z�Ó:ˆBØ�{‰{˜4Ó  2Õ%Ð%r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   rÞ   rr   rß   rà   r   r   )r   s   "r   r   r  �  s,   € ÷ (;ñ (;Ø)ð(;Ø8Dð(;ØUað(;à	ñ(;r   c                	óÆ  € \         P                  ! V\         P                  R 7      p\         P                  ! 4       ;_uu_ 4        \         P                  P
                  P                  W P                  ^,           R7      pVP                  ^^^ 4      p\         P                  ! VP                  R	R7      ^ 4      P                  RR7      P                  R4      pRRR4       X EFÌ  pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
V P                  R8X  d   V P                  V,          MRpT P!                  T
V P"                  '       d   V P$                  V,          MV P&                  V,          V P"                  '       d   V P(                  V,          MV P*                  V,          VR7      pV P"                  '       d   V P-                  V4      MV P/                  V4      pV P                  R8X  d   V P0                  V,          MRpV P!                  VV P2                  V,          V P4                  V,          VR7      pW9VR3,          pWÎP7                  VP8                  4      ,          pVP;                  ^ WŸP7                  VP8                  4      4       EKÏ  	  VP7                  VP8                  4      #   + '       g   i     ELÿ; i)
r`   )Únum_classesrâ   F)Úas_tupleNr¡   r{   rË   )rË   rÊ   )r!   Ú
zeros_liker¦   Úno_gradrd   r   Úone_hotrV   ÚpermuteÚgreaterrô   ÚnonzerorÍ   Úwherer˜   r.  r²   ré   rê   rë   rì   rí   rð   rñ   r/  rî   rï   rò   ra   Ú
index_add_)r¨   rÞ   rß   rà   rÿ   Úexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgate_up_act_scalerW   Údown_act_scaleÚrouting_weightsrþ   s   &&&&            r   r´   ÚFP8Experts.forward�  sG  € ô
 $×.Ò.¨}ÄEÇMÁMÔRÐä�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_ÐbcÕOcÐ5ÓdˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÐZ_ÐPÓ`×eÑeÐfhÓiˆJ÷ ô
 %ˆJØ×-Ñ-Ô-Ùä#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMàBF×BXÑBXÐ\dÔBd�×2Ñ2°:Ö>Ðjnð ð —{‘{ØØ15··°�×!Ñ! *Ö-ÀDÇLÁLÐQ[ÕD\Ø;?¿=¿=¸=�×+Ñ+¨JÖ7Èd×NdÑNdÐeoÕNpØ!2ð	 #ó ˆHð 6:·]·]°]�t×'Ñ'¨Ô1ÈÏÉÐT\ÓH]ˆHà?C×?UÑ?UÐYaÔ?a�×/Ñ/°
Ö;Ðgkð ð —{‘{ØØ—‘˜zÕ*Ø×(Ñ(¨Õ4Ø!/ð	 #ó ˆHð ,°yÀ$Ð,FÕGˆOØ#×&8Ñ&8¸¿¹Ó&HÕHˆLØ×*Ñ*¨1¨i¿¹ÐI\×IbÑIbÓ9c×dñ7 %ð8 #×%Ñ% m×&9Ñ&9Ó:Ð:÷C �_ˆ_ús   ÁBKËK 	c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   rq   rr   ri   rs   rw   rv   r   r   )r   s   "r   r   r  º  s<   € ÷ 
ñ 
àð
ð ð
ð 'ð	
ð
 .ð
ð 
ñ
r   c           	     	ó¤   € VP                  4       ^8”  d   \        P                  ! WR4      # \        VVVV P                  VVP
                  R7      # )r¯   N)rw   rx   )r°   r±   r²   r‘   rt   ra   )r¨   rq   ri   rs   rw   s   &&&&&r   r²   ÚFP8Experts.linearº  sM   € ð ×ÑÓ  1Ô$Ü—8’8˜E¨4Ó0Ð0äØØØØ�O‰OØ-ØŸ™ô
ð 	
r   )rñ   r˜   rt   r)  rî   r/  rï   rê   r.  rì   r›   ré   r×   r+  r,  rV   r  r   rë   rí   )Nr·   r¸   FTrI   )r)   r4   r5   r6   r  r8   r£   rð   r´   r²   r9   r¹   rº   s   @r   r  r  .  sM   ø‡ ð 	ØÐ/ØÐ)ð
ð;ÐÐ7ó ÷Dnò DnõL&õ(;÷T
õ 
r   r  c                  ó.   € ] tR tRtRtR]R]R]R]/t	Rt
R# )	ÚFP8ExpertsInterfaceiÎ  z?Interface for registering custom FP8 experts forward functions.Ú
batched_mmÚ
grouped_mmÚdeepgemmr  r   N)r)   r4   r5   r6   r7   r   r  r   r   Ú_global_mappingr9   r   r   r   rX  rX  Î  s&   † ÙIð 	Ð4ØÐ4ØÐ4ØÐDð	„Or   rX  c               ó   € V ^8„  d   QhRR/# )r   Úmodules_to_not_convertzlist[str] | Noner   )r   s   "r   r   r   Ü  s   € ÷ Pñ PØ#3ñPr   c                óª  € VP                   '       d   V # RpV P                  4        EFõ  w  rV\        WQ4      '       g   K  Rp\        P                  ! R4      ;_uu_ 4        VP                  R4      '       d…   \        VRR4      p\        VRR4      p	\        VRV P                  P                  4       4      p
\        \        \        V	VR	7      pV! V
VP                  VP                  VP                  V	VR
7      pEM\        V4      \         P"                  J dQ   \%        VP&                  VP(                  VP                  VP                  VP                  VP*                  RJR7      pM”\-        V\         P"                  4      '       du   R\        V4      P.                  9   d[   \1        VP&                  VP(                  VP2                  VP                  VP                  VP                  VP*                  RJR7      pVe   V P5                  WW4       RpRRR4       EKø  	  V'       g   \6        P9                  R4       V #   + '       g   i     EK+  ; i)a‚  
A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.

Parameters:
    model (`torch.nn.Module`):
        Input model or `torch.nn.Module` as the function is run recursively.
    modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
        Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons.
    quantization_config (`FineGrainedFP8Config`):
        The quantization config object that contains the quantization parameters.
    pre_quantized (`book`, defaults to `False`):
        Whether the model is pre-quantized or not
FNÚmetaz.expertsré   Tr›   r)  )Úexperts_classÚexperts_interfacer›   ré   )r)  rt   r˜   rš   r›   ré   rÂ   ÚGroupedLinear)r¾   r–   r¿   rt   r˜   rš   r›   z�You are loading your model using fp8 but no linear modules were found in your model. Please double check your model architecture.)Ú
dequantizeÚnamed_modulesr
   r!   r†   Úendswithr&   r)  Úget_text_configr   r  ÚALL_FP8_EXPERTS_FUNCTIONSÚweight_block_sizer˜   rš   r(   rd   ÚLinearr“   r•   r–   ru   Ú
isinstancer)   r¼   r¿   Úset_submoduler�   Úwarning)Úmodelr^  Úquantization_configÚpre_quantizedÚhas_been_replacedÚmodule_nameÚmoduleÚ
new_moduleré   r›   r)  Ú	new_classs   &&&&        r   Úreplace_with_fp8_linearrv  Ü  sö  € ð" ×%×%Ð%ØˆàÐØ$×2Ñ2×4ÑˆÜ$ [×IÒIÙàˆ
Ü�\Š\˜&×!Õ!Ø×#Ñ# J×/Ò/Ü" 6¨:°tÓ<�Ü" 6¨:°uÓ=�Ü  ¨°5·<±<×3OÑ3OÓ3QÓR�Ü6Ü",Ü&?Ø%Ø%ô	�	ñ 'Ø!Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø%Ø%ô’
ô �f“¤§¡Ó*ä&Ø &× 2Ñ 2Ø!'×!4Ñ!4Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø#Ÿ[™[°Ð4ô‘
ô ˜F¤B§I¡I×.Ò.°?ÄdÈ6Ãl×F[ÑF[Ô3[ô .Ø*0×*<Ñ*<Ø!'×!4Ñ!4Ø#Ÿ_™_Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø#Ÿ[™[°Ð4ô�
ð Ò%Ø×#Ñ# KÔ<Ø$(Ð!÷_ "Ò!ñ  5÷l Ü�‰ð<ô	
ð €L÷m "×!Ñ!ús   Á F4I É Ic                  óZ   € ] tR tRtRtR tR R ltR R ltR R	 lt]	R
 R l4       t
RtR# )ÚFp8Quantizei/  zV
A quantization operation that creates two tensors, weight and scale out of a weight.
c                	ó   € Wn         R # rI   ©Úhf_quantizer©r¨   r{  s   &&r   r£   ÚFp8Quantize.__init__4  ó   € Ø(Ör   c               ó    € V ^8„  d   QhRRRR/# )r   Úvaluerr   r   ztuple[int, int]r   )r   s   "r   r   ÚFp8Quantize.__annotate__7  s   € ÷ 	!ñ 	!¨ð 	!¸/ñ 	!r   c                	ó‚  € R pV P                   P                  er   \        V P                   P                  \        4      '       d'   V P                   P                  P	                  R4      pM!\        V P                   P                  RR 4      pVf'   VP                  R,          VP                  R,          3p\        V4      # )Nri  rÊ   rË   )r{  ro  rk  ÚdictrŒ   r&   rÌ   Útuple)r¨   r€  rt   s   && r   Ú_resolve_block_sizeÚFp8Quantize._resolve_block_size7  s–   € Øˆ
Ø×Ñ×0Ñ0Ò<Ü˜$×+Ñ+×?Ñ?Ä×FÒFØ!×.Ñ.×BÑB×FÑFÐGZÓ[‘
ä$ T×%6Ñ%6×%JÑ%JÐL_ÐaeÓf�
ØÒØŸ+™+ b�/¨5¯;©;°r­?Ð;ˆJÜ�ZÓ Ð r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úkeyr™   r€  rr   r   údict[str, torch.Tensor]r   )r   s   "r   r   r�  B  s"   € ÷ &7ñ &7 ð &7¨\ð &7Ð>Uñ &7r   c                	óÚ  € VP                   ^8  d   W/# V P                  V4      w  r4VP                  R,          VP                  R,          reWS,          ^ 8w  g   Wd,          ^ 8w  d   W/# VP                  RR pWS,          pWd,          p	VP                  p
VP                  \        P
                  4      pVP                  ! . VOVNVNV	NVN5!  pVP                  4       P                  RR7      p\        P                  ! V^ 8„  V\        P                  ! V4      4      p\        V,          p\        P                  ! V^ 8„  V\        P                  ! V4      4      pVP                  R4      P                  R4      pVV,          p\        P                  ! V\        \        R7      P                  \        4      pVP                  V
4      pRV,          P                  \        P
                  4      pV P                   P"                  P$                  R8X  d•   \        P&                  ! R\        P(                  ! \        P*                  ! VP                  \        P,                  ! \        P
                  4      P.                  R7      4      4      4      pVP                  \1        4       4      pVP3                  R4      '       d!   VP5                  R	^4      ^ ,          R
,           MVR,           pVVVV/# )r   Nrâ   r6  rŸ   r    ç       @)r  ú.weightÚ.ú.weight_scale_invÚ
_scale_invrÊ   rË   éýÿÿÿ)r�  rË   )Úndimr…  rÌ   rò   r!   r¦   rÏ   ÚabsÚamaxrH  Ú	ones_likeÚ_FP8_MAXrè   r8  Ú_FP8_MINr¤   r{  ro  rš   ÚpowÚceilÚlog2ÚfinfoÚtinyr$   rf  Úrsplit)r¨   rˆ  r€  Úblock_mÚblock_nÚrowsÚcolsÚleading_shapeÚ
rows_tilesÚ
cols_tilesÚoriginal_shapeÚ
value_fp32ÚreshapedÚmax_absÚsafe_max_absÚscalesÚscales_broadcastÚscaledÚ	quantizedÚ
inv_scalesÚ	scale_keys   &&&                  r   Ú_quantize_oneÚFp8Quantize._quantize_oneB  sE  € ð �:‰:˜Œ>Ø�<ÐØ×3Ñ3°EÓ:ÑˆØ—[‘[ •_ e§k¡k°"¥oˆdØ�>˜QÔ $¥.°AÔ"5Ø�<Ðð Ÿ™ C RÐ(ˆØ•_ˆ
Ø•_ˆ
ØŸ™ˆØ—X‘XœeŸm™mÓ,ˆ
à×%Ò%Ð_ }Ð_°jÐ_À'Ð_È:Ð_ÐW^Ó_ˆà—,‘,“.×%Ñ%¨(Ð%Ó3ˆÜ—{’{ 7¨Q¡;°¼¿ºÈÓ9QÓRˆä˜LÕ(ˆÜ—’˜W q™[¨&´%·/²/À&Ó2IÓJˆà!×+Ñ+¨BÓ/×9Ñ9¸"Ó=ÐØÐ,Õ,ˆÜ—K’K ¬H¼(ÔC×FÑFÄzÓRˆ	Ø×%Ñ% nÓ5ˆ	Ø˜F•l×&Ñ&¤u§}¡}Ó5ˆ
ð ×Ñ×0Ñ0×:Ñ:¸gÔEÜŸš 3¬¯
ª
´5·:²:¸j×>NÑ>NÔSX×S^ÒS^Ô_d×_lÑ_lÓSm×SrÑSrÐ>NÓ>sÓ3tÓ(uÓvˆJØ#Ÿ™Ô'7Ó'9Ó:ˆJØCFÇ<Á<ÐPY×CZÒCZ�C—J‘J˜s AÓ& qÕ)Ð,?Ö?Ð`cÐfrÕ`rˆ	Ø�Y 	¨:Ð6Ð6r   c               ó    € V ^8„  d   QhRRRR/# )r   Ú
input_dictrr   r   r‰  r   )r   s   "r   r   r�  j  s   € ÷ ñ  ,ð Ð=Tñ r   c                	ó¾   € / pVP                  4        FF  w  rE\        V\        4      '       d
   V^ ,          MTpVP                  V P	                  WF4      4       KH  	  V# )r   )Úitemsrk  ÚlistÚupdater¯  )r¨   r²  ÚkwargsÚresultrˆ  r€  r¥   s   &&,    r   ÚconvertÚFp8Quantize.convertj  sS   € ð +-ˆØ$×*Ñ*Ö,‰JˆCÜ!+¨E´4×!8Ò!8�U˜1–X¸eˆFØ�M‰M˜$×,Ñ,¨SÓ9Ö:ñ -ð ˆr   c               ó   € V ^8„  d   QhRR/# ©r   r   r   r   )r   s   "r   r   r�  u  s   € ÷ 0ñ 0˜Mñ 0r   c                	ó,   € \        V P                  4      # rI   )ÚFp8Dequantizer{  ©r¨   s   &r   Ú
reverse_opÚFp8Quantize.reverse_opt  s   € ä˜T×.Ñ.Ó/Ð/r   rz  N)r)   r4   r5   r6   r7   r£   r…  r¯  r¹  ÚpropertyrÀ  r9   r   r   r   rx  rx  /  s0   † ñò)õ	!õ&7õPð ô0ó ô0r   rx  c                  ó~   € ] tR tRtRtR tR R ltRtR R ltRR	 R
 llt	R R lt
RR R llt]R R l4       tRtR# )r¾  iy  ux  Dequantize FP8 weights using their per-block ``weight_scale_inv``.

Designed to run as the *first* op in any :class:`WeightConverter` chain when
loading with ``dequantize=True`` â€” :meth:`update_weight_conversions` on the
FP8 quantizer attaches it to each existing model-specific converter so that
per-expert (weight, scale) pairs are folded into full-precision tensors before
the chain's merge / concat ops collapse the per-expert structure.

Pattern semantics
    Input ``input_dict`` carries one entry per source pattern; each value is a
    list of tensors (one per ``*`` match). For every weight pattern that has a
    sibling ``*.weight_scale_inv`` pattern in the dict, this op pairs them up by
    index, dequantizes per-pair, and emits the dequantized list under the
    original *weight* key. Scale entries are dropped from the output so the
    remaining ops only see weights.
c                	ó   € Wn         R # rI   rz  r|  s   &&r   r£   ÚFp8Dequantize.__init__‹  r~  r   c               ó    € V ^8„  d   QhRRRR/# )r   Úweight_patternr™   r   r   )r   s   "r   r   ÚFp8Dequantize.__annotate__Ž  s   € ÷ 
2ñ 
2°ð 
2¸ñ 
2r   c                	óê   € VP                  R 4      pV'       d   VRR MTpVP                  R4      '       d   VR\        R4      )  R,           pMVR8X  d   RpM	VR,           pV'       d
   VR ,           # T# )Ú$NrŒ  rŽ  ri   rs   r�  rË   )rf  Úlen)r¨   rÇ  ÚanchoredÚbaseÚscales   &&   r   Ú_scale_pattern_forÚ Fp8Dequantize._scale_pattern_forŽ  sr   € à!×*Ñ*¨3Ó/ˆß&.ˆ~˜c˜rÑ"°NˆØ�=‰=˜×#Ò#ØÐ*œC 	›N˜?Ð+Ð.AÕA‰EØ�XÔØ&‰Eà˜<Õ'ˆEß&ˆu�s�{Ð1¨EÐ1r   c               ó    € V ^8„  d   QhRRRR/# )r   Úpackedrr   r   r   )r   s   "r   r   rÈ  Ÿ  s   € ÷ Jñ J ,ð J°<ñ Jr   c                óü  € \         P                  ! V P                  \         P                  VP                  R7      pVP                  4       P                  \         P                  4      pV^,          P                  4       pV^,	          ^,          P                  4       p\         P                  ! W$,          W%,          .RR7      pVP                  ! . VP                  RR O^VP                  R,          ,          N5!  # )uR   Two ``e2m1`` FP4 values per byte â†’ float32 tensor twice as wide on the last dim.)ra   r†   râ   NrË   )r!   r¥   Ú_FP4_E2M1_LUTr¦   r†   Ú
contiguousrÍ   Úuint8ÚlongÚstackrÏ   rÌ   )r¨   rÒ  ÚlutÚu8ÚlowÚhighÚunpackeds   &&     r   Ú_unpack_fp4ÚFp8Dequantize._unpack_fp4Ÿ  s°   € ä�lŠl˜4×-Ñ-´U·]±]È6Ï=É=ÔYˆØ×ÑÓ ×%Ñ%¤e§k¡kÓ2ˆØ�C�x�o‰oÓˆØ�q•˜C•×%Ñ%Ó'ˆÜ—;’; ¥¨#­)Ð4¸"Ô=ˆØ×ÒÐI §¡¨c¨rÐ!2ÐI°A¸¿¹ÀRÕ8HÕ4HÓIÐIr   Nc               ó(   € V ^8„  d   QhRRRRRRRR/# )r   r¬  rr   r©  rx   ry   r   r   )r   s   "r   r   rÈ  ¨  s2   € ÷ +@ñ +@Ø%ð+@Ø/;ð+@ØK]ð+@à	ñ+@r   c                	óX  € \        \        R R4      pVP                  \        P                  8X  g   Ve$   VP                  V8X  d   V P	                  V4      pMVP                  \        P                  4      pVP                  RR w  rg VP                  RR w  r‰Wh,          '       g   Wy,          '       d   \        RV RV RV RV	 R2	4      hWh,          p
Wy,          pVfN   VP                  P                  '       d"   VP                  4       ^8¼  d   VP                  M\        P                  pVP                  \        P                  8X  d6   VP                  \        P                  4      R,
          P                  4       pMVP                  \        P                  4      pVP                  pVP                  RWŠW›4      pVP                  RW‰4      P!                  R4      P!                  ^4      pWï,          P                  V4      P                  V4      #   \         d    ^^r˜ ELŽi ; i)	Úfloat4_e2m1fn_x2NzWeight shape (r?   z) not divisible by scale grid (z).ç     À_@rÊ   rË   )r&   r!   ra   r‰   rÞ  rò   r¦   rÌ   Ú	ExceptionÚ
ValueErrorrf   r°   Úbfloat16rÖ  Úexp2rÏ   rè   )r¨   r¬  r©  rx   Ú	fp4_dtypeÚquantized_fp32rŸ  r   Ú
scale_rowsÚ
scale_colsr�  rž  Ús_fp32r¤  ÚqÚss   &&&&            r   Ú_dequantize_oneÚFp8Dequantize._dequantize_one¨  sÑ  € ô
 œEÐ#5°tÓ<ˆ	Ø�?‰?œeŸj™jÔ(¨YÒ-BÀyÇÁÐZcÔGcØ!×-Ñ-¨iÓ8‰Nà&Ÿ\™\¬%¯-©-Ó8ˆNØ#×)Ñ)¨"¨#Ð.‰
ˆð	*Ø%+§\¡\°"°#Ð%6Ñ"ˆJð ×Ô × 1Ô 1ÜØ    b¨¨Ð.MÈjÈ\ÐY[Ð\fÐ[gÐgiÐjóð ð Õ$ˆØÕ$ˆð
 Òà &§¡× >× >Ð >À6×CVÑCVÓCXÐ\]ÔC]�—’Ôch×cqÑcqð ð �<‰<œ5Ÿ;™;Ô&Ø—i‘i¤§¡Ó.°Õ6×<Ñ<Ó>‰Fà—Y‘YœuŸ}™}Ó-ˆFØ'×-Ñ-ˆØ×"Ñ" 2 z¸JÓPˆØ�N‰N˜2˜zÓ6×@Ñ@ÀÓD×NÑNÈqÓQˆØ•�z‰z˜,Ó'×/Ñ/°Ó?Ð?øô7 ô 	*à%&¨›
ð	*ús   Â	H ÈH)È(H)c               ó$   € V ^8„  d   QhRRRRRR/# )r   rn  útorch.nn.Module | NoneÚfull_layer_nameú
str | Noner   ry   r   )r   s   "r   r   rÈ  Õ  s$   € ÷ -ñ -Ð'=ð -ÐPZð -Ð_qñ -r   c                	ób   € Ve   Vf   R # \        W4      w  r4\        W4R 4      p\        VRR 4      # )Nra   )r	   r&   )r¨   rn  ró  rs  Útensor_nameÚparams   &&&   r   Ú_get_target_dtypeÚFp8Dequantize._get_target_dtypeÕ  s7   € ØŠ=˜OÒ3ÙÜ2°5ÓJÑˆÜ˜¨TÓ2ˆÜ�u˜g tÓ,Ð,r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   r²  ú,dict[str, list[torch.Tensor] | torch.Tensor]ró  rô  rn  rò  r   r   )r   s   "r   r   rÈ  Ü  s2   € ÷ ,ñ ,à@ð,ð $ð,ð &ð	,ð 
6ñ,r   c                	ó&  € V P                  W24      pR V9   d{   Ve   TMRpVR ,          p\        V\        4      '       d
   V^ ,          MTpRV9   d?   VR,          p\        V\        4      '       d
   V^ ,          MTpW`P                  WxVR7      /# Wg/# / p	VP	                  4        Fâ  w  r«RV
9   g   RV
9   d   K  V P                  V
4      pWÁ9  d   W¹V
&   K2  \        V\        4      '       d   TMV.pW,          p\        V\        4      '       d   TMV.p\        V4      \        V4      8w  d(   \        RV
 R\        V4       R\        V4       R24      h\        WØ4       UUu. uF  w  rïV P                  WïVR7      NK  	  uppWš&   Kä  	  V	# u uppi )	zweight$ri   rs   )rx   rw   z/Fp8Dequantize: weight/scale count mismatch for z (z weights vs z	 scales).)	rø  rk  rµ  rï  r´  rÏ  rË  rå  Úzip)r¨   r²  ró  rn  r·  rx   Ú
target_keyr¬  r©  r¸  rˆ  r€  r®  ÚweightsrØ   rî  s   &&&&,           r   r¹  ÚFp8Dequantize.convertÜ  s¥  € ð ×-Ñ-¨eÓEˆð
 ˜
Ô"ð -<Ò,G™ÈXˆJØ" 9Õ-ˆIÜ(2°9¼d×(CÒ(C˜	 !žÈˆIØ! ZÔ/Ø#Ð$6Õ7�Ü&0°¼×&>Ò&>˜ žÀF�Ø"×$8Ñ$8¸ÐYeÐ$8Ó$fÐgÐgØÐ*Ð*ð @BˆØ$×*Ñ*Ö,‰JˆCØ! SÔ(Ð,>À#Ô,EÙØ×/Ñ/°Ó4ˆIØÔ*à#�s‘ÙÜ)¨%´×6Ò6‘e¸U¸GˆGØÕ*ˆFÜ)¨&´$×7Ò7‘V¸f¸XˆFÜ�7‹|œs 6›{Ô*Ü ØEÀcÀUð KÜ˜G›�~ \´#°f³+°¸iðIóð ô ^aÐahÔ]qÔrÑ]qÑUYÐUV˜4×/Ñ/°À<Ð/ÖPÑ]qÒrˆF‹Kñ! -ð" ˆùó ss   Å#Fc               ó   € V ^8„  d   QhRR/# r¼  r   )r   s   "r   r   rÈ    s   € ÷ .ñ .˜Mñ .r   c                	ó,   € \        V P                  4      # rI   )rx  r{  r¿  s   &r   rÀ  ÚFp8Dequantize.reverse_op
  s   € ô
 ˜4×,Ñ,Ó-Ð-r   rz  )rå   g      à?rŸ   g      ø?r‹  g      @g      @g      @g       €g      à¿g      ð¿g      ø¿g       Àg      Àg      Àg      ÀrI   )NN)r)   r4   r5   r6   r7   r£   rÏ  rÔ  rÞ  rï  rø  r¹  rÂ  rÀ  r9   r   r   r   r¾  r¾  y  sE   † ñò")õ
2ð m€MõJ÷+@õZ-÷,ð\ ô.ó ô.r   r¾  c                  óB   € ] tR tRtRtR t]R R l4       tR R ltRt	R	# )
ÚFp8DecodeScalei  a7  Decode MXFP8 ``ue8m0`` per-block scales (stored as ``uint8`` exponents) into the
float32 multiplicative scales the FP8 compute path expects.

Native MXFP8 loading (``dequantize=False``) keeps weights in ``float8_e4m3fn`` and only
needs the sibling ``*.weight_scale_inv`` tensors turned from raw E8M0 bytes into real
scales (``2 ** (byte - 127)``). Prepended to each weight converter, this op runs before
any merge/concat collapses the per-expert structure: it rewrites only the ``uint8`` scale
entries and passes weights (and already-float scales) through untouched.
c                	ó   € Wn         R # rI   rz  r|  s   &&r   r£   ÚFp8DecodeScale.__init__  r~  r   c               ó    € V ^8„  d   QhRRRR/# )r   r¥   rr   r   r   )r   s   "r   r   ÚFp8DecodeScale.__annotate__!  s   € ÷ dñ d˜ð d¨ñ dr   c                	ó¬   € V P                   \        P                  8X  d5   V P                  \        P                  4      R ,
          P                  4       # T # )rã  )ra   r!   rÖ  rò   r¦   rç  )r¥   s   &r   Ú_decodeÚFp8DecodeScale._decode   s;   € ð =C¿L¹LÌEÏKÉKÔ<W�—	‘	œ%Ÿ-™-Ó(¨5Õ0×6Ñ6Ó8ÐcÐ]cÐcr   c               ó   € V ^8„  d   QhRR/# )r   r²  rû  r   )r   s   "r   r   r	  %  s   € ÷ 
ñ 
Ð"Nñ 
r   c                	óò   € VP                  4        UUUu/ uFM  w  r4T\        V\        4      '       d!   V Uu. uF  qPP                  V4      NK  	  upMV P                  V4      bKO  	  uppp# u upi u upppi rI   )r´  rk  rµ  r  )r¨   r²  r·  rˆ  r€  Úts   &&,   r   r¹  ÚFp8DecodeScale.convert%  sl   € ð )×.Ñ.Ô0õ
á0‘
�ð ´ZÀÄt×5LÒ5L©5Ó1©5 a—,‘,˜q–/©5Ò1ÐRV×R^ÑR^Ð_dÓReÒeÙ0ó
ð 	
ùÚ1ùô
s   •"A2·A-ÁA2Á-A2rz  N)
r)   r4   r5   r6   r7   r£   Ústaticmethodr  r¹  r9   r   r   r   r  r    s,   † ñò)ð ôdó ðd÷
ñ 
r   r  )r¯   NNr¯   )NNNN)NNF)HÚ
__future__r   Ú	functoolsrŠ   Úcollections.abcr   Údataclassesr   r!   Útorch.nnrd   r   r±   Úactivationsr   Úcore_model_loadingr   Úquantizers.quantizers_utilsr	   r
   Úutilsr   Úutils.import_utilsr   r   r[  r   r   r   Úhub_kernelsr   Úmoer   r   Útensor_parallelr   Ú
get_loggerr)   r�   Úfloat8_e4m3fnr¤   rš  r  r–  rg   r•  Úcacher$   r-   r0   rE   Ú_dynamoÚallow_in_graphrK   rN   rT   rn   r   r‘   rj  r“   r¼   r   r  ÚModuler  rX  rh  rv  rx  r¾  r  r   r   r   Ú<module>r%     s£  ðõ #ã Û 	Ý $Ý !ã Ý Ý $å  Ý .ß UÝ ß O÷ñ õ
 *ß =Ý %ð 
×	Ò	˜HÓ	%€ð × Ñ €
Ø�;Š;�zÓ"×&Ñ&€Ø�;Š;�zÓ"×&Ñ&€ð ‡�ô ó ð òGñ �$Ô÷ð ó ðð ‡�ô+ó ð+ð\ ‡�×Ñôó ðõ
*õ÷
÷8÷:BuôJ3
�—	‘	ô 3
ôlD�yô DõNG7õT[7ô|]
�—‘ô ]
ô@Ð*ô ñ 0Ó1Ð ÷PôfG0�-ô G0ôTV.�Mô V.ôr
�]ö 
r   