+
    G-jE  ã                   ó\   € ^ RI Ht ^ RIHt ^ RIHt ^ RIHt ]! ]4      t	 ! R R]4      t
R# )é    )Ú	getLogger)ÚFusion)Úhelper©Ú	OnnxModelc                   ó˜   a a€ ] tR t^t oV3R lV 3R lltV3R lR ltV3R lR ltV3R lR ltV3R	 lR
 ltV3R lR lt	Rt
VtV ;t# )ÚFusionFastGeluc                ó    <€ V ^8„  d   QhRS[ /# )é   Úmodelr   )ÚformatÚ__classdict__s   "€Úy/Volumes/fast/ai/experiments/nudenet-smoke/.venv/lib/python3.14/site-packages/onnxruntime/transformers/fusion_fastgelu.pyÚ__annotate__ÚFusionFastGelu.__annotate__   s   ø€ ÷ 4ñ 4™iñ 4ó    c                ó*   <€ \         SV `  VR R4       R# )ÚFastGeluÚTanhN)ÚsuperÚ__init__)Úselfr   Ú	__class__s   &&€r   r   ÚFusionFastGelu.__init__   s   ø€ Ü‰Ñ˜ 
¨FÖ3r   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Úinput_name_to_nodesÚoutput_name_to_node)Údict)r   r   s   "€r   r   r      s   ø€ ÷ ñ ±4ð Édñ r   c                óÖ   € V P                  WV4      '       d   R # V P                  WV4      '       d   R # V P                  WV4      '       d   R # V P                  WV4      '       d   R # R # )N)Úfuse_1Úfuse_2Úfuse_3Úfuse_4)r   Ú	tanh_noder   r   s   &&&&r   ÚfuseÚFusionFastGelu.fuse   s\   € Ø�;‰;�yÐ7J×KÒKÙà�;‰;�yÐ7J×KÒKÙà�;‰;�yÐ7J×KÒKÙà�;‰;�yÐ7J×KÒKÙñ Lr   c                ó.   <€ V ^8„  d   QhRS[ R,          /# )r   ÚreturnN)Úbool)r   r   s   "€r   r   r      s   ø€ ÷ jñ jÉTÐTXÍ[ñ jr   c                ó0  € VP                   ^ ,          V9  d   R# W!P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP                   ^ ,          V9  d   R# W%P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P                  VRRV4      pVf   R# V P                  P                  VR4      pV^ 8  d   R# VP                  V^8X  d   ^ M^,          p	V P                  P                  Yx^8X  d   ^ M^V4      p
V P                  P                  VR^ V4      pVf   R# V P                  P                  VRRR7      pV^ 8  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# V P                  P                  TRRTV
'       d   V
.M. R	7      pVf   R# V P                  P                  VR
RR7      pV^ 8  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# V P                  P	                  VR4      '       g   R# VP                  ^ ,          V	8w  d   R# VVVVVVVV.pV P                  P                  VVP                   ^ ,          .VV4      '       g   R# V P                  P                  V4       \        P                  ! RV	.VP                   V P                  P                  R4      R7      pRVn        V P                   P#                  V4       V P$                  V P&                  VP(                  &   R# )a  
Fuse Gelu with tanh into one node:
      +---------------------------+
      |                           |
      |                           v
    [root] --> Pow --> Mul -----> Add  --> Mul --> Tanh --> Add --> Mul
      |       (Y=3)   (B=0.0447...)       (B=0.7978...)    (B=1)     ^
      |                                                              |
      +------> Mul(B=0.5)--------------------------------------------+
Note that constant input for Add and Mul could be first or second input: like either A=0.5 or B=0.5 is fine.
NÚAddç      ð?ÚMulç      à?ç+ö—Ý“‡é?ç-Cëâ6?©Údelta©Úexcludeç¥,Cëâ¦?ÚPowç      @r   ©ÚinputsÚoutputsÚnameúcom.microsoftT)ÚoutputÚlenÚop_typer   Úhas_constant_inputÚmatch_parentÚfind_constant_inputÚinputÚ
get_parentÚis_safe_to_fuse_nodesÚnodes_to_removeÚextendr   Ú	make_nodeÚcreate_node_nameÚdomainÚnodes_to_addÚappendÚthis_graph_nameÚnode_name_to_graph_namer;   )r   r$   r   r   ÚchildrenÚadd_after_tanhÚmul_after_tanhÚmul_halfÚiÚ
root_inputÚ	root_nodeÚmul_before_tanhÚadd_before_tanhÚmul_after_powÚpowÚsubgraph_nodesÚ
fused_nodes   &&&&             r   r    ÚFusionFastGelu.fuse_1   sG  € ð ×Ñ˜AÕÐ&9Ô9ÙØ&×'7Ñ'7¸Õ':Õ;ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ! !�ˆà�z‰z×,Ñ,¨^¸S×AÒAÙà× Ñ  Õ#Ð+>Ô>ÙØ&×'<Ñ'<¸QÕ'?Õ@ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ! !�ˆà—:‘:×*Ñ*¨>¸5À$ÐH[Ó\ˆØÒÙà�J‰J×*Ñ*¨8°SÓ9ˆØˆqŒ5Ùà—^‘^¨¨a¬¡A°QÕ7ˆ
ð —J‘J×)Ñ)¨(¸a¼±AÀQÐH[Ó\ˆ	àŸ*™*×1Ñ1°)¸UÀAÐGZÓ[ˆØÒ"Ùà�J‰J×*Ñ*¨?¸FÈ&Ð*ÓQˆØˆqŒ5ÙàŸ*™*×1Ñ1°/À5ÈqÐTUÌvÉ!Ð[\Ð^qÓrˆØÒ"ÙàŸ
™
×/Ñ/ØØØØß#,�Y‘K°"ð 0ó 
ˆð Ò Ùà�J‰J×*Ñ*¨=¸&ÈÐ*ÓOˆØˆqŒ5Ùà�j‰j×%Ñ% m°UÀÀaÄ¹AÈQÐPcÓdˆØŠ;Ùà�z‰z×,Ñ,¨S°#×6Ò6Ùà�9‰9�Q�<˜:Ô%Ùð ØØØØØØØð	
ˆð �z‰z×/Ñ/ØØ×"Ñ" 1Õ%Ð&ØØ÷	
ò 
ñ à×Ñ×#Ñ# NÔ3Ü×%Ò%ØØ�<Ø"×)Ñ)Ø—‘×,Ñ,¨ZÓ8ô	
ˆ
ð ,ˆ
ÔØ×Ñ× Ñ  Ô,Ø8<×8LÑ8Lˆ×$Ñ$ Z§_¡_Ñ5Ùr   c                ó:   <€ V ^8„  d   QhRS[ RS[ RS[R,          /# ©r   r   r   r(   N©r   r)   )r   r   s   "€r   r   r   ‹   ó/   ø€ ÷ rñ r±Tð rÑPTð rÑY]Ð`dÕYdñ rr   c                ó>  € VP                   ^ ,          V9  d   R# W!P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP                   ^ ,          V9  d   R# W%P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P                  VR4      pV^ 8  d   R# VP                   ^ ,          V9  d   R# W&P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P                  TVP                  ^,          VP                   ^ ,          8X  d   ^ M^V4      p	V P                  P                  VR^ V4      p
V
f   R# V P                  P                  V
RRR7      pV^ 8  d   R# V P                  P                  T
RV^8X  d   ^ M^V4      pVf   R# V P                  P                  TRRTV	'       d   V	.M. R	7      pVf   R# V P                  P                  VR
RR7      pV^ 8  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# V P                  P	                  VR4      '       g   R# VP                  VP                  ^,          VP                   ^ ,          8X  d   ^ M^,          pVP                  ^ ,          V8w  d   R# VVVVV
VVV.pV P                  P                  VVP                   ^ ,          .VV4      '       g   R# V P                  P                  V4       \        P                  ! RV.VP                   V P                  P                  R4      R7      pRVn        V P                   P#                  V4       V P$                  V P&                  VP(                  &   R# )ax  
This pattern is from Tensorflow model.
Fuse Gelu with tanh into one node:
      +---------------------------+
      |                           |
      |                           v
    [root] --> Pow --> Mul -----> Add  --> Mul --> Tanh --> Add --> Mul(B=0.5)-->Mul-->
      |       (Y=3)   (B=0.0447...)       (B=0.7978...)    (B=1)                  ^
      |                                                                           |
      +---------------------------------------------------------------------------+
Note that constant input for Add and Mul could be first or second input: like either A=0.5 or B=0.5 is fine.
Nr+   r,   r-   r.   r/   r0   r1   r3   r5   r6   r7   r   r8   r<   T)r=   r>   r?   r   r@   rB   rD   rC   rA   rE   rF   rG   r   rH   rI   rJ   rK   rL   rM   rN   r;   )r   r$   r   r   rO   rP   rR   rS   Úmul_after_mul_halfrU   rV   rW   rX   rY   rT   rZ   r[   s   &&&&             r   r!   ÚFusionFastGelu.fuse_2‹   s­  € ð ×Ñ˜AÕÐ&9Ô9ÙØ&×'7Ñ'7¸Õ':Õ;ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ! !�ˆà�z‰z×,Ñ,¨^¸S×AÒAÙà× Ñ  Õ#Ð+>Ô>ÙØ&×'<Ñ'<¸QÕ'?Õ@ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ˜A•;ˆà�J‰J×*Ñ*¨8°SÓ9ˆØˆqŒ5Ùà�?‰?˜1ÕÐ%8Ô8ÙØ&§¡°qÕ'9Õ:ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ% a�[Ðð —J‘J×)Ñ)ØØ#×)Ñ)¨!Õ,°·±ÀÕ0BÔB‰AÈØó
ˆ	ð Ÿ*™*×1Ñ1°)¸UÀAÐGZÓ[ˆØÒ"Ùà�J‰J×*Ñ*¨?¸FÈ&Ð*ÓQˆØˆqŒ5ÙàŸ*™*×1Ñ1°/À5ÈqÐTUÌvÉ!Ð[\Ð^qÓrˆØÒ"ÙàŸ
™
×/Ñ/ØØØØß#,�Y‘K°"ð 0ó 
ˆð Ò Ùà�J‰J×*Ñ*¨=¸&ÈÐ*ÓOˆØˆqŒ5Ùà�j‰j×%Ñ% m°UÀÀaÄ¹AÈQÐPcÓdˆØŠ;Ùà�z‰z×,Ñ,¨S°#×6Ò6Ùà'×-Ñ-Ð3E×3KÑ3KÈAÕ3NÐRZ×RaÑRaÐbcÕRdÔ3d©aÐjkÕlˆ
à�9‰9�Q�<˜:Ô%Ùð ØØØØØØØð	
ˆð �z‰z×/Ñ/ØØ×&Ñ& qÕ)Ð*ØØ÷	
ò 
ñ à×Ñ×#Ñ# NÔ3Ü×%Ò%ØØ�<Ø&×-Ñ-Ø—‘×,Ñ,¨ZÓ8ô	
ˆ
ð ,ˆ
ÔØ×Ñ× Ñ  Ô,Ø8<×8LÑ8Lˆ×$Ñ$ Z§_¡_Ñ5Ùr   c                ó:   <€ V ^8„  d   QhRS[ RS[ RS[R,          /# r^   r_   )r   r   s   "€r   r   r   ÿ   r`   r   c           	     óÎ  € VP                   ^ ,          V9  d   R# W!P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP                   ^ ,          V9  d   R# W%P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P                  VRRV4      pVf   R# V P                  P                  VR4      pV^ 8  d   R# VP                  V^8X  d   ^ M^,          p	V P                  P                  VR^ V4      p
V
f   R# V P                  P                  V
RRV4      pVf   R# V P                  P                  VR4      pV^ 8  d   R# V P                  P                  V
RRV4      pVf   R# V P                  P                  VRRR7      pV^ 8  d   R# VP                  V^8X  d   ^ M^,          V	8w  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# VP                  ^ ,          V	8X  d   ^pMVP                  ^,          V	8X  d   ^ pMR# V P                  P                  VRVV4      pVf   R# V P                  P                  VR	RR7      pV^ 8  d   R# VP                  V^8X  d   ^ M^,          V	8w  d   R# VVVV
VVVVV.	pV P                  P                  VVP                   ^ ,          .VV4      '       g   R# V P                  P                  V4       \        P                  ! R
V	.VP                   V P                  P                  R
4      R7      pRVn        V P                  P!                  V4       V P"                  V P$                  VP&                  &   R# )a6  
OpenAI's gelu implementation, also used in Megatron:
   Gelu(x) = x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1.0 + 0.044715 * x * x)))

Fuse subgraph into a FastGelu node:
    +------------ Mul (B=0.79788456) -------------------+
    |                                                   |
    +-------------------------------+                   |
    |                               |                   |
    |                               v                   v
  [root] --> Mul (B=0.044715) --> Mul --> Add(B=1) --> Mul --> Tanh --> Add(B=1) --> Mul-->
    |                                                                                 ^
    |                                                                                 |
    +-----------> Mul (B=0.5) --------------------------------------------------------+
Nr+   r,   r-   r.   r/   r0   r1   r5   r   r8   r<   T)r=   r>   r?   r   r@   rA   rB   rC   rE   rF   rG   r   rH   rI   rJ   rK   rL   rM   rN   r;   )r   r$   r   r   rO   rP   Úmul_lastrR   rS   rT   rV   Úadd_1ÚjÚmul_7978ÚkÚmul_before_add_1ÚanotherÚmul_0447ÚmrZ   r[   s   &&&&                 r   r"   ÚFusionFastGelu.fuse_3ÿ   s‹  € ð  ×Ñ˜AÕÐ&9Ô9Ùà&×'7Ñ'7¸Õ':Õ;ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ! !�ˆà�z‰z×,Ñ,¨^¸S×AÒAÙà× Ñ  Õ#Ð+>Ô>ÙØ&×'<Ñ'<¸QÕ'?Õ@ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ˜A•;ˆà—:‘:×*Ñ*¨8°U¸DÐBUÓVˆØÒÙà�J‰J×*Ñ*¨8°SÓ9ˆØˆqŒ5Ùà—^‘^¨¨a¬¡A°QÕ7ˆ
àŸ*™*×1Ñ1°)¸UÀAÐGZÓ[ˆØÒ"Ùà—
‘
×'Ñ'¨¸ÀÐFYÓZˆØŠ=ÙØ�J‰J×*Ñ*¨5°#Ó6ˆØˆqŒ5Ùà—:‘:×*Ñ*¨?¸EÀ4ÐI\Ó]ˆØÒÙØ�J‰J×*Ñ*¨8°VÀ6Ð*ÓJˆØˆqŒ5ÙØ�>‰>˜q Aœv™!¨1Õ-°Ô;ÙàŸ:™:×2Ñ2°5¸%ÀaÈ1ÄfÁÐRSÐUhÓiÐØÒ#Ùà×!Ñ! !Õ$¨
Ô2Ø‰GØ×#Ñ# AÕ&¨*Ô4Ø‰Gáà—:‘:×*Ñ*Ð+;¸UÀGÐM`ÓaˆØÒÙØ�J‰J×*Ñ*¨8°VÀ6Ð*ÓJˆØˆqŒ5Ùà�>‰>˜q Aœv™!¨1Õ-°Ô;Ùð ØØØØØØØØð

ˆð �z‰z×/Ñ/ØØ�_‰_˜QÕÐ ØØ÷	
ò 
ñ à×Ñ×#Ñ# NÔ3Ü×%Ò%ØØ�<Ø—O‘OØ—‘×,Ñ,¨ZÓ8ô	
ˆ
ð ,ˆ
ÔØ×Ñ× Ñ  Ô,Ø8<×8LÑ8Lˆ×$Ñ$ Z§_¡_Ñ5Ùr   c                ó:   <€ V ^8„  d   QhRS[ RS[ RS[R,          /# r^   r_   )r   r   s   "€r   r   r   s  s/   ø€ ÷ yñ y±Tð yÑPTð yÑY]Ð`dÕYdñ yr   c           	     óÂ  € VP                   ^ ,          V9  d   R# W!P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP                   ^ ,          V9  d   R# W%P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pVP                   ^ ,          V9  d   R# W&P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP
                  VP
                  ^,          VP                   ^ ,          8X  d   ^ M^,          pV P                  P                  VR^ V4      p	V	f   R# V P                  P                  V	RRR7      p
V
^ 8  d   R# V P                  P                  T	RV
^8X  d   ^ M^V4      pVf   R# VP
                  ^ ,          V8X  d   ^pMVP
                  ^,          V8X  d   ^ pMR# V P                  P                  VRWÃ4      pVf   R# V P                  P                  VR	RR7      pV^ 8  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# VP
                  ^ ,          V8X  d   ^pMVP
                  ^,          V8X  d   ^ pMR# V P                  P                  VRWÃ4      pVf   R# VP
                  ^ ,          V8w  g   VP
                  ^,          V8w  d   R# VVVVV	VVVV.	pV P                  P                  VVP                   ^ ,          .VV4      '       g   R# V P                  P                  V4       \        P                  ! R
V.VP                   V P                  P                  R
4      R7      pRVn        V P                  P!                  V4       V P"                  V P$                  VP&                  &   V P)                  R
4       R# )aò  
PyTorch's gelu implementation with tanh approximation:
   Gelu(x) = 0.5 * x * (1 + torch.tanh(0.7978845834732056 * (x + 0.044714998453855515 * x * x * x)))

Fuse Gelu with tanh into one node:
      +-----------------+------------------+
      |                 |                  |
      |                 v                  v
    [root] ==> Mul --> Mul --> Mul -----> Add  --> Mul --> Tanh --> Add -----> Mul --> Mul -->
      |                       (A=0.0447)          (A=0.7978)        (A=1)       ^     (A=0.5)
      |                                                                         |
      +-------------------------------------------------------------------------+
Note that constant input for Add and Mul could be first or second input.
Nr+   r,   r-   r.   r/   g{®Gáz„?r1   r5   r   r8   r<   T)r=   r>   r?   r   r@   rC   rA   rB   rE   rF   rG   r   rH   rI   rJ   rK   rL   rM   rN   r;   Úincrease_counter)r   r$   r   r   rO   rP   rQ   rR   rT   rV   rj   rW   rl   rX   rn   Ú	mul_cubedÚmul_squaredrZ   r[   s   &&&&               r   r#   ÚFusionFastGelu.fuse_4s  sß  € ð ×Ñ˜AÕÐ&9Ô9Ùà&×'7Ñ'7¸Õ':Õ;ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ! !�ˆà�z‰z×,Ñ,¨^¸S×AÒAÙà× Ñ  Õ#Ð+>Ô>ÙØ&×'<Ñ'<¸QÕ'?Õ@ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ! !�ˆà× Ñ  Õ#Ð+>Ô>ÙØ&×'<Ñ'<¸QÕ'?Õ@ˆÜˆx‹=˜AÔ ¨!¥×!4Ñ!4¸Ô!=ÙØ˜A•;ˆà�z‰z×,Ñ,¨X°s×;Ò;Ùà#×)Ñ)¨~×/CÑ/CÀAÕ/FÈ.×J_ÑJ_Ð`aÕJbÔ/b©!ÐhiÕjˆ
àŸ*™*×1Ñ1°)¸UÀAÐGZÓ[ˆØÒ"Ùà�J‰J×*Ñ*¨?¸FÈ$Ð*ÓOˆØˆqŒ5ÙàŸ*™*×1Ñ1°/À5ÈqÐTUÌvÉ!Ð[\Ð^qÓrˆØÒ"Ùà× Ñ  Õ# zÔ1Ø‰GØ×"Ñ" 1Õ%¨Ô3Ø‰GáàŸ
™
×/Ñ/°ÀÈÓeˆØÒ Ùà�J‰J×*Ñ*¨=¸&ÈÐ*ÓMˆØˆqŒ5Ùà—J‘J×+Ñ+¨M¸5ÀqÈAÄvÁ!ÐSTÐViÓjˆ	ØÒÙà�?‰?˜1Õ Ô+Ø‰GØ�_‰_˜QÕ :Ô-Ø‰Gáà—j‘j×-Ñ-¨i¸ÀÓ]ˆØÒÙà×Ñ˜QÕ :Ô-°×1BÑ1BÀ1Õ1EÈÔ1SÙð ØØØØØØØØð

ˆð �z‰z×/Ñ/ØØ�_‰_˜QÕÐ ØØ÷	
ò 
ñ à×Ñ×#Ñ# NÔ3Ü×%Ò%ØØ�<Ø—O‘OØ—‘×,Ñ,¨ZÓ8ô	
ˆ
ð ,ˆ
ÔØ×Ñ× Ñ  Ô,Ø8<×8LÑ8Lˆ×$Ñ$ Z§_¡_Ñ5Ø×Ñ˜jÔ)Ùr   © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r%   r    r!   r"   r#   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r   r   s   @@r   r	   r	      sR   ù‡ € ÷4ó 4÷ð ÷jð j÷Xrð r÷hrð r÷hy÷ yð yr   r	   N)Úloggingr   Úfusion_baser   Úonnxr   Ú
onnx_modelr   rw   Úloggerr	   rv   r   r   Ú<module>rƒ      s+   ðõ
 å Ý Ý  á	�8Ó	€ô^�Vö ^r   