
    (ti@                     n   d dl Z d dlZd dlZd dlmZ d dlmZ d dlZd dlm	Z
 ej                  j                         r	d dlm	c mZ ddlmZmZmZmZmZ ddlmZ ddlmZmZ dd	lmZmZ  ee      Zd
ZdZ e G d d             Z!dejD                  jF                  dede$e%ef   ddfdZ&dejD                  jF                  de$e%ef   ddfdZ' G d de      Z( G d de      Z) G d dejT                  jV                        Z, G d d      Z- G d dejT                  jV                        Z. G d d      Z/ ej`                  d !      d"e1e2   d#e1e2   d$e2d%e2de3e3e2      f
d&       Z4ed'ejj                  d$e2d(e
jl                  jn                  dejj                  fd)       Z8d*ejD                  jF                  d+e%dejD                  jF                  e3ejD                  jF                     z  fd,Z9d*ejD                  jF                  d+e%dejD                  jF                  e3ejD                  jF                     z  fd-Z:y).    N)	dataclass)Type   )ContextParallelConfigContextParallelInputContextParallelModelPlanContextParallelOutputgather_size_by_comm)
get_logger)maybe_allow_in_graphunwrap_module   )HookRegistry	ModelHookzcp_input---{}zcp_output---{}c                   B    e Zd ZU dZeeef   ed<   dZe	ed<   ddefdZ
y)ModuleForwardMetadataNcached_parameter_indices_cls
identifierc                    |xs i }||v r||   dd fS | j                   5| j                   j                  |d       }|t        d| d      ||   d|fS | j                  t        d      t	        t        j                  | j                  j                        j                  j                               }|dd  }t        |      D ci c]  \  }}||
 c}}| _         || j                   vrt        d| d      | j                   |   }|t        |      k\  rt        d| d	t        |       d
      ||   d|fS c c}}w )NTzParameter 'z' not found in cached indices.Fz$Model class is not set for metadata.r   z4' not found in function signature but was requested.z	Expected z arguments but got .)r   get
ValueErrorr   listinspect	signatureforward
parameterskeys	enumeratelen)selfr   argskwargsindexr   iparams           o/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/diffusers/hooks/context_parallel.py_get_parameter_from_args_kwargsz5ModuleForwardMetadata._get_parameter_from_args_kwargs3   s^   2*%tT11((41155j$GE} ;zl:X!YZZ;u,,99CDD'++DII,=,=>IINNPQ
^
BKJBW(XBWhaBW(X%T:::{:,6jkll--j9CIy/B3t9+QOPPE{E5(( )Ys   D;) N)__name__
__module____qualname__r   dictstrint__annotations__r   r   r)   r*       r(   r   r   .   s+    /3d38n3D$)# )r2   r   moduleparallel_configplanreturnc           	         t         j                  d|j                   d|        |j                         D ]1  \  }}t	        | |      }t        |t              s|g}t         j                  d|dt        |       d       |D ]  }t        |t              r"t        ||      }t        j                  |      }nt        |t        t        t        f      rUt        |t              r|g}t        d |D              st        d|       t!        ||      }t"        j                  |      }nt        dt%        |             t'        j(                  |      }	|	j+                  ||        4 y	)
z"Apply context parallel on a model.z(Applying context parallel with CP mesh: z and plan: z*Applying ContextParallelHook to module_id=z identifying a total of z modulesc              3   <   K   | ]  }t        |t                y wN)
isinstancer	   .0xs     r(   	<genexpr>z)apply_context_parallel.<locals>.<genexpr>g   s     WA:a)>?s   z?Expected all elements of cp_model_plan to be CPOutput, but got .Unsupported context parallel model plan type: N)loggerdebug_meshitems_get_submodule_by_namer:   r   r!   r.   ContextParallelSplitHook%_CONTEXT_PARALLEL_INPUT_HOOK_TEMPLATEformatr	   tupleallr   ContextParallelGatherHook&_CONTEXT_PARALLEL_OUTPUT_HOOK_TEMPLATEtyper   check_if_exists_or_initializeregister_hook)
r3   r4   r5   	module_idcp_model_plan	submodulemhook	hook_nameregistrys
             r(   apply_context_parallelrV   Q   sN    LL;O<Q<Q;RR]^b]cde$(JJL 	=*69=	)T*"IB	|C[\_`i\j[kkstuA-./OAHHS	M,A4+OPm-BC%2OMWWW$'fgtfu%vww0PBII)T	 #QRVWdReQf!ghh#AA!DH""43  %1r2   c                    |j                         D ]  \  }}t        | |      }t        |t              s|g}|D ]  }t	        j
                  |      }t        |t              rt        j                  |      }nHt        |t        t        t        f      rt        j                  |      }nt        dt        |             |j                  |         y )Nr?   )rC   rD   r:   r   r   rM   r.   rF   rG   r	   rH   rK   r   rL   remove_hook)r3   r5   rO   rP   rQ   rR   rU   rT   s           r(   remove_context_parallelrY   q   s    $(JJL 	=*69=	)T*"IA#AA!DH-.AHHS	M,A4+OPBII)T	 #QRVWdReQf!ghh  +  %1r2   c                   z     e Zd Zdededdf fdZd Zd Zd Zd	e	j                  d
ede	j                  fdZ xZS )rE   metadatar4   r6   Nc                 L    t         |           || _        || _        d | _        y r9   )super__init__r[   r4   module_forward_metadatar"   r[   r4   	__class__s      r(   r^   z!ContextParallelSplitHook.__init__   s%     .'+$r2   c                 R    t        |      j                  }t        |      | _        |S )N)r   )r   ra   r   r_   )r"   r3   clss      r(   initialize_hookz(ContextParallelSplitHook.initialize_hook   s$    F#--'<#'F$r2   c           	      D   t        |      }| j                  j                         D ]k  \  }}t        |t              r|j
                  r$| j                  j                  |||      \  }}}	|Ht        |t        j                        r| j                  ||      }nt        |t         t        f      rt        |      t        |      k7  r$t        dt        |       dt        |       d      g }
t        |      D ]O  \  }}t        j                  |      r$||   j
                  s| j                  |||         }|
j!                  |       Q |
}nt        dt#        |             |r|||<   H|	|	t        |      k  r|||	<   _t        d| d       t        |      |fS )Nz"Expected input model plan to have  elements, but got r   zUnsupported input type: z9An unexpected error occurred while processing the input 'z'. Please open an issue at https://github.com/huggingface/diffusers/issues and provide a minimal reproducible example along with the full stack trace.)r   r[   rC   r:   r   split_outputr_   r)   torchTensor_prepare_cp_inputrH   r!   r   r    	is_tensorappendrL   )r"   r3   r#   r$   	args_listnamecpm	input_valis_kwargr%   sharded_input_valr&   r=   s                r(   pre_forwardz$ContextParallelSplitHook.pre_forward   s   J	,,.ID##349I9I *.)E)E)e)ei*&Ix   )U\\2 229cB	Ie}5y>SX-$<S^<LL_`cdg`h_iijk  %'!%i0DAqq)#a&2E2E 221c!f=%,,Q/ 1 .	 #;DO;L!MNN(t"us9~'=#,	%  OPTv V? @ G /R Y''r2   c           	         t        |t        j                        }t        |t        t        f      xr t        d |D              }|s|st        dt        |       d      |r|gn
t        |      }| j                  j                         D ]g  \  }}t        |t              r|j                  s#|t        |      k\  rt        d| dt        |       d      ||   }| j                  ||      }|||<   i |r|d   S t	        |      S )Nc              3   P   K   | ]  }t        |t        j                           y wr9   r:   rh   ri   r;   s     r(   r>   z8ContextParallelSplitHook.post_forward.<locals>.<genexpr>   s"     Bohncd:aQVQ]Q]C^hn   $&CExpected output to be a tensor or a list/tuple of tensors, but got r   zIndex z$ out of bounds for output of length r   )r:   rh   ri   r   rH   rI   r   rL   r[   rC   r   rg   r!   rj   )r"   r3   outputrk   is_tensor_listr%   ro   current_outputs           r(   post_forwardz%ContextParallelSplitHook.post_forward   s   vu||4	#FT5M:osBohnBo?obcghncobppqrss&&DL----/JE3c#78@P@PF# 6%0TUXY_U`Taab!cdd#E]N!33NCHN*F5M 0 &vay85=8r2   r=   cp_inputc                    |j                   S|j                         |j                   k7  r6t        j                  d|j                    d|j                          d       |S | j                  j
                  r5t        j                  ||j                  | j                  j                        S t        j                  ||j                  | j                  j                        S )NzExpected input tensor to have z dimensions, but got z' dimensions, split will not be applied.)expected_dimsdimr@   warning_oncer4   ulysses_anythingPartitionAnythingShardershard_anything	split_dim_flattened_meshEquipartitionShardershard)r"   r=   r}   s      r(   rj   z*ContextParallelSplitHook._prepare_cp_input   s    !!-!%%'X=S=S2S01G1G0HH]^_^c^c^e]f  gN  O H##44/>>x))4+?+?+O+O  (--a1C1CTEYEYEiEijjr2   )r+   r,   r-   r   r   r^   rd   rs   r|   rh   ri   r   rj   __classcell__ra   s   @r(   rE   rE      sZ    ,!9 ,La ,fj ,
,(\9&k5<< k;O kTYT`T` kr2   rE   c                   2     e Zd Zdededdf fdZd Z xZS )rJ   r[   r4   r6   Nc                 >    t         |           || _        || _        y r9   )r]   r^   r[   r4   r`   s      r(   r^   z"ContextParallelGatherHook.__init__   s     .r2   c                    t        |t        j                        }|r|g}n@t        |t        t        f      rt        d |D              st        dt        |       d      t        |      }t        |      t        | j                        k7  r.t        dt        | j                         dt        |       d      t        | j                        D ]  \  }}|	| j                  j                  r<t        j                  ||   |j                  | j                  j                         ||<   [t"        j%                  ||   |j                  | j                  j                         ||<    |r|d   S t	        |      S )Nc              3   P   K   | ]  }t        |t        j                           y wr9   rv   r;   s     r(   r>   z9ContextParallelGatherHook.post_forward.<locals>.<genexpr>   s     ;hag\]Jq%,,<Wagrw   rx   r   zExpected output to have rf   r   )r:   rh   ri   r   rH   rI   r   rL   r!   r[   r    r4   r   r   unshard_anything
gather_dimr   r   unshard)r"   r3   ry   rk   r&   ro   s         r(   r|   z&ContextParallelGatherHook.post_forward   sH   vu||4	XFVdE]3;hag;h8hbcghncobppqrssfv;#dmm,,7DMM8J7KK^_bci_j^kklmnn.FAs{##444EE1Is~~t/C/C/S/Sq	 1881Is~~t/C/C/S/Sq	 / &vay85=8r2   )r+   r,   r-   r   r   r^   r|   r   r   s   @r(   rJ   rJ      s%    /!9 /La /fj /
9r2   rJ   c                   ,    e Zd Zed        Zed        Zy)AllGatherFunctionc                     || _         || _        t        j                  j	                  |      | _        t        j                  j                  |      | _        t        j                  |||      S )Ngroup)
r   r   rh   distributedget_world_size
world_sizeget_rankrankfuncolall_gather_tensor)ctxtensorr   r   s       r(   r   zAllGatherFunction.forward   sU    	**99%@$$--e4''5AAr2   c                 ~    t        j                  || j                  | j                        }|| j                     d d fS Nr   )rh   chunkr   r   r   )r   grad_outputgrad_chunkss      r(   backwardzAllGatherFunction.backward  s2    kk+s~~377K388$dD00r2   N)r+   r,   r-   staticmethodr   r   r*   r2   r(   r   r      s*    B B 1 1r2   r   c            	          e Zd Zedej
                  dedej                  j                  j                  dej
                  fd       Z
edej
                  dedej                  j                  j                  dej
                  fd       Zy)r   r   r   meshr6   c                     |j                         |   |j                         z  dk(  sJ d       |j                  |j                         |      t        j                  j	                  |j                                  S )Nr   zHTensor size along dimension to be sharded must be divisible by mesh sizer   )sizer   rh   r   r   	get_grouprc   r   r   r   s       r(   r   zEquipartitionSharder.shard  sj     {{}S!DIIK/14 	
V	
4 ||DIIKS|1%2C2C2L2LT^^M]2^__r2   c                 p    |j                         }t        j                  |||j                               }|S r9   )
contiguousr   applyr   r   s       r(   r   zEquipartitionSharder.unshard  s0    ""$"((dnn6FGr2   N)r+   r,   r-   classmethodrh   ri   r0   r   device_mesh
DeviceMeshr   r   r*   r2   r(   r   r     s    
`5<< 
`c 
`9J9J9V9V9a9a 
`fkfrfr 
` 
` U\\  5;L;L;X;X;c;c hmhtht  r2   r   c                   v    e Zd Zedej
                  dedej                  j                  fd       Z
ed        Zy)AllGatherAnythingFunctionr   r   r   c                     || _         || _        t        j                  |      | _        t        j
                  |      | _        t        |||      }|S r9   )r   r   distr   r   r   r   _all_gather_anything)r   r   r   r   gathered_tensors        r(   r   z!AllGatherAnythingFunction.forward$  sF    	,,U3=='.vsEBr2   c                 ~    t        j                  || j                  | j                        }|| j                     d d fS r   )rh   tensor_splitr   r   r   )r   r   grad_splitss      r(   r   z"AllGatherAnythingFunction.backward-  s6     ((cnn#''R388$dD00r2   N)r+   r,   r-   r   rh   ri   r0   r   r   r   r   r   r*   r2   r(   r   r   #  sM    U\\  D<L<L<W<W   1 1r2   r   c            	          e Zd Zedej
                  dedej                  j                  j                  dej
                  fd       Z
edej
                  dedej                  j                  j                  dej
                  fd       Zy)r   r   r   r   r6   c                 .   |j                         |   |j                         k\  s-J d|j                          d| d|j                          d       |j                  |j                         |      t        j                  |j	                                  S )NzCannot shard tensor of size z along dim z across mesh of size r   r   )r   r   r   r   r   r   s       r(   r   z'PartitionAnythingSharder.shard_anything6  s     {{}S!TYY[0 	
*6;;=/SEI^_c_h_h_j^kklm	
0
 ""499;C"8t~~GW9XYYr2   c                 p    |j                         }t        j                  |||j                               }|S r9   )r   r   r   r   r   s       r(   r   z)PartitionAnythingSharder.unshard_anythingA  s2     ""$*00dnn>NOr2   N)r+   r,   r-   r   rh   ri   r0   r   r   r   r   r   r*   r2   r(   r   r   5  s    Z\\Z(+Z383D3D3P3P3[3[Z	Z Z \\(+383D3D3P3P3[3[	 r2   r   @   )maxsizeshapegather_dimsr   r   c                     g }t        |      D ]9  }t        t        j                  |             }||   ||<   |j	                  |       ; |S r9   )ranger   copydeepcopyrl   )r   r   r   r   gather_shapesr&   
rank_shapes          r(   _fill_gather_shapesr   J  sL    M:$--./
%a.
3Z(  r2   r   r   c                    t        j                  |      }| j                         } | j                  }||   }t	        ||      }t        t        |      t        |      ||      }|D cg c].  }t        j                  || j                  | j                        0 }}t        j                  || |       t        j                  ||      }	|	S c c}w )Nr   )devicedtyper   )r   r   r   r   r
   r   rH   rh   emptyr   r   
all_gathercat)
r   r   r   r   r   rank_dimr   r   gathered_tensorsr   s
             r(   r   r   T  s    $$51J FLLESzH%h6K'eeK6H#zZMbopboY^E&--v||TbopOO$fE:ii 0c:O	 qs   (3Cmodelrn   c                 X    |j                  d      dkD  rt        d      t        | |      S )N*r   z.Wildcard '*' can only be used once in the name)countr   _find_submodule_by_name)r   rn   s     r(   rD   rD   f  s+    zz#IJJ"5$//r2   c                    |dk(  r| S d|v r|j                  dd      n|df\  }}|dk(  rjt        | t        j                  j                        st        d      g }| D ]2  }t        ||      }t        |t              s|g}|j                  |       4 |S t        | |      rt        | |      }t        ||      S t        d| d| j                  j                   d      )N r   r   r   z-Wildcard '*' can only be used with ModuleList'z' is not a submodule of ')splitr:   rh   nn
ModuleListr   r   r   extendhasattrgetattrra   r+   )r   rn   
first_atomremaining_name
submodulesrQ   subsubmoduless          r(   r   r   l  s    rz7:d{C!3r
JS%!4!45LMM
I3I~NMmT2!.m,	 
 5*%z2I*9nEEq,EeooF^F^E__`abbr2   );r   	functoolsr   dataclassesr   typingr   rh   torch.distributedr   r   is_available)torch.distributed._functional_collectives_functional_collectivesr   models._modeling_parallelr   r   r   r	   r
   utilsr   utils.torch_utilsr   r   hooksr   r   r+   r@   rF   rK   r   r   Moduler.   r/   rV   rY   rE   rJ   autogradFunctionr   r   r   r   	lru_cacherH   r0   r   r   ri   r   r   r   rD   r   r*   r2   r(   <module>r      so      !     	!!#>>   C * 
H	(7 %)9 & ) ) )D4HHOO4*4 s,,
-4 
	4@,EHHOO ,4E]@];^ ,cg ,"Xky Xkv9	 9D1// 1 *1 7 7 1$ * R uSz c
  Z] bfgklogpbq  !  C @P@P@[@[ `e`l`l  "0%((// 0 0SWX]X`X`XgXgShAh 0c588?? c# c%((//TXY^YaYaYhYhTiBi cr2   