
    (ti.              	          d dl Z d dlmZmZ d dlmZ d dlZddlmZ ddl	m
Z
 ddlmZmZmZmZ dd	lmZmZ dd
lmZmZ  ee      ZdZe G d d             Z G d dej4                  j6                        Z G d de      Z G d de      Z G d de      Zdej@                  jB                  deddfdZ"ddej@                  jB                  dede#dz  ddfdZ$y)    N)asdict	dataclass)Callable   )
get_logger)unwrap_module   )"_ALL_TRANSFORMER_BLOCK_IDENTIFIERS_ATTENTION_CLASSES_FEEDFORWARD_CLASSES_get_submodule_from_fqn)AttentionProcessorRegistryTransformerBlockRegistry)HookRegistry	ModelHooklayer_skip_hookc                       e Zd ZU dZee   ed<   dZeed<   dZ	e
ed<   dZe
ed<   dZe
ed	<   d
Zeed<   d Zd Zededd fd       Zy)LayerSkipConfiga  
    Configuration for skipping internal transformer blocks when executing a transformer model.

    Args:
        indices (`list[int]`):
            The indices of the layer to skip. This is typically the first layer in the transformer block.
        fqn (`str`, defaults to `"auto"`):
            The fully qualified name identifying the stack of transformer blocks. Typically, this is
            `transformer_blocks`, `single_transformer_blocks`, `blocks`, `layers`, or `temporal_transformer_blocks`.
            For automatic detection, set this to `"auto"`. "auto" only works on DiT models. For UNet models, you must
            provide the correct fqn.
        skip_attention (`bool`, defaults to `True`):
            Whether to skip attention blocks.
        skip_ff (`bool`, defaults to `True`):
            Whether to skip feed-forward blocks.
        skip_attention_scores (`bool`, defaults to `False`):
            Whether to skip attention score computation in the attention blocks. This is equivalent to using `value`
            projections as the output of scaled dot product attention.
        dropout (`float`, defaults to `1.0`):
            The dropout probability for dropping the outputs of the skipped layers. By default, this is set to `1.0`,
            meaning that the outputs of the skipped layers are completely ignored. If set to `0.0`, the outputs of the
            skipped layers are fully retained, which is equivalent to not skipping any layers.
    indicesautofqnTskip_attentionFskip_attention_scoresskip_ff      ?dropoutc                     d| j                   cxk  rdk  sn t        d| j                    d      t        j                  | j                   d      s| j                  rt        d      y y )Nr   r	   z6Expected `dropout` to be between 0.0 and 1.0, but got .r   bCannot set `skip_attention_scores` to True when `dropout` is not 1.0. Please set `dropout` to 1.0.)r   
ValueErrormathiscloser   selfs    i/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/diffusers/hooks/layer_skip.py__post_init__zLayerSkipConfig.__post_init__I   sd    T\\&Q&UVZVbVbUccdeff||DLL#.43M3Mt  4N.    c                     t        |       S N)r   r#   s    r%   to_dictzLayerSkipConfig.to_dictQ   s    d|r'   datareturnc                     t        di | S )N )r   )r+   s    r%   	from_dictzLayerSkipConfig.from_dictT   s    &&&r'   N)__name__
__module____qualname____doc__listint__annotations__r   strr   boolr   r   r   floatr&   r*   staticmethoddictr/   r.   r'   r%   r   r   (   ss    0 #YCND"'4'GTGU ' '!2 ' 'r'   r   c                       e Zd ZddZy)AttentionScoreSkipFunctionModeNc                 J   |i }|t         j                  j                  j                  u rr|j	                  dd       }|j	                  dd       }|j	                  dd       }||n|d   }||n|d   }||n|d   }|j
                  d   |j
                  d   k(  r|S  ||i |S )Nquerykeyvaluer   r	   r   )torchnn
functionalscaled_dot_product_attentiongetshape)r$   functypesargskwargsr?   r@   rA   s           r%   __torch_function__z1AttentionScoreSkipFunctionMode.__torch_function__Z   s    >F588&&CCCJJw-E**UD)CJJw-E".EDGE#d1gC".EDGE {{1~Q/T$V$$r'   )r.   N)r0   r1   r2   rL   r.   r'   r%   r=   r=   Y   s    %r'   r=   c                   V    e Zd ZddededefdZdej                  j                  fdZ
y)	AttentionProcessorSkipHookskip_processor_output_fnr   r   c                 .    || _         || _        || _        y r)   )rO   r   r   )r$   rO   r   r   s       r%   __init__z#AttentionProcessorSkipHook.__init__o   s    (@%%:"r'   modulec                    | j                   r\t        j                  | j                  d      st	        d      t               5   | j                  j                  |i |}d d d        |S t        j                  | j                  d      r | j                  |g|i |}|S  | j                  j                  |i |}t        j                  j                  j                  || j                        }|S # 1 sw Y   S xY w)Nr   r   p)r   r!   r"   r   r    r=   fn_reforiginal_forwardrO   rB   rC   rD   r$   rR   rJ   rK   outputs        r%   new_forwardz&AttentionProcessorSkipHook.new_forwardt   s    %%<<c2 x  01555tFvF 2  ||DLL#.666vOOO  655tFvF,,44Vt||4L 2 s   C44C>N)Fr   )r0   r1   r2   r   r8   r9   rQ   rB   rC   ModulerZ   r.   r'   r%   rN   rN   n   s2     RV in 
%((// r'   rN   c                   X     e Zd Zdef fdZdej                  j                  fdZ xZ	S )FeedForwardSkipHookr   c                 0    t         |           || _        y r)   superrQ   r   r$   r   	__class__s     r%   rQ   zFeedForwardSkipHook.__init__       r'   rR   c                 b   t        j                  | j                  d      r=|j                  dd       }||j                  dd       }|t	        |      dkD  r|d   }|S  | j
                  j                  |i |}t        j                  j                  j                  || j                        }|S )Nr   hidden_statesxr   rT   )
r!   r"   r   rF   lenrV   rW   rB   rC   rD   rX   s        r%   rZ   zFeedForwardSkipHook.new_forward   s    <<c*ZZ6F~C.~#d)a-a  2T[[114B6BFXX((004<<0HFr'   )
r0   r1   r2   r9   rQ   rB   rC   r[   rZ   __classcell__rb   s   @r%   r]   r]      s#     
%((// 
r'   r]   c                   ^     e Zd Zdef fdZd Zdej                  j                  fdZ	 xZ
S )TransformerBlockSkipHookr   c                 0    t         |           || _        y r)   r_   ra   s     r%   rQ   z!TransformerBlockSkipHook.__init__   rc   r'   c                 `    t        j                  t        |      j                        | _        |S r)   )r   rF   r   rb   	_metadata)r$   rR   s     r%   initialize_hookz(TransformerBlockSkipHook.initialize_hook   s$    155mF6K6U6UVr'   rR   c                    t        j                  | j                  d      rZ| j                  j	                  d||      }| j                  j
                  |}|S | j                  j	                  d||      }||f}|S  | j                  j                  |i |}t        j                  j                  j                  || j                        }|S )Nr   re   encoder_hidden_statesrT   )r!   r"   r   rn   _get_parameter_from_args_kwargs"return_encoder_hidden_states_indexrV   rW   rB   rC   rD   )r$   rR   rJ   rK   original_hidden_statesrY   original_encoder_hidden_statess          r%   rZ   z$TransformerBlockSkipHook.new_forward   s    <<c*%)^^%S%STceikq%r"~~@@H/  261_1_+T62. 12PQ  2T[[114B6BFXX((004<<0HFr'   )r0   r1   r2   r9   rQ   ro   rB   rC   r[   rZ   rh   ri   s   @r%   rk   rk      s(     %((// r'   rk   rR   configr,   c                     t        | |       y)a  
    Apply layer skipping to internal layers of a transformer.

    Args:
        module (`torch.nn.Module`):
            The transformer model to which the layer skip hook should be applied.
        config (`LayerSkipConfig`):
            The configuration for the layer skip hook.

    Example:

    ```python
    >>> from diffusers import apply_layer_skip_hook, CogVideoXTransformer3DModel, LayerSkipConfig

    >>> transformer = CogVideoXTransformer3DModel.from_pretrained("THUDM/CogVideoX-5b", torch_dtype=torch.bfloat16)
    >>> config = LayerSkipConfig(layer_index=[10, 20], fqn="transformer_blocks")
    >>> apply_layer_skip_hook(transformer, config)
    ```
    N)_apply_layer_skip_hook)rR   rv   s     r%   apply_layer_skipry      s    ( 66*r'   namec                    |xs t         }|j                  r|j                  rt        d      t	        j
                  |j                  d      s|j                  rt        d      |j                  dk(  r+t        D ]  }t        | |      s||_         n t        d      t        | |j                        }|$t        |t        j                  j                        st        d|j                   d      t        |j                         dk(  rt        d	      d
}t#        |      D ]   \  }}||j                   vrd}|j                  ro|j$                  rct&        j)                  d|j                   d| d       t+        j,                  |      }t/        |j                        }	|j1                  |	|       n|j                  s|j                  r|j3                         D ]  \  }
}t        |t4              s|j6                  r$t&        j)                  d|j                   d| d|
 d       t9        j:                  |j<                  j>                        j@                  }t+        j,                  |      }tC        ||j                  |j                        }	|j1                  |	|        |j$                  s|j3                         D ]{  \  }
}t        |tD              st&        j)                  d|j                   d| d|
 d       t+        j,                  |      }tG        |j                        }	|j1                  |	|       } # |s&t        d|j                    d|j                   d      y )NzXCannot set both `skip_attention` and `skip_attention_scores` to True. Please choose one.r   r   r   zCould not find a suitable identifier for the transformer blocks automatically. Please provide a valid `fqn` (fully qualified name) that identifies a stack of transformer blocks.zCould not find z in the provided module, or configured `fqn` (fully qualified name) does not identify a `torch.nn.ModuleList`. Please provide a valid `fqn` that identifies a stack of transformer blocks.r   zTLayer index list is empty. Please provide a non-empty list of layer indices to skip.FTz&Applying TransformerBlockSkipHook to 'r   'z(Applying AttentionProcessorSkipHook to 'z!Applying FeedForwardSkipHook to 'zDCould not find any transformer blocks matching the provided indices z and fully qualified name 'z4'. Please check the indices and fqn for correctness.)$_LAYER_SKIP_HOOKr   r   r    r!   r"   r   r   r
   hasattrr   
isinstancerB   rC   
ModuleListrg   r   	enumerater   loggerdebugr   check_if_exists_or_initializerk   register_hooknamed_modulesr   is_cross_attentionr   rF   	processorrb   rO   rN   r   r]   )rR   rv   rz   
identifiertransformer_blocksblocks_foundiblockregistryhooksubmodule_name	submodule	output_fns                r%   rx   rx      s   ##D!=!=stt<<,1M1Mp
 	
 zzV<Jvz*'
 =
 ^ 
 1D!4FH[H[)\fjj\ *s t
 	
 6>>aoppL015FNN"  V^^LLA&**QqcQRST#AA%HH+FNN;D""4.""f&B&B-2-@-@-B)	i);<YEaEaLL#KFJJ<WXYZX[[\]k\llm!no : > >y?R?R?\?\ ] v vI+II)TH5iA]A]_e_m_mnD**46 .C >>-2-@-@-B)	i)=>LL#DVZZLPQRSQTTUVdUeef!gh+II)TH.v~~>D**46 .C- 2: RSYSaSaRb c%%+ZZL0df
 	
 r'   r)   )%r!   dataclassesr   r   typingr   rB   utilsr   utils.torch_utilsr   _commonr
   r   r   r   _helpersr   r   hooksr   r   r0   r   r}   r   	overridesTorchFunctionModer=   rN   r]   rk   rC   r[   ry   r7   rx   r.   r'   r%   <module>r      s     )    -  K * 
H	$ 
 -' -' -'`%U__%F%F %* .) $y 2+UXX__ +o +$ +.@
588?? @
O @
SVY]S] @
im @
r'   