
      i                         d dl mZmZ d dlmZmZmZ d dlZd dlm	Z	 d dlm
Z
 d dlmZ d dlmZmZ d dlmZ d d	lmZmZ d d
lmZ erd dlmZ ed   Z G d de      Zy)    )AbstractContextManagernullcontext)TYPE_CHECKINGAnyLiteralN)apply_to_collection)Tensor)Module)get_argsoverride)	Precision)_convert_fp_tensor_DtypeContextManager)	Steppable)DeepSpeedEngine)32-true16-true	bf16-true16-mixed
bf16-mixedc                       e Zd ZdZdeddfdZededefd       Zede	fd       Z
ede	fd	       Zed
edefd       Zed
edefd       Zededddededdf
d       Zedededefd       Zy)DeepSpeedPrecisiona  Precision plugin for DeepSpeed integration.

    Args:
        precision: Full precision (32-true), half precision (16-true, bf16-true) or
            mixed precision (16-mixed, bf16-mixed).

    Raises:
        ValueError:
            If unsupported ``precision`` is provided.

    	precisionreturnNc                     t        t              }||vrt        d|d| d      || _        t        j
                  t        j                  t        j
                  t        j                  t        j                  d}|| j                     | _        y )Nz`precision=z>)` is not supported in DeepSpeed. `precision` must be one of: .)r   r   r   r   r   )	r   _PRECISION_INPUT
ValueErrorr   torchbfloat16float16float32_desired_dtype)selfr   supported_precisionprecision_to_types       /Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning/fabric/plugins/precision/deepspeed.py__init__zDeepSpeedPrecision.__init__.   s    &'78//i] +00C/DAG  #  ..}}}}
 0?    modulec                 Z    d| j                   v r|j                  | j                        S |S )Ntrue)dtype)r   tor#   )r$   r*   s     r'   convert_modulez!DeepSpeedPrecision.convert_module@   s*    T^^#994#6#6977r)   c                 \    d| j                   vr
t               S t        | j                        S )Nr,   )r   r   r   r#   r$   s    r'   tensor_init_contextz&DeepSpeedPrecision.tensor_init_contextF   s'    '= #D$7$788r)   c                 "    | j                         S )N)r2   r1   s    r'   module_init_contextz&DeepSpeedPrecision.module_init_contextL   s    ''))r)   datac                 D    t        |t        t        | j                        S N)functionr-   dst_type)r   r   r	   r#   r$   r5   s     r'   convert_inputz DeepSpeedPrecision.convert_inputP   s    "42DF]a]p]pqqr)   c                 T    t        |t        t        t        j                               S r7   )r   r   r	   r   get_default_dtyper:   s     r'   convert_outputz!DeepSpeedPrecision.convert_outputT   s    "42DF]b]t]t]vwwr)   tensormodelr   argskwargsc                 0     |j                   |g|i | y)z3Performs back-propagation using DeepSpeed's engine.N)backward)r$   r?   r@   rA   rB   s        r'   rD   zDeepSpeedPrecision.backwardX   s     	v///r)   	optimizerc                 &     |j                   di |S )N )step)r$   rE   rB   s      r'   optimizer_stepz!DeepSpeedPrecision.optimizer_step]   s     y~~'''r)   )__name__
__module____qualname____doc__r   r(   r   r
   r/   r   r2   r4   r   r;   r>   r	   rD   r   rI   rG   r)   r'   r   r   !   s.   
@"2 @t @$ V   
 9%; 9 9
 *%; * * r# r# r r x3 x3 x x 0v 0.? 0 0WZ 0_c 0 0 (( ( 
	( (r)   r   )
contextlibr   r   typingr   r   r   r   #lightning_utilities.core.apply_funcr   r	   torch.nnr
   typing_extensionsr   r   ,lightning.fabric.plugins.precision.precisionr   (lightning.fabric.plugins.precision.utilsr   r    lightning.fabric.utilities.typesr   	deepspeedr   r   r   rG   r)   r'   <module>rW      sH    ; . .  C   0 B ] 6)VW C( C(r)   