
      iD              	          U d dl Z d dlmZmZmZ d dlmZ d dlmZm	Z	 d dl
mZ d dlmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/  ede0e1ef         Z2dZ3dZ4e5e6d<    G d d      Z7 G d de-      Z8 G d d       Z9d!ed"efd#Z:d$eeef   d"e;eeejx                  f   ee0e1ef      f   fd%Z=d&ejx                  d'e0e1ef   d"efd(Z>d)e5d*ed"dfd+Z?d$e@d"e5fd,ZAd-ed"efd.ZB eBej                        e_C        y)/    N)	GeneratorIteratorMapping)deepcopy)partialwraps)
MethodType)AnyCallableOptionalTypeVarUnionoverload)is_overridden)apply_to_collection)Tensor)nn)OptimizedModule)_IncompatibleKeys)	Optimizer)
DataLoader)override)	Precision)Strategy)move_data_to_device)_set_sampler_epoch)_DeviceDtypeModuleMixin)OptimizableT_destination)bound)training_stepvalidation_step	test_steppredict_stepF_in_fabric_backwardc            	           e Zd Zddededeee      ddfdZe	defd       Z
deeef   fdZd	eeef   ddfd
Zddee   defdZdedefdZy)_FabricOptimizerN	optimizerstrategy	callbacksreturnc                     || _         || _        |xs g | _        t        d|j                  j
                  z   | j                  |j                  fi       | _        y)a  FabricOptimizer is a thin wrapper around the :class:`~torch.optim.Optimizer` that delegates the optimizer
        step calls to the strategy.

        The underlying wrapped optimizer object can be accessed via the property :attr:`optimizer`.

        Args:
            optimizer: The optimizer to wrap
            strategy: Reference to the strategy for handling the optimizer step

        FabricN)
_optimizer	_strategy
_callbackstype	__class____name__)selfr(   r)   r*   s       n/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning/fabric/wrappers.py__init__z_FabricOptimizer.__init__5   sO     $!#/rh)<)<)E)EEXaXkXkGlnpq    c                     | j                   S N)r.   r4   s    r5   r(   z_FabricOptimizer.optimizerF   s    r7   c                 L    | j                   j                  | j                        S r9   )r/   get_optimizer_stater(   r:   s    r5   
state_dictz_FabricOptimizer.state_dictJ   s    ~~11$..AAr7   r=   c                 :    | j                   j                  |       y r9   )r(   load_state_dict)r4   r=   s     r5   r?   z _FabricOptimizer.load_state_dictM   s    &&z2r7   closurec                    |d|ini }t        | j                  d      r;t        | j                  j                  t              r| j                  j                  }n| j
                  } | j                  j                  |fi |}| j                  D ]/  }t        |dd       }t        |      s || j                  |       1 |S )Nr@   modelon_after_optimizer_step)r)   r(   )
hasattrr/   
isinstancerB   r   r(   optimizer_stepr0   getattrcallable)r4   r@   kwargsr(   outputcallbackhooks          r5   stepz_FabricOptimizer.stepP   s    )0)<)W%"4>>7+
4>>;O;OQ\0],,II...

 H8%>ED~dnn	B ( r7   itemc                 .    t        | j                  |      S r9   )rG   r.   r4   rN   s     r5   __getattr__z_FabricOptimizer.__getattr__a   s    t--r7   r9   )r3   
__module____qualname__r   r   r   listr   r6   propertyr(   dictstrr   r=   r?   r
   rM   rQ    r7   r5   r'   r'   4   s    r) rx rHUYZbUcLd rpt r" 9  BDf- B3$sF{*; 3 3HX. # ". . .r7   r'   c                       e Zd Z	 d"dej                  dedeej                     ddf fdZedej                  fd       Z	e
ded	edefd
       Zeddddedededefd       Zeddddededeeef   fd       Ze
	 d#dee   dededeeeef      fd       Ze
	 d$deeef   ded	edefd       Zdeeef   ddfdZdedefdZdededefdZdedefdZe
dedef fd       Ze
ded eddf fd!       Z xZ S )%_FabricModuleNforward_moduler)   original_moduler+   c                     t         |           || _        |xs || _        || _        t        t              | _        d| _        y)a  The FabricModule is a thin wrapper around the :class:`torch.nn.Module` and handles precision / autocast
        automatically for the forward pass.

        The underlying wrapped module can be accessed via the property :attr:`module`.

        Args:
            forward_module: The module to wrap the ``forward`` method on.
            strategy: Reference to the strategy for handling precision etc.
            original_module: The original, unmodified module as passed into the
                :meth:`lightning.fabric.fabric.Fabric.setup` method. This is needed when attribute lookup
                on this wrapper should pass through to the original module.

        TN)	superr6   _forward_module_original_moduler/   set_LIGHTNING_MODULE_STEP_METHODS_forward_methods_fabric_module_initialized)r4   r[   r)   r\   r2   s       r5   r6   z_FabricModule.__init__f   sB      	- / A>! #$B C*.'r7   c                 6    | j                   xs | j                  S r9   )r`   r_   r:   s    r5   modulez_FabricModule.module}   s    $$<(<(<<r7   argsrI   c                 &   | j                   j                  }|j                  ||f      \  }}|j                         5   | j                  |i |}ddd       |j                        }t        |t        | j                         |S # 1 sw Y   8xY w)ziCasts all inputs to the right precision and handles autocast for operations in the module forward method.Ndtypefunction)	r/   	precisionconvert_inputforward_contextr_   convert_outputr   r   _register_backward_hook)r4   rg   rI   rl   rJ   s        r5   forwardz_FabricModule.forward   s     NN,,	 ..f~>f&&()T))4:6:F ) ))&1F&4;W;WX )(s   BB.)prefix	keep_varsdestinationrr   rs   c                     y r9   rX   r4   rt   rr   rs   s       r5   r=   z_FabricModule.state_dict   s    twr7   c                     y r9   rX   )r4   rr   rs   s      r5   r=   z_FabricModule.state_dict   s    Y\r7   c                 >    | j                   j                  |||      S )N)rt   rr   rs   )r`   r=   rv   s       r5   r=   z_FabricModule.state_dict   s+     $$//# 0 
 	
r7   r=   strictc                 @     | j                   j                  d||d|S )N)r=   ry   rX   )r`   r?   )r4   r=   ry   rI   s       r5   r?   z_FabricModule.load_state_dict   s)     5t$$44d
SYd]cddr7   methodc           	         t        |t        t        f      s!t        dt	        |      j
                         t        |t              r|n|j
                  }|dk(  rt        d      t        t        | j                  |d      t              s2t        d| dt	        | j                        j
                   d| d      | j                  j                  |       y)	z]Mark a method as a 'forward' method to prevent it bypassing the strategy wrapper (e.g., DDP).z(Expected a method or a string, but got: rq   z>You cannot mark the forward method itself as a forward method.NzYou marked 'z' as a forward method, but `.z$` does not exist or is not a method.)rE   r	   rW   	TypeErrorr1   r3   
ValueErrorrG   r`   AttributeErrorrc   add)r4   r{   names      r5   mark_forward_methodz!_FabricModule.mark_forward_method   s    &:s"34FtF|G\G\F]^__#FC0vfoo9]^^'$"7"7tDjQ tf$@dF[F[A\AeAe@ffghlgm n- .  	!!$'r7   method_namec                      dk7  sJ  j                   j                  dt        dt        dt        f fddt        dt        dt        f fd}|S )Nrq   rg   rI   r+   c                  b    j                   _        t        j                         } || i |S r9   )r`   rq   rG   )rg   rI   r{   r   original_forwardr4   s      r5   wrapped_forwardzC_FabricModule._redirection_through_forward.<locals>.wrapped_forward   s5     -=D!!)T22K@F4*6**r7   c                  J    j                   _         j                  | i |S r9   )r`   rq   )rg   rI   r4   r   s     r5   call_forward_modulezG_FabricModule._redirection_through_forward.<locals>.call_forward_module   s'    ,;D!!)4<<000r7   )r`   rq   r
   )r4   r   r   r   r   s   `` @@r5   _redirection_through_forwardz*_FabricModule._redirection_through_forward   s_    i'''0088	+3 	+# 	+# 	+	1s 	1c 	1c 	1
 #"r7   r   c                      ddt         dt         ddffdt              dt         dt         dt         f fd	       }|S )
zTracks whether any submodule in ``self._original_module`` was called during the execution of ``method`` by
        registering forward hooks on all submodules.F___r+   Nc                  
    dy )NTrX   )r   r   module_calleds     r5   rL   zA_FabricModule._wrap_method_with_module_call_tracker.<locals>.hook   s     Mr7   rg   rI   c            	      0   g }
j                   j                         D ]"  }|j                  |j                               $  | i |}r2t	        dt        
j                         j                   d	 d	d      |D ]  }|j                           |S )NzYou are calling the method `r}   z()` from outside the model. To avoid issues with the currently selected strategy, explicitly mark it as a forward method with `fabric_model.mark_forward_method(z)` after `fabric.setup()`.)r`   modulesappendregister_forward_hookRuntimeErrorr1   r3   remove)rg   rI   handlesrf   rJ   handlerL   r{   r   r   r4   s         r5   _wrapped_methodzL_FabricModule._wrap_method_with_module_call_tracker.<locals>._wrapped_method   s    G//779v;;DAB : T,V,F"248M8M3N3W3W2XXYZ^Y_ `NNRXUoq 
 " "Mr7   )r
   r   )r4   r{   r   r   rL   r   s   ``` @@r5   %_wrap_method_with_module_call_trackerz3_FabricModule._wrap_method_with_module_call_tracker   s^     	!S 	! 	! 	! 
v	3 	# 	# 	 	 
	" r7   tensorc                      |j                   s|S t        d j                  t              }t	         fddD              }t        t        |xs |      }|j                  |       |S )Nbackwardparentc              3   j   K   | ]*  }t        |j                  j                  t                , yw)r   N)r   r/   rl   r   ).0r{   r4   s     r5   	<genexpr>z8_FabricModule._register_backward_hook.<locals>.<genexpr>   s/      !
G &$..":":9MMGs   03)pre_backwardr   post_backward)requires_gradr   r/   r   anyr   _backward_hookregister_hook)r4   r   strategy_requiresprecision_requiresrL   s   `    r5   rp   z%_FabricModule._register_backward_hook   sd    ##M)*dnnXV  !
G!
 
 ~(9(O=OQT"r7   rN   c                 v   |dk7  r8|| j                   v r*| j                  | j                  k7  r| j                  |      S 	 t        |   |      S # t        $ r` t        |   d      }t        ||      }t        j                  |      r+| j                  | j                  k7  r| j                  ||      }|cY S w xY w)Nrc   r`   )rc   r_   r`   r   r^   rQ   r   rG   inspectismethodr   )r4   rN   r\   attrr2   s       r5   rQ   z_FabricModule.__getattr__   s     &&---$$(=(== 44T::	 7&t,, 	#g12DEO?D1D%$*>*>$BWBW*WAA$MK	s    A A&B87B8valuec                     t        | dd      st        | 	  ||       y | j                  }t	        ||      }|t        |       v }|s|st        |||       y |rt        |||       |rt        | 	  ||       y y )Nrd   F)rG   r^   __setattr__r`   rD   dirsetattr)r4   r   r   r\   original_has_attrfabric_has_attrr2   s         r5   r   z_FabricModule.__setattr__
  s    t95AGe, //#OT: #d)+!_OT51
 OT51Ge, r7   r9   )N F)T)!r3   rR   rS   r   Moduler   r   r6   rU   rf   r   r
   rq   r   r   rW   boolr=   rV   r   r   r?   r   r	   r   r   r   r   r   rp   rQ   r   __classcell__)r2   s   @r5   rZ   rZ   e   s   dh/ ii/3;/NVWYW`W`Na/	/. =		 = = S C C   FI]`wwwVZwerw w*-\C\$\cSVh\ \_d
#M2
CF
X\
	$sCx.	!
 
 <@e!#s(+e59eLOe	e e
(%
C*@ (T (# # #,H C T\ :f      , - -C -D - -r7   rZ   c                       e Zd Zd	dedeej                     ddfdZedeej                     fd       Zde	fdZ
deee   eeddf   f   fdZy)
_FabricDataLoaderN
dataloaderdevicer+   c                 x    | j                   j                  |j                          || _        || _        d| _        y)a  The FabricDataLoader is a wrapper for the :class:`~torch.utils.data.DataLoader`. It moves the data to the
        device automatically if the device is specified.

        Args:
            dataloader: The dataloader to wrap
            device: The device to which the data should be moved. By default the device is `None` and no data
                transfers will be made (identical behavior as :class:`~torch.utils.data.DataLoader`).

        r   N)__dict__update_dataloader_device_num_iter_calls)r4   r   r   s      r5   r6   z_FabricDataLoader.__init__&  s3     	Z001% r7   c                     | j                   S r9   )r   r:   s    r5   r   z_FabricDataLoader.device5  s    ||r7   c                 ,    t        | j                        S r9   )lenr   r:   s    r5   __len__z_FabricDataLoader.__len__9  s    4##$$r7   c              #      K   t        | j                  | j                         | xj                  dz  c_        | j                  t	        | j                        E d {    y | j                  D ]  }t        || j                          y 7 /w)N   )r   r   r   r   iterr   rP   s     r5   __iter__z_FabricDataLoader.__iter__<  ss      	4++T-A-AB!<<D,,---(()$== ) .s   ABB0Br9   )r3   rR   rS   r   r   torchr   r6   rU   intr   r   r   r
   r   r   rX   r7   r5   r   r   %  s{    !: !x7M !Y] ! .  % %>%ydD/I IJ >r7   r   
collectionr+   c                     dt         t        t        t        f   dt         t        j
                  t        t        f   fd}t        t        t        g}|j                  t               t        | t        |      |      S )Nobjr+   c                     t        t        |       d   x}t              rt        |j                        d   S t        | t              r| j
                  S t        | t              r| j                  S | S Nr   )rE   _unwrap_compiledrZ   r_   r'   r(   r   r   )r   	unwrappeds     r5   _unwrapz _unwrap_objects.<locals>._unwrapK  sc     #3C#8#;;i]K#I$=$=>qAAc+,== c,-??"
r7   ri   )r   rZ   r'   r   r   r   r   r   r   r   r   tuple)r   r   typess      r5   _unwrap_objectsr   J  sd    	="24EEF		ryy)Z/	0	 ,.?@E	LL!zuPPr7   r   c                 z    t        | t              r(t        | dd      x}t        d      | j                  |fS | dfS )zRemoves the :class:`torch._dynamo.OptimizedModule` around the object if it is wrapped.

    Use this function before instance checks against e.g. :class:`_FabricModule`.

    _compile_kwargsNzFailed to determine the arguments that were used to compile the module. Make sure to import lightning before `torch.compile` is used.)rE   r   rG   r   	_orig_mod)r   compile_kwargss     r5   r   r   \  sP     #'%c+<dCCNL=  }}n,,9r7   rf   r   c                 .    t        j                  | fi |S r9   )r   compile)rf   r   s     r5   _to_compiledr   l  s    ==2>22r7   requires_backwardr   c                 ,    | rt         st        d      y y )NzwThe current strategy and precision selection requires you to call `fabric.backward(loss)` instead of `loss.backward()`.)r%   r   )r   r   s     r5   r   r   p  s"    !4-
 	
 "5r7   c                 T    t        |       \  } }t        | t        t        t        f      S )aZ  Checks if an object was set up by Fabric.

    A :class:`~torch.nn.Module` may be wrapped by a :class:`_FabricModule`, a :class:`~torch.optim.Optimizer`
    may be wrapped by a :class:`_FabricOptimizer`, or a :class:`~torch.utils.data.DataLoader` may be wrapped by
    :class:`_FabricDataLoader`.

    Args:
        obj: The object to test.

    )r   rE   rZ   r'   r   )r   r   s     r5   
is_wrappedr   x  s(     c"FCcM+;=NOPPr7   
compile_fnc                 T     t               dt        dt        dt        f fd       }|S )aZ  Wraps the ``torch.compile`` function and captures the compile arguments.

    We extract the compile arguments so that we can reapply ``torch.compile`` in ``Fabric.setup()`` with the
    same arguments as the user passed to the original call. The arguments get stored in a dictionary
    ``_compile_kwargs`` on the returned compiled module.

    rg   rI   r+   c                      | rt        | d   t        j                        s | i |S | d   } |fi |}t        |      |_        |S r   )rE   r   r   r   r   )rg   rI   rB   compiled_modelr   s       r5   _capturez)_capture_compile_kwargs.<locals>._capture  sR    :d1gryy9t.v..Q#E4V4)1&)9&r7   )r   r
   )r   r   s   ` r5   _capture_compile_kwargsr     s8     : s s   Or7   )Dr   collections.abcr   r   r   copyr   	functoolsr   r   r   r	   typingr
   r   r   r   r   r   r   lightning_utilitiesr   #lightning_utilities.core.apply_funcr   r   r   torch._dynamor   torch.nn.modules.moduler   torch.optimr   torch.utils.datar   typing_extensionsr   lightning.fabric.pluginsr   lightning.fabric.strategiesr   lightning.fabric.utilitiesr   lightning.fabric.utilities.datar   -lightning.fabric.utilities.device_dtype_mixinr    lightning.fabric.utilities.typesr   rV   rW   r   rb   r%   r   __annotations__r'   rZ   r   r   r   r   r   r   r   objectr   r   r   rX   r7   r5   <module>r      sx    8 8  $    - C   ) 5 ! ' & . 0 : > Q 8tCH~>!b ! T !.. ..b}-+ }-@"> ">JQ Q Q$%_ 45 %c299n@UW_`dehjmem`nWo@o:p  3 3DcN 3 3
d 
 
 
QF Qt Q X 2 (6r7   