
      ih                     j   d dl Z d dlZd dlmZ d dlmZmZ d dlmZ d dl	m
Z
 d dlmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlm Z  d dl!m"Z" d dl#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3m4Z4m5Z5m6Z6m7Z7 d dl2m8Z9 d dl:m;Z;m<Z< d dl=m>Z> d dl?m@Z@mAZAmBZB d dlCmZ d dlDmEZE d dlFmGZGmHZH erd dlImJZJ  ede      ZK G d de,      ZL G d de0      ZM G d  d!e      ZN	 d;d"e
d#eOePeeeef   f   d$eQd%eRd&eeOePeePegeQf   f      d'dfd(ZS	 	 	 d<d"e
d#eOePeeeef   f   d)eQd*eQd+eeQ   d'eOePef   fd,ZTd-eRd.eRd/eRd0ej                  d'd1f
d2ZVd3eWd'ee   fd4ZXd=d"e
d3ed/eRd)eQd'df
d5ZY	 d>d6eOePef   d3ed/eRd)eQd'df
d7ZZd3ed'efd8Z[d9eOePef   d3ed'eOePef   fd:Z\y)?    N)	Generator)AbstractContextManager	ExitStack)	timedelta)Path)TYPE_CHECKINGAnyCallableLiteralOptionalTypeVarUnion)rank_zero_only)Tensor)Module)	Optimizer)	TypeGuardoverride)CheckpointIO)default_pg_timeout)_distributed_checkpoint_load_distributed_checkpoint_save_get_full_state_dict_context_is_full_checkpoint_is_sharded_checkpoint)_SubprocessScriptLauncher)ParallelStrategy)
TBroadcast_apply_filter_BackwardSyncControl!_validate_keys_for_strict_loading)ReduceOp_distributed_is_initialized-_get_default_process_group_backend_for_device_init_dist_connection_sync_ddp_if_availablegroup)_TORCH_GREATER_EQUAL_2_3_TORCH_GREATER_EQUAL_2_4)_materialize_distributed_module)_METADATA_FILENAME
_lazy_load_move_state_into)
reset_seed)_PATH	_Stateful)
DeviceMeshTModel)boundc                       e Zd ZdZddddefdeedgef   deed   e	f   deed   e	f   d	e
d
ee   dee   ddf fdZed5d       Zeedefd              Zej(                  ededdfd              Zeedej,                  fd              Zede	fd       Zej(                  de	ddfd       Zede	fd       Zeedeeef   fd              Zedee   fd       Zed6d       Zed6 fd       Zede de fd       Z!ede ddfd       Z"ed7dee
   de#fd       Z$e	 d8d e%d!ee   d"eee&ef      de%fd#       Z'ed$ed%eddfd&       Z(ed9d'e)d(e	de)fd)       Z*e	 	 d:d*e+d+eeee e,ef   f   d,ee   d-eeeeeege
f   f      ddf
d.       Z-e	 	 	 d;d*e+d+eee e,eeee e,ef   f   f      d/e
d0ee
   deeef   f
d1       Z.d6d2Z/defd3Z0d6d4Z1 xZ2S )<ModelParallelStrategya  Enables user-defined parallelism applied to a model.

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Currently supports up to 2D parallelism. Specifically, it supports the combination of
    Fully Sharded Data-Parallel 2 (FSDP2) with Tensor Parallelism (DTensor). These PyTorch APIs are currently still
    experimental in PyTorch. Requires PyTorch 2.4 or newer.

    Arguments:
        parallelize_fn: A function that applies parallelisms to a module. The strategy will provide the
            model and device mesh as input.
        data_parallel_size: The number of devices within a data-parallel group. Defaults to ``"auto"``, which
            sets this size to the number of nodes in the cluster.
        tensor_parallel_size: The number of devices within a tensor-parallel group. Defaults to ``"auto"``, which
            sets this size to the number of GPUs in a single node.
        save_distributed_checkpoint: If ``True``, each rank saves its shard of weights and optimizer states to a file.
            The checkpoint is a folder with as many files as the world size.
            If ``False``, the full weights and optimizer states get assembled on rank 0 and saved to a single file.

    autoTNparallelize_fnr2   data_parallel_sizetensor_parallel_sizesave_distributed_checkpointprocess_group_backendtimeoutreturnc                     t         |           t        s!t        t	        |       j
                   d      || _        || _        || _        d| _	        || _
        || _        || _        t               | _        d | _        y )Nz  requires PyTorch 2.4 or higher.   )super__init__r*   ImportErrortype__name___parallelize_fn_data_parallel_size_tensor_parallel_size
_num_nodes_save_distributed_checkpoint_process_group_backend_timeout_ParallelBackwardSyncControl_backward_sync_control_device_mesh)selfr8   r9   r:   r;   r<   r=   	__class__s          /Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning/fabric/strategies/model_parallel.pyrB   zModelParallelStrategy.__init__Y   sz     	'd!4!4 55UVWW-#5 %9",G)5J#-4&B&D#26    c                 H    | j                   t        d      | j                   S )NzKAccessing the device mesh before processes have initialized is not allowed.)rO   RuntimeErrorrP   s    rR   device_meshz!ModelParallelStrategy.device_meshp   s&    $lmm   rS   c                 F    t        dt        |       j                   d      )NThe `z3` does not use the `CheckpointIO` plugin interface.NotImplementedErrorrD   rE   rV   s    rR   checkpoint_ioz#ModelParallelStrategy.checkpoint_iov   $     "E$t**=*=)>>q"rssrS   ioc                 F    t        dt        |       j                   d      )NrY   z3` does not support setting a `CheckpointIO` plugin.rZ   )rP   r^   s     rR   r\   z#ModelParallelStrategy.checkpoint_io{   r]   rS   c                 P    | j                   J | j                   | j                     S N)parallel_devices
local_rankrV   s    rR   root_devicez!ModelParallelStrategy.root_device   s+     $$000$$T__55rS   c                     | j                   S ra   rI   rV   s    rR   	num_nodeszModelParallelStrategy.num_nodes   s    rS   rg   c                     || _         y ra   rf   )rP   rg   s     rR   rg   zModelParallelStrategy.num_nodes   s	    #rS   c                 H    | j                   t        | j                         S dS )Nr   )rb   lenrV   s    rR   num_processesz#ModelParallelStrategy.num_processes   s$    -1-B-B-Ns4(()UTUUrS   c                 ~    | j                   J | j                   d   }|j                         |j                         dS )Ndata_parallel)num_replicasrank)rW   sizeget_local_rank)rP   data_parallel_meshs     rR   distributed_sampler_kwargsz0ModelParallelStrategy.distributed_sampler_kwargs   sE     +++!--o> 2 7 7 9CUCdCdCfggrS   c                     | j                   S ra   )rK   rV   s    rR   r<   z+ModelParallelStrategy.process_group_backend   s    ***rS   c                     | j                   J | j                   j                  s1t        | j                   | j                  | j                        | _        y y ra   )cluster_environmentcreates_processes_externallyr   rk   rg   	_launcherrV   s    rR   _configure_launcherz)ModelParallelStrategy._configure_launcher   sM    ''333''DD6t7O7OQUQcQceiesestDN ErS   c                 8   t         |           | j                          | j                  dk(  r| j                  | _        | j
                  dk(  r| j                  | _        t        | j                  | j
                  | j                  | j                        | _
        y )Nr7   )rA   setup_environment_setup_distributedrG   rg   rH   rk   _setup_device_mesh
world_sizerd   rO   )rP   rQ   s    rR   r{   z'ModelParallelStrategy.setup_environment   s}    !#!##v-'+~~D$%%/)-););D&.$$d&@&@$//SWScSc
rS   modulec                 d   ddl m t        fd|j                         D              r#t	        d| j
                  j                   d      | j                  || j                        }t        |t              s!t	        dt        |      j                         t        || j                         |S )Nr   FullyShardedDataParallelc              3   6   K   | ]  }t        |        y wra   
isinstance).0modr   s     rR   	<genexpr>z5ModelParallelStrategy.setup_module.<locals>.<genexpr>   s     UDTSz#78DT   z\Found modules that are wrapped with `torch.distributed.fsdp.FullyShardedDataParallel`. The `z5` only supports the new FSDP2 APIs in PyTorch >= 2.4.zBThe `parallelize_fn` must return a `nn.Module` instance, but got: )torch.distributed.fsdpr   anymodules	TypeErrorrQ   rE   rF   rW   r   r   rD   r+   rd   )rP   r   r   s     @rR   setup_modulez"ModelParallelStrategy.setup_module   s    CUFNNDTUU0011fh 
 %%fd.>.>?&&)TUYZ`UaUjUjTkl  	(0@0@ArS   c                      y ra    )rP   r   s     rR   module_to_devicez&ModelParallelStrategy.module_to_device   s    rS   
empty_initc                     | j                   j                         }t               }|r$|j                  t	        j
                  d             |j                  |       |S )Nmeta)	precisionmodule_init_contextr   enter_contexttorchdevice)rP   r   precision_init_ctxstacks       rR   r   z)ModelParallelStrategy.module_init_context   sL    !^^??A V 45./rS   tensorr(   	reduce_opc                 B    t        |t              rt        |||      S |S )N)r   )r   r   r&   )rP   r   r(   r   s       rR   
all_reducez ModelParallelStrategy.all_reduce   s"     ff%)&%9MMrS   argskwargsc                    t               sy t        j                  j                         dk(  r6t        j                  j	                  | j
                  j                  g       y t        j                  j	                          y )Nnccl)
device_ids)r#   r   distributedget_backendbarrierrd   index)rP   r   r   s      rR   r   zModelParallelStrategy.barrier   sZ    *,((*f4%%$2B2B2H2H1I%J%%'rS   objsrcc                     t               s|S |g}t        j                  j                  ||t        j
                         |d   S )Nr'   r   )r#   r   r   broadcast_object_list_groupWORLD)rP   r   r   s      rR   	broadcastzModelParallelStrategy.broadcast   s<    *,Je//S/M1vrS   pathstatestorage_optionsfilterc                 T   |8t        dt        |       j                   dt        |       j                   d      |-| j                  r!t	        t        |       j                   d      t        | j                  |            }t        ||| j                   | j                  |       y)aN  Save model, optimizer, and other state to a checkpoint on disk.

        If distributed checkpointing is enabled (default), the checkpoint gets saved as a directory containing one file
        per process, with model- and optimizer shards stored per file. Additionally, it creates a metadata file
        `meta.pt` with the rest of the user's state (only saved from rank 0).
        If distributed checkpointing is disabled (``save_distributed_checkpoint=False``), the checkpoint will be
        written to a single file containing the weights, optimizer state and other metadata.

        N`zF.save_checkpoint(..., storage_options=...)` is not supported because `z"` does not use the `CheckpointIO`.zV doesn't support loading distributed filtered checkpoints, so saving them is disabled.)r   r   full_state_dictro   r   )	r   rD   rE   rJ   r[   r   r   _save_checkpointglobal_rank)rP   r   r   r   r   s        rR   save_checkpointz%ModelParallelStrategy.save_checkpoint   s    " &DJ''( )$Z(())KM  $"C"C%:&&' (/ / 
 DNN4()!%!B!BB!!	
rS   strictweights_onlyc           
         |s;t        dt        |       j                   d|dt        |       j                   d      t        | j	                  |            }t        |t              rt        ||| j                  |       i S t        |t              r"t        dt        |       j                   d      t        ||||      S )	zOLoad the contents from a checkpoint and restore the state of the given objects.zGot z.load_checkpoint(..., state=zY) but a state with at least  a model instance to reload is required. Pass it in like so: z2.load_checkpoint(..., state={'model': model, ...}))r   r~   r   zNLoading a single optimizer object from a checkpoint is not supported yet with .)r   r   r   r   )
ValueErrorrD   rE   r   r   r   r    _load_raw_module_state_from_pathr~   r   r[   _load_checkpoint)rP   r   r   r   r   s        rR   load_checkpointz%ModelParallelStrategy.load_checkpoint  s     tDz**++Gy QJ''((\^  DNN4()eV$,T%DOOdjkIeY'%`aefjakatat`uuvw   TvT`aarS   c                 <   t                | j                          | j                         | _        | j                  J d| j
                  i}t        r*| j                  j                  dk7  r| j                  nd |d<   t        | j                  | j                  fi | y )Nr=   cpu	device_id)
r/   _set_world_ranks_get_process_group_backendrK   rv   rL   r)   rd   rD   r%   )rP   r   s     rR   r|   z(ModelParallelStrategy._setup_distributed-  s    &*&E&E&G#''333"+T]]!;#6:6F6F6K6Ku6T$"2"2Z^F;d668S8S^W]^rS   c                 H    | j                   xs t        | j                        S ra   )rK   r$   rd   rV   s    rR   r   z0ModelParallelStrategy._get_process_group_backend7  s     **m.[\`\l\l.mmrS   c                 >   | j                   q| j                   j                  | j                  | j                  z  | j                  z          | j                   j                  | j                  | j                  z         | j                  xt        _	        t        _	        y ra   )rv   set_global_rank	node_rankrk   rc   set_world_sizerg   r   r   ro   utils_rank_zero_onlyrV   s    rR   r   z&ModelParallelStrategy._set_world_ranks:  sy    ##/$$44T^^dFXFX5X[_[j[j5jk$$33DNNTEWEW4WX ;?:J:JJ27rS   )r>   r2   r>   Nra   )Nmean)r   )NN)NTN)3rE   
__module____qualname____doc__r   r
   r3   r   r   intboolr   strr   rB   propertyrW   r   r   r\   setterr   r   rd   rg   rk   dictr	   rs   r<   ry   r{   r   r   r   r   r   r   r"   r   r   r   r   r0   r   r   r   r|   r   r   __classcell__)rQ   s   @rR   r6   r6   C   s7   0 ;A<B,0/3'97 &,!7!?@7 "'&/3"677 $GFOS$89	7
 &*7  (}7 )$7 
7. ! !
 t| t  t t t t  t 6U\\ 6  6 3   $3 $4 $ $ Vs V V hDcN h  h
 +x} + + u u
 	
 	
 6 f  " v $   htn H^   gm%-c]FNuU]_bUbOcFd	  (S (C (D ( ( Z c *   
 *.BF#
#
 Cvy#5667#
 "#	#

 c8S#J,<#==>?#
 
#
 #
J  _c'+bb fic5TWAW;X6X1YYZ[b 	b
 tnb 
c3hb b8_nC nKrS   r6   c                   *    e Zd Zedededefd       Zy)rM   r   enabledr>   c                     t        ||      S )z9Blocks gradient synchronization inside the FSDP2 modules.)r   r   )_FSDPNoSyncrP   r   r   s      rR   no_backward_syncz-_ParallelBackwardSyncControl.no_backward_syncD  s     &'::rS   N)rE   r   r   r   r   r   r   r   r   rS   rR   rM   rM   C  s*    ;v ; ;AW ; ;rS   rM   c                   P    e Zd ZdededdfdZdeddfdZddZd	ed
ededdfdZ	y)r   r   r   r>   Nc                      || _         || _        y ra   )_module_enabledr   s      rR   rB   z_FSDPNoSync.__init__K  s    rS   requires_grad_syncc                     ddl m} | j                  j                         D ]"  }t	        ||      s|j                  |d       $ y )Nr   )
FSDPModuleFrecurse)"torch.distributed._composable.fsdpr   r   r   r   set_requires_gradient_sync)rP   r   r   r   s       rR   _set_requires_grad_syncz#_FSDPNoSync._set_requires_grad_syncO  s:    A<<'')C#z*../A5.Q *rS   c                 <    | j                  | j                          y ra   r   r   rV   s    rR   	__enter__z_FSDPNoSync.__enter__V  s    $$%67rS   exc_type	exc_value	tracebackc                 :    | j                  | j                         y ra   r   )rP   r   r   r   s       rR   __exit__z_FSDPNoSync.__exit__Y  s    $$T]]3rS   r   )
rE   r   r   r   r   rB   r   r   r	   r   r   rS   rR   r   r   J  sX     v      R$ R4 R84 4 4 4 4rS   r   r   r   r   ro   r   r>   c                    | j                         r|rt        |       st        d|        |j                         D cg c]  }t	        |      s| }}t        |      dk(  rt        d      t        |      dkD  rt        d      |d   }ddlm}m	}m
}	  ||d      }
i }i }|j                         D ]v  \  }}t        |t              r |||
	      }|}nBt        |t              r |	|||
	      }|}n$t        |t              r|j!                         n|}|}t#        ||xs i ||       x |rNt        |       rt%        j&                  |        |j)                  |       |dk(  rt+        j,                  ||        y y | j/                         r| j1                          | j3                  dd
       t5        ||        |dk(  rt+        j,                  || t6        z         y y c c}w )Nz/The checkpoint path exists and is a directory: r   a  Could not find a distributed model in the provided checkpoint state. Please provide the model as part of the state like so: `save_checkpoint(..., state={'model': model, ...})`. Make sure you set up the model (and optimizers if any) through the strategy before saving the checkpoint.r@   zFound multiple distributed models in the given state. Saving distributed checkpoints is currently limited to a single model per checkpoint. To save multiple models, call the save method for each model separately with a different path.)StateDictOptionsget_model_state_dictget_optimizer_state_dictT)r   cpu_offloadoptions)parentsexist_ok)is_dirr   IsADirectoryErrorvalues_has_dtensor_modulesrj   r   'torch.distributed.checkpoint.state_dictr   r   r   itemsr   r   r   r1   
state_dictr   shutilrmtreeupdater   saveis_fileunlinkmkdirr   r,   )r   r   r   ro   r   r   r   r   r   r   state_dict_optionsconverted_statemetadatakeyr   	convertedtarget_dicts                    rR   r   r   ]  s    {{}1G1M"QRVQW XYY$)LLNSN&6J66RvNGS
7|qo
 	

 7|aL
 	

 QZFxx)/W[\ ')O!HKKMSc6",S:LMI)KY'0FXYI)K,6sI,F(CI"Kc6<RK@ " !$'MM$x(19JJ-  <<>KKM

4$
/$_d;19JJx(:!:; _ Ts   G G r   optimizer_states_from_listr   c                    ddl m}m}m}m} |j                         D 	
ci c]  \  }	}
t        |
      s|	|
 }}	}
t        |      dk(  rt        d      |j                         D 	ci c]  \  }	}t        |t              s|	| }}	}t        |      dkD  rt        d      t        |j                               d   \  }}
t        |       r |d      }| ||
      i}t        ||        |
j                  ||   |       |j                         D ]+  \  }}| ||
|      i}t        ||         ||
|||   |	       - t        j                   | t"        z  |
      }|j%                         |j%                         z
  |j%                         z
  }t'        ||j%                         |       |D ]  }	|	|vr|j)                  |	      ||	<    |S t+        |       rt        j                   | dd|      }t-        |j)                  |      |
|        |dd|      }t/        |j                               D ]<  \  }\  }}|r	|d   |   }n|j)                  |      }t1        ||
      } ||
|||	       > |j%                         |j%                         z
  |j%                         z
  }t'        ||j%                         |       t3        |||       |S t        dt5        |       d      c c}
}	w c c}}	w )Nr   )r   r   r   set_optimizer_state_dicta  Could not find a distributed model in the provided checkpoint state. Please provide the model as part of the state like so: `load_checkpoint(..., state={'model': model, ...})`. Make sure you set up the model (and optimizers if any) through the strategy before loading the checkpoint.r@   zFound multiple distributed models in the given state. Loading distributed checkpoints is currently limited to a single model per checkpoint. To load multiple models, call the load method for each model separately with a different path.T)r   r   )optim_state_dictr   )r   r   )mmapmap_locationr   broadcast_from_rank0r   r   optimizer_states)sourcedestinationkeysz	The path z does not point to a valid checkpoint. Make sure the path points to either a directory with distributed checkpoint shards, or a single file with a full checkpoint.)r   r   r   r   r  r  r   rj   r   r   r   listr   r   load_state_dictr   loadr,   r  r!   popr   _load_raw_module_state	enumerate _rekey_optimizer_state_if_neededr.   r   )r   r   r   r  r   r   r   r   r  r  r   r   optim
optimizers
module_keyr
  module_state	optim_keyoptim_stater  requested_metadata_keys
checkpointoptimizer_idxoptimizer_name	optimizeroptimizer_states                             rR   r   r     s      /4kkm\m{sF?STZ?[sF{mG\
7|qp
 	

 05{{}]}e
5R[@\#u*}J]
7|aL
 	

 gmmo.q1Jd#-$?"$8$@A$\48|J7G !+ 0 0 2Iu$&>vu&MNK(d;$VU[QZE[ewx !3 ::d%77lS"'**,"?*//BS"S)*A8==?[ab*C("!c*E#J + 4 ZZ4eR^_
z~~j96&Q-!% 

 ;DJDTDTDV:W6M6NI) #--?"@"O",.."@>PVWO$!0*	 ;X  #(**,"?*//BS"S)*A:??CT]cd 	
D[\ 

CI= !b 	b W ] ^s   K(K((K.K.r9   r:   r~   r   r2   c           	      v    ddl m} | |z  |k7  rt        d|  d| d| d       ||j                  | |fd      S )	Nr   )init_device_meshzThe sizes `data_parallel_size=z` and `tensor_parallel_size=z*` multiplied should equal the world size (z).)rm   tensor_parallel)device_type
mesh_shapemesh_dim_names)torch.distributed.device_meshr1  rU   rD   )r9   r:   r~   r   r1  s        rR   r}   r}     sj     ?00J>,-?,@ A&&:%; <B 
 	

 KK&(<=; rS   r   c                 x    ddl m t        | t              xr" t	        fd| j                         D              S )Nr   )DTensorc              3   6   K   | ]  }t        |        y wra   r   )r   tr8  s     rR   r   z'_has_dtensor_modules.<locals>.<genexpr>  s     -bNajG.DNar   )torch.distributed._tensorr8  r   r   r   
parameters)r   r8  s    @rR   r   r     s,    1ff%b#-bfN_N_Na-b*bbrS   c                     t        |       st        d|        t        rt        j                  | dd      n
t        |       }t        ||||       y)z;Loads the state dict from a file path into the FSDP module.zxFailed to load checkpoint directly into the model. The given path must be a single file containing the full state dict: Tr   )r  r  )r  r   r~   r   N)r   r   r)   r   r  r-   r!  )r   r   r~   r   r  s        rR   r   r     sV    t$!!%(
 	

 E]Dt%@blmqbrJjJ_efrS   r  c                    ddl m} t        |      rsddlm}m}  |ddd      }|j                         D ]L  \  }}	t        |	      D ]9  \  }
}| |rdnd |
 }|| vr|st        d	| d
      |
| |   i} ||	||       ; N yt        ||      r+t        ||d      5  |j                  | |       ddd       y|j                  | |       y# 1 sw Y   yxY w)zlLoads the state dict into the module by gathering all weights first and then and writing back to each shard.r   r   )r   set_model_state_dictTFr  r    zThe model contains a key 'z^' that does not exist in the loaded checkpoint. To disable strict loading, set `strict=False`.r   )r~   
rank0_onlyr  N)r   r   r   r   r   r?  named_modules%_named_parameters_and_buffers_to_loadKeyErrorr   r   r  )r  r   r~   r   FSDPr   r?  r
  submodule_name	submodule
param_name_full_param_namelocal_state_dicts                 rR   r!  r!     s    HF#b-!% 	
 *0)=)=)?%NI!Fy!Q
A%3$4NSPR4ST^S_"`"*4! "4_4E FJ J  %/
?0K#L $Y0@J\] "R *@ 
FD	!)&ZTYZ"":f"= [Z 	z&9 [Zs   CCc              #      K   t        j                  | j                  d      | j                  d            D ]  \  }}|| j                  v r||f  yw)zEReturns parameters and buffers, with non-persistent buffers excluded.Fr   N)	itertoolschainnamed_buffersnamed_parameters_non_persistent_buffers_set)r   rH  params      rR   rC  rC  D  s^     &__U+.
E ;;;%s   AAoptimizer_state_dictc                     ddl m} ddl m} t        t	        | d   j                               d   t              r|j                  | |j                  |      } | S )zyHandles the case where the optimizer state is saved from a normal optimizer and converts the keys to parameter
    names.r   r   )OptimStateKeyTyper   )	r   r   rU  r   r  r  r   rekey_optim_state_dict
PARAM_NAME)rS  r   rE  rU  s       rR   r#  r#  O  sR     H8$+G499;<Q?E#::;OQbQmQmouvrS   ra   )TFN)T)r@   T)]rM  r  collections.abcr   
contextlibr   r   datetimer   pathlibr   typingr   r	   r
   r   r   r   r   r   "lightning_utilities.core.rank_zeror   r   r   torch.nnr   torch.optimr   typing_extensionsr   r   lightning.fabric.pluginsr   5lightning.fabric.plugins.collectives.torch_collectiver    lightning.fabric.strategies.fsdpr   r   r   r   r   7lightning.fabric.strategies.launchers.subprocess_scriptr   $lightning.fabric.strategies.parallelr   $lightning.fabric.strategies.strategyr   r   r    r!   &lightning.fabric.utilities.distributedr"   r#   r$   r%   r&   r(   r   "lightning.fabric.utilities.importsr)   r*   lightning.fabric.utilities.initr+   lightning.fabric.utilities.loadr,   r-   r.   $lightning.fabric.utilities.rank_zerolightning.fabric.utilities.seedr/    lightning.fabric.utilities.typesr0   r1   r6  r2   r3   r6   rM   r   r   r   r   r   r   r   r   r}   objectr   r   r!  rC  r#  r   rS   rR   <module>ro     s     % 8   R R R  U   ! 1 1 T  ^ A   C a K \ \ ? 6 =8		(}K, }K@;#7 ;4( 40 ?C:<
:<U69c1223:< :< 	:<
 T#xc
D(899:;:< 
:<@ ',#'\
\U69c1223\ \ !%	\
 4.\ 
#s(^\~  LL	
 *c cIf,= c	g4 	g 	gS 	gZ^ 	gjn 	g UY!:S#X!:(.!:<?!:MQ!:	!:H &  Y   4S>  SY  ^bcfhkck^l  rS   