Ë
      çiEN  ã                   ó  — d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	m
Z
mZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlZd dlmZmZ d dlm Z  d dl!m"Z" d dl#m$Z$m%Z%m&Z&m'Z' d dl#m(Z) d dl*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6m7Z7m8Z8 d dl9m:Z: d dl;m<Z<m=Z= d dl>m?Z? d dl@mAZAmBZB d dlCmDZD d dlEmFZF d dlGmHZHmIZImZ erd dlJmKZK  e j˜                  eM«      ZNdZO G d„ d e?«      ZP G d!„ d"eB«      ZQy)#é    N)Únullcontext)Ú	timedelta)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteralÚOptionalÚUnion)Úrank_zero_only)ÚTensor)ÚModule)ÚDistributedDataParallel)Ú	Optimizer)Úoverride)ÚCheckpointIOÚClusterEnvironment)Údefault_pg_timeout)Ú_StrategyRegistry)Ú_distributed_is_initializedÚ-_get_default_process_group_backend_for_deviceÚ_init_dist_connectionÚ_sync_ddp_if_available©Úgroup)Ú_IS_WINDOWSÚ_TORCH_GREATER_EQUAL_2_3)Ú_optimizers_to_device)Ú
reset_seed)ÚReduceOp)ÚLightningOptimizer)Ú_register_ddp_comm_hookÚ_sync_module_statesÚprepare_for_backward)Ú	Precision)Ú_MultiProcessingLauncherÚ_SubprocessScriptLauncher)ÚParallelStrategy)Ú
TBroadcastÚ_ForwardRedirection)Ú	TrainerFn©Ú_augment_message)Úrank_zero_deprecationÚrank_zero_infor   )ÚModelAverager)Úddp_forkÚ%ddp_fork_find_unused_parameters_falseÚ$ddp_fork_find_unused_parameters_trueÚddp_notebookÚ)ddp_notebook_find_unused_parameters_falseÚ(ddp_notebook_find_unused_parameters_truec                   ó¾  ‡ — e Zd ZdZddddddddddedfded   deeej                        dee	   dee
   d	ee   d
ee   dee   dee   dee   dee   dee   ded   deddfˆ fd„Zedefd„«       Zeedej                  fd„«       «       Zedefd„«       Zej4                  deddfd„«       Zedefd„«       Zeedeeef   fd„«       «       Zedee   fd„«       Zed?d„«       Zed?ˆ fd„«       Z ed@d„«       Z!ed e"de#fd!„«       Z$d?d"„Z%defd#„Z&d?d$„Z'd?d%„Z(d?d&„Z)e	 dAd'e*d(eg ef   d ee+d)e"f      dedef
ˆ fd*„«       Z,d?d+„Z-deee      fd,„Z.ed-ededdfd.„«       Z/edBd/e0d0ede0fd1„«       Z1ed2e2ddfd3„«       Z3ed2e2ddfd4„«       Z4ed?d5„«       Z5e	 dCd6e2d7ee   d8ee+e6ef      de2fd9„«       Z7e8ed:e9ddfd;„«       «       Z:ed<e;ddfd=„«       Z<ed?ˆ fd>„«       Z=ˆ xZ>S )DÚDDPStrategyzKStrategy for multi-process single-device training on one or multiple nodes.NÚpopenÚacceleratorzpl.accelerators.AcceleratorÚparallel_devicesÚcluster_environmentÚcheckpoint_ioÚprecision_pluginÚddp_comm_stateÚddp_comm_hookÚddp_comm_wrapperÚmodel_averaging_periodÚprocess_group_backendÚtimeoutÚstart_method)r8   ÚspawnÚforkÚ
forkserverÚkwargsÚreturnc                 ó>  •— t         ‰| �  |||||¬«       t        j                  | j                  j
                  › d�«       t        «       | _        d| _        || _	        || _
        || _        || _        |	| _        d | _        |
| _        || _        || _        d| _        y )N)r9   r:   r;   r<   r=   z: initializing DDP strategyé   F)ÚsuperÚ__init__ÚlogÚdebugÚ	__class__Ú__name__Ú_DDPForwardRedirectionÚ_forward_redirectionÚ
_num_nodesÚ_ddp_kwargsÚ_ddp_comm_stateÚ_ddp_comm_hookÚ_ddp_comm_wrapperÚ_model_averaging_periodÚ_model_averagerÚ_process_group_backendÚ_timeoutÚ_start_methodÚ_pl_static_graph_delay_done)Úselfr9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rH   rP   s                 €úu/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/pytorch_lightning/strategies/ddp.pyrM   zDDPStrategy.__init__G   s¨   ø€ ô  	‰ÑØ#Ø-Ø 3Ø'Ø-ð 	ô 	
ô 	�	‰	�T—^‘^×,Ñ,Ð-Ð-HÐIÔJÜ$:Ó$<ˆÔ!ØˆŒØ!ˆÔØ-ˆÔØ+ˆÔØ!1ˆÔØ'=ˆÔ$Ø8<ˆÔØ5JˆÔ#Ø-4ˆŒØ)ˆÔØ+0ˆÕ(ó    c                 óL   — t        dt        | «      j                  › d�d¬«       y)z1Legacy property kept for backwards compatibility.Ú`z3.is_distributed` is deprecated. Use is discouraged.é   )Ú
stacklevelT)r-   ÚtyperQ   ©r_   s    r`   Úis_distributedzDDPStrategy.is_distributedl   s,   € ô 	Ø”�T“
×#Ñ#Ð$Ð$WÐXÐefõ	
ð ra   c                 óP   — | j                   €J ‚| j                   | j                     S ©N)r:   Ú
local_rankrg   s    r`   Úroot_devicezDDPStrategy.root_devicet   s+   € ð ×$Ñ$Ð0Ð0Ð0Ø×$Ñ$ T§_¡_Ñ5Ð5ra   c                 ó   — | j                   S rj   ©rT   rg   s    r`   Ú	num_nodeszDDPStrategy.num_nodesz   s   € à�‰Ðra   ro   c                 ó   — || _         y rj   rn   )r_   ro   s     r`   ro   zDDPStrategy.num_nodes~   s   € ð $ˆ�ra   c                 óH   — | j                   �t        | j                   «      S dS ©Nr   )r:   Úlenrg   s    r`   Únum_processeszDDPStrategy.num_processesƒ   s$   € à-1×-BÑ-BÐ-NŒs�4×(Ñ(Ó)ÐUÐTUÐUra   c                 óN   — | j                   | j                  z  | j                  dœS )N)Únum_replicasÚrank)ro   rt   Úglobal_rankrg   s    r`   Údistributed_sampler_kwargsz&DDPStrategy.distributed_sampler_kwargs‡   s$   € ð "&§¡°$×2DÑ2DÑ!DÈt×O_ÑO_Ñ`Ð`ra   c                 ó   — | j                   S rj   )r[   rg   s    r`   rB   z!DDPStrategy.process_group_backendŒ   s   € à×*Ñ*Ð*ra   c                 óØ   — | j                   €J ‚| j                  dk(  r1t        | j                   | j                  | j                  «      | _        y t        | | j                  ¬«      | _        y )Nr8   )rD   )r;   r]   r&   rt   ro   Ú	_launcherr%   rg   s    r`   Ú_configure_launcherzDDPStrategy._configure_launcher�   sZ   € à×'Ñ'Ð3Ð3Ð3Ø×Ñ Ò(Ü6°t×7OÑ7OÐQU×QcÑQcÐei×esÑesÓtˆD�Nä5°dÈ×I[ÑI[Ô\ˆD�Nra   c                 óB   •— t         ‰| �  «        | j                  «        y rj   )rL   Úsetup_environmentÚsetup_distributed)r_   rP   s    €r`   r   zDDPStrategy.setup_environment˜   s   ø€ ä‰Ñ!Ô#Ø×ÑÕ ra   c                 ó6  — | j                   €J ‚| j                   j                  |«       |j                  j                  }| j                  €J ‚|t
        j                  k(  r6| j                  r*| j                  j                  | j                  «      | _        | j                  j                  | j                  «       | j                  «        |t
        j                  k(  r"| j                  «        | j                  |«       nt        | j                  «       | j                  «        |t
        j                  k(  rat!        | j"                  | j$                  «       dd lmc mc mc m} t1        | j2                  |j4                  «      r| j7                  «        y y y rr   )r9   ÚsetupÚstateÚfnÚmodelr*   ÚFITTINGÚ_layer_syncÚapplyr=   Úconvert_moduleÚmodel_to_deviceÚconfigure_ddpÚsetup_optimizersr"   Úsetup_precision_pluginr   Ú
optimizersrl   Ú>torch.distributed.algorithms.ddp_comm_hooks.post_localSGD_hookÚdistributedÚ
algorithmsÚddp_comm_hooksÚpost_localSGD_hookÚ
isinstancerV   ÚPostLocalSGDStateÚ_enable_model_averaging)r_   ÚtrainerÚ
trainer_fnÚpost_localSGDs       r`   r‚   zDDPStrategy.setup�   s6  € à×ÑÐ+Ð+Ð+Ø×Ñ×Ñ˜wÔ'à—]‘]×%Ñ%ˆ
Ø�z‰zÐ%Ð%Ð%Øœ×*Ñ*Ò*¨t×/?Ò/?Ø×)Ñ)×/Ñ/°·
±
Ó;ˆDŒJà×Ñ×,Ñ,¨T¯Z©ZÔ8Ø×ÑÔàœ×*Ñ*Ò*à×ÑÔ ð ×!Ñ! 'Õ*ô   §
¡
Ô+Ø×#Ñ#Ô%Øœ×*Ñ*Ò*Ü! $§/¡/°4×3CÑ3CÔDçbÖbä˜$×.Ñ.°×0OÑ0OÔPØ×,Ñ,Õ.ð Qð +ra   r…   c                 óZ  — | j                  «       }t        j                  d|› d| j                  › �«       |�;t        j
                  j                  t        j
                  j                  «       «      n	t        «       }|5  t        d||dœ| j                  ¤Žcddd«       S # 1 sw Y   yxY w)z^Wraps the model into a :class:`~torch.nn.parallel.distributed.DistributedDataParallel` module.z&setting up DDP model with device ids: z
, kwargs: N)ÚmoduleÚ
device_ids© )
Údetermine_ddp_device_idsrN   rO   rU   ÚtorchÚcudaÚstreamÚStreamr   r   )r_   r…   rœ   Úctxs       r`   Ú_setup_modelzDDPStrategy._setup_model¼   s†   € ð ×2Ñ2Ó4ˆ
Ü�	‰	Ð:¸:¸,ÀjÐQU×QaÑQaÐPbÐcÔdà8BÐ8NŒe�j‰j×Ñ¤§
¡
× 1Ñ 1Ó 3Ô4ÔT_ÓTaˆÚÜ*Ðc°%ÀJÑcÐRV×RbÑRbÑc÷ �SŠSús   Á?B!Â!B*c                 ó”  — t         j                  | j                  j                  › d�«       t	        «        | j                  «        | j                  «       | _        | j                  €J ‚d| j                  i}t        r*| j                  j                  dk7  r| j                  nd |d<   t        | j                  | j                  fi |¤Ž y )Nz: setting up distributed...rC   ÚcpuÚ	device_id)rN   rO   rP   rQ   r   Úset_world_ranksÚ_get_process_group_backendr[   r;   r\   r   rl   rf   r   )r_   rH   s     r`   r€   zDDPStrategy.setup_distributedÆ   s¨   € Ü�	‰	�T—^‘^×,Ñ,Ð-Ð-HÐIÔJÜŒØ×ÑÔØ&*×&EÑ&EÓ&GˆÔ#Ø×'Ñ'Ð3Ð3Ð3Ø"+¨T¯]©]Ð!;ˆÝ#Ø6:×6FÑ6F×6KÑ6KÈuÒ6T $×"2Ò"2ÐZ^ˆF�;ÑÜ˜d×6Ñ6¸×8SÑ8SÑ^ÐW]Ó^ra   c                 óH   — | j                   xs t        | j                  «      S rj   )r[   r   rl   rg   s    r`   r©   z&DDPStrategy._get_process_group_backendÑ   s    € Ø×*Ñ*ÒmÔ.[Ð\`×\lÑ\lÓ.mÐmra   c                 ó>  — | j                   �q| j                   j                  | j                  | j                  z  | j                  z   «       | j                   j                  | j                  | j                  z  «       | j                  xt        _	        t        _	        y rj   )r;   Úset_global_rankÚ	node_rankrt   rk   Úset_world_sizero   rx   r   rw   Úutils_rank_zero_onlyrg   s    r`   r¨   zDDPStrategy.set_world_ranksÔ   sy   € Ø×#Ñ#Ð/Ø×$Ñ$×4Ñ4°T·^±^Àd×FXÑFXÑ5XÐ[_×[jÑ[jÑ5jÔkØ×$Ñ$×3Ñ3°D·N±NÀT×EWÑEWÑ4WÔXð ;?×:JÑ:JÐJŒÔÔ2Õ7ra   c                 ó6  — t         j                  | j                  j                  › d�«       | j                  j
                  dk(  rTt        | j                  t        «      sJ ‚t        | j                  | j                  | j                  | j                  ¬«       y y )Nz: registering ddp hooksr    )r…   r>   r?   r@   )rN   rO   rP   rQ   rl   rf   r”   r…   r   r!   rV   rW   rX   rg   s    r`   Ú_register_ddp_hookszDDPStrategy._register_ddp_hooksÜ   s{   € Ü�	‰	�T—^‘^×,Ñ,Ð-Ð-DÐEÔFð ×Ñ× Ñ  FÒ*Ü˜dŸj™jÔ*AÔBÐBÐBÜ#Ø—j‘jØ#×3Ñ3Ø"×1Ñ1Ø!%×!7Ñ!7ö	ð +ra   c                 óf  — t         j                  | j                  j                  › d�«       | j                  €t        d«      ‚ddlm}m}m	} | j                  D ]e  }t        |t        «      r|j                  }t        st        ||«      nd}t        |||f«      s|sŒDt        d|j                  j                  › d�«      ‚ | j                  €J ‚t         j"                  j$                  j&                  j(                  j+                  | j                  | j                  j,                  ¬«      | _        y )	Nz.: reinitializing optimizers with post localSGDz\Post-localSGD algorithm is used, but model averaging period is not provided to DDP strategy.r   )ÚDistributedOptimizerÚPostLocalSGDOptimizerÚZeroRedundancyOptimizerFzKCurrently model averaging cannot work with a distributed optimizer of type Ú.)ÚperiodÚwarmup_steps)rN   rO   rP   rQ   rY   Ú
ValueErrorÚtorch.distributed.optimr³   r´   rµ   rŽ   r”   r    Ú
_optimizerr   rV   rŸ   r�   r‘   Úmodel_averagingÚ	averagersÚPeriodicModelAveragerÚstart_localSGD_iterrZ   )r_   r³   r´   rµ   Ú	optimizerÚis_distributed_optimizers         r`   r–   z#DDPStrategy._enable_model_averagingé   s  € Ü�	‰	�T—^‘^×,Ñ,Ð-Ð-[Ð\Ô]Ø×'Ñ'Ð/ÜØnóð ÷ 	qÑpàŸœˆIÜ˜)Ô%7Ô8Ø%×0Ñ0�	åZe¤z°)Ð=QÔ'RÐkpÐ$Ü˜)Ð&=Ð?TÐ%UÔVÒZrÜ ØaØ ×*Ñ*×3Ñ3Ð4°Að7óð ð )ð ×#Ñ#Ð/Ð/Ð/Ü$×0Ñ0×;Ñ;×KÑK×UÑU×kÑkØ×/Ñ/¸d×>RÑ>R×>fÑ>fð  ló  
ˆÕra   rÀ   Úclosureúpl.LightningModulec                 ó   •— t        ‰	| �  |||fi |¤Ž}| j                  €|S |j                  D ��cg c]  }|d   D ]  }|j                  €Œ|‘Œ Œ }}}| j                  j                  t        |«      «       |S c c}}w )aI  Performs the actual optimizer step.

        Args:
            optimizer: the optimizer performing the step
            closure: closure calculating the loss value
            model: reference to the model, optionally defining optimizer step related hooks
            **kwargs: Any extra arguments to ``optimizer.step``

        Úparams)rL   Úoptimizer_steprZ   Úparam_groupsÚgradÚaverage_parametersÚiter)
r_   rÀ   rÂ   r…   rH   Úoptimizer_outputr   ÚparamrÅ   rP   s
            €r`   rÆ   zDDPStrategy.optimizer_step  sˆ   ø€ ô" !™7Ñ1°)¸WÀeÑVÈvÑVÐà×ÑÐ'Ø#Ð#à%.×%;Ò%;ÔsÑ%;˜EÈÈxÌÀÐ\a×\fÑ\fÑ\r’%È�%Ð%;ˆÑsØ×Ñ×/Ñ/´°V³Ô=àÐùó ts   ±A:ÁA:c                 ó  — t         j                  | j                  j                  › d�«       t	        | j
                  t        j                  «      sJ ‚| j                  | j
                  «      | _        | j                  «        y )Nz%: configuring DistributedDataParallel)
rN   rO   rP   rQ   r”   r…   ÚplÚLightningModuler¤   r±   rg   s    r`   r‹   zDDPStrategy.configure_ddp  s]   € Ü�	‰	�T—^‘^×,Ñ,Ð-Ð-RÐSÔTÜ˜$Ÿ*™*¤b×&8Ñ&8Ô9Ð9Ð9Ø×&Ñ& t§z¡zÓ2ˆŒ
Ø× Ñ Õ"ra   c                 ód   — | j                   j                  dk(  ry | j                   j                  gS )Nr¦   )rl   rf   Úindexrg   s    r`   rž   z$DDPStrategy.determine_ddp_device_ids"  s.   € Ø×Ñ× Ñ  EÒ)ØØ× Ñ ×&Ñ&Ð'Ð'ra   Úargsc                 óö   — t        «       sy t        j                  j                  «       dk(  r/t        j                  j	                  | j                  «       ¬«       y t        j                  j	                  «        y )NÚnccl)rœ   )r   rŸ   r�   Úget_backendÚbarrierrž   )r_   rÒ   rH   s      r`   rÖ   zDDPStrategy.barrier'  sT   € ä*Ô,Øä×Ñ×(Ñ(Ó*¨fÒ4Ü×Ñ×%Ñ%°×1NÑ1NÓ1PÐ%ÕQä×Ñ×%Ñ%Õ'ra   ÚobjÚsrcc                 óŠ   — t        «       s|S |g}t        j                  j                  ||t        j
                  ¬«       |d   S )Nr   r   )r   rŸ   r�   Úbroadcast_object_listÚ_groupÚWORLD)r_   r×   rØ   s      r`   Ú	broadcastzDDPStrategy.broadcast1  s<   € ä*Ô,ØˆJàˆeˆÜ×Ñ×/Ñ/°°SÄÇÁÐ/ÔMØ�1‰vˆra   Úclosure_lossc                 ó°   — t        | j                  t        «      sy| j                  €J ‚| j                  j                  st        | j                  |«       yy)z.Run before precision plugin executes backward.N)r”   r…   r   Úlightning_moduleÚautomatic_optimizationr#   )r_   rÞ   s     r`   Úpre_backwardzDDPStrategy.pre_backward:  sJ   € ô ˜$Ÿ*™*Ô&=Ô>ØØ×$Ñ$Ð0Ð0Ð0Ø×$Ñ$×;Ò;Ü  §¡¨\Õ:ð <ra   c                 óð   — | j                   }| j                  }t        |t        «      sy |�|j                  ry t        |dd«      sy | j                  ry |j                  }|j                  «        d| _        y )NÚstatic_graphFT)	r…   rà   r”   r   rá   Úgetattrr^   ÚreducerÚ_delay_all_reduce)r_   rÞ   r…   Úlmræ   s        r`   Úpost_backwardzDDPStrategy.post_backwardC  sp   € ð —
‘
ˆØ×"Ñ"ˆÜ˜%Ô!8Ô9ØØˆ:˜×2Ò2ØÜ�u˜n¨eÔ4ØØ×+Ò+Øð —-‘-ˆØ×!Ñ!Ô#à+/ˆÕ(ra   c                 óÜ   — t         j                  | j                  j                  › d| j                  › d�«       | j
                  €J ‚| j
                  j                  | j                  «       y )Nz: moving model to device [z]...)rN   rO   rP   rQ   rl   r…   Útorg   s    r`   rŠ   zDDPStrategy.model_to_deviceX  sU   € ä�	‰	�T—^‘^×,Ñ,Ð-Ð-GÈ×HXÑHXÐGYÐY]Ð^Ô_Ø�z‰zÐ%Ð%Ð%Ø�
‰
�‰�d×&Ñ&Õ'ra   Útensorr   Ú	reduce_opc                 óB   — t        |t        «      rt        |||¬«      S |S )a  Reduces a tensor from several distributed processes to one aggregated tensor.

        Args:
            tensor: the tensor to sync and reduce
            group: the process group to gather results from. Defaults to all processes (world)
            reduce_op: the reduction operation. Defaults to 'mean'/'avg'.
                Can also be a string 'sum' to calculate the sum during reduction.

        Return:
            reduced value, except when the input was not a tensor the output remains is unchanged

        )rí   )r”   r   r   )r_   rì   r   rí   s       r`   ÚreducezDDPStrategy.reduce^  s"   € ô  �fœfÔ%Ü)¨&°%À9ÔMÐMØˆra   Ústrategy_registryc           
      ó¢   — d}|D ]  \  }}|j                  || d|› d�|¬«       Œ  d}|D ]#  \  }}}|j                  || d|› d|› d�||¬«       Œ% y )	N))Úddpr8   )Ú	ddp_spawnrE   )r0   rF   )r3   rF   z"DDP strategy with `start_method` 'Ú')ÚdescriptionrD   ))Ú ddp_find_unused_parameters_falseFr8   )Úddp_find_unused_parameters_trueTr8   )Ú&ddp_spawn_find_unused_parameters_falseFrE   )Ú%ddp_spawn_find_unused_parameters_trueTrE   )r1   FrF   )r2   TrF   )r4   FrF   )r5   TrF   z.DDP strategy with `find_unused_parameters` as z and `start_method` ')rõ   Úfind_unused_parametersrD   )Úregister)Úclsrð   ÚentriesÚnamerD   Úfups         r`   Úregister_strategieszDDPStrategy.register_strategiesr  s‘   € ð
ˆó #*ÑˆD�,Ø×&Ñ&ØØØ@ÀÀÈaÐPØ)ð	 'õ ð #*ð	
ˆó (/Ñ#ˆD�#�|Ø×&Ñ&ØØØLÈSÈEÐQfÐgsÐftÐtuÐvØ'*Ø)ð 'õ ñ (/ra   Ú	exceptionc                 ó    — t        |dd¬«       y )Nz>.*Expected to have finished reduction in the prior iteration.*ay  It looks like your LightningModule has parameters that were not used in producing the loss returned by training_step. If this is intentional, you must enable the detection of unused parameters in DDP, either by setting the string value `strategy='ddp_find_unused_parameters_true'` or by setting the flag in the strategy with `strategy=DDPStrategy(find_unused_parameters=True)`.)ÚpatternÚnew_messager+   )r_   r  s     r`   Úon_exceptionzDDPStrategy.on_exception–  s   € äØØTðtö			
ra   c                 óž  •— t         j                  | j                  j                  › d�«       | j                  }t        | j                  t        «      ri| j                  j                  sL| j                  j                  «       j                  d«      r#t        d| j                  j                  › d�«       || _        |��|j                  �u|j                  j                  j                  t        j                   k(  rD| j"                  r8| j                  €J ‚| j"                  j%                  | j                  «      | _        t&        ‰| �Q  «        y )Nz: tearing down strategyÚcan_set_static_graphzyYour model can run with static graph optimizations. For future training runs, we suggest you pass `Trainer(..., strategy=z%(static_graph=True))` to enable them.)rN   rO   rP   rQ   rà   r”   r…   r   rä   Ú_get_ddp_logging_dataÚgetr.   Ú_trainerrƒ   r„   r*   r†   r‡   ÚrevertrL   Úteardown)r_   Ú	pl_modulerP   s     €r`   r  zDDPStrategy.teardown£  s	  ø€ ä�	‰	�T—^‘^×,Ñ,Ð-Ð-DÐEÔFà×)Ñ)ˆ	Ü�d—j‘jÔ"9Ô:Ø—:‘:×*Ò*¨t¯z©z×/OÑ/OÓ/Q×/UÑ/UÐVlÔ/mÜð4Ø48·N±N×4KÑ4KÐ3LÐLqðsôð
 #ˆDŒJð Ð!ð ×"Ñ"Ð.Ø×"Ñ"×(Ñ(×+Ñ+¬y×/@Ñ/@Ò@Ø× Ò à—:‘:Ð)Ð)Ð)Ø×)Ñ)×0Ñ0°·±Ó<ˆDŒJä‰ÑÕra   )rI   N)r—   z
pl.TrainerrI   Nrj   )r   )NÚmean)?rQ   Ú
__module__Ú__qualname__Ú__doc__r   r	   ÚlistrŸ   Údevicer   r   r$   Úobjectr   ÚintÚstrr   r   r   rM   ÚpropertyÚboolrh   r   rl   ro   Úsetterrt   Údictry   rB   r}   r   r‚   r   r   r¤   r€   r©   r¨   r±   r–   r   r
   rÆ   r‹   rž   rÖ   r(   rÝ   r   râ   ré   rŠ   r   rï   Úclassmethodr   r   ÚBaseExceptionr  r  Ú__classcell__)rP   s   @r`   r7   r7   D   sS  ø„ ÙUð @DØ9=Ø<@Ø04Ø04Ø+/Ø,0Ø/3Ø04Ø/3Ø'9ØHOñ#1àÐ;Ñ<ð#1ð # 4¨¯©Ñ#5Ñ6ð#1ð &Ð&8Ñ9ð	#1ð
   Ñ-ð#1ð # 9Ñ-ð#1ð ! Ñ(ð#1ð   Ñ)ð#1ð # 8Ñ,ð#1ð !)¨¡ð#1ð  (¨™}ð#1ð ˜)Ñ$ð#1ð ÐDÑEð#1ð ð#1ð 
õ#1ðJ ð ò ó ðð Øð6˜UŸ\™\ò 6ó ó ð6ð ð˜3ò ó ðð ×Ñð$ 3ð $¨4ò $ó ð$ð ðV˜sò Vó ðVð Øða¨D°°c°©Nò aó ó ðað ð+ x°¡}ò +ó ð+ð ò]ó ð]ð ô!ó ð!ð ò/ó ð/ð< ðd &ð dÐ-Dò dó ðdó	_ðn¨Có nóKóó
ð0 ð
 @Dñ	 àð ð ˜"˜c˜'Ñ"ð ð ˜Ð2°FÐ:Ñ;Ñ<ð	 ð
 ð ð 
ô ó ð ó4#ð(¨(°4¸±9Ñ*=ó (ð
 ð(˜Sð (¨Cð (°Dò (ó ð(ð ñ˜Zð ¨cð ¸*ò ó ðð ð;¨ð ;°Dò ;ó ð;ð ð0¨&ð 0°Tò 0ó ð0ð( ò(ó ð(ð
 àgmñØðØ%-¨c¡]ðØFNÈuÐU]Ð_bÐUbÑOcÑFdðà	òó ðð& Øð Ð4Eð  È$ò  ó ó ð ðD ð

 mð 

¸ò 

ó ð

ð ôó ôra   r7   c                   óH   — e Zd Zededdddfd„«       Zededdddfd„«       Zy)rR   Úwrapper_moduleÚoriginal_modulerÃ   rI   Nc                 óN   — t        |t        «      r|j                  sd|_        y y y )NF©r”   r   rá   Úrequire_backward_grad_sync©r_   r  r   s      r`   Úon_after_inner_forwardz-_DDPForwardRedirection.on_after_inner_forwardÀ  s)   € ô �nÔ&=Ô>À×GmÒGmØ8=ˆNÕ5ð HnÐ>ra   c                 óN   — t        |t        «      r|j                  sd|_        y y y )NTr"  r$  s      r`   Úon_after_outer_forwardz-_DDPForwardRedirection.on_after_outer_forwardÇ  s'   € ä�nÔ&=Ô>À×GmÒGmØ8<ˆNÕ5ð HnÐ>ra   )rQ   r  r  r   r   r%  r'  r�   ra   r`   rR   rR   ¿  sV   „ Øð>°Vð >ÐNbð >Ðgkò >ó ð>ð ð=°Vð =ÐNbð =Ðgkò =ó ñ=ra   rR   )RÚloggingÚ
contextlibr   Údatetimer   Útypingr   r   r   r   r	   r
   rŸ   Útorch.distributedÚ"lightning_utilities.core.rank_zeror   r¯   r   Útorch.nnr   Útorch.nn.parallel.distributedr   Útorch.optim.optimizerr   Útyping_extensionsr   Úpytorch_lightningrÎ   Úlightning_fabric.pluginsr   r   Ú5lightning_fabric.plugins.collectives.torch_collectiver   Úlightning_fabric.strategiesr   Ú&lightning_fabric.utilities.distributedr   r   r   r   r   rÛ   Ú"lightning_fabric.utilities.importsr   r   Ú$lightning_fabric.utilities.optimizerr   Úlightning_fabric.utilities.seedr   Ú lightning_fabric.utilities.typesr   Ú pytorch_lightning.core.optimizerr    Ú'pytorch_lightning.overrides.distributedr!   r"   r#   Ú#pytorch_lightning.plugins.precisionr$   Ú&pytorch_lightning.strategies.launchersr%   r&   Ú%pytorch_lightning.strategies.parallelr'   Ú%pytorch_lightning.strategies.strategyr(   r)   Ú pytorch_lightning.trainer.statesr*   Ú&pytorch_lightning.utilities.exceptionsr,   Ú%pytorch_lightning.utilities.rank_zeror-   r.   Ú6torch.distributed.algorithms.model_averaging.averagersr/   Ú	getLoggerrQ   rN   Ú_DDP_FORK_ALIASESr7   rR   r�   ra   r`   Ú<module>rG     s´   ðó Ý "Ý ß I× Iã Û Ý UÝ Ý Ý AÝ +Ý &ã ß EÝ TÝ 9÷ó õ Cß TÝ FÝ 6Ý 5Ý ?ß vÑ vÝ 9ß fÝ Bß QÝ 6Ý Cß gÑ gáÝTà€g×Ñ˜Ó!€ðÐ ôxÐ"ô xôv=Ð0õ =ra   