Ë
      çid¤  ã                   óD  — d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZ d dlZd dlmZ d d	lmZ d d
lmZmZ d dlmZ d dlZ d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z< d dl=m>Z> d dl?m@Z@ d dlAmBZB d dlCmDZD d dlEmFZFmGZGmHZH d dlImJZJ  ej–                  eL«      ZM eF«       ZNerd dlOZOdej                   j.                  ddfd „ZQ G d!„ d"e<«      ZRy)#é    N)ÚOrderedDict)Ú	GeneratorÚMapping)Úcontextmanager)Ú	timedelta)ÚPath)ÚTYPE_CHECKINGÚAnyÚOptionalÚUnion)ÚModule)Ú	Optimizer)ÚLRSchedulerÚReduceLROnPlateau)Úoverride)ÚClusterEnvironment)Údefault_pg_timeout)Ú_StrategyRegistry)Ú_DEEPSPEED_AVAILABLEÚ_DEEPSPEED_GREATER_EQUAL_0_16Ú_format_precision_configÚ_validate_checkpoint_directoryÚ _validate_device_index_selection)Ú_TORCH_GREATER_EQUAL_2_6)Ú_optimizers_to_device)Ú
reset_seed)Ú_PATH)ÚCUDAAccelerator)Ú"_init_optimizers_and_lr_schedulers)Ú	Precision)ÚDDPStrategy©Ú	TrainerFn)ÚGradClipAlgorithmType)ÚMisconfigurationException)Úis_overridden)ÚWarningCacheÚrank_zero_infoÚrank_zero_warn)ÚLRSchedulerConfigÚmodelÚreturnc                 óÒ   — | j                  «       D ]T  }t        «       |_        d |_        t        «       |_        t        «       |_        t        «       |_        t        «       |_        ŒV y ©N)Úmodulesr   Ú_backward_hooksÚ_is_full_backward_hookÚ_forward_hooksÚ_forward_pre_hooksÚ_state_dict_hooksÚ_load_state_dict_pre_hooks)r+   Úmodules     ú{/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/pytorch_lightning/strategies/deepspeed.pyÚremove_module_hooksr8   A   sL   € à—-‘-–/ˆÜ!,£ˆÔØ(,ˆÔ%Ü +£ˆÔÜ$/£MˆÔ!Ü#.£=ˆÔ Ü,7«MˆÕ)ñ "ó    c            _       óÈ  ‡ — e Zd ZdZdZddddddddd	d
dddddddddddddddddddej                  ddddddddddddddedf.ded   de	de
dee   de	de	dedede
d e
d!e
d"ed#e
d$e
d%e
d&e	d'e	d(e
d)e	d*e
d+e	d,e	d-e	d.e	d/e
d0e
d1e	d2eee
f   d3eeeeeef   f      d4e
d5eeej$                        d6ee   d7ed8e
d9e
d:e
d;e
d<e	d=e	d>e	d?e	d@e	dAee   dBee   dCee   dDe	dEdf^ˆ fdF„Zedzˆ fdG„«       ZedzdH„«       Zed{dI„«       ZdzdJ„ZdzdK„ZeedEe	fdL„«       «       ZedMe dNee!   dEe"dOee!   f   fdP„«       Z#	 d|dMe dQee!   dReee$e%f      dEe"dOe!f   fdS„Z&dzdT„Z'dEe"e!ee(   f   fdU„Z)edEe	fdV„«       Z*dMe dEdfdW„Z+e,ed|dXee	   dEe-dY   fˆ fdZ„«       «       Z.e,edEe-dY   fd[„«       «       Z/dzd\„Z0dMe dEdfd]„Z1eedEeee
f   fd^„«       «       Z2ed{d_„«       Z3dMe dEe fd`„Z4eedEe	fda„«       «       Z5ed}db„«       Z6edEe	fdc„«       Z7ed|ddedeedfee   dEdfdg„«       Z8ed|dhediee	   dEeeef   fˆ fdj„«       Z9eedEe	fdk„«       «       Z:ed~dde;eef   dle	dEdfdm„«       Z<dne;eef   dle	dEdfdo„Z=edde;eef   dEdfdp„«       Z>e?edqe@dEdfdr„«       «       ZAd3eeeeeef   f      dEeeeef      fds„ZBdzdt„ZCdzdu„ZDde	d1e	d2eee
f   d<e	d=e	d>e	d?e	de	de	dedede
d e
d!e
d"ed#e
d)e	d$e
d%e
d&e	d'e	d(e
dvedEef0dw„ZEdzdx„ZFdEe
fdy„ZGˆ xZHS )ÚDeepSpeedStrategyÚ	deepspeedÚPL_DEEPSPEED_CONFIG_PATHNTé   FÚcpuz/local_nvmeé   i áõi Êš;é   i   é   é   l    J)£i ÂëÚautor   é   iè  Úacceleratorzpl.accelerators.AcceleratorÚzero_optimizationÚstageÚremote_deviceÚoffload_optimizerÚoffload_parametersÚoffload_params_deviceÚ	nvme_pathÚparams_buffer_countÚparams_buffer_sizeÚ
max_in_cpuÚoffload_optimizer_deviceÚoptimizer_buffer_countÚ
block_sizeÚqueue_depthÚsingle_submitÚoverlap_eventsÚthread_countÚ
pin_memoryÚsub_group_sizeÚcontiguous_gradientsÚoverlap_commÚallgather_partitionsÚreduce_scatterÚallgather_bucket_sizeÚreduce_bucket_sizeÚzero_allow_untested_optimizerÚlogging_batch_size_per_gpuÚconfigÚlogging_levelÚparallel_devicesÚcluster_environmentÚ
loss_scaleÚinitial_scale_powerÚloss_scale_windowÚ
hysteresisÚmin_loss_scaleÚpartition_activationsÚcpu_checkpointingÚcontiguous_memory_optimizationÚsynchronize_checkpoint_boundaryÚload_full_weightsÚprecision_pluginÚprocess_group_backendÚtimeoutÚexclude_frozen_parametersr,   c/                 ó®  •— t         st        d«      ‚t        r%t        sddl}/|/j
                  }0t        d|0› d�«      ‚t        ‰1| �!  ||| |+|,¬«       |-| _	        | j                  |«      | _        | j                  €i | j                  |||fi d|“d|“d	|“d
|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|&“d|'“d|(“d|)“d|“d|“d|“d|“d|“d|“d |“d!|“Ž| _        ddl}/d"| _        |/j                  j                  j                   j#                  |«       || _        |*| _        |.| _        |!| _        |"| _        |#| _        |$| _        |%| _        y)#a·  Provides capabilities to run training using the DeepSpeed library, with training optimizations for large
        billion parameter models. *For more information:* :ref:`deepspeed_advanced`.

        .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

        Defaults have been set to enable ZeRO-Offload and some have been taken from the link below.
        These defaults have been set generally, but may require tuning for optimum performance based on your model size.
        *For more information:* https://www.deepspeed.ai/docs/config-json/#zero-optimizations-for-fp16-training.

        Arguments:

            zero_optimization: Enable ZeRO optimization. This is compatible with either `precision="16-mixed"` or
                `precision="bf16-mixed"`.

            stage: Different stages of the ZeRO Optimizer. 0 is disabled,
                1 is optimizer state partitioning, 2 is optimizer+gradient state partitioning,
                3 is optimizer+gradient_parameter partitioning using the infinity engine.

            remote_device: Device to instantiate the model on initially (``cpu`` or ``nvme``). Defaults to GPU.

            offload_optimizer: Enable offloading optimizer memory and computation to CPU or NVMe
                based on ``offload_optimizer_device``.

            offload_parameters: When using ZeRO Stage 3, Enable offloading parameter memory and computation
                to CPU or NVMe based on ``offload_params_device``.

            offload_params_device: When offloading parameters choose the device to offload to, ``cpu`` or ``nvme``.

            offload_optimizer_device: When offloading optimizer state choose the device to offload to,
                ``cpu`` or ``nvme``.

            params_buffer_count: Number of buffers in buffer pool for
                parameter offloading when ``offload_params_device`` is ``nvme``.

            params_buffer_size: Size of buffers in buffer pool for parameter offloading
                when ``offload_params_device`` is ``nvme``.

            max_in_cpu: Number of parameter elements to maintain in CPU memory when offloading to NVMe is enabled.

            nvme_path: Filesystem path for NVMe device for optimizer/parameter state offloading.

            optimizer_buffer_count: Number of buffers in buffer pool for optimizer state offloading
                when ``offload_optimizer_device`` is set to ``nvme``.
                This should be at least the number of states maintained per parameter by the optimizer.
                For example, Adam optimizer has 4 states (parameter, gradient, momentum, and variance).

            block_size: When using NVMe Offloading, the I/O block size in bytes.

            queue_depth: When using NVMe Offloading, the I/O queue depth.

            single_submit: When using NVMe Offloading,
                submit requests to storage device as multiple individual requests,
                as opposed to one block of requests.

            overlap_events: When using NVMe Offloading,
                submit requests to storage device in an overlapped fashion
                without waiting for completion of earlier requests.

            thread_count: When using NVMe Offloading,
                Intra-request parallelism for each read/write submitted by a user thread.

            pin_memory: When using ZeRO stage 3, pin optimizer state memory on CPU.
                This could boost throughput at the cost of extra memory overhead.

            sub_group_size: When using ZeRO stage 3, defines the number of parameters
                within a sub group to offload at a time.
                Smaller numbers require more communication, but improve memory efficiency.

            contiguous_gradients: Copies gradients to a continuous buffer as they are produced.
                Avoids memory fragmentation during backwards. Useful when training large models.

            overlap_comm: Overlap the reduction (synchronization) of gradients with the backwards computation.
                This is a speed optimization when training across multiple GPUs/machines.

            allgather_partitions: All gather updated parameters at the end of training step,
                instead of using a series of broadcast collectives.

            reduce_scatter: Use reduce/scatter instead of allreduce to average gradients.

            allgather_bucket_size: Number of elements to allgather at once.
                Used to limit the memory required for larger model sizes, with a tradeoff with speed.

            reduce_bucket_size: Number of elements to reduce at once.
                Used to limit the memory required for larger model sizes, with a tradeoff with speed.

            zero_allow_untested_optimizer: Allow untested optimizers to be used with ZeRO. Currently only Adam is a
                DeepSpeed supported optimizer when using ZeRO.

            logging_batch_size_per_gpu: Config used in DeepSpeed to calculate verbose timing for logging
                on a per sample per second basis (only displayed if logging=logging.INFO).
                If set to "auto", the strategy tries to infer this from
                the train DataLoader's BatchSampler, else defaults to 1.
                To obtain accurate logs when using datasets that do not support batch samplers,
                set this to the actual per gpu batch size (trainer.batch_size).

            config: Pass in a deepspeed formatted config dict,
                or path to a deepspeed config: https://www.deepspeed.ai/docs/config-json.
                All defaults will be ignored if a config is passed in.

            logging_level: Set logging level for deepspeed.

            loss_scale: Loss scaling value for FP16 training.
                0.0 results in dynamic loss scaling, otherwise static.

            initial_scale_power: Power of the initial dynamic loss scale value. Loss scale is computed
                by ``2^initial_scale_power``.

            loss_scale_window: Window in which to raise/lower the dynamic FP16 loss scaling value.

            hysteresis: FP16 Delay shift in Dynamic Loss scaling.

            min_loss_scale: The minimum FP16 dynamic loss scaling value.

            partition_activations: Enables partition activation when used with ZeRO stage 3 and model parallelism.
                Still requires you to wrap your forward functions in deepspeed.checkpointing.checkpoint.
                See `deepspeed tutorial
                <https://www.deepspeed.ai/tutorials/megatron/#deepspeed-activation-checkpoints-optional>`_.

            cpu_checkpointing: Offloads partitioned activations to CPU if ``partition_activations`` is enabled.

            contiguous_memory_optimization: Copies partitioned activations so that they are contiguous in memory.
                Not supported by all models.

            synchronize_checkpoint_boundary: Insert :func:`torch.cuda.synchronize` at each checkpoint boundary.

            load_full_weights: True when loading a single checkpoint file containing the model state dict
                when using ZeRO Stage 3. This differs from the DeepSpeed checkpoint which contains shards
                per worker.

            exclude_frozen_parameters: Exclude frozen parameters when saving checkpoints.

        ztTo use the `DeepSpeedStrategy`, you must have DeepSpeed installed. Install it by running `pip install -U deepspeed`.r   NzIPyTorch >= 2.6 requires DeepSpeed >= 0.16.0. Detected DeepSpeed version: zA. Please upgrade by running `pip install -U 'deepspeed>=0.16.0'`.)rF   rd   re   rp   rq   rJ   rK   rM   rL   rN   rO   rP   rX   rQ   rR   rS   rT   rU   rV   rW   rk   rl   rm   rn   rH   rZ   r[   r\   r]   r^   r_   rY   F)r   r%   r   r   r<   Ú__version__ÚImportErrorÚsuperÚ__init__Ú_timeoutÚ_load_configrb   Ú_create_default_configÚ_config_initializedÚutilsÚloggingÚloggerÚsetLevelrI   ro   rs   rf   rg   rh   ri   rj   )2ÚselfrF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   r<   Údeepspeed_versionÚ	__class__s2                                                    €r7   rx   zDeepSpeedStrategy.__init__P   s<  ø€ õj $Ü+ðEóð õ
 $Õ,Ió à )× 5Ñ 5Ðäð/Ø/@Ð.Að BRðRóð ô 	‰ÑØ#Ø-Ø 3Ø-Ø"7ð 	ô 	
ð .5ˆŒà×'Ñ'¨Ó/ˆŒØ�;‰;Ðà5˜$×5Ñ5Ø!Ø-Ø*òñ #4ð	ñ
 $6ðñ $ðñ '<ðñ %8ðñ $6ðñ &ðñ &ðñ *Bðñ (>ðñ &ðñ (ðñ  ,ð!ñ"  .ð#ñ$ *ð%ñ& '<ð'ñ( #4ð)ñ* 0Nð+ñ, 1Pð-ñ. ð/ñ0 &:ð1ñ2 *ð3ñ4 &:ð5ñ6  .ð7ñ8 '<ð9ñ: $6ð;ñ<  .ð=ˆDŒKó@ 	à#(ˆÔ Ø�‰×Ñ×&Ñ&×/Ñ/°Ô>à*ˆÔØ!2ˆÔØ)BˆÔ&ð %ˆŒØ#6ˆÔ Ø!2ˆÔØ$ˆŒØ,ˆÕr9   c                 ó°   •— t        | j                  t        «      s-t        d| j                  j                  j
                  › d�«      ‚t        ‰| �  «        y )Nz;The DeepSpeed strategy is only supported on CUDA GPUs but `z
` is used.)Ú
isinstancerF   r   ÚRuntimeErrorrƒ   Ú__name__rw   Úsetup_environment)r�   rƒ   s    €r7   rˆ   z#DeepSpeedStrategy.setup_environmentT  sR   ø€ ä˜$×*Ñ*¬OÔ<ÜØMÈd×N^ÑN^×NhÑNh×NqÑNqÐMrð sð óð ô 	‰Ñ!Õ#r9   c                 óž   — | j                   €J ‚t        | j                   «       t        «        | j                  «        | j	                  «        y r.   )rd   r   r   Úset_world_ranksÚ_init_deepspeed_distributed©r�   s    r7   Úsetup_distributedz#DeepSpeedStrategy.setup_distributed]  s?   € à×$Ñ$Ð0Ð0Ð0Ü(¨×)>Ñ)>Ô?ÜŒØ×ÑÔØ×(Ñ(Õ*r9   c                 ó„  — | j                  «        | j                  €J ‚| j                  j                  |«       | j                  €J ‚| j                  j                  | j                  «      | _        | j                  | j                  «      | _        |j                  j                  t        j                  k(  r| j                  |«       | j                  «        |j                  j                  t        j                  k(  r t        | j                  | j                  «       | j!                  «        | j#                  «        y r.   )Ú_init_config_if_neededrF   Úsetupr+   rp   Úconvert_moduleÚ_setup_modelÚstateÚfnr#   ÚFITTINGÚsetup_optimizersÚsetup_precision_pluginr   Ú
optimizersÚroot_deviceÚinit_deepspeedÚbarrier©r�   Útrainers     r7   r�   zDeepSpeedStrategy.setupe  sé   € à×#Ñ#Ô%Ø×ÑÐ+Ð+Ð+Ø×Ñ×Ñ˜wÔ'à�z‰zÐ%Ð%Ð%Ø×*Ñ*×9Ñ9¸$¿*¹*ÓEˆŒ
Ø×&Ñ& t§z¡zÓ2ˆŒ
à�=‰=×Ñœy×0Ñ0Ò0Ø×!Ñ! 'Ô*Ø×#Ñ#Ô%Ø�=‰=×Ñœy×0Ñ0Ò0Ü! $§/¡/°4×3CÑ3CÔDà×ÑÔØ�‰�r9   c                 ó˜  — dd l }| j                  €J ‚t        j                  «       dk7  rO| j	                  «        t
        j                  d| j                  › d| j                  dz   › d| j                  › �«       | j                  «       | _
         |j                  | j                  | j                  j                  | j                  ¬«       y )Nr   ÚWindowsz1initializing deepspeed distributed: GLOBAL_RANK: z
, MEMBER: rC   Ú/)Údistributed_portrr   )r<   re   ÚplatformÚsystemÚ_set_node_environment_variablesÚlogÚinfoÚglobal_rankÚ
world_sizeÚ_get_process_group_backendÚ_process_group_backendÚinit_distributedÚ	main_portry   ©r�   r<   s     r7   r‹   z-DeepSpeedStrategy._init_deepspeed_distributedx  s¹   € Ûà×'Ñ'Ð3Ð3Ð3Ü�?‰?Ó 	Ò)à×0Ñ0Ô2Ü�H‰Hð Ø $× 0Ñ 0Ð1ð 2Ø×+Ñ+¨aÑ/Ð0°°$·/±/Ð1BðDôð
 '+×&EÑ&EÓ&GˆÔ#Ø"ˆ	×"Ñ"Ø×'Ñ'¸$×:RÑ:R×:\Ñ:\Ðfj×fsÑfsö	
r9   c                 ó²  — | j                   €J ‚| j                   j                  t        j                  d<   t	        | j                   j
                  «      t        j                  d<   t	        | j                  «      t        j                  d<   t	        | j                  «      t        j                  d<   t	        | j                  «      t        j                  d<   y )NÚMASTER_ADDRÚMASTER_PORTÚRANKÚ
WORLD_SIZEÚ
LOCAL_RANK)	re   Úmain_addressÚosÚenvironÚstrr¬   r§   r¨   Ú
local_rankrŒ   s    r7   r¤   z1DeepSpeedStrategy._set_node_environment_variables‰  s”   € Ø×'Ñ'Ð3Ð3Ð3Ø$(×$<Ñ$<×$IÑ$IŒ�
‰
�=Ñ!Ü$'¨×(@Ñ(@×(JÑ(JÓ$KŒ�
‰
�=Ñ!Ü  ×!1Ñ!1Ó2Œ�
‰
�6ÑÜ#& t§¡Ó#7Œ�
‰
�<Ñ Ü#& t§¡Ó#7Œ�
‰
�<Ò r9   c                  ó   — y©NT© rŒ   s    r7   Úrestore_checkpoint_after_setupz0DeepSpeedStrategy.restore_checkpoint_after_setup‘  s   € ð r9   r+   r˜   údeepspeed.DeepSpeedEnginec                 ó  — t        |«      dk7  rt        dt        |«      › d�«      ‚| j                  €J ‚| j                  j                  dd«       | j	                  ||d   «      \  | _        }| j                  «        | j
                  |gfS )a  Setup a model and multiple optimizers together.

        Currently only a single optimizer is supported.

        Return:
            The model wrapped into a :class:`deepspeed.DeepSpeedEngine` and a list with a single
            deepspeed optimizer.

        rC   z>Currently only one optimizer is supported with DeepSpeed. Got z optimizers instead.Útrain_micro_batch_size_per_gpur   )ÚlenÚ
ValueErrorrb   Ú
setdefaultÚ_setup_model_and_optimizerr+   Ú'_set_deepspeed_activation_checkpointing)r�   r+   r˜   Ú	optimizers       r7   Ú_setup_model_and_optimizersz-DeepSpeedStrategy._setup_model_and_optimizers–  s”   € ô ˆz‹?˜aÒÜØPÔQTÐU_ÓQ`ÐPaÐauÐvóð ð �{‰{Ð&Ð&Ð&Ø�‰×ÑÐ?ÀÔCØ $× ?Ñ ?ÀÀzÐRSÁ}Ó UÑˆŒ
�IØ×4Ñ4Ô6Ø�z‰z˜I˜;Ð&Ð&r9   rÅ   Úlr_schedulerc           	      óè   — ddl }t        d„ |j                  «       «      } |j                  t	        j
                  | j                  j                  ¬«      | j                  ||||d¬«      \  }}}}||fS )z“Initialize one model and one optimizer with an optional learning rate scheduler.

        This calls ``deepspeed.initialize`` internally.

        r   Nc                 ó   — | j                   S r.   )Úrequires_grad)Úps    r7   Ú<lambda>z>DeepSpeedStrategy._setup_model_and_optimizer.<locals>.<lambda>¾  s   € ¨A¯OªOr9   ©Údevice_rankF)Úargsrb   r+   Úmodel_parametersrÅ   rÇ   Údist_init_required)	r<   ÚfilterÚ
parametersÚ
initializeÚargparseÚ	Namespacer™   Úindexrb   )	r�   r+   rÅ   rÇ   r<   rÐ   Údeepspeed_engineÚdeepspeed_optimizerÚ_s	            r7   rÃ   z,DeepSpeedStrategy._setup_model_and_optimizer±  sy   € ó 	ä!Ñ";¸U×=MÑ=MÓ=OÓPÐØ6J°i×6JÑ6JÜ×#Ñ#°×0@Ñ0@×0FÑ0FÔGØ—;‘;ØØ-ØØ%Ø$ô7
Ñ3ÐÐ-¨q°!ð  Ð!4Ð4Ð4r9   c                 ó  — | j                   €J ‚t        d| j                   t        j                  «      rt	        d«       | j                   j
                  j                  t        j                  k(  rt        d«      ‚t        | j                  t        j                  «      sJ ‚| j                   j
                  r<| j                   j
                  j                  r| j                  | j                  «       y | j                  | j                  «       y )NÚconfigure_gradient_clippinga8  Since DeepSpeed handles gradient clipping internally, the default `LightningModule.configure_gradient_clipping` implementation will not actually clip gradients. The hook will still be called. Consider setting `Trainer(gradient_clip_val=..., gradient_clip_algorithm='norm')` which will use the internal mechanism.z7DeepSpeed does not support clipping gradients by value.)Úlightning_moduler&   ÚplÚLightningModuler)   r�   Úgradient_clip_algorithmr$   ÚVALUEr%   r…   r+   ÚtrainingÚ_initialize_deepspeed_trainÚ_initialize_deepspeed_inferencerŒ   s    r7   rš   z DeepSpeedStrategy.init_deepspeedÊ  sÊ   € Ø×$Ñ$Ð0Ð0Ð0äÐ6¸×8MÑ8MÌr×OaÑOaÔbÜð:ôð × Ñ ×(Ñ(×@Ñ@ÔDY×D_ÑD_Ò_Ü+Ð,eÓfÐfä˜$Ÿ*™*¤b×&8Ñ&8Ô9Ð9Ð9Ø× Ñ ×(Ò(¨T×-BÑ-B×-JÑ-J×-SÒ-SØ×,Ñ,¨T¯Z©ZÕ8à×0Ñ0°·±Õ<r9   c                 óº   — | j                   €J ‚t        | j                   «      \  }}t        |«      dkD  st        |«      dkD  rt        d«      ‚|d   |r|d   fS d fS )NrC   zNDeepSpeed currently only supports single optimizer, single optional scheduler.r   )rÝ   r   rÀ   r%   )r�   r˜   Úlr_schedulerss      r7   Ú_init_optimizersz"DeepSpeedStrategy._init_optimizersß  sq   € Ø×$Ñ$Ð0Ð0Ð0Ü$FÀt×G\ÑG\Ó$]Ñ!ˆ
�MÜˆz‹?˜QÒ¤# mÓ"4°qÒ"8Ü+Ø`óð ð ˜!‰}±-˜m¨AÑ.ÐIÐIÀTÐIÐIr9   c                 ó¤   — t        | j                  t        «      sJ ‚| j                  j                  d«      }|d uxr |j                  d«      dk(  S )NrG   rH   é   )r…   rb   ÚdictÚget)r�   rG   s     r7   Úzero_stage_3zDeepSpeedStrategy.zero_stage_3è  sL   € ä˜$Ÿ+™+¤tÔ,Ð,Ð,Ø ŸK™KŸO™OÐ,?Ó@ÐØ ¨Ð,ÒTÐ1B×1FÑ1FÀwÓ1OÐSTÑ1TÐTr9   c                 ó’  — d\  }}t        | j                  t        «      sJ ‚d| j                  v rt        d«       d }n!| j	                  «       \  }}|�|j
                  }| j                  |||«      \  }}| j                  «        |g| _        |j                  }|�&d |_	        |€t        |d¬«      }n||_        |g| _        || _        y )N)NNrÅ   z—You have specified an optimizer and/or scheduler within the DeepSpeed config. It is recommended to define it in `LightningModule.configure_optimizers`.Ústep)Úinterval)r…   rb   rê   r(   rç   Ú	schedulerrÃ   rÄ   r˜   rÇ   r*   Úlr_scheduler_configsr+   )r�   r+   rÅ   rð   rÇ   rÙ   Údeepspeed_schedulers          r7   rã   z-DeepSpeedStrategy._initialize_deepspeed_trainî  sä   € Ø)Ñˆ	�9Ü˜$Ÿ+™+¤tÔ,Ð,Ð,Ø˜$Ÿ+™+Ñ%Üð]ôð  ‰Lð
 ×%Ñ%Ó'ñØØàÐ'Ø(×2Ñ2�	à%)×%DÑ%DÀUÈIÐW`Ó%aÑ"ˆÐ"Ø×4Ñ4Ô6ð /Ð/ˆŒà#×0Ñ0ÐØÐ*à!%ˆEÔØÐ#Ü0Ð1DÈvÔV‘à)<�Ô&Ø)5¨ˆDÔ%Øˆ�
r9   Ú
empty_init)NNNc              #   ó¨   •K  — | j                   r|du rt        d|›d�«      ‚d –— y t        ‰| �  |¬«      5  d –— d d d «       y # 1 sw Y   y xY w­w)NFz`empty_init=zN` is not a valid choice with `DeepSpeedStrategy` when ZeRO stage 3 is enabled.)ró   )rì   ÚNotImplementedErrorrw   Útensor_init_context)r�   ró   rƒ   s     €r7   rö   z%DeepSpeedStrategy.tensor_init_context  s[   øè ø€ ð ×ÒØ˜UÑ"Ü)Ø#˜
�}Ð$rÐsóð ó ØÜ‰WÑ(°JÐ(Õ?Û÷ @×?Ñ?üs   ƒ5A¸A½	AÁAÁAc              #   óÞ   K  — dd l }| j                  «        |j                  j                  | j                  | j
                  | j                  ¬«      5  d –— d d d «       y # 1 sw Y   y xY w­w)Nr   )ÚenabledrI   Úconfig_dict_or_path)r<   r�   ÚzeroÚInitrì   rI   rb   r­   s     r7   Úmodel_sharded_contextz'DeepSpeedStrategy.model_sharded_context  s[   è ø€ ó 	à×#Ñ#Ô%Ø�^‰^× Ñ Ø×%Ñ%Ø×,Ñ,Ø $§¡ð !õ 
ó
 ÷
÷ 
ñ 
üs   ‚AA-ÁA!Á	A-Á!A*Á&A-c           	      óR  — dd l }t        | j                  t        «      sJ ‚| j                  j	                  d«      rl| j                  d   }|j
                  j                  d |j	                  d«      |j	                  d«      |j	                  d«      |j	                  d«      ¬«       y y )Nr   Úactivation_checkpointingrk   rm   rl   Úprofile)Úmpu_rk   Úcontiguous_checkpointingÚcheckpoint_in_cpurÿ   )r<   r…   rb   rê   rë   ÚcheckpointingÚ	configure)r�   r<   Úcheckpoint_configs      r7   rÄ   z9DeepSpeedStrategy._set_deepspeed_activation_checkpointing*  s–   € Ûä˜$Ÿ+™+¤tÔ,Ð,Ð,Ø�;‰;�?‰?Ð5Ô6Ø $§¡Ð,FÑ GÐØ×#Ñ#×-Ñ-ØØ&7×&;Ñ&;Ð<SÓ&TØ):×)>Ñ)>Ð?_Ó)`Ø"3×"7Ñ"7Ð8KÓ"LØ)×-Ñ-¨iÓ8ð .õ ð 7r9   c           	      ó(  — dd l }t        | j                  t        «      sJ ‚ddi}d| j                  v r |j	                  d| j                  d   i«       d| j                  v r |j	                  d| j                  d   i«       | j
                  r.|j	                  | j                  d   | j                  d   dœ«       t        |«        |j                  t        j                  | j                  j                  ¬	«      ||d d g d
¬«      \  }}}}|| _        y )Nr   r¿   rC   Úfp16Úbf16r`   rG   ©r`   rG   rÍ   F)rÏ   rb   r+   rÅ   rÇ   rÐ   rÑ   )r<   r…   rb   rê   Úupdaterì   r8   rÔ   rÕ   rÖ   r™   r×   r+   )r�   r+   r<   Úinference_configrÚ   s        r7   rä   z1DeepSpeedStrategy._initialize_deepspeed_inference8  s  € Ûä˜$Ÿ+™+¤tÔ,Ð,Ð,ð =¸aÐ@ÐØ�T—[‘[Ñ Ø×#Ñ# V¨T¯[©[¸Ñ-@Ð$AÔBØ�T—[‘[Ñ Ø×#Ñ# V¨T¯[©[¸Ñ-@Ð$AÔBØ×ÒØ×#Ñ#Ø15·±Ð=\Ñ1]Ø%)§[¡[Ð1DÑ%Eñ%ô ô
 	˜EÔ"Ø-˜×-Ñ-Ü×#Ñ#°×0@Ñ0@×0FÑ0FÔGØ#ØØØØØ$ô
‰ˆˆq�!�Qð ˆ�
r9   c                 ó4   — | j                   | j                  dœS )N)Únum_replicasÚrank)r¨   r§   rŒ   s    r7   Údistributed_sampler_kwargsz,DeepSpeedStrategy.distributed_sampler_kwargsU  s   € ð !%§¡¸×9IÑ9IÑJÐJr9   c                 ó    — g | _         g | _        y)z…Creates optimizers and schedulers.

        Args:
            trainer: the Trainer, these optimizers should be connected to

        N)r˜   rñ   rœ   s     r7   r–   z"DeepSpeedStrategy.setup_optimizersZ  s   € ð ˆŒØ$&ˆÕ!r9   c                 ó   — |S r.   r»   )r�   r+   s     r7   r’   zDeepSpeedStrategy._setup_modeli  s   € Øˆr9   c                  ó   — y)z>Whether the strategy handles gradient accumulation internally.Tr»   rŒ   s    r7   Úhandles_gradient_accumulationz/DeepSpeedStrategy.handles_gradient_accumulationl  s   € ð r9   c                 ó   — | j                   S r.   )r+   rŒ   s    r7   rØ   z"DeepSpeedStrategy.deepspeed_enginer  s   € à�z‰zÐr9   c                 óB   — | j                   dkD  xs | j                  dkD  S )NrC   )Únum_processesÚ	num_nodesrŒ   s    r7   Ú_multi_devicezDeepSpeedStrategy._multi_devicev  s!   € à×!Ñ! AÑ%Ò;¨¯©¸!Ñ);Ð;r9   Ú
checkpointÚfilepathÚstorage_optionsc                 ó˜  — | j                  |«      }|�#t        d| j                  j                  › d�«      ‚| j                  r-| j
                  r!| j                  rt        j                  d«       ddg}|j                  «       D ��ci c]  \  }}||vsŒ||“Œ }}}| j                  j                  ||d| j                  ¬«       yc c}}w )	a’  Save model/training states as a checkpoint file through state-dump and file-write.

        Args:
            checkpoint: The checkpoint state dictionary
            filepath: write-target file's path
            storage_options: not used for ``DeepSpeedStrategy`` as ``CheckpointIO`` is not used

        Raises:
            TypeError:
                If ``storage_options`` arg is passed in

        Nze`Trainer.save_checkpoint(..., storage_options=...)` with `storage_options` arg is not supported for `z ` as `CheckpointIO` is not used.a  When saving the DeepSpeed Stage 3 checkpoint, each worker will save a shard of the checkpoint within a directory. If a single file is required after training, see https://lightning.ai/docs/pytorch/stable/advanced/model_parallel.html#deepspeed-zero-stage-3-single-file for instructions.Ú
state_dictÚoptimizer_statesr  )Úclient_stateÚtagrs   )Ú	broadcastÚ	TypeErrorrƒ   r‡   rì   r  Úis_global_zeroÚwarning_cacheÚwarnÚitemsrØ   Úsave_checkpointrs   )r�   r  r  r  Ú_exclude_keysÚkÚvs          r7   r'  z!DeepSpeedStrategy.save_checkpointz  sÞ   € ð —>‘> (Ó+ˆàÐ&Üð*Ø*.¯.©.×*AÑ*AÐ)BÐBbðdóð ð
 ×Ò ×!3Ò!3¸×8KÒ8KÜ×ÑðGôð &Ð'9Ð:ˆØ'1×'7Ñ'7Ô'9ÔTÑ'9™t˜q !¸QÀmÒ=S�a˜‘dÐ'9ˆ
ÑTØ×Ñ×-Ñ-ØØ#ØØ&*×&DÑ&Dð	 	.õ 	
ùó Us   ÂCÂCÚcheckpoint_pathÚweights_onlyc                 ó¬  •— | j                   r-| j                  r!| j                  |«      }t        ‰| �  ||«      S t        |«       | j                  €J ‚ddlm} | j                  j                  j                  j                  |j                  k(  }| j                  j	                  ||d| j                  j                  ¬«      \  }}|€t        d«      ‚|S )Nr   r"   F)Úload_optimizer_statesÚload_lr_scheduler_statesÚload_module_strictzÃDeepSpeed was unable to load the checkpoint. Ensure you passed in a DeepSpeed compatible checkpoint or a single checkpoint file with `Trainer(strategy=DeepSpeedStrategy(load_full_weights=True))`.)ro   rì   r!  rw   Úload_checkpointr   rÝ   Ú pytorch_lightning.trainer.statesr#   r�   r“   r”   r•   rØ   Ústrict_loadingr%   )r�   r+  r,  r#   Ú
is_fittingrÚ   r  rƒ   s          €r7   r1  z!DeepSpeedStrategy.load_checkpoint¤  sØ   ø€ à×!Ò! d×&7Ò&7ð #Ÿn™n¨_Ó=ˆOÜ‘7Ñ*¨?¸LÓIÐIä& Ô7ð ×$Ñ$Ð0Ð0Ð0å>à×*Ñ*×2Ñ2×8Ñ8×;Ñ;¸y×?PÑ?PÑPˆ
à×/Ñ/×?Ñ?ØØ",Ø%*Ø#×4Ñ4×CÑCð	 @ó 
‰ˆˆ<ð ÐÜ+ðróð ð Ðr9   c                 óÜ   — | j                   €J ‚| j                  rR| j                  rF| j                   j                  j                  j
                  t        j                  k(  rt        d«       y)Na  A single checkpoint file has been given. This means optimizer states cannot be restored. If you'd like to restore these states, you must provide a path to the originally saved DeepSpeed checkpoint. When using ZeRO 3, the original path should be a directory.F)	rÝ   ro   rì   r�   r“   r”   r#   r•   r)   rŒ   s    r7   Úlightning_restore_optimizerz-DeepSpeedStrategy.lightning_restore_optimizerÂ  s`   € ð ×$Ñ$Ð0Ð0Ð0à×!Ò! d×&7Ò&7¸D×<QÑ<Q×<YÑ<Y×<_Ñ<_×<bÑ<bÔfo×fwÑfwÒ<wÜð[ôð
 r9   Ústrictc                 ó~   — | j                   r1| j                  r$| j                  «        | j                  ||¬«       y y y )N)r7  )ro   rì   Úmodel_to_deviceÚ_restore_zero_state)r�   r  r7  s      r7   Úload_model_state_dictz'DeepSpeedStrategy.load_model_state_dictÏ  s<   € ð ×!Ò! d×&7Ò&7Ø× Ñ Ô"Ø×$Ñ$ Z¸Ð$Õ?ð '8Ð!r9   Úckptc                 ó´   ‡ ‡‡‡‡— ddl Š‰ j                  €J ‚d	dt        j                  j                  dt
        ddfˆˆˆˆ ˆfd„Š ‰‰ j                  d¬«       y)
a8  Overrides the normal load_state_dict behaviour in PyTorch to ensure we gather parameters that may be sharded
        across processes before loading the state dictionary when using ZeRO stage 3. This is then automatically synced
        across processes.

        Args:
            ckpt: The ckpt file.

        r   NÚ r6   Úprefixr,   c           
      óà  •— g }g }g }‰
d   }t        |dd «      }|j                  «       }|�||_        |€i n|j                  |d d i «      }‰j                  j                  t        | j                  d¬«      «      d¬«      5  ‰j                  r| j                  |||‰|||¬«       d d d «       | j                  j                  «       D ]  \  }}	|	€Œ	 ‰|	||z   d	z   «       Œ y # 1 sw Y   Œ>xY w)
Nr  Ú	_metadataéÿÿÿÿF)Úrecurser   )Úmodifier_rank)r  r?  Úlocal_metadatar7  Úmissing_keysÚunexpected_keysÚ
error_msgsÚ.)ÚgetattrÚcopyrA  rë   rú   ÚGatheredParametersÚlistrÓ   r#  Ú_load_from_state_dictÚ_modulesr&  )r6   r?  rF  rG  rH  r  ÚmetadatarE  ÚnameÚchildr<  r<   Úloadr�   r7  s             €€€€€r7   rS  z3DeepSpeedStrategy._restore_zero_state.<locals>.loadã  s	  ø€ Ø&(ˆLØ)+ˆOØ$&ˆJØ˜lÑ+ˆJô ˜z¨;¸Ó=ˆHØ#Ÿ™Ó*ˆJØÐ#Ø'/�
Ô$à#+Ð#3™R¸¿¹ÀfÈSÈbÀkÐSUÓ9VˆNð —‘×2Ñ2´4¸×8IÑ8IÐRWÐ8IÓ8XÓ3YÐijÐ2ÕkØ×&Ò&Ø×0Ñ0Ø#-Ø%Ø'5Ø%Ø%1Ø(7Ø#-ð 1ô ÷ lð  &Ÿ™×4Ñ4Ö6‘��eØÑ$Ù˜ ¨¡°Ñ 3Õ4ñ  7÷ lÐkús   Â%C$Ã$C-)r?  )r>  )r<   rÝ   ÚtorchÚnnr   r·   )r�   r<  r7  r<   rS  s   ```@@r7   r:  z%DeepSpeedStrategy._restore_zero_stateÖ  sT   ü€ ó 	à×$Ñ$Ð0Ð0Ð0ñ	5œŸ™Ÿ™ð 	5´#ð 	5¸t÷ 	5ñ 	5ñ@ 	ˆT×"Ñ"¨2Ö.r9   c                  ó   — y r.   r»   )r�   r  s     r7   Úload_optimizer_state_dictz+DeepSpeedStrategy.load_optimizer_state_dict  s   € ð 	r9   Ústrategy_registryc                 ó8  — |j                  d| d¬«       |j                  d| dd¬«       |j                  d| d	d
¬«       |j                  d| dd
d¬«       |j                  d| dd¬«       |j                  d| dddd¬«       |j                  d| ddddddd¬«	       y )Nr<   zDefault DeepSpeed Strategy)ÚdescriptionÚdeepspeed_stage_1z#DeepSpeed with ZeRO Stage 1 enabledrC   )rZ  rH   Údeepspeed_stage_2z#DeepSpeed with ZeRO Stage 2 enabledr>   Údeepspeed_stage_2_offloadz&DeepSpeed ZeRO Stage 2 and CPU OffloadT)rZ  rH   rJ   Údeepspeed_stage_3zDeepSpeed ZeRO Stage 3ré   Údeepspeed_stage_3_offloadz&DeepSpeed ZeRO Stage 3 and CPU Offload)rZ  rH   rJ   rK   Údeepspeed_stage_3_offload_nvmez'DeepSpeed ZeRO Stage 3 and NVMe OffloadÚnvme)rZ  rH   rJ   rK   rI   rL   rQ   )Úregister)ÚclsrX  s     r7   Úregister_strategiesz%DeepSpeedStrategy.register_strategies
  sá   € ð 	×"Ñ" ;°ÐA]Ð"Ô^Ø×"Ñ"Ð#6¸ÐInÐvwÐ"ÔxØ×"Ñ"Ð#6¸ÐInÐvwÐ"ÔxØ×"Ñ"Ø'ØØ@ØØ"ð 	#ô 	
ð 	×"Ñ"Ð#6¸ÐIaÐijÐ"ÔkØ×"Ñ"Ø'ØØ@ØØ"Ø#ð 	#ô 	
ð 	×"Ñ"Ø,ØØAØØ"Ø#Ø Ø"(Ø%+ð 	#õ 
	
r9   c                 óÆ  — |€R| j                   t        j                  v r6t        d| j                   › d�«       t        j                  | j                      }t	        |t
        t        f«      rVt        j                  j                  |«      st        d|› �«      ‚t        |«      5 }t        j                  |«      }d d d «       t	        |t        «      s|�J ‚|S # 1 sw Y   ŒxY w)Nz"Loading DeepSpeed config from set z environment variablezHYou passed in a path to a DeepSpeed config but the path does not exist: )ÚDEEPSPEED_ENV_VARrµ   r¶   r(   r…   r·   r   ÚpathÚisfiler%   ÚopenÚjsonrS  rê   )r�   rb   Úfs      r7   rz   zDeepSpeedStrategy._load_config,  s»   € Øˆ>˜d×4Ñ4¼¿
¹
ÑBÜÐ?À×@VÑ@VÐ?WÐWlÐmÔnÜ—Z‘Z × 6Ñ 6Ñ7ˆFÜ�fœs¤D˜kÔ*Ü—7‘7—>‘> &Ô)Ü/Ø^Ð_eÐ^fÐgóð ô �f” ÜŸ™ 1›�÷ ä˜&¤$Ô'¨6¨>Ð9Ð9Øˆ÷ �ús   Â#CÃC c                 óL   — | j                   s| j                  «        d| _         y y rº   )r|   Ú_format_configrŒ   s    r7   r�   z(DeepSpeedStrategy._init_config_if_needed:  s$   € Ø×'Ò'Ø×ÑÔ!Ø'+ˆDÕ$ð (r9   c           	      ó  — | j                   €t        d«      ‚| j                  «        t        | j                   | j                  j
                  | j                  | j                  | j                  | j                  | j                  ¬«       y )Nz«To use DeepSpeed you must pass in a DeepSpeed config dict, or a path to a JSON config. See: https://lightning.ai/docs/pytorch/stable/advanced/model_parallel.html#deepspeed)rb   Ú	precisionrf   rh   rj   rg   ri   )rb   r%   Ú(_format_batch_size_and_grad_accum_configr   rp   ro  rf   rh   rj   rg   ri   rŒ   s    r7   rm  z DeepSpeedStrategy._format_config?  st   € Ø�;‰;ÐÜ+ðhóð ð 	×5Ñ5Ô7Ü Ø—;‘;Ø×+Ñ+×5Ñ5Ø—‘Ø"×4Ñ4Ø×.Ñ.Ø $× 8Ñ 8Ø—‘ö	
r9   Úzero_kwargsc                 ó†   — ||||dœ|||||dœdœ}|r#|}|r
||
||dœ|d<   |	r||
||||dœ|d<   ||dœ|¥}|d	k7  rd
|i|¥}|S )N)rk   rl   rm   rn   )rS   rT   rU   rV   rW   )rþ   Úaio)ÚdevicerM   Úbuffer_countrX   rJ   )rt  rM   ru  Úbuffer_sizerP   rX   Úoffload_paramr	  rD   r¿   r»   )r�   rG   r`   ra   rk   rl   rm   rn   rJ   rK   rM   rL   rN   rO   rP   rQ   rR   rX   rS   rT   rU   rV   rW   rq  ÚcfgÚzero_configs                             r7   r{   z(DeepSpeedStrategy._create_default_configP  sµ   € ð8 *?Ø%6Ø2PØ3Rñ	)ð )Ø*Ø!.Ø"0Ø ,ññ
ˆñ Ø%ˆKá à6Ø!*Ø$:Ø",ñ	4�Ð/Ñ0ñ "à3Ø!*Ø$7Ø#5Ø",Ø",ñ0�˜OÑ,ð 2OØ%0ñð ðˆCð
 &¨Ò/Ø3Ð5OÐWÐSVÐWˆCØˆ
r9   c                 ó¼  — t        | j                  t        «      sJ ‚| j                  €y d| j                  v rt	        d«      ‚| j                  j
                  j                  | j                  d<   d| j                  vr| j                  «       }|| j                  d<   d| j                  vr2| j                  j
                  j                  xs d| j                  d<   y y )NÚgradient_accumulation_stepsz¢Do not set `gradient_accumulation_steps` in the DeepSpeed config as this will be set with the `accumulate_grad_batches` argument passed via the Lightning Trainer.r¿   Úgradient_clippingg        )	r…   rb   rê   rÝ   r%   r�   Úaccumulate_grad_batchesÚ_auto_select_batch_sizeÚgradient_clip_val)r�   Ú
batch_sizes     r7   rp  z:DeepSpeedStrategy._format_batch_size_and_grad_accum_config•  sÈ   € ä˜$Ÿ+™+¤tÔ,Ð,Ð,Ø× Ñ Ð(Øà(¨D¯K©KÑ7Ü+ðuóð ð 6:×5JÑ5J×5RÑ5R×5jÑ5jˆ�‰Ð1Ñ2Ø+°4·;±;Ñ>Ø×5Ñ5Ó7ˆJØ<FˆD�K‰KÐ8Ñ9Ø d§k¡kÑ1Ø/3×/DÑ/D×/LÑ/L×/^Ñ/^Ò/eÐbeˆD�K‰KÐ+Ò,ð 2r9   c                 óþ   — | j                   €J ‚d}| j                   j                  j                  j                  }|j	                  «       r2|j                  «       }t        |d«      r|j                  j                  }|S )NrC   Úbatch_sampler)	rÝ   r�   Úfit_loopÚ_data_sourceÚ
is_definedÚ
dataloaderÚhasattrr‚  r€  )r�   r€  Údata_sourceÚtrain_dataloaders       r7   r~  z)DeepSpeedStrategy._auto_select_batch_size§  su   € ð ×$Ñ$Ð0Ð0Ð0Øˆ
Ø×+Ñ+×3Ñ3×<Ñ<×IÑIˆØ×!Ñ!Ô#Ø*×5Ñ5Ó7ÐÜÐ'¨Ô9Ø-×;Ñ;×FÑF�
ØÐr9   )r,   N)r�   z
pl.Trainerr,   Nr.   )r,   r½   )T)Ir‡   Ú
__module__Ú__qualname__Ústrategy_namerf  r~   ÚWARNr   r   ÚboolÚintr·   r   r   rê   r
   rM  rT  rt  r   Úfloatr    r   rx   r   rˆ   r�   r�   r‹   r¤   Úpropertyr¼   r   r   ÚtuplerÆ   r   r   rÃ   rš   r*   rç   rì   rã   r   r   rö   rü   rÄ   rä   r  r–   r’   r  rØ   r  r'  r1  r6  r   r;  r:  rW  Úclassmethodr   rd  rz   r�   rm  r{   rp  r~  Ú__classcell__)rƒ   s   @r7   r;   r;   L   s9  ø„ Ø€MØ2Ðð @DØ"&ØØ'+Ø"'Ø#(Ø%*Ø&Ø#$Ø"-Ø'Ø(-Ø&'Ø!ØØ#Ø#ØØ Ø/Ø%)Ø!Ø%)Ø#Ø%0Ø"-Ø.2Ø6<Ø9=Ø$Ÿ\™\Ø9=Ø<@ØØ#%Ø!%ØØØ&+Ø"'Ø/4Ø05Ø"'Ø04Ø/3Ø'9Ø*/ñ_B-àÐ;Ñ<ðB-ð  ðB-ð ð	B-ð
   ‘}ðB-ð  ðB-ð !ðB-ð  #ðB-ð ðB-ð !ðB-ð  ðB-ð ðB-ð #&ðB-ð !$ðB-ð ðB-ð  ð!B-ð" ð#B-ð$ ð%B-ð& ð'B-ð( ð)B-ð* ð+B-ð, #ð-B-ð. ð/B-ð0 #ð1B-ð2 ð3B-ð4  #ð5B-ð6  ð7B-ð8 (,ð9B-ð: %*¨#¨s¨(¡Oð;B-ð< ˜˜u d¨3°¨8¡nÐ4Ñ5Ñ6ð=B-ð> ð?B-ð@ # 4¨¯©Ñ#5Ñ6ðAB-ðB &Ð&8Ñ9ðCB-ðD ðEB-ðF !ðGB-ðH ðIB-ðJ ðKB-ðL ðMB-ðN  $ðOB-ðP  ðQB-ðR )-ðSB-ðT *.ðUB-ðV  ðWB-ðX # 9Ñ-ðYB-ðZ  (¨™}ð[B-ð\ ˜)Ñ$ð]B-ð^ $(ð_B-ð` 
õaB-ðH ô$ó ð$ð ò+ó ð+ð òó ðó$
ó"8ð Øð°ò ó ó ðð ð'Øð'Ø)-¨i©ð'à	Ð*¨D°©OÐ;Ñ	<ò'ó ð'ð< IMñ	5àð5ð ˜IÑ&ð5ð ˜u [Ð2CÐ%CÑDÑEð	5ð
 
Ð*¨IÐ5Ñ	6ó5ó2=ð*J %¨	°8Ð<MÑ3NÐ(NÑ"Oó Jð ðU˜dò Uó ðUð
 °ð  ¸Dó  ðD Øñ	¨h°t©nð 	È	ÐRbÑHcô 	ó ó ð	ð Øð	 yÐ1AÑ'Bò 	ó ó ð	óð°Vð Àó ð: ØðK¨D°°c°©Nò Kó ó ðKð ò'ó ð'ð &ð ¨Vó ð Øð¨tò ó ó ðð òó ðð ð<˜tò <ó ð<ð ñ'
¨$ð '
¸%ð '
ÐRZÐ[^ÑR_ð '
Ðkoò '
ó ð'
ðR ñ¨uð ÀHÈTÁNð Ð^bÐcfÐhkÐckÑ^lô ó ðð: Øð	¨Tò 	ó ó ð	ð ñ@°¸¸S¸Ñ0Að @È4ð @Ð[_ò @ó ð@ð-/¨°°S°Ñ(9ð -/À4ð -/ÈDó -/ð^ ð°G¸CÀ¸HÑ4Eð È$ò ó ðð Øð
Ð4Eð 
È$ò 
ó ó ð
ð@ 8¨E°%¸¸cÀ3¸h¹Ð2GÑ,HÑ#Ið ÈhÐW[Ð\_ÐadÐ\dÑWeÑNfó ó,ó

ð"CàðCð (,ðCð %*¨#¨s¨(¡Oð	Cð
  $ðCð  ðCð )-ðCð *.ðCð  ðCð !ðCð ðCð  #ðCð !ðCð  ðCð ðCð  #&ð!Cð" !$ð#Cð$ ð%Cð& ð'Cð( ð)Cð* ð+Cð, ð-Cð. ð/Cð0 ð1Cð2 
ó3CóJfð$
¨÷ 
r9   r;   )SrÕ   rj  r~   rµ   r¢   Úcollectionsr   Úcollections.abcr   r   Ú
contextlibr   Údatetimer   Úpathlibr   Útypingr	   r
   r   r   rT  Útorch.nnr   Útorch.optimr   Útorch.optim.lr_schedulerr   r   Útyping_extensionsr   Úpytorch_lightningrÞ   Úlightning_fabric.pluginsr   Ú5lightning_fabric.plugins.collectives.torch_collectiver   Úlightning_fabric.strategiesr   Ú%lightning_fabric.strategies.deepspeedr   r   r   r   r   Ú"lightning_fabric.utilities.importsr   Ú$lightning_fabric.utilities.optimizerr   Úlightning_fabric.utilities.seedr   Ú lightning_fabric.utilities.typesr   Ú#pytorch_lightning.accelerators.cudar   Ú pytorch_lightning.core.optimizerr   Ú#pytorch_lightning.plugins.precisionr    Ú pytorch_lightning.strategies.ddpr!   r2  r#   Úpytorch_lightning.utilitiesr$   Ú&pytorch_lightning.utilities.exceptionsr%   Ú)pytorch_lightning.utilities.model_helpersr&   Ú%pytorch_lightning.utilities.rank_zeror'   r(   r)   Ú!pytorch_lightning.utilities.typesr*   Ú	getLoggerr‡   r¥   r$  r<   rU  r8   r;   r»   r9   r7   Ú<module>r²     sÈ   ðó Û Û Û 	Û Ý #ß .Ý %Ý Ý ß 6Ó 6ã Ý Ý !ß CÝ &ã Ý 7Ý TÝ 9÷õ õ HÝ FÝ 6Ý 2Ý ?Ý OÝ 9Ý 8Ý 6Ý =Ý LÝ Cß ^Ñ ^Ý ?à€g×Ñ˜Ó!€Ù“€áÛð:˜uŸx™xŸ™ð :°4ó :ôe˜õ er9   