Ë
      çi4  ã                   óÂ   — d dl mZmZmZ d dlZd dlmZ d dlmZmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlmZ d dlmZ  G d„ de«      Zy)é    )ÚAnyÚOptionalÚUnionN)ÚTensor)ÚDataParallelÚModule)Úoverride)ÚAccelerator)ÚCheckpointIO)Ú	Precision)ÚParallelStrategy)Ú_StrategyRegistry)Ú
TBroadcastÚTReduce)Úapply_to_collection)ÚReduceOpc                   ó\  ‡ — e Zd ZdZ	 	 	 	 d$dee   deeej                        dee	   dee
   fˆ fd„Zeedej                  fd	„«       «       Zeed%d
„«       «       Zededefd„«       Zededdfd„«       Zed&dedeej                     defd„«       Ze	 d'dedee   deeeef      defd„«       Zedededdfd„«       Zed(dededefd„«       Zed)dededefd„«       Z edede!eeee"f   f   fˆ fd„«       Z#e	 d)dede!eeee"f   f   d eddfˆ fd!„«       Z$e%ed"e&ddfd#„«       «       Z'ˆ xZ(S )*ÚDataParallelStrategyzŒImplements data-parallel training in a single process, i.e., the model gets replicated to each device and each
    gets a split of the data.NÚacceleratorÚparallel_devicesÚcheckpoint_ioÚ	precisionc                 ó.   •— t         ‰| �  ||d ||¬«       y )N)r   r   Úcluster_environmentr   r   )ÚsuperÚ__init__)Úselfr   r   r   r   Ú	__class__s        €ús/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning_fabric/strategies/dp.pyr   zDataParallelStrategy.__init__#   s&   ø€ ô 	‰ÑØ#Ø-Ø $Ø'Øð 	õ 	
ó    Úreturnc                 ó<   — | j                   €J ‚| j                   d   S )Nr   )r   ©r   s    r   Úroot_devicez DataParallelStrategy.root_device2   s'   € ð ×$Ñ$Ð0Ð0Ð0Ø×$Ñ$ QÑ'Ð'r    c                  ó   — y ©N© r#   s    r   Údistributed_sampler_kwargsz/DataParallelStrategy.distributed_sampler_kwargs8   s   € ð r    Úmodulec                 ó0   — t        || j                  ¬«      S )zDWraps the given model into a :class:`~torch.nn.DataParallel` module.)r)   Ú
device_ids)r   r   ©r   r)   s     r   Úsetup_modulez!DataParallelStrategy.setup_module=   s   € ô  6°d×6KÑ6KÔLÐLr    c                 ó:   — |j                  | j                  «       y r&   )Útor$   r,   s     r   Úmodule_to_devicez%DataParallelStrategy.module_to_deviceB   s   € à�	‰	�$×"Ñ"Õ#r    ÚbatchÚdevicec                 ó   — |S r&   r'   )r   r1   r2   s      r   Úbatch_to_devicez$DataParallelStrategy.batch_to_deviceF   s	   € ð ˆr    Ú
collectionÚgroupÚ	reduce_opc                 óD   — dt         dt         fd„}t        |t         |«      S )NÚtr!   c                 ót   — | j                   }| j                  «       j                  «       j                  |«      S r&   )ÚdtypeÚfloatÚmeanr/   )r9   Úoriginal_dtypes     r   r=   z-DataParallelStrategy.all_reduce.<locals>.meanO   s)   € ØŸW™WˆNØ—7‘7“9—>‘>Ó#×&Ñ& ~Ó6Ð6r    )r   r   )r   r5   r6   r7   r=   s        r   Ú
all_reducezDataParallelStrategy.all_reduceK   s&   € ð	7”Fð 	7œvó 	7ô # :¬v°tÓ<Ð<r    ÚargsÚkwargsc                  ó   — y r&   r'   )r   r@   rA   s      r   ÚbarrierzDataParallelStrategy.barrierU   s   € àr    ÚobjÚsrcc                 ó   — |S r&   r'   )r   rD   rE   s      r   Ú	broadcastzDataParallelStrategy.broadcastY   s   € àˆ
r    ÚdecisionÚallc                 ó   — |S r&   r'   )r   rH   rI   s      r   Úreduce_boolean_decisionz,DataParallelStrategy.reduce_boolean_decision]   s   € àˆr    c                 óZ   •— t        |t        «      r|j                  }t        ‰| �  |«      S r&   )Ú
isinstancer   r)   r   Úget_module_state_dict)r   r)   r   s     €r   rN   z*DataParallelStrategy.get_module_state_dicta   s&   ø€ ä�fœlÔ+Ø—]‘]ˆFÜ‰wÑ,¨VÓ4Ð4r    Ú
state_dictÚstrictc                 ób   •— t        |t        «      r|j                  }t        ‰| �  |||¬«       y )N)r)   rO   rP   )rM   r   r)   r   Úload_module_state_dict)r   r)   rO   rP   r   s       €r   rR   z+DataParallelStrategy.load_module_state_dictg   s-   ø€ ô �fœlÔ+Ø—]‘]ˆFÜ‰Ñ&¨fÀÐTZÐ&Õ[r    Ústrategy_registryc                 ó@   — |j                  d| | j                  ¬«       y )NÚdp)Údescription)ÚregisterÚ__name__)ÚclsrS   s     r   Úregister_strategiesz(DataParallelStrategy.register_strategieso   s   € ð 	×"Ñ" 4¨¸#¿,¹,Ð"ÕGr    )NNNN)r!   Nr&   )Nr=   )r   )T))rX   Ú
__module__Ú__qualname__Ú__doc__r   r
   ÚlistÚtorchr2   r   r   r   Úpropertyr	   r$   r(   r   r   r-   r0   r   r4   r   r   r   Ústrr?   rC   r   ÚintrG   ÚboolrK   Údictr   rN   rR   Úclassmethodr   rZ   Ú__classcell__)r   s   @r   r   r      s˜  ø„ ñ!ð
 .2Ø9=Ø04Ø)-ñ
à˜kÑ*ð
ð # 4¨¯©Ñ#5Ñ6ð
ð   Ñ-ð	
ð
 ˜IÑ&õ
ð Øð(˜UŸ\™\ò (ó ó ð(ð Øòó ó ðð ðM 6ð M¨lò Mó ðMð ð$ vð $°$ò $ó ð$ð ñ Sð °(¸5¿<¹<Ñ2Hð ÐTWò ó ðð àlrñ=Ø!ð=Ø*2°3©-ð=ØKSÐTYÐZbÐdgÐZgÑThÑKið=à	ò=ó ð=ð ð˜Sð ¨Cð °Dò ó ðð ñ˜Zð ¨cð ¸*ò ó ðð ñ°ð ¸4ð È4ò ó ðð ð5¨Fð 5°t¸CÀÀsÈFÀ{ÑASÐ<SÑ7Tô 5ó ð5ð
 àX\ñ\Øð\Ø*.¨s°E¸#¸v¸+Ñ4FÐ/FÑ*Gð\ØQUð\à	ô\ó ð\ð ØðHÐ4Eð HÈ$ò Hó ó ôHr    r   )Útypingr   r   r   r_   r   Útorch.nnr   r   Útyping_extensionsr	   Úlightning_fabric.acceleratorsr
   Ú)lightning_fabric.plugins.io.checkpoint_ior   Ú"lightning_fabric.plugins.precisionr   Ú$lightning_fabric.strategies.parallelr   Ú$lightning_fabric.strategies.registryr   Ú$lightning_fabric.strategies.strategyr   r   Ú%lightning_fabric.utilities.apply_funcr   Ú&lightning_fabric.utilities.distributedr   r   r'   r    r   Ú<module>rr      sB   ð÷ (Ñ 'ã Ý ß )Ý &å 5Ý BÝ 8Ý AÝ Bß DÝ EÝ ;ôSHÐ+õ SHr    