Ë
      çi´3  ã                   óB  — d dl Z d dlZd dlmZmZmZmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlZd dlmZmZ d dlmZmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z/ erd dl0m1Z1  G d„ de$«      Z2y)é    N)ÚTYPE_CHECKINGÚAnyÚOptionalÚUnion)ÚTensor)ÚModule)Úoverride)Ú_XLA_AVAILABLEÚ_XLA_GREATER_EQUAL_2_1)ÚCheckpointIOÚ	PrecisionÚXLACheckpointIO)ÚXLAEnvironment)Ú_StrategyRegistry)Ú_optimizers_to_device)Ú_PATHÚReduceOp)ÚXLAPrecision)Ú_WrappingCheckpointIO)ÚDDPStrategy)Ú_XLALauncher)Ú
TBroadcast)Ú	TrainerFn)Úfind_shared_parametersÚset_shared_parameters)Úrank_zero_only©ÚMpDeviceLoaderc                   óF  ‡ — e Zd ZdZdZ	 	 	 	 	 	 d;ded   deeej                        dee	e
ef      dee   d	ed
ededdfˆ fd„Zeede	e
ef   fd„«       «       Zej&                  edee   ddfd„«       «       Zeedefd„«       «       Zej&                  edee   ddfd„«       «       Zeedej                  fd„«       «       Zeedefˆ fd„«       «       Zeedefˆ fd„«       «       Zeedefˆ fd„«       «       Zeedefˆ fd„«       «       Zed<d„«       Zed=d„«       Zededefd„«       Z eede!e"ef   fd„«       «       Z#ede$ddfd„«       Z%ed<d „«       Z&ed<d!„«       Z'ed>d"ee"   d#ed$eddfd%„«       Z(ed?d&e)d'ede)fd(„«       Z*e	 	 d@d)e	e+ef   d*ee   d+ee	e,e"f      de+fd,„«       Z-ed<ˆ fd-„«       Z.ed<d.„«       Z/ed<d/„«       Z0e	 d>d0e!e"ef   d1e1d2ee   ddfˆ fd3„«       Z2ed1e1ddfd4„«       Z3edAd5e+d*ee   d6ede+fd7„«       Z4ed<ˆ fd8„«       Z5e6ed9e7ddfd:„«       «       Z8ˆ xZ9S )BÚXLAStrategyzxStrategy for training multiple TPU devices using the :func:`torch_xla.distributed.xla_multiprocessing.spawn`
    method.ÚxlaNÚacceleratorzpl.accelerators.AcceleratorÚparallel_devicesÚcheckpoint_ioÚprecision_pluginÚdebugÚsync_module_statesÚ_Úreturnc                 ó¦   •— t         st        t        t         «      «      ‚t        ‰| �  ||t        «       ||d¬«       || _        d| _        || _        y )NÚfork)r"   r#   Úcluster_environmentr$   r%   Ústart_methodF)	r
   ÚModuleNotFoundErrorÚstrÚsuperÚ__init__r   r&   Ú	_launchedÚ_sync_module_states)	Úselfr"   r#   r$   r%   r&   r'   r(   Ú	__class__s	           €úu/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/pytorch_lightning/strategies/xla.pyr1   zXLAStrategy.__init__1   sW   ø€ õ Ü%¤c¬.Ó&9Ó:Ð:Ü‰ÑØ#Ø-Ü .Ó 0Ø'Ø-Øð 	ô 	
ð ˆŒ
ØˆŒØ#5ˆÕ ó    c                 óf   — | j                   }|�t        |t        t        f«      sJ ‚|S t        «       S ©N)Ú_checkpoint_ioÚ
isinstancer   r   ©r4   Úplugins     r6   r$   zXLAStrategy.checkpoint_ioI   s9   € ð ×$Ñ$ˆØÐÜ˜f¤Ô8MÐ&NÔOÐOÐOØˆMÜÓ Ð r7   Úioc                 ó^   — |�$t        |t        t        f«      st        d|› �«      ‚|| _        y )NzHThe XLA strategy can only work with the `XLACheckpointIO` plugin, found )r;   r   r   Ú	TypeErrorr:   )r4   r>   s     r6   r$   zXLAStrategy.checkpoint_ioR   s5   € ð ˆ>¤*¨R´/ÔCXÐ1YÔ"ZÜÐfÐgiÐfjÐkÓlÐlØ ˆÕr7   c                 óZ   — | j                   }|�t        |t        «      sJ ‚|S t        «       S r9   )Ú_precision_pluginr;   r   r<   s     r6   r%   zXLAStrategy.precision_pluginY   s2   € ð ×'Ñ'ˆØÐÜ˜f¤lÔ3Ð3Ð3ØˆMÜ‹~Ðr7   c                 óR   — |�t        |t        «      st        d|› �«      ‚|| _        y )NzEThe XLA strategy can only work with the `XLAPrecision` plugin, found )r;   r   r@   rB   )r4   r%   s     r6   r%   zXLAStrategy.precision_pluginb   s1   € ð Ð'´
Ð;KÌ\Ô0ZÜÐcÐdtÐcuÐvÓwÐwØ!1ˆÕr7   c                 ób   — | j                   st        d«      ‚dd lmc m} |j                  «       S )NzFAccessing the XLA device before processes have spawned is not allowed.r   )r2   ÚRuntimeErrorÚtorch_xla.core.xla_modelÚcoreÚ	xla_modelÚ
xla_device)r4   Úxms     r6   Úroot_devicezXLAStrategy.root_devicei   s)   € ð �~Š~ÜÐgÓhÐhß-Ð-à�}‰}‹Ðr7   c                 ó4   •— | j                   r
t        ‰| �  S dS ©Nr   )r2   r0   Úglobal_rank©r4   r5   s    €r6   rN   zXLAStrategy.global_rankr   s   ø€ ð '+§n¢nŒu‰wÑ"Ð;¸!Ð;r7   c                 ó4   •— | j                   r
t        ‰| �  S dS rM   )r2   r0   Ú
local_rankrO   s    €r6   rQ   zXLAStrategy.local_rankw   ó   ø€ ð &*§^¢^Œu‰wÑ!Ð:¸Ð:r7   c                 ó4   •— | j                   r
t        ‰| �  S dS rM   )r2   r0   Ú	node_rankrO   s    €r6   rT   zXLAStrategy.node_rank|   s   ø€ ð %)§N¢NŒu‰wÑ Ð9¸Ð9r7   c                 ó4   •— | j                   r
t        ‰| �  S dS )Né   )r2   r0   Ú
world_sizerO   s    €r6   rW   zXLAStrategy.world_size�   rR   r7   c                 ó$   — t        | «      | _        y r9   )r   Ú	_launcher©r4   s    r6   Ú_configure_launcherzXLAStrategy._configure_launcher†   s   € ä% dÓ+ˆ�r7   c                 ó2  — | j                   €J ‚| j                   j                  |«       | j                  rdt        j                  d<   | j
                  €J ‚| j                  j                  | j
                  «       t        | j
                  «      }| j                  «        t        | j
                  |«       | j                  | j
                  «      | _        | j                  r%t        rddlm} nddlm}  || j
                  «       |j"                  j$                  t&        j(                  k(  r| j+                  |«       | j-                  «        |j"                  j$                  t&        j(                  k(  r!t/        | j0                  | j2                  «       y y )NÚ1ÚPT_XLA_DEBUGr   )Úbroadcast_master_param)r"   Úsetupr&   ÚosÚenvironÚmodelr%   Úconvert_moduler   Úmodel_to_devicer   Ú_setup_modelr3   r   rF   r_   Útorch_xla.experimental.pjrtÚstateÚfnr   ÚFITTINGÚsetup_optimizersÚsetup_precision_pluginr   Ú
optimizersrK   )r4   ÚtrainerÚshared_paramsr_   s       r6   r`   zXLAStrategy.setupŠ   s$  € à×ÑÐ+Ð+Ð+Ø×Ñ×Ñ˜wÔ'à�:Š:Ø),ŒB�J‰J�~Ñ&à�z‰zÐ%Ð%Ð%Ø×Ñ×,Ñ,¨T¯Z©ZÔ8ä.¨t¯z©zÓ:ˆØ×ÑÔÜ˜dŸj™j¨-Ô8à×&Ñ& t§z¡zÓ2ˆŒ
à×#Ò#Ý%ÞKåNá" 4§:¡:Ô.à�=‰=×Ñœy×0Ñ0Ò0Ø×!Ñ! 'Ô*Ø×#Ñ#Ô%Ø�=‰=×Ñœy×0Ñ0Ò0Ü! $§/¡/°4×3CÑ3CÕDð 1r7   rc   c                 ó   — |S r9   © )r4   rc   s     r6   rf   zXLAStrategy._setup_model©   s   € àˆr7   c                 ó4   — | j                   | j                  dœS )N)Únum_replicasÚrank)rW   rN   rZ   s    r6   Údistributed_sampler_kwargsz&XLAStrategy.distributed_sampler_kwargs­   s   € ð !%§¡¸×9IÑ9IÑJÐJr7   Ú
dataloaderr   c                 óÂ   — ddl m} t        ||«      r|S  ||| j                  «      }|j                  j
                  |_        t        |j                  dd «      |_        |S )Nr   r   Úbatch_sampler)Ú%torch_xla.distributed.parallel_loaderr   r;   rK   Ú_loaderÚdatasetÚgetattrrx   )r4   rv   r   s      r6   Úprocess_dataloaderzXLAStrategy.process_dataloader²   sY   € åHä�j .Ô1àÐá# J°×0@Ñ0@ÓAˆ
à'×/Ñ/×7Ñ7ˆ
ÔÜ#*¨:×+=Ñ+=¸ÐPTÓ#Uˆ
Ô ØÐr7   c                  ó   — y r9   rq   rZ   s    r6   Úconfigure_ddpzXLAStrategy.configure_ddpÀ   s   € àr7   c                 ót   — | j                   €J ‚| j                   j                  | j                  «      | _         y r9   )rc   ÚtorK   rZ   s    r6   re   zXLAStrategy.model_to_deviceÄ   s,   € à�z‰zÐ%Ð%Ð%Ø—Z‘Z—]‘] 4×#3Ñ#3Ó4ˆ�
r7   ÚnameÚargsÚkwargsc                 óZ   — | j                   sy dd lmc m} |€d}|j	                  |«       y )Nr   Ú )r2   rF   rG   rH   Ú
rendezvous)r4   r‚   rƒ   r„   rJ   s        r6   ÚbarrierzXLAStrategy.barrierÉ   s(   € à�~Š~Øç-Ð-àˆ<àˆDØ
�‰�dÕr7   ÚobjÚsrcc                 óÚ  — | j                   s|S dd lmc m} t	        |t
        «      }|rL|j                  «       dk(  r|j                  d«      }|j                  }|j                  | j                  «      }nqt        j                  «       }t        j                  ||«       t        j                  t!        |j#                  «       «      | j                  t        j$                  ¬«      }|g}|j'                  ||¬«       |d   }|sVt        j                  |j)                  «       j+                  «       j-                  «       «      }t        j.                  |«      }|S |j                  «      }|S )Nr   )ÚdeviceÚdtype)Úroot_ordinal)r2   rF   rG   rH   r;   r   ÚdimÚ	unsqueezerŒ   r�   rK   r>   ÚBytesIOÚtorchÚsaveÚtensorÚ	bytearrayÚ	getbufferÚfloatÚcollective_broadcastÚcpuÚbyteÚnumpyÚload)r4   r‰   rŠ   rJ   Ú	is_tensorÚoriginal_deviceÚbuffers          r6   Ú	broadcastzXLAStrategy.broadcastÕ   s  € à�~Š~ØˆJç-Ð-ä˜s¤FÓ+ˆ	ÙØ�w‰w‹y˜AŠ~Ø—m‘m AÓ&�Ø!Ÿj™jˆOà—&‘&˜×)Ñ)Ó*‰Cô —Z‘Z“\ˆFÜ�J‰J�s˜FÔ#Ü—,‘,Ü˜&×*Ñ*Ó,Ó-°d×6FÑ6FÌeÏkÉkôˆCð ˆeˆØ
×Ñ °#ÐÔ6Ø�!‰fˆáä—Z‘Z §¡£	§¡Ó 0× 6Ñ 6Ó 8Ó9ˆFÜ—*‘*˜VÓ$ˆCð ˆ
ð —&‘&˜Ó)ˆCàˆ
r7   ÚoutputÚgroupÚ	reduce_opc                 óÀ  — t        |t        «      s!t        j                  || j                  ¬«      }t        |t
        «      xr |t
        j                  k7  }t        |t        «      xr |j                  «       dv}|s|rt        d|› �«      ‚dd l
mc m} |j                  d|t        «      }t        |t        «      r!|j                  «       dv r|| j                  z  }|S )N)rŒ   )ÚsumÚmeanÚavgz]Currently, the XLAStrategy only supports `sum`, `mean`, `avg` for the reduce operation, got: r   Úreduce)r§   r¦   )r;   r   r’   r”   rK   r   ÚSUMr/   ÚlowerÚ
ValueErrorrF   rG   rH   Úmesh_reducer¥   rW   )r4   r¡   r¢   r£   Úinvalid_reduce_opÚinvalid_reduce_op_strrJ   s          r6   r¨   zXLAStrategy.reduceø   sÅ   € ô ˜&¤&Ô)Ü—\‘\ &°×1AÑ1AÔBˆFä& y´(Ó;ÒYÀ	ÌXÏ\É\Ñ@YÐÜ *¨9´cÓ :Ò n¸y¿¹Ó?PÐXnÐ?nÐÙÑ 5ÜðØ�;ð óð ÷
 	.Ð-à—‘ ¨&´#Ó6ˆä�i¤Ô%¨)¯/©/Ó*;¸Ñ*NØ˜dŸo™oÑ-ˆFàˆr7   c                 ó0   •— d| _         t        ‰| �	  «        y )NT)r2   r0   Úsetup_environmentrO   s    €r6   r°   zXLAStrategy.setup_environment  s   ø€ àˆŒÜ‰Ñ!Õ#r7   c                 ó�   — | j                   €J ‚t        | j                   «      dk(  rt        d«      ‚| j                  t        _        y )NrV   z”The `XLAStrategy` does not support running on a single device with the PjRT runtime. Try using all devices or the `SingleDeviceXLAStrategy` strategy)r#   ÚlenÚNotImplementedErrorrN   r   rt   rZ   s    r6   Úsetup_distributedzXLAStrategy.setup_distributed  sL   € à×$Ñ$Ð0Ð0Ð0Üˆt×$Ñ$Ó%¨Ò*ô &ðSóð ð #×.Ñ.ŒÕr7   c                  ó   — y r9   rq   rZ   s    r6   Úset_world_rankszXLAStrategy.set_world_ranks$  s   € ð
 	r7   Ú
checkpointÚfilepathÚstorage_optionsc                 ó\   •— dd l mc m} |j                  «        t        ‰| �  |||¬«       y )Nr   )r¹   )rF   rG   rH   Ú	mark_stepr0   Úsave_checkpoint)r4   r·   r¸   r¹   rJ   r5   s        €r6   r¼   zXLAStrategy.save_checkpoint+  s*   ø€ ÷ 	.Ð-ð 	�‰Œä‰Ñ 
¨HÀoÐÕVr7   c                 óZ   — | j                   dk(  r| j                  j                  |«       yy)zqRemove checkpoint filepath from the filesystem.

        Args:
            filepath: Path to checkpoint

        r   N)rQ   r$   Úremove_checkpoint)r4   r¸   s     r6   r¾   zXLAStrategy.remove_checkpoint6  s)   € ð �?‰?˜aÒØ×Ñ×0Ñ0°Õ:ð  r7   r”   Ú
sync_gradsc                 ó®  — | j                   s|S t        |t        «      s$t        dt	        | «      j
                  › d|› �«      ‚|j                  «       dk(  r|j                  d«      }|j                  }|j                  | j                  «      }ddlmc m} ddlmc m} |r|j!                  |«      n|j!                  |«      }|j                  |«      }|S )aC  Function to gather a tensor from several distributed processes.

        Args:
            tensor: tensor to all-gather.
            group: unused.
            sync_grads: flag that allows users to synchronize gradients for the all-gather operation.
        Return:
            A tensor of shape (world_size, ...)

        Ú`z4.all_gather` is only implemented for tensors. Given r   N)r2   r;   r   r³   ÚtypeÚ__name__r�   r�   rŒ   r�   rK   Útorch_xla.core.functionsrG   Ú	functionsrF   rH   Ú
all_gather)r4   r”   r¢   r¿   rž   ÚxfrJ   s          r6   rÆ   zXLAStrategy.all_gatherA  s¼   € ð �~Š~ØˆMÜ˜&¤&Ô)Ü%Ø”D˜“J×'Ñ'Ð(Ð(\Ð]cÐ\dÐeóð ð �:‰:‹<˜1ÒØ×%Ñ% aÓ(ˆFØ Ÿ-™-ˆØ—‘˜4×+Ñ+Ó,ˆç-Ð-ß-Ð-á*4�—‘˜vÔ&¸"¿-¹-ÈÓ:OˆØ—‘˜?Ó+ˆØˆr7   c                 óp   •— t         ‰| �  «        d| _        t        j                  j                  dd «       y )NFr^   )r0   Úteardownr2   ra   rb   ÚpoprO   s    €r6   rÉ   zXLAStrategy.teardown_  s'   ø€ ä‰ÑÔØˆŒÜ
�
‰
�‰�~ tÕ,r7   Ústrategy_registryc                 ó~   — |j                  d| dd¬«       |j                  | j                  | | j                  ¬«       y )NÚ	xla_debugz!XLA strategy with `debug` as TrueT)Údescriptionr&   )rÎ   )ÚregisterÚstrategy_namerÃ   )ÚclsrË   s     r6   Úregister_strategieszXLAStrategy.register_strategiese  sC   € ð 	×"Ñ" ;°ÐAdÐlpÐ"ÔqØ×"Ñ"Ø×ÑØØŸ™ð 	#õ 	
r7   )NNNNFT)r)   N)rn   z
pl.Trainerr)   Nr9   )r   )Nr¦   )NF):rÃ   Ú
__module__Ú__qualname__Ú__doc__rÐ   r   Úlistr’   rŒ   r   r   r   r   Úboolr   r1   Úpropertyr	   r$   Úsetterr   r%   r   rK   ÚintrN   rQ   rT   rW   r[   r`   r   rf   Údictr/   ru   Úobjectr}   r   re   rˆ   r   r    r   r   r¨   r°   r´   r¶   r   r¼   r¾   rÆ   rÉ   Úclassmethodr   rÒ   Ú__classcell__)r5   s   @r6   r    r    +   s­  ø„ ñð €Mð @DØ9=ØQUØ37ØØ#'ñ6àÐ;Ñ<ð6ð # 4¨¯©Ñ#5Ñ6ð6ð    oÐ7LÐ&LÑ MÑNð	6ð
 # <Ñ0ð6ð ð6ð !ð6ð ð6ð 
õ6ð0 Øð!˜u _Ð6KÐ%KÑLò !ó ó ð!ð ×ÑØð! ¨Ñ 6ð !¸4ò !ó ó ð!ð
 Øð ,ò ó ó ðð ×ÑØð2°¸)Ñ1Dð 2Èò 2ó ó ð2ð
 Øð˜UŸ\™\ò ó ó ðð Øð<˜Sô <ó ó ð<ð Øð;˜Cô ;ó ó ð;ð Øð:˜3ô :ó ó ð:ð Øð;˜Cô ;ó ó ð;ð ò,ó ð,ð òEó ðEð< ð &ð ¨Vò ó ðð ØðK¨D°°c°©Nò Kó ó ðKð ð¨Vð Ð8Hò ó ðð òó ðð ò5ó ð5ð ñ	˜H S™Mð 	¸ð 	Èð 	ÐPTò 	ó ð	ð ñ ˜Zð  ¨cð  ¸*ò  ó ð ðD ð  $Ø4:ñ	à�f˜c�kÑ"ðð ˜‰}ðð ˜E (¨C -Ñ0Ñ1ð	ð
 
òó ðð4 ô$ó ð$ð ò	/ó ð	/ð òó ðð à\`ñWØ˜s C˜x™.ðWØ49ðWØLTÐUXÉMðWà	ôWó ðWð ð;¨%ð ;°Dò ;ó ð;ð ñ ð °¸±ð ÐRVð Ðciò ó ðð: ô-ó ð-ð
 Øð
Ð4Eð 
È$ò 
ó ó ô
r7   r    )3r>   ra   Útypingr   r   r   r   r’   r   Útorch.nnr   Útyping_extensionsr	   Úpytorch_lightningÚplÚ!lightning_fabric.accelerators.xlar
   r   Úlightning_fabric.pluginsr   r   r   Ú%lightning_fabric.plugins.environmentsr   Úlightning_fabric.strategiesr   Ú$lightning_fabric.utilities.optimizerr   Ú lightning_fabric.utilities.typesr   r   Úpytorch_lightning.pluginsr   Ú$pytorch_lightning.plugins.io.wrapperr   Ú pytorch_lightning.strategies.ddpr   Ú*pytorch_lightning.strategies.launchers.xlar   Ú%pytorch_lightning.strategies.strategyr   Ú pytorch_lightning.trainer.statesr   Úpytorch_lightning.utilitiesr   r   Ú%pytorch_lightning.utilities.rank_zeror   ry   r   r    rq   r7   r6   Ú<module>rò      sc   ðó 
Û 	ß 6Ó 6ã Ý Ý Ý &ã ß Tß MÑ MÝ @Ý 9Ý Fß <Ý 2Ý FÝ 8Ý CÝ <Ý 6ß UÝ @áÝDôB
�+õ B
r7   