
      i3                     F   d dl Z d dlZd dlmZmZmZmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z-m.Z. d dl/m0Z0 erd dl1m2Z2  G d de%      Z3y)    N)TYPE_CHECKINGAnyOptionalUnion)Tensor)Module)override)_XLA_AVAILABLE_XLA_GREATER_EQUAL_2_1)CheckpointIO	PrecisionXLACheckpointIO)XLAEnvironment)_StrategyRegistry)_optimizers_to_device)_PATHReduceOp)XLAPrecision)_WrappingCheckpointIO)DDPStrategy)_XLALauncher)
TBroadcast)	TrainerFn)find_shared_parametersset_shared_parameters)rank_zero_onlyMpDeviceLoaderc                   F    e Zd ZdZdZ	 	 	 	 	 	 d;ded   deeej                        dee	e
ef      dee   d	ed
ededdf fdZeede	e
ef   fd              Zej&                  edee   ddfd              Zeedefd              Zej&                  edee   ddfd              Zeedej                  fd              Zeedef fd              Zeedef fd              Zeedef fd              Zeedef fd              Zed<d       Zed=d       Zededefd       Z eede!e"ef   fd              Z#ede$ddfd       Z%ed<d        Z&ed<d!       Z'ed>d"ee"   d#ed$eddfd%       Z(ed?d&e)d'ede)fd(       Z*e	 	 d@d)e	e+ef   d*ee   d+ee	e,e"f      de+fd,       Z-ed< fd-       Z.ed<d.       Z/ed<d/       Z0e	 d>d0e!e"ef   d1e1d2ee   ddf fd3       Z2ed1e1ddfd4       Z3edAd5e+d*ee   d6ede+fd7       Z4ed< fd8       Z5e6ed9e7ddfd:              Z8 xZ9S )BXLAStrategyzxStrategy for training multiple TPU devices using the :func:`torch_xla.distributed.xla_multiprocessing.spawn`
    method.xlaNacceleratorzpl.accelerators.Acceleratorparallel_devicescheckpoint_ioprecision_plugindebugsync_module_states_returnc                     t         st        t        t                     t        |   ||t               ||d       || _        d| _        || _        y )Nfork)r"   r#   cluster_environmentr$   r%   start_methodF)	r
   ModuleNotFoundErrorstrsuper__init__r   r&   	_launched_sync_module_states)	selfr"   r#   r$   r%   r&   r'   r(   	__class__s	           u/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning/pytorch/strategies/xla.pyr1   zXLAStrategy.__init__1   sW     %c.&9::#- . 0'- 	 	
 
#5     c                 f    | j                   }|t        |t        t        f      sJ |S t               S N)_checkpoint_io
isinstancer   r   r4   plugins     r6   r$   zXLAStrategy.checkpoint_ioI   s9     $$f8M&NOOOM  r7   ioc                 ^    |$t        |t        t        f      st        d|       || _        y )NzHThe XLA strategy can only work with the `XLACheckpointIO` plugin, found )r;   r   r   	TypeErrorr:   )r4   r>   s     r6   r$   zXLAStrategy.checkpoint_ioR   s5     >*R/CX1Y"Zfgifjkll r7   c                 Z    | j                   }|t        |t              sJ |S t               S r9   )_precision_pluginr;   r   r<   s     r6   r%   zXLAStrategy.precision_pluginY   s2     ''fl333M~r7   c                 R    |t        |t              st        d|       || _        y )NzEThe XLA strategy can only work with the `XLAPrecision` plugin, found )r;   r   r@   rB   )r4   r%   s     r6   r%   zXLAStrategy.precision_pluginb   s1     '
;K\0Zcdtcuvww!1r7   c                 b    | j                   st        d      dd lmc m} |j                         S )NzFAccessing the XLA device before processes have spawned is not allowed.r   )r2   RuntimeErrortorch_xla.core.xla_modelcore	xla_model
xla_device)r4   xms     r6   root_devicezXLAStrategy.root_devicei   s)     ~~ghh--}}r7   c                 4    | j                   r
t        |   S dS Nr   )r2   r0   global_rankr4   r5   s    r6   rN   zXLAStrategy.global_rankr   s     '+nnuw";!;r7   c                 4    | j                   r
t        |   S dS rM   )r2   r0   
local_rankrO   s    r6   rQ   zXLAStrategy.local_rankw        &*^^uw!::r7   c                 4    | j                   r
t        |   S dS rM   )r2   r0   	node_rankrO   s    r6   rT   zXLAStrategy.node_rank|   s     %)NNuw 99r7   c                 4    | j                   r
t        |   S dS )N   )r2   r0   
world_sizerO   s    r6   rW   zXLAStrategy.world_size   rR   r7   c                 $    t        |       | _        y r9   )r   	_launcherr4   s    r6   _configure_launcherzXLAStrategy._configure_launcher   s    %d+r7   c                 2   | j                   J | j                   j                  |       | j                  rdt        j                  d<   | j
                  J | j                  j                  | j
                         t        | j
                        }| j                          t        | j
                  |       | j                  | j
                        | _        | j                  r%t        rddlm} nddlm}  || j
                         |j"                  j$                  t&        j(                  k(  r| j+                  |       | j-                          |j"                  j$                  t&        j(                  k(  r!t/        | j0                  | j2                         y y )N1PT_XLA_DEBUGr   )broadcast_master_param)r"   setupr&   osenvironmodelr%   convert_moduler   model_to_devicer   _setup_modelr3   r   rF   r_   torch_xla.experimental.pjrtstatefnr   FITTINGsetup_optimizerssetup_precision_pluginr   
optimizersrK   )r4   trainershared_paramsr_   s       r6   r`   zXLAStrategy.setup   s$   +++w'::),BJJ~&zz%%%,,TZZ8.tzz:djj-8&&tzz2
##%KN"4::.==y000!!'*##%==y000!$//43C3CD 1r7   rc   c                     |S r9    )r4   rc   s     r6   rf   zXLAStrategy._setup_model   s    r7   c                 4    | j                   | j                  dS )N)num_replicasrank)rW   rN   rZ   s    r6   distributed_sampler_kwargsz&XLAStrategy.distributed_sampler_kwargs   s     !%9I9IJJr7   
dataloaderr   c                     ddl m} t        ||      r|S  ||| j                        }|j                  j
                  |_        t        |j                  dd       |_        |S )Nr   r   batch_sampler)%torch_xla.distributed.parallel_loaderr   r;   rK   _loaderdatasetgetattrrx   )r4   rv   r   s      r6   process_dataloaderzXLAStrategy.process_dataloader   sY    Hj.1#J0@0@A
'//77
#*:+=+=PT#U
 r7   c                      y r9   rq   rZ   s    r6   configure_ddpzXLAStrategy.configure_ddp   s    r7   c                 t    | j                   J | j                   j                  | j                        | _         y r9   )rc   torK   rZ   s    r6   re   zXLAStrategy.model_to_device   s,    zz%%%ZZ]]4#3#34
r7   nameargskwargsc                 Z    | j                   sy dd lmc m} |d}|j	                  |       y )Nr    )r2   rF   rG   rH   
rendezvous)r4   r   r   r   rJ   s        r6   barrierzXLAStrategy.barrier   s(    ~~--<D
dr7   objsrcc                    | j                   s|S dd lmc m} t	        |t
              }|rL|j                         dk(  r|j                  d      }|j                  }|j                  | j                        }nqt        j                         }t        j                  ||       t        j                  t!        |j#                               | j                  t        j$                        }|g}|j'                  ||       |d   }|sVt        j                  |j)                         j+                         j-                               }t        j.                  |      }|S |j                        }|S )Nr   )devicedtype)root_ordinal)r2   rF   rG   rH   r;   r   dim	unsqueezer   r   rK   r>   BytesIOtorchsavetensor	bytearray	getbufferfloatcollective_broadcastcpubytenumpyload)r4   r   r   rJ   	is_tensororiginal_devicebuffers          r6   	broadcastzXLAStrategy.broadcast   s   ~~J--sF+	wwyA~mmA&!jjO&&))*C ZZ\FJJsF#,,&**,-d6F6FekkC e
#6!fZZ	 0 6 6 89F**V$C 
 &&)C
r7   outputgroup	reduce_opc                    t        |t              s!t        j                  || j                        }t        |t
              xr |t
        j                  k7  }t        |t              xr |j                         dv}|s|rt        d|       dd l
mc m} |j                  d|t              }t        |t              r!|j                         dv r|| j                  z  }|S )N)r   )summeanavgz]Currently, the XLAStrategy only supports `sum`, `mean`, `avg` for the reduce operation, got: r   reduce)r   r   )r;   r   r   r   rK   r   SUMr/   lower
ValueErrorrF   rG   rH   mesh_reducer   rW   )r4   r   r   r   invalid_reduce_opinvalid_reduce_op_strrJ   s          r6   r   zXLAStrategy.reduce   s     &&)\\&1A1ABF&y(;Y	X\\@Y *9c : ny?PXn?n 5;  
 	.-&#6i%)//*;*Ndoo-Fr7   c                 0    d| _         t        | 	          y )NT)r2   r0   setup_environmentrO   s    r6   r   zXLAStrategy.setup_environment  s    !#r7   c                     | j                   J t        | j                         dk(  rt        d      | j                  t        _        y )NrV   zThe `XLAStrategy` does not support running on a single device with the PjRT runtime. Try using all devices or the `SingleDeviceXLAStrategy` strategy)r#   lenNotImplementedErrorrN   r   rt   rZ   s    r6   setup_distributedzXLAStrategy.setup_distributed  sL    $$000t$$%* &S  #..r7   c                      y r9   rq   rZ   s    r6   set_world_rankszXLAStrategy.set_world_ranks$  s    
 	r7   
checkpointfilepathstorage_optionsc                 \    dd l mc m} |j                          t        |   |||       y )Nr   )r   )rF   rG   rH   	mark_stepr0   save_checkpoint)r4   r   r   r   rJ   r5   s        r6   r   zXLAStrategy.save_checkpoint+  s*     	.- 	
HoVr7   c                 Z    | j                   dk(  r| j                  j                  |       yy)zqRemove checkpoint filepath from the filesystem.

        Args:
            filepath: Path to checkpoint

        r   N)rQ   r$   remove_checkpoint)r4   r   s     r6   r   zXLAStrategy.remove_checkpoint6  s)     ??a00:  r7   r   
sync_gradsc                    | j                   s|S t        |t              s$t        dt	        |       j
                   d|       |j                         dk(  r|j                  d      }|j                  }|j                  | j                        }ddlmc m} ddlmc m} |r|j!                  |      n|j!                  |      }|j                  |      }|S )aC  Function to gather a tensor from several distributed processes.

        Args:
            tensor: tensor to all-gather.
            group: unused.
            sync_grads: flag that allows users to synchronize gradients for the all-gather operation.
        Return:
            A tensor of shape (world_size, ...)

        `z4.all_gather` is only implemented for tensors. Given r   N)r2   r;   r   r   type__name__r   r   r   r   rK   torch_xla.core.functionsrG   	functionsrF   rH   
all_gather)r4   r   r   r   r   xfrJ   s          r6   r   zXLAStrategy.all_gatherA  s     ~~M&&)%DJ''((\]c\de  ::<1%%a(F --4++,----*4v&"--:O?+r7   c                 p    t         |           d| _        t        j                  j                  dd        y )NFr^   )r0   teardownr2   ra   rb   poprO   s    r6   r   zXLAStrategy.teardown_  s'    


~t,r7   strategy_registryc                 ~    |j                  d| dd       |j                  | j                  | | j                         y )N	xla_debugz!XLA strategy with `debug` as TrueT)descriptionr&   )r   )registerstrategy_namer   )clsr   s     r6   register_strategieszXLAStrategy.register_strategiese  sC     	"";Adlp"q"" 	# 	
r7   )NNNNFT)r)   N)rn   z
pl.Trainerr)   Nr9   )r   )Nr   )NF):r   
__module____qualname____doc__r   r   listr   r   r   r   r   r   boolr   r1   propertyr	   r$   setterr   r%   r   rK   intrN   rQ   rT   rW   r[   r`   r   rf   dictr/   ru   objectr}   r   re   r   r   r   r   r   r   r   r   r   r   r   r   r   r   classmethodr   r   __classcell__)r5   s   @r6   r    r    +   s    M @D9=QU37#'6;<6 #4#566  o7L&L MN	6
 #<06 6 !6 6 
60 !u_6K%KL !  ! ! 6 !4 !  !
 ,    2)1D 2 2  2
 U\\    <S <  < ;C ;  ; :3 :  : ;C ;  ; , , E E< & V   KDcN K  K V 8H     5 5 	HSM 	 	 	PT 	 	  Z  c  *    D   $4:	fck" } E(C-01	
 
 4 $ $ 	/ 	/   \`WsCx.W49WLTUXMW	W W ;% ;D ; ;   RV ci  : - -
 
4E 
$ 
  
r7   r    )4r>   ra   typingr   r   r   r   r   r   torch.nnr   typing_extensionsr	   lightning.pytorchpytorchpl!lightning.fabric.accelerators.xlar
   r   lightning.fabric.pluginsr   r   r   %lightning.fabric.plugins.environmentsr   lightning.fabric.strategiesr   $lightning.fabric.utilities.optimizerr    lightning.fabric.utilities.typesr   r   lightning.pytorch.pluginsr   $lightning.pytorch.plugins.io.wrapperr    lightning.pytorch.strategies.ddpr   *lightning.pytorch.strategies.launchers.xlar   %lightning.pytorch.strategies.strategyr    lightning.pytorch.trainer.statesr   lightning.pytorch.utilitiesr   r   %lightning.pytorch.utilities.rank_zeror   ry   r   r    rq   r7   r6   <module>r      sc    
 	 6 6    &  T M M @ 9 F < 2 F 8 C < 6 U @DB
+ B
r7   