
      io                        d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZmZmZmZmZmZmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 d dl:m;Z; d dl<m=Z=m>Z> d dl?m@Z@ d dlAmBZBmCZC d dlDmEZE d dlFmGZG eee!ef   ZH G d d      ZIdee,   dee.   fdZJdeKfdZLy)    N)Counter)Iterable)AnyOptionalUnioncast)get_args)ACCELERATOR_REGISTRY)Accelerator)CUDAAccelerator)MPSAccelerator)XLAAccelerator)BitsandbytesPrecisionCheckpointIODeepSpeedPrecisionHalfPrecisionMixedPrecision	PrecisionTransformerEnginePrecisionXLAPrecision)ClusterEnvironmentLightningEnvironmentLSFEnvironmentMPIEnvironmentSLURMEnvironmentTorchElasticEnvironment)DoublePrecision)FSDPPrecision)_PRECISION_INPUT_PRECISION_INPUT_INT_PRECISION_INPUT_STR_PRECISION_INPUT_STR_ALIAS%_PRECISION_INPUT_STR_ALIAS_CONVERSION)STRATEGY_REGISTRYDeepSpeedStrategyParallelStrategySingleDeviceStrategySingleDeviceXLAStrategyStrategyXLAFSDPStrategyXLAStrategy)_DDP_FORK_ALIASES)_FSDP_ALIASESFSDPStrategy)ModelParallelStrategy)rank_zero_inforank_zero_warn)_determine_root_gpu_device)_IS_INTERACTIVEc                      e Zd ZdZ	 	 	 	 	 	 ddeeef   deeef   deee	   ee	f   de	de
e   de
eeee   f      d	dfd
Zdeeef   deeef   de
e   de
eeee   f      d	df
dZdeee	   ee	f   de	d	dfdZed	efd       Zed	efd       ZddZddZd	efdZd	eeef   fdZddZddZd	efdZddZedededed	efd       Zy)
_Connectoraf  The Connector parses several Fabric arguments and instantiates the Strategy including its owned components.

        A. accelerator flag could be:
            1. accelerator class
            2. accelerator str
            3. accelerator auto

        B. strategy flag could be:
            1. strategy class
            2. strategy str registered with STRATEGY_REGISTRY
            3. strategy str in _strategy_type enum which listed in each strategy as
               backend (registered these too, and _strategy_type could be deprecated)

        C. plugins flag could be:
            1. precision class (should be removed, and precision flag should allow user pass classes)
            2. checkpoint_io class
            3. cluster_environment class

    priorities which to take when:
        A. Class > str
        B. Strategy > Accelerator/precision/plugins

    Nacceleratorstrategydevices	num_nodes	precisionpluginsreturnc                    | j                  d|d      }| j                  d|d      }| j                  d|d      }t        | j                  d|d            }| j                  d|d       }t        j                         | _        t        j                         | _        d| _        d| _	        d	| _
        d | _        d | _        g | _        d | _        | j                  ||||
       | j!                  ||       | j                  dk(  r| j#                         | _	        n$| j                  dk(  r| j%                         | _	        | j'                          | j)                         | _        | j                  dk(  r| j-                         | _        | j/                          | j1                          | j3                         | _        | j7                          y )Nr6   auto)defaultr7   r8   r9      r:   32-true)r7   r6   r:   r;   )r8   r9   gpu)_argument_from_envintr$   available_strategies_registered_strategiesr
   available_accelerators_registered_accelerators_strategy_flag_accelerator_flag_precision_input_precision_instance_cluster_environment_flag_parallel_devicescheckpoint_io!_check_config_and_set_final_flags(_check_device_config_and_set_final_flags_choose_auto_accelerator_choose_gpu_accelerator_backend*_set_parallel_devices_and_init_accelerator$_choose_and_init_cluster_environmentcluster_environment_choose_strategy_check_strategy_and_fallback_init_strategy_check_and_init_precisionr:   _lazy_init_strategy)selfr6   r7   r8   r9   r:   r;   s          o/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning/fabric/connector.py__init__z_Connector.__init__c   s    --m[RX-Y**:x*P)))Wf)M//YPQ/RS	++KD+Q	 '8&L&L&N#(<(S(S(U%
 5;:@6?8< SW&FH59..#	 	/ 	
 	55gQZ5[ !!V+%)%B%B%DD"##u,%)%I%I%KD"779 8<7`7`7b  &("&"7"7"9D))+ 779 	  "    c                 
   |t        |t              s|gn|}t        |t              r|j                         }|| _        |dk7  r-|| j
                  vrt        |t              st        d|d      || j                  vr?|dvr;t        |t              s+t        d|ddj                  | j                         d	      t        |t              xr d
|v }t        |t              xr d|v }t        |t              xr d|v }t        |t              xs
 |xs |xs |}t        j                         xr |dv xs t        |t              }	|	r|rt        d| d      || _        t        |      }
|r=t!               }|D ]  }t        |t"              r#|| _        |t"        j&                  xx   dz  cc<   6t        |t(              r#|| _        |t(        j&                  xx   dz  cc<   it        |t,              r#|| _        |t,        j&                  xx   dz  cc<   t1        d| d       |j3                         D cg c]  \  }}|dkD  s| }}}|rt        ddj                  |       d      |j5                  t"        j&                        r|
t        d|
 d| j$                   d      |
dn|
| _        t        | j                  t              rr| j                  j8                  r5| j                  dk7  rt        d      | j                  j8                  | _        | j                  j:                  r2| j$                  rt        d      | j                  j:                  | _        | j                  j<                  r2| j*                  rt        d      | j                  j<                  | _        t?        | j                  dd      r2| j.                  rt        d      t?        | j                  d      | _        tA        | j                  d      r6| j                  jB                  r| j                  jB                  d   jD                  d k(  r[| j                  rH| j                  d!vr:t        d"| j                  jF                  j&                   d#| j                   d$      d | _        | j                  jB                  d   jD                  d%k(  r[| j                  rH| j                  d&vr:t        d'| j                  jF                  j&                   d#| j                   d$      d%| _        | j                  jB                  | _$        yyyyc c}}w )(a  This method checks:

        1. strategy: whether the strategy name is valid, and sets the internal flags if it is.
        2. accelerator: if the value of the accelerator argument is a type of accelerator (instance or string),
            set self._accelerator_flag accordingly.
        3. precision: The final value of the precision flag may be determined either by the precision argument or
            by a plugin instance.
        4. plugins: The list of plugins may contain a Precision plugin, CheckpointIO, ClusterEnvironment and others.
            Additionally, other flags such as `precision` can populate the list with the
            corresponding plugin instances.

        Nr>   z1You selected an invalid strategy name: `strategy=z`. It must be either a string or an instance of `lightning.fabric.strategies.Strategy`. Example choices: auto, ddp, ddp_spawn, deepspeed, dp, ... Find a complete list of options in our documentation at https://lightning.ai)r>   rB   z7You selected an invalid accelerator name: `accelerator=z`. Available names are: auto, , .ddpdp	deepspeed)mpsr>   rB   NzYou set `strategy=z` but strategies from the DDP family are not supported on the MPS accelerator. Either explicitly set `accelerator='cpu'` or change the strategy.r@   zFound invalid type for plugin z?. Expected one of: Precision, CheckpointIO, ClusterEnvironment.zReceived multiple values for z> flags in `plugins`. Expected one value for each type at most.zReceived both `precision=z` and `plugins=z`. Choose one.rA   zLaccelerator set through both strategy class and accelerator flag, choose onezAprecision set through both strategy class and plugins, choose onezEcheckpoint_io set through both strategy class and plugins, choose onerV   zKcluster_environment set through both strategy class and plugins, choose oneparallel_devicesr   cpu)r>   rh   z!CPU parallel_devices set through z class, but accelerator set to z, please choose one device typecuda)r>   ri   rB   z!GPU parallel_devices set through )%
isinstancer   strlowerrI   rF   r)   
ValueErrorrH   r   joinr&   r   is_availablerJ   "_convert_precision_to_unified_argsr   r   rL   __name__r   rO   r   rM   	TypeErroritemsgetrK   _accelerator
_precision_checkpoint_iogetattrhasattrrg   type	__class__rN   )r\   r7   r6   r:   r;   
is_ddp_str	is_dp_stris_deepspeed_stris_parallel_strategyis_mps_acceleratorprecision_inputplugins_flags_typespluginkvduplicated_plugin_keys                   r]   rP   z,_Connector._check_config_and_set_final_flags   sM   & '1'8'Dwi'Gh$~~'H&v($2M2M"MV`aiksVtCH< P` `  t<<<?2{K8I+ Y//3yy9V9V/W.XXY[   #.D5H3D
x-B$(2B	%h4P9P)(4DEttW`tdt+88: 
77b:kSa;b 	 "6$XJ /f g 
 "-<YG29)!fi0/5D,'	(:(:;q@;5)/D&'(=(=>!C>(:;5;D2'(:(C(CDID#8 A< <  "  4G3L3L3N$X3N41aRSVWRWQ3N!$X$ 3DII>S4T3U VA A 
 #&&y'9'9:?Z //@PTPhPhOiiwx  .=-D	/
 d))84""//))V3$%stt)-)<)<)I)I&""--++$%hii+/+>+>+I+I(""11%%$%lmm%)%8%8%G%G"t**,A4H11$%rss189L9LNc1d.t**,>?DDWDWDhDh&&77:??5H--$2H2HP_2_(?@S@S@]@]@f@f?g h77;7M7M6NNmo  .3D*&&77:??6I--$2H2HPg2g(?@S@S@]@]@f@f?g h77;7M7M6NNmo  .4D*)-)<)<)M)M& Ei?' 5# %Ys   ;U	Uc                 N   t        |t              r|dk  rt        d| d      || _        || _        | j                  g ddfv rbt        | j
                  t              r | j
                  j                  j                  n| j
                  }t        d| j                  d| d      y )	Nr@   z0`num_nodes` must be a positive integer, but got rb   r   0z`Fabric(devices=z$)` value is not a valid input using z accelerator.)	rj   rD   rm   _num_nodes_flag_devices_flagrJ   r   r{   __qualname__)r\   r8   r9   accelerator_names       r]   rQ   z3_Connector._check_device_config_and_set_final_flags*  s    )S)Y]OPY{Z[\]]($"a- d44kB &&00==++ 
 "4#5#5"8 9*+=:  .r_   c                      t        j                         ryt        j                         ryt        j                         ryy)zTChoose the accelerator type (str) based on availability when ``accelerator='auto'``.tpurf   ri   rh   )r   ro   r   r    r_   r]   rR   z#_Connector._choose_auto_accelerator<  s4     &&(&&('')r_   c                  l    t        j                         ryt        j                         ryt        d      )Nrf   ri   zNo supported gpu backend found!)r   ro   r   RuntimeErrorr   r_   r]   rS   z*_Connector._choose_gpu_accelerator_backendG  s,    &&('')<==r_   c                 n   t        | j                  t              r| j                  | _        n2| j                  J t	        j
                  | j                        | _        | j                  j                  }|j                         sO| j                  D cg c]  }t        |   d   j                         r|  }}t        d|j                   d| d      | j                          |j                  | j                        | _        | j                  s!|j                  | j                        | _        y y c c}w )Nr6   `z` can not run on your system since the accelerator is not available. The following accelerator(s) is available and can be passed into `accelerator` argument of `Fabric`: rb   )rj   rJ   r   r6   r
   rt   r{   ro   rH   r   r    _set_devices_flag_if_auto_passedparse_devicesr   rN   get_parallel_devices)r\   accelerator_clsacc_stravailable_accelerators       r]   rT   z5_Connector._set_parallel_devices_and_init_acceleratorO  s*   d,,k:,0,B,BD))5553778N8NOD**44++-  $<<%<G'0?LLN < " %
 O001 2 44A7  	--/,::4;M;MN%%%4%I%I$J\J\%]D" &%s   #D2c                 p   | j                   dk7  ry t        rt        | j                  t              rg| j                  j                         dkD  rJd| _         t        d| j                  j                          d| j                  j                          d       y | j                  j                         | _         y )Nr>   r@   zFabric will use only 1 of zm GPUs because it is running inside an interactive / notebook environment. You may try to set `Fabric(devices=z)` but please note that multi-GPU inside interactive / notebook environments is considered experimental and unstable. Your mileage may vary.)r   r3   rj   r6   r   auto_device_countr0   r\   s    r]   r   z+_Connector._set_devices_flag_if_auto_passedj  s    '4++_=  224q8!"D,T-=-=-O-O-Q,R S^##5578 9ii "&!1!1!C!C!EDr_   c                     t        | j                  t              r| j                  S t        t        t
        t        fD ]  }|j                         s |       c S  t               S )N)	rj   rM   r   r   r   r   r   detectr   )r\   env_types     r]   rU   z/_Connector._choose_and_init_cluster_environment|  sX    d446HI111 $
H  z!
 $%%r_   c                 8   | j                   dk(  st        | j                   t              r>| j                  rt	        | j                        dkD  ryt        | j                  d         S | j                  dkD  ryt	        | j                        dk  rlt        | j                   t        t        f      s(t        | j                   t              r$| j                   dv rt        | j                        }nd}t        |      S t	        | j                        dkD  rt        ry	y)
Nr   r@   xlar   )devicerc   )ri   rB   rf   rh   ddp_fork)rJ   rj   r   rN   lenr(   r   r   r   rk   r2   r'   r3   )r\   r   s     r]   rW   z_Connector._choose_strategy  s    !!U*j9O9OQ_.`%%#d.D.D*E*I*$2H2H2KLL!#t%%&!+$00?N2ST41137D<R<RVl<l3D4J4JK'v66t%%&*r_   c                    t        | j                  t              rdn| j                  }|dk(  r| j                  dk(  rd}|dk(  r| j                  dk(  rt	        |d       d}|t
        v r/d	t        j                  j                         vrt        d
| d      |t        v st        | j                        t        u r| j                  dvrt        d      |r|| _        yy)zChecks edge cases when the strategy selection was a string input, and we need to fall back to a different
        choice depending on other parameters or the environment. fsdpr   xla_fsdprd   rh   z: is not supported on CPUs, hence setting `strategy='ddp'`.rc   forkzYou selected `Fabric(strategy='zo')` but process forking is not supported on this platform. We recommend `Fabric(strategy='ddp_spawn')` instead.)ri   rB   zYou selected the FSDP strategy but FSDP is only available on GPU. Set `Fabric(accelerator='gpu', ...)` to continue or select a different strategy.N)rj   rI   r)   rJ   r1   r,   torchmultiprocessingget_all_start_methodsrm   r-   rz   r.   )r\   strategy_flags     r]   rX   z'_Connector._check_strategy_and_fallback  s    
 ))<)<hGTM`M` F"t'='='F&MD T%;%;u%Dm..hij!M--&@U@U@k@k@m2m1- AR S 
 ]*d43F3F.G<.W$$O;?  "/D r_   c                     t        | j                  t        t        f      sJ t        | j                  t              r%t	        j
                  | j                        | _        y| j                  | _        y)zNInstantiate the Strategy given depending on the setting of ``_strategy_flag``.N)rj   rI   rk   r)   r$   rt   r7   r   s    r]   rY   z_Connector._init_strategy  sS     $--X???d))3/-11$2E2EFDM //DMr_   c                 &   t        | j                  t              rKt        | j                  t              r%t        | j                  t
              st        d      | j                  S t        | j                  t        t        t        f      rt        | j                        S t        | j                  t              rt        | j                        S t        | j                  t              rt!        | j                        S d}t        | j                  t"              r9| j                  |vr+t%        d| j                  ddj'                  |       d      | j                  dv rt)        | j                        S | j                  d	k(  r
t               S | j                  d
k(  r
t+               S | j                  dk(  rt-        t.        j0                        S | j                  dk(  rt-        t.        j2                        S | j                  dk(  r!| j4                  dk(  rt7        d       d| _        | j                  dv rOt9        | j                  dk(  rdnd       | j4                  dv r| j4                  nd}t;        | j                  |      S t        d      )Nz,Bitsandbytes is only supported on CUDA GPUs.)r:   )rA   
bf16-mixed	bf16-true16-truezDThe `ModelParallelStrategy` does not support `Fabric(..., precision=z()`. Choose a different precision among: ra   rb   )r   r   rA   z64-trueztransformer-engine)weights_dtypeztransformer-engine-float1616-mixedrh   zYou passed `Fabric(accelerator='cpu', precision='16-mixed')` but AMP with fp16 is not supported on CPU. Using `precision='bf16-mixed'` instead.r   )r   r   z,Using 16-bit Automatic Mixed Precision (AMP)z.Using bfloat16 Automatic Mixed Precision (AMP))rh   rf   ri   )r:   r   zNo precision set)rj   rL   r   r   r6   r   r   r7   r(   r+   r*   r   rK   r%   r   r.   r   r/   rm   rn   r   r   r   r   bfloat16float16rJ   r1   r0   r   )r\   mp_precision_supportedr   s      r]   rZ   z$_Connector._check_and_init_precision  s;   d..	:$224IJS]  /T ##QRR+++dmm&={O%\] 5 566dmm%67%d&;&;<<dmm\2 4+@+@AA!Rdmm%:;@U@U]s@sVW[WlWlVo p88<		BX8Y7ZZ[]    $<< !6!677  I-;  I-"$$  $88-ENNKK  $@@-EMMJJ  J.43I3IU3R? %1D!  $>>((J6 ?E
 04/E/E/WT++]cF!D,A,A&QQ-..r_   c                    | j                   | j                  _         | j                  r| j                  | j                  _        | j                  r| j                  | j                  _        t	        | j                  d      rL| j                  j
                  | j
                  | j                  _        | j                  j
                  | _        t	        | j                  d      rM| j                  j                  r| j                  j                  | _        n| j                  | j                  _        t	        | j                  d      r| j                  | j                  _	        t	        | j                  d      r| j                  j                          | j                  j                          t        rO| j                  j                  r9| j                  j                  j                  st        d| j                   d      t#        | j                   t$              rSt#        | j                  t&        t(        t*        f      s-t-        d| j                  j.                  j0                   d	      yy)
zFLazily set missing attributes on the previously instantiated strategy.rV   Nrg   r9   _set_world_ranksz`Fabric(strategy=a  )` is not compatible with an interactive environment. Run your code as a script, or choose one of the compatible strategies: `Fabric(strategy='dp'|'ddp_notebook')`. In case you are spawning processes yourself, make sure to include the Fabric creation inside the worker function.zsThe `XLAAccelerator` can only be used with a `SingleDeviceXLAStrategy`, `XLAStrategy`, or `XLAFSDPStrategy`. Found rb   )r6   r7   r:   rO   ry   rV   rg   rN   r   
_num_nodesr   _configure_launcherr3   launcheris_interactive_compatibler   rI   rj   r   r(   r+   r*   rm   r{   rq   r   s    r]   r[   z_Connector._lazy_init_strategy  s   $($4$4!>>&*nnDMM#*.*<*<DMM'4=="78}}008484L4L1'+}}'H'HD$4=="45}}--)-)G)G&151G1G.4==+.'+';';DMM$4=="45MM**,))+t}}55dmm>T>T>n>n#D$7$7#: ;8 8  d&&7
MM3[/RA
 --1]]-D-D-M-M,NaQ A
7r_   namecurrentr?   c                     t         j                  j                  d| j                         z         }||S |G|t	        |      k7  r9t	        |      t	        |      k7  r"t               rt        d|  d|d|  d| d	      |S )NLT_zYour code has `Fabric(=z+, ...)` but it conflicts with the value `--zZ` set through the CLI.  Remove it either from the CLI or from the Lightning Fabric object.)osenvironrt   upperrk   _is_using_clirm   )r   r   r?   	env_values       r]   rC   z_Connector._argument_from_env  s    #%::>>%$**,2F#G	N Y#g,%>3w<SVW^S_C_dqds(a{ ;V1YK (VV 
 r_   )r>   r>   r>   r@   NN)r<   N) rq   
__module__r   __doc__r   rk   r   r)   listrD   r   r   _PLUGIN_INPUTr   r^   rP   rQ   staticmethodrR   rS   rT   r   r   rU   rW   rX   rY   r   rZ   r[   r   rC   r   r_   r]   r5   r5   J   s   4 06)/.404KO?#3+,?# X&?# tCy#s*+	?#
 ?# ,-?# %x/F FGH?# 
?#BDNX&DN 3+,DN ,-	DN
 %x/F FGHDN 
DNLd3iQTVYFY@Z gj os $ c   >S > >^6F$&6H &%#"6 *080./9 ./`'R  s S S  r_   r5   r:   r<   c                 ^   | y t        t              t        t              z   t        t              z   }| |vrt	        dt        |        d|       t        |       } | t        t              v r2t        |       d d dvrt        d|  dt        |     d       t        |    } t        t        |       S )Nz
Precision z' is invalid. Allowed precision values:    )3264z`precision=za` is supported for historical reasons but its usage is discouraged. Please set your precision to z	 instead!)
r	   r!   r    r"   rm   reprrk   r1   r#   r   )r:   supported_precisions     r]   rp   rp   -  s     	%&2F)GG(SmJnn  ++:d9o%66]^q]rsttIIH788y>"1\1i[ )00UV_0`/aajl :)D	$i00r_   c                  f    t        t        t        j                  j	                  dd                  S )NLT_CLI_USEDr   )boolrD   r   r   rt   r   r_   r]   r   r   C  s!    BJJNN=#6788r_   )Mr   collectionsr   collections.abcr   typingr   r   r   r   r   typing_extensionsr	   lightning.fabric.acceleratorsr
   )lightning.fabric.accelerators.acceleratorr   "lightning.fabric.accelerators.cudar   !lightning.fabric.accelerators.mpsr   !lightning.fabric.accelerators.xlar   lightning.fabric.pluginsr   r   r   r   r   r   r   r   %lightning.fabric.plugins.environmentsr   r   r   r   r   r   )lightning.fabric.plugins.precision.doubler   'lightning.fabric.plugins.precision.fsdpr   ,lightning.fabric.plugins.precision.precisionr   r    r!   r"   r#   lightning.fabric.strategiesr$   r%   r&   r'   r(   r)   r*   r+   lightning.fabric.strategies.ddpr,    lightning.fabric.strategies.fsdpr-   r.   *lightning.fabric.strategies.model_parallelr/   lightning.fabric.utilitiesr0   r1   (lightning.fabric.utilities.device_parserr2   "lightning.fabric.utilities.importsr3   r   r5   rp   r   r   r   r_   r]   <module>r      s    
  $ - -  & > A > < <	 	 	  F A 	 	 	 > H L E O >i!3\AB` `F1(;K2L 1QYZnQo 1,9t 9r_   