i"ddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZddlmZej$eZGdd e Zd efd Zd efd Zy) N)Optional)override)ClusterEnvironment) _IS_WINDOWS)rank_zero_warn)PossibleUserWarningceZdZdZddedeejddffd Ze e defdZ e e de fdZ e e defd Zee defd Zedee fd Zedeefd Ze defd Ze deddfdZe defdZe deddfdZe defdZe defdZe dededdfdZede de fdZeddZeddZxZS)SLURMEnvironmenta'Cluster environment for training on a cluster managed by SLURM. You can configure the `main_address` and `main_port` properties via the env variables `MASTER_ADDR` and `MASTER_PORT`, respectively. Args: auto_requeue: Whether automatic job resubmission is enabled or not. How and under which conditions a job gets rescheduled gets determined by the owner of this plugin. requeue_signal: The signal that SLURM will send to indicate that the job should be requeued. Defaults to SIGUSR1 on Unix. N auto_requeuerequeue_signalreturnct|||_|tstj }||_|j|jy)N) super__init__r rsignalSIGUSR1r _validate_srun_used_validate_srun_variables)selfr r __class__s /Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning_fabric/plugins/environments/slurm.pyrzSLURMEnvironment.__init__/sG (  !+#^^N,   " %%'cy)NTrs rcreates_processes_externallyz-SLURMEnvironment.creates_processes_externally8src"tjjd}|Dtjjdd}|j|}|tjd<tj dtjd|S)N MASTER_ADDRSLURM_NODELISTz 127.0.0.1z MASTER_ADDR: )osenvirongetresolve_root_node_addresslogdebug)r root_nodenodelists r main_addresszSLURMEnvironment.main_address=srJJNN=1  zz~~&6 DH66x@I(1BJJ} % M"**]";!<=>rc\tjjd}||dd}t|dz}nd}dtjvrttjd}nt |tjd<t j dtjd|S)N SLURM_JOB_IDi:in2 MASTER_PORTz MASTER_PORT: )r r!r"intstrr$r%)rjob_id default_ports r main_portzSLURMEnvironment.main_portIs/  !"#;L|,u4L L BJJ &rzz-89L(+L(9BJJ} % M"**]";!<=>rc>tjtS)aReturns ``True`` if the current process was launched on a SLURM cluster. It is possible to use the SLURM scheduler to request resources and then launch processes manually using a different environment. For this, the user can set the job name in SLURM to 'bash' or 'interactive' (srun --job- name=interactive). This will then avoid the detection of ``SLURMEnvironment`` and another environment can be detected automatically. )r r _is_srun_usedrrrdetectzSLURMEnvironment.detectes ,,.rc@tjjdS)NSLURM_JOB_NAME)r r!r"rrrjob_namezSLURMEnvironment.job_namesszz~~.//rctrytjjd}|y t |S#t $rYywxYw)Nr*_is_slurm_interactive_moder r!r"r- ValueError)r/s rr/zSLURMEnvironment.job_idwsH & '/ > v;   s : AAc:ttjdSN SLURM_NTASKSr-r r!rs r world_sizezSLURMEnvironment.world_size2::n-..rsizec.tjdy)Nz[SLURMEnvironment.set_world_size was called, but setting world size is not allowed. Ignored.r$r%)rrBs rset_world_sizezSLURMEnvironment.set_world_sizes oprc:ttjdS)N SLURM_PROCIDr?rs r global_rankzSLURMEnvironment.global_rankrArrankc.tjdy)Nz]SLURMEnvironment.set_global_rank was called, but setting global rank is not allowed. Ignored.rD)rrIs rset_global_rankz SLURMEnvironment.set_global_ranks qrrc:ttjdS)N SLURM_LOCALIDr?rs r local_rankzSLURMEnvironment.local_ranks2::o.//rc:ttjdS)N SLURM_NODEIDr?rs r node_rankzSLURMEnvironment.node_rankrAr num_devices num_nodesc ,trytjjd}|#t ||k7rt d|d|d|dtjjd}|$t ||k7rt d|d|d |dyy) NSLURM_NTASKS_PER_NODEzYou set `devices=zX` in Lightning, but the number of tasks per node configured in SLURM `--ntasks-per-node=z%` does not match. HINT: Set `devices=z`. SLURM_NNODESzYou set `num_nodes=zE` in Lightning, but the number of nodes configured in SLURM `--nodes=z'` does not match. HINT: Set `num_nodes=r9)rrRrSntasks_per_nodennodess rvalidate_settingsz"SLURMEnvironment.validate_settingss % ' **..)@A  &3+?;+N#K=1''6&77\]l\mmoq /  #f+":%i[1#H$KF8SUW #; rnodesctjdd|}tjdd|}|jddjddS)aThe node selection format in SLURM supports several formats. This function selects the first host name from - a space-separated list of host names, e.g., 'host0 host1 host3' yields 'host0' as the root - a comma-separated list of host names, e.g., 'host0,host1,host3' yields 'host0' as the root - the range notation with brackets, e.g., 'host[5-9]' yields 'host5' as the root z\[(.*?)[,-].*\]z\1z \[(.*?)\] r,)resubsplit)rZs rr#z*SLURMEnvironment.resolve_root_node_addresssM)5%8|UE2{{3"((-a00rc(trytjddu}|rrtsgdj dt j jtjgtjdd}td|dtyyy)aUChecks if the `srun` command is available and used. Parallel jobs (multi-GPU, multi-node) in SLURM are launched by prepending `srun` in front of the Python command. Not doing so will result in processes hanging, which is a frequent user error. Lightning will emit a warning if `srun` is found but not used. Nsrunr\@zThe `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: z ...)category) rshutilwhichr3joinr pathbasenamesys executableargvrr) srun_existshints rrz$SLURMEnvironment._validate_srun_useds  ll6*$6 }88VRWW%5%5cnn%EQQRSVTVWD YY]X^^bd,  /;rcttjjdd}|dkDr%dtjvrt d|d|dyy) a2Checks for conflicting or incorrectly set variables set through `srun` and raises a useful error message. Right now, we only check for the most common user errors. See `the srun docs `_ for a complete list of supported srun variables. r>1rUzYou set `--ntasks=z^` in your SLURM bash script, but this variable is not supported. HINT: Use `--ntasks-per-node=z ` instead.N)r-r r!r" RuntimeError)ntaskss rrz)SLURMEnvironment._validate_srun_variabless^RZZ^^NC89 A:1C$VH-117 D D:r)TN)r N)__name__ __module__ __qualname____doc__boolrrSignalsrpropertyrrr.r(r-r1 staticmethodr4r7r/r@rErHrKrNrQrYr#rr __classcell__)rs@rr r !s! (T((6>>BZ(fj( d c 34  D  0hsm00 HSM  /C//q3q4qq/S//sCsDss0C00/3//SST  1 1 1 1(  rr r c@dtjvxr t Sr=)r r!r:rrrr3r3s RZZ ' L0J0L,LLrc.tjdvS)N)bash interactive)r r7rrrr:r:s  $ $ &*A AAr)loggingr r^rerrjtypingrtyping_extensionsr9lightning_fabric.plugins.environments.cluster_environmentr"lightning_fabric.utilities.importsr$lightning_fabric.utilities.rank_zeror#lightning_fabric.utilities.warningsr getLoggerrtr$r rxr3r:rrrrsi &X:?Cg!)DMtMBDBr