+
    NV-jaE  ã                   ó"  € ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	H
t
 ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHtHtHt ^ RIHt ^R	IHtHtHtHt  ! R
 R4      t  ! R R] 4      t!R t"R t#R t$R t%R t&R t'R t(R# )é    N)ÚCounter)Úchain)ÚPath)ÚEmpty)ÚQueue)Úselect)ÚPIPEÚPopenÚrun)ÚHostfileÚlogÚlog_warningÚpositive_numberc                   ó\   a € ] tR t^t o ]R 4       t]R 4       tRV 3R lR lltR tRt	V t
R# )	ÚCommandProcessc                ó   € \        4       h)z;Return the Popen object that refers to the current command.©ÚNotImplementedError©Úselfs   &Ún/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx/_distributed_utils/launch.pyÚprocessÚCommandProcess.process   ó   € ô "Ó#Ð#ó    c                ó   € \        4       h)zuReturn a tuple (returncode, killed) for the command. It should be
(None, None) while the command is running normally.r   r   s   &r   Úexit_statusÚCommandProcess.exit_status   s   € ô "Ó#Ð#r   c                ó    <€ V ^8„  d   QhRS[ /# )é   Údata)Ústr)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚCommandProcess.__annotate__%   s   ø€ ÷ $ñ $¡cñ $r   c                ó   € \        4       h)zhPreprocess the output of the command so that extra data can be
capture or the format changed on the fly.r   )r   r!   Ú	is_stdouts   &&&r   Úpreprocess_outputÚ CommandProcess.preprocess_output%   r   r   c                ó   € \        4       h)z"Terminate or return the exit code.r   r   s   &r   Ú	terminateÚCommandProcess.terminate*   s   € ä!Ó#Ð#r   © N©F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úpropertyr   r   r)   r,   Ú__static_attributes__Ú__classdictcell__©r$   s   @r   r   r      s?   ø‡ € Øñ$ó ð$ð ñ$ó ð$÷
$ò $÷
$ð $r   r   c                   óv   a € ] tR t^/t o R t]R 4       t]R 4       tR
R ltR t	]
R 4       t]
R 4       tRtV tR	# )ÚRemoteProcessc           	     óþ   € VR 8H  p\         P                  WWVWx4      p	V'       g   RV R\        P                  ! V	4       2p	W n        RV n        W€n        \        V	RR\        \        \        R7      V n	        RV n
        R# )ú	127.0.0.1zssh -tt -o LogLevel=QUIET Ú NTú	/bin/bash)ÚshellÚ
executableÚstdinÚstdoutÚstderrF)r9   Úmake_launch_scriptÚshlexÚquoteÚ_hostÚ_pidfileÚ	_is_localr
   r	   Ú_processÚ_killed)
r   ÚrankÚhostÚpythonÚcwdÚfilesÚenvÚcommandÚis_localÚcmds
   &&&&&&&&  r   Ú__init__ÚRemoteProcess.__init__0   sw   € Ø˜;Ñ&ˆÜ×.Ñ.¨t¸%ÀgÓXˆßØ.¨t¨f°A´e·k²kÀ#Ó6FÐ5GÐHˆCàŒ
ØˆŒØ!ŒÜØØØ"ÜÜÜô
ˆŒð ˆŽr   c                ó   € V P                   # ©N)rI   r   s   &r   r   ÚRemoteProcess.processD   s   € à�}‰}Ðr   c                óN   € V P                   P                  4       V P                  3# rW   )rI   ÚpollrJ   r   s   &r   r   ÚRemoteProcess.exit_statusH   s   € à�}‰}×!Ñ!Ó# T§\¡\Ð1Ð1r   c                ó|   € V P                   f.   VP                  R^R7      vr4W0n         V'       d
   V^ ,          # R# V# )NÚ
©ÚmaxsplitÚ )rG   Úsplit)r   r!   r(   ÚpidfileÚrests   &&&  r   r)   ÚRemoteProcess.preprocess_outputL   s;   € Ø�=‰=Ò Ø!ŸZ™Z¨°q˜ZÓ9ˆNˆGØ#ŒMß"�4˜•7Ð*¨Ð*àˆr   c           	     óª  € V P                   '       d   R # V P                  P                  4        V P                  P                  4        \        P                  V P                  4      pV P                  '       g'   RV P                   R\        P                  ! V4       2p\        VRRRRRR7      pVP                  P                  4       R8H  V n         R # )Nzssh r<   Tr=   )Úcheckr>   r?   Úcapture_outputÚtextÚ1)rJ   rI   r,   Úwaitr9   Úmake_kill_scriptrG   rH   rF   rD   rE   r   rA   Ústrip)r   rS   Úcs   &  r   r,   ÚRemoteProcess.terminateT   s¢   € Ø�<�<ˆ<Ùà�‰×ÑÔ!Ø�‰×ÑÔô ×,Ñ,¨T¯]©]Ó;ˆØ�~�~ˆ~Ø˜Ÿ™˜ A¤e§k¢k°#Ó&6Ð%7Ð8ˆCÜØØØØ"ØØô
ˆð —x‘x—~‘~Ó'¨3Ñ.ˆŽr   c                óÄ  € R pV'       g   RpVR,          pVR,          pVR,          pT;'       g    \         P                  ! 4       pVR\        V4       R2,          pVR\        V4       R2,          pVe    VR	,          pVR
\        V4       R2,          pVR,          pV F¡  pVP                  R^R7      vrš\	        V
4      ^ 8”  d   \
        P                  ! V
^ ,          4      MR p
\        ;QJ d    R V	 4       F  '       d   K   RM	  RM! R V	 4       4      '       g   \        RV R24       K‘  VRV	 RV
 R2,          pK£  	  VP                  4        F<  w  r¼VR,          pVR\
        P                  ! V4       R2,          pVRV R2,          pK>  	  VRV  R2,          pVRRP                  \        \
        P                  V4      4       R2,          pVR,          pV# )r`   zstty -echo; zpidfile=$(mktemp); zecho $$ > $pidfile; zprintf "%s\n" $pidfile; z	if [[ -d z
 ]]; then z  cd z; úelse z& echo 'Failed to change directory to' z >2; úfi; Ú=r^   c              3   óZ   "  € T F!  qP                  4       ;'       g    VR 8H  x € K#  	  R# 5i)Ú_N)Úisalnum)Ú.0rm   s   & r   Ú	<genexpr>Ú3RemoteProcess.make_launch_script.<locals>.<genexpr>…   s#   é € Ð<¹°1—y‘y“{×.Ð. a¨3¡hÔ.»ùs   ‚+�+FTÚ'z5' is an invalid environment variable so it is ignoredzexport zfname=$(mktemp); zecho z
 >$fname; z	=$fname; zexport MLX_RANK=zcmd=(r<   ú); zexec "${cmd[@]}")ÚosÚgetcwdÚreprra   ÚlenrD   rE   Úallr   ÚitemsÚjoinÚmap)rK   rN   rO   rP   rQ   rR   ÚscriptÚdÚeÚkeyÚvalueÚenv_nameÚcontents   &&&&&&       r   rC   Ú RemoteProcess.make_launch_scriptj   sÚ  € àˆ÷ Ø#ˆFð 	Ð'Õ'ˆØÐ(Õ(ˆØÐ-Õ-ˆð ×Ð”2—9’9“;ˆØ�Iœd 1›g˜Y jÐ1Õ1ˆØ�Eœ$˜q›'˜ "Ð%Õ%ˆØŠ?Ø�gÕˆFØÐ>¼tÀA»w¸iÀuÐMÕMˆFØ�&Õˆó ˆAØŸ'™' #°˜'Ó2ˆKˆCÜ-0°«Z¸!¬^”E—K’K  a¥Ô)ÀˆEß“3Ñ<¹Ó<—3—3’3Ñ<¹Ó<×<Ò<ÜØ˜�sÐOÐPôñ Ø˜ ˜u A e W¨BÐ/Õ/ŠFñ ð "'§¡¦ÑˆHØÐ)Õ)ˆFØ˜œeŸkšk¨'Ó2Ð3°:Ð>Õ>ˆFØ˜ ˜z¨Ð3Õ3ŠFñ "/ð 	Ð$ T F¨"Ð-Õ-ˆð 	�E˜#Ÿ(™(¤3¤u§{¡{°GÓ#<Ó=Ð>¸cÐBÕBˆØÐ$Õ$ˆàˆr   c                ó¨   € R pVRV  R2,          pVR,          pVR,          pVR,          pVR,          pVR,          pVR,          pVR	V  2,          pV# )
r`   z
pid=$(cat rz   zif ps -p $pid >/dev/null; then z    kill $pid; z    echo 1; rp   z    echo 0; rq   zrm r.   )rb   rƒ   s   & r   rk   ÚRemoteProcess.make_kill_script›   sm   € àˆØ�J˜w˜i sÐ+Õ+ˆØÐ3Õ3ˆØÐ#Õ#ˆØ�.Õ ˆØ�'ÕˆØ�.Õ ˆØ�&ÕˆØ�C˜�y�/Õ!ˆàˆr   )rF   rH   rJ   rG   rI   Nr/   )r0   r1   r2   r3   rT   r4   r   r   r)   r,   ÚstaticmethodrC   rk   r5   r6   r7   s   @r   r9   r9   /   se   ø‡ € òð( ñó ðð ñ2ó ð2ôò/ð, ñ.ó ð.ð` ñó ör   r9   c                 ó
  a aaa€ R oR.\        V4      ,          oV VVV3R lp. p. p. p. p\        V4       F¯  w  pw  ršVP                  \        4       4       VP                  \        4       4       VP                  \        4       4       \        P
                  ! VV	V
RVR,          RVR,          RVR,          /,          R7      pVP                  4        VP                  V4       K±  	  \        P                  ! \        P                  P                  4       R 4       \        P                  ! \        P                  P                  4       R4       \        P                  ! \        P                  P                  4       R4       S'       dN   \        ;QJ d%    R \        WV4       4       F  '       g   K   RM	  R M! R \        WV4       4       4      '       Edñ   \!        \        P                  P                  4       .. . R	4      w  p pV F4  p\        P"                  ! VR
4      pV F  pVP%                  V4       K  	  K6  	  V FS  p VP'                  4       '       g:   \        P                  P(                  P+                  VP-                  4       4       KO  KU  	  V FS  p VP'                  4       '       g:   \        P                  P(                  P+                  VP-                  4       4       KO  KU  	  \        P                  P(                  P1                  4        \        P                  P(                  P1                  4        \        ;QJ d    R V 4       F  '       g   K   RM	  R M! R V 4       4      '       dK   \        V4       F8  w  r‹VP3                  4       '       d   K  SV,          ^ ,          ^ 8w  g   K4  Ro EK?  	  EKD   V F  pVP5                  4        K  	  V FR  pVP'                  4       '       d   K  \        P                  P(                  P+                  VP7                  4       4       KQ  	  V FR  pVP'                  4       '       d   K  \        P                  P(                  P+                  VP7                  4       4       KQ  	  \        P                  P(                  P1                  4        \        P                  P(                  P1                  4        R#   \.         d     EK¯  i ; i  \.         d     EKi  i ; i)FNc                 óÚ  <€ VP                  R 4      pVP                  R4      pVP                  R4      pS! V .VO5/ VB pVP                  p\        P                  ! VP                  P                  4       R4       \        P                  ! VP                  P                  4       R4       \        P                  ! VP                  P                  4       R4       VP                  P                  4       VP                  P                  4       .pVP                  P                  4       .p	Rp
VP                  4       Ef     W£P                  4       ,          p
\        W‰. R4      w  r¼pV F   pWçP                  P                  4       8H  p\        P                  ! VR4      P                  RR	7      pVP                  VV4      pV'       d"   VP                  VP!                  4       4       K�  VP                  VP!                  4       4       K¢  	  V F0  p\#        V
4      ^ 8”  g   K  \        P$                  ! Wê4      pV
VR p
K2  	  S'       g   EK!  VP'                  4         VP(                  SV &   SV ,          ^,          '       d   \+        R
V  R24       R# SV ,          ^ ,          ^ 8w  d"   \+        R
V  RSV ,          ^ ,           24       R# \-        SR
V  R24       R#   \         d     EL”i ; i)Ústdin_queueÚstdout_queueÚstderr_queueFr   Ng      ð?é    Úignore)ÚerrorszNode with rank z was killedz exited with code z
 completed)Úpopr   r{   Úset_blockingrA   ÚfilenorB   r@   rZ   Ú
get_nowaitÚ
QueueEmptyr   ÚreadÚdecoder)   ÚputÚencoder~   Úwriter,   r   r   r   )rK   ÚargsÚkwargsr�   r‘   r’   rQ   ÚpÚto_readÚto_writeÚstdin_bufferÚrlistÚwlistrt   Úfdr(   ÚmsgÚnÚcommand_classÚ
exit_codesÚstopÚverboses   &*,               €€€€r   Ú
_thread_fnÚ#_launch_with_io.<locals>._thread_fn®   sP  ø€ Ø—j‘j Ó/ˆØ—z‘z .Ó1ˆØ—z‘z .Ó1ˆá Ð6 tÒ6¨vÑ6ˆØ�O‰OˆÜ
�Š˜Ÿ™Ÿ™Ó)¨5Ô1Ü
�Š˜Ÿ™Ÿ™Ó)¨5Ô1Ü
�Š˜Ÿ™Ÿ™Ó(¨%Ô0à—8‘8—?‘?Ó$ a§h¡h§o¡oÓ&7Ð8ˆØ—G‘G—N‘NÓ$Ð%ˆàˆØ�f‰f‹hÓðØ× 6Ñ 6Ó 8Õ8�ô % W¸¸CÓ@‰OˆE˜!Û�Ø§(¡(§/¡/Ó"3Ñ3�	Ü—g’g˜b $Ó'×.Ñ.°hÐ.Ó?�Ø×/Ñ/°°YÓ?�ßØ ×$Ñ$ S§Z¡Z£\Ö2à ×$Ñ$ S§Z¡Z£\Ö2ñ ó �Ü�|Ó$ qÖ(ÜŸš Ó2�AØ#/°°Ð#3’Lñ ÷ ŠtØ×!Ñ!Ô#ØØ"×.Ñ.ˆ
�4Ñà�dÕ˜A×ÔÜ˜/¨$¨¨{Ð;Ö<Ø˜Õ˜aÕ  AÔ%Ü˜/¨$¨Ð/AÀ*ÈTÕBRÐSTÕBUÐAVÐWÖXä�˜?¨4¨&°
Ð;Ö<øô3 ô Úðús   Ä?K ËK*Ë)K*r�   r‘   r’   )Útargetr    r¡   Tc              3   óJ   "  € T F  qP                  4       '       * x € K  	  R # 5irW   )Úempty)rv   Úqs   & r   rw   Ú"_launch_with_io.<locals>.<genexpr>ó   s   é € ÐUÑ1T¨AŸg™g›iŸ-™-Ó1Tùs   ‚!#gš™™™™™¹?r“   c              3   ó@   "  € T F  qP                  4       x € K  	  R # 5irW   )Úis_alive)rv   Úts   & r   rw   rµ     s   é € Ð-¡W �z‰z�|ˆ|£Wùs   ‚)NNéÿÿÿÿ)r~   Ú	enumerateÚappendr   Ú	threadingÚThreadÚstartr{   r—   Úsysr@   r˜   rA   rB   Úanyr   r   r›   r�   r³   ÚbufferrŸ   r™   rš   Úflushr·   r�   Úget)r«   Ú	argumentsr®   r¯   Ústdin_queuesÚstdout_queuesÚstderr_queuesÚthreadsÚir    r¡   r¸   r¦   rt   r¨   r¥   r´   r¬   r­   s   f&f              @@r   Ú_launch_with_iorÊ   ª   s‹  û€ Ø€DØ�¤# i£.Õ0€J÷+=ð +=ðZ €LØ€MØ€MØ€GÜ& yÖ1Ñˆ‰>ˆDØ×ÑœE›GÔ$Ø×ÑœU›WÔ%Ø×ÑœU›WÔ%Ü×ÒØØØà˜|¨BÕ/Ø ¨bÕ 1Ø ¨bÕ 1ðõô	
ˆð 	
�‰Œ	Ø�‰�qÖñ 2ô" ‡O‚O”C—I‘I×$Ñ$Ó&¨Ô.Ü‡O‚O”C—J‘J×%Ñ%Ó'¨Ô.Ü‡O‚O”C—J‘J×%Ñ%Ó'¨Ô.ß—c“cÑU´°}Ô1TÓU—c—c’cÑU´°}Ô1TÓU×UÓUäœcŸi™i×.Ñ.Ó0Ð1°2°r¸3Ó?‰ˆˆq�!ÛˆBÜŸ7š7 2 tÓ,ˆLÛ!�Ø—‘�lÖ#ó "ñ ó ˆAðØŸ'™'Ÿ)š)Ü—J‘J×%Ñ%×+Ñ+¨A¯L©L«NÖ;ñ $ñ ó ˆAðØŸ'™'Ÿ)š)Ü—J‘J×%Ñ%×+Ñ+¨A¯L©L«NÖ;ñ $ñ ô 	�
‰
×Ñ×ÑÔ!Ü�
‰
×Ñ×ÑÔ!÷ ‹3Ñ-¡WÓ-�3�3Š3Ñ-¡WÓ-×-Ò-Ü! 'Ö*‘�Ø—z‘z—|”|Ø! !•} QÕ'¨1Ö,Ø#˜Ûô	 +ð ó ˆØ	�‰Žñ ó ˆØ—'‘'—)”)Ü�J‰J×Ñ×#Ñ# A§E¡E£GÖ,ñ ó ˆØ—'‘'—)”)Ü�J‰J×Ñ×#Ñ# A§E¡E£GÖ,ñ ô ‡J�J×Ñ×ÑÔÜ‡J�J×Ñ×ÑÖøôC ô Ûðûô ô Ûðús0   ÉS"É9S"ÊS5Ê59S5Ó"S2Ó1S2Ó5TÔTc                 ó`  € \         ;QJ d    R  V 4       F  '       g   K   RM	  RM! R  V 4       4      '       d   V P                  R4       VP                  p. pV Fd  p. pVP                   F>  p\	        VP
                  4       F"  p	VP                  V RV 24       V^,          pK$  	  K@  	  VP                  V4       Kf  	  \        V4      ^8”  d   \        P                  ! V4      MRp
RV
/pVP                  pVP                  '       d   VP                  R4       VP                  p\        VP                  R\        P                  ! V4      4       \!        \"        \%        V4       UUu. uF!  w  ræWæP&                  VP(                  WÛWÃ3/ 3NK#  	  uppVP                  4       R	# u uppi )
c              3   óR   "  € T F  p\        VP                  4      ^ 8H  x € K  	  R# 5i)r   N)r~   Úips©rv   Úhs   & r   rw   Úlaunch_ring.<locals>.<genexpr>%  s   é € Ð
*¡E˜qŒ3ˆq�u‰u‹:˜Ž?£Eùs   ‚%'TFzAThe ring backend requires IPs to be provided instead of hostnamesÚ:r`   ÚMLX_HOSTFILEzMLX_RING_VERBOSE=1ÚRunningN)rÀ   ÚerrorÚstarting_portrÍ   ÚrangeÚconnections_per_ipr»   r~   ÚjsonÚdumpsrP   r®   rN   r   rD   r�   rÊ   r9   rº   Ússh_hostnamerM   )ÚparserÚhostsr    rQ   ÚportÚ
ring_hostsrÏ   ÚnodeÚiprÉ   ÚhostfilerO   rP   rN   rK   s   &&&&           r   Úlaunch_ringrâ   $  s`  € ß
ƒsÑ
*¡EÓ
*‡s‡s‚sÑ
*¡EÓ
*×*Ò*Ø�‰ØOô	
ð ×Ñ€DØ€JÛˆØˆØ—%”%ˆBÜ˜4×2Ñ2Ö3�Ø—‘˜r˜d ! D 6˜NÔ+Ø˜•	’ó 4ñ ð 	×Ñ˜$Öñ ô *-¨Z«¸1Ô)<Œt�zŠz˜*Ô%À"€Hà˜XÐ&€EØ
�(‰(€CØ‡|‡|€|Ø�
‰
Ð'Ô(Ø
�(‰(€Cäˆ�‰�i¤§¢¨GÓ!4Ô5äÜô % UÔ+ô	
á+‘�ð —N‘N D§K¡K°¸SÐJÈBÓOÙ+ò	
ð 	�‰öùó	
s   Å.'F*c                 óà  € V^ ,          P                   '       g   \        R4      hV^ ,          P                   ^ ,          pVP                  p\        V4      pVP                  pVP
                  pVP                  '       d   VP                  R4       VP                  RV 24       VP                  RV 24       VP                  RV 24       \        VP                  R\        P                  ! V4      4       \        \        \        V4       U	U
u. uF?  w  ršV	V
P                  VP                  V/ VRW’P                   ,           2.,           V3/ 3NKA  	  up
p	VP                  4       R# u up
p	i )	r   ú7Rank 0 should have an IP reachable from all other rankszNCCL_DEBUG=INFOzNCCL_HOST_IP=z
NCCL_PORT=zMLX_WORLD_SIZE=rÓ   zCUDA_VISIBLE_DEVICES=N)rÍ   Ú
ValueErrorÚ	nccl_portr~   rP   rN   r®   r»   r   rD   r�   rÊ   r9   rº   rÚ   rM   Úrepeat_hosts)rÛ   rÜ   r    rQ   Úmaster_hostÚmaster_portÚ
world_sizerP   rN   rK   rÏ   s   &&&&       r   Úlaunch_ncclrë   G  s<  € Ø��8�<�<ˆ<ÜÐRÓSÐSà˜•(—,‘,˜q•/€KØ—.‘.€KÜ�U“€Jà
�(‰(€CØ
�(‰(€CØ‡|‡|€|Ø�
‰
Ð$Ô%Ø‡J�J�˜{˜mÐ,Ô-Ø‡J�J�˜K˜=Ð)Ô*Ø‡J�J�  Ð-Ô.äˆ�‰�i¤§¢¨GÓ!4Ô5äÜô % UÔ+ô	
ñ ,‘�ð Ø—N‘NØ—K‘KØØØÐ2°4×:KÑ:KÕ3KÐ2LÐMÐNÕNØðð óñ ,ò	
ð 	�‰ö#ùó	
s   ÄAE*c                 óî  a€ S^ ,          P                   '       g   \        R4      hVP                  R8H  p\        ;QJ d    V3R lS 4       F  '       d   K   RM	  RM! V3R lS 4       4      p\        ;QJ d%    R \	        S4       4       F  '       d   K   RM	  RM! R \	        S4       4       4      pV'       d	   V'       g   \        R4      hS^ ,          P                   ^ ,          pVP
                  pVP                  p	VP                  RV R	VP                   24       V'       d   VP                  R
4       R\        P                  ! S U
u. uF  qªP                  NK  	  up
4      /p\        VP                  R\        P                  ! V4      4       \!        \"        \	        S4       UU
u. uF!  w  rÊWÊP$                  VP&                  W›Wƒ3/ 3NK#  	  up
pVP                  4       R# u up
i u up
pi )r   rä   ú
jaccl-ringc              3   óf   <"  € T F&  p\        VP                  4      \        S4      8H  x € K(  	  R # 5irW   )r~   Úrdma)rv   rÏ   rÜ   s   & €r   rw   Úlaunch_jaccl.<locals>.<genexpr>s  s"   øé € Ð>¹°1”S˜Ÿ™“[¤C¨£JÖ.»ùs   ƒ.1FTc              3   óP   "  € T F  w  rVP                   V,          R J x € K  	  R # 5irW   )rï   )rv   rÉ   rÏ   s   &  r   rw   rð   t  s!   é € ÐDÑ3C©4¨1�Q—V‘V˜A•Y $Õ&Ó3Cùs   ‚$&z$Malformed hostfile for jaccl backendzMLX_JACCL_COORDINATOR=rÑ   zMLX_JACCL_RING=1ÚMLX_IBV_DEVICESrÓ   N)rÍ   rå   Úbackendr   rº   rP   rN   r»   rÕ   rØ   rÙ   rï   r   r®   rD   r�   rÊ   r9   rÚ   rM   )rÛ   rÜ   r    rQ   Ú
jaccl_ringÚ
have_rdmasÚ
have_nullsÚcoordinatorrP   rN   rÏ   rO   rK   s   &f&&         r   Úlaunch_jacclrø   n  so  ø€ Ø��8�<�<ˆ<ÜÐRÓSÐSà—‘ Ñ-€Jß“Ô>¹Ó>——’Ô>¹Ó>Ó>€Jß“ÑD´9¸UÔ3CÓD——’ÑD´9¸UÔ3CÓDÓD€JßŸZÜÐ?Ó@Ð@à˜•(—,‘,˜q•/€KØ
�(‰(€CØ
�(‰(€CØ‡J�JÐ'¨ }°A°d×6HÑ6HÐ5IÐJÔKßØ�
‰
Ð%Ô&Ø¤§
¢
¹EÓ+B¹E°q¯F¬F¹EÑ+BÓ CÐD€Eäˆ�‰�i¤§¢¨GÓ!4Ô5äÜô % UÔ+ô	
á+‘�ð —N‘N D§K¡K°¸SÐJÈBÓOÙ+ò	
ð 	�‰öùò	 ,Cùó	
s   ÅG,
Æ0'G1c                  óÞ  €  \        R R.RRR7      p V P                  P                  4       P                  4       p \        P                  4       R8X  d   \        RRV .RRR7      pM\        RV .RRR7      pVP                  P                  4       p\        \        R VP                  4       4      4      pVP                  4       P                  4       ^ ,          P                  R	4      #     R
# ; i)ÚwhichÚ	ompi_infoT©rf   rg   ÚDarwinÚotoolz-LÚlddc                 ó   € R V 9   # )Úlibmpir.   )Úlines   &r   Ú<lambda>Ú!get_mpi_libname.<locals>.<lambda>›  s	   €  ¨DÒ 0r   z@rpath/N)r   rA   rl   rœ   ÚplatformÚsystemÚnextÚfilterÚ
splitlinesra   Úremoveprefix)rû   Úotool_outputÚlibmpi_lines      r   Úget_mpi_libnamer  Œ  sÛ   € ðÜ˜ +Ð.°dÈ4ÔPˆ	Ø×$Ñ$×*Ñ*Ó,×3Ñ3Ó5ˆ	ä�?‰?Ó Ô(ÜØ˜$ 	Ð*°$Àtô‰Lô   yÐ1¸ÈdÔSˆLØ#×*Ñ*×1Ñ1Ó3ˆô ÜÑ0°,×2IÑ2IÓ2KÓLó
ˆð × Ñ Ó"×(Ñ(Ó*¨1Õ-×:Ñ:¸9ÓEÐEøðÚús   ‚C$C' Ã'C,c                 ó"  € \        R R.RRR7      pVP                  P                  4       P                  4       p\	        4       pVeP   \        V4      P                  P                  R,          pR\        V4       2RV 2.VP                  ,           Vn        \        VP                  RV R	24       \        P                  ! R
R7      ;_uu_ 4       p\        R V 4       4      pVP                  4        F  w  r‰\        V RV	 2VR7       K  	  VP!                  4        VRRRVP"                  .VP$                  '       d   RVP$                  .M. O\'        R VP                   4       . 4      O\'        VP(                   U
u. uF  p
\*        P,                  ! V
4      NK  	  up
. 4      ORNVOp\        VP                  RRP/                  V4      4        \        V4       RRR4       R# u up
i   \0         d     Li ; i  + '       g   i     R# ; i)rú   ÚmpirunTrü   NÚlibzDYLD_LIBRARY_PATH=zMLX_MPI_LIBNAME=zUsing 'ry   Úw)Úmodec              3   ó8   "  € T F  qP                   x € K  	  R # 5irW   )rÚ   rÎ   s   & r   rw   Úlaunch_mpi.<locals>.<genexpr>±  s   é € Ð7±¨AŸž³ùs   ‚z slots=)Úfilez--outputz:rawú
--hostfilez-cwdc              3   ó*   "  € T F	  pR V.x € K  	  R# 5i)z-xNr.   )rv   r…   s   & r   rw   r  ½  s   é € Ð.¡X �4˜•)£Xùs   ‚ú--rÓ   r<   )r   rA   rl   rœ   r  r   Úparentr"   rP   r   r®   ÚtempfileÚNamedTemporaryFiler   r€   ÚprintrÂ   ÚnamerN   ÚsumÚmpi_argrD   ra   r�   ÚKeyboardInterrupt)rÛ   rÜ   r    rQ   r  Úmpi_libnameÚdyldÚfrÏ   rª   ÚargrS   s   &&&&        r   Ú
launch_mpir%  ¢  sÝ  € Ü�'˜8Ð$¨DÀÔF€FØ�]‰]× Ñ Ó"×)Ñ)Ó+€Fô "Ó#€KØÒÜ�F‹|×"Ñ"×)Ñ)¨EÕ1ˆà ¤ T£ Ð,Ø˜{˜mÐ,ð
ð �H‰HõˆŒô
 ˆ�‰˜ ˜x qÐ)Ô*Ü	×	$Ò	$¨#×	.Õ	.°!ÜÑ7±Ó7Ó8ˆØ—K‘K–M‰DˆAÜ�Q�C�w˜q˜cÐ"¨×+ñ "à	�‰Œ	ð ØØØØ�F‰Fð
ð %)§H§H Hˆv�t—x‘xÑ °"ð
ô Ñ. T§X¢XÓ.°Ó3ð
ô ¨d¯lªlÓ;©l s”%—+’+˜cÖ"©lÑ;¸RÓ@ð
ð ð
ð ð
ˆô 	ˆD�L‰L˜) S§X¡X¨c£]Ô3ð	Ü�ŒH÷) 
/Ñ	.ùò <øô !ô 	Ùð	ú÷+ 
/×	.Ð	.úsD   ÃA/G=Ä?A G=Å?G'Æ3G=ÇG,Ç'G=Ç,G:Ç7G=Ç9G:Ç:G=Ç=H	c                  ód  € \         P                  ! R R7      p V P                  RRRR7       V P                  RRRR7       V P                  RR	R
R7       V P                  RR\        ^RR7       V P                  RRR7       V P                  RRR7       V P                  RR. RR7       V P                  RR. RR7       V P                  R^\        RR7       V P                  RR\        R R!R7       V P                  R"R#R7       V P                  R$\        R%R&R7       V P                  R'R(R)R*R+7       V P                  R,\
        P                  R-R7       V P                  4       w  rVP                  '       d   \        VP                  4       R.# \        V4      ^ 8X  d   V P                  R/4       V^ ,          R08X  d   VP                  ^ 4       VP                  e"   \        P                   ! VP                  4      pM+\        P"                  ! VP$                  VP&                  4      pVP(                  R18w  d    VP(                  f   VP(                  Vn        VP(                  f.   \*        P,                  P/                  4       '       d   R2MR3Vn        VP0                  VP2                  ,           Vn        \5        V^ ,          4      ;pP7                  4       '       d@   VP9                  4       '       d*   VP                  \;        VP=                  4       4      .VR4&   MM\>        P@                  ! V^ ,          4      ;pe   WR^ &   M'VPB                  '       d   \E        R5V^ ,           24      hVP(                  R38X  d   \G        WP$                  W4       R.# VP(                  R68X  d   \I        WP$                  W4       R.# VP(                  R28X  d   \K        WP$                  W4       R.# VP(                  R78X  g   VP(                  R88X  d   \M        WP$                  W4       R.# V P                  R94       R.# ):z!Launch an MLX distributed program)Údescriptionz--print-pythonÚ
store_truez8Print the path to the current python executable and exit)ÚactionÚhelpz	--verbosezPrint debug messages in stdoutz--hostsr;   zA comma separated list of hosts)Údefaultr*  z--repeat-hostsz-nz(Repeat each host a given number of times)Útyper+  r*  r  zThe file containing the hosts)r*  z	--backendz#Which distributed backend to launchz--envr»   z&Set environment variables for the jobs)r)  r+  r*  z	--mpi-argz$Arguments to pass directly to mpirunz--connections-per-ipz7How many connections per ip to use for the ring backend)r+  r,  r*  z--starting-portz-piC~  zHFor the ring backend listen on this port increasing by 1 per rank and IPz--cwdz:Set the working directory on each node to the provided onez--nccl-porti90  zBThe port to use for the NCCL communication (only for nccl backend)z--no-verify-scriptÚstore_falseÚverify_scriptz$Do not verify that the script exists)r)  Údestr*  z--pythonz#Use this python on the remote hostsNzNo script is providedr  r`   ÚncclÚring:r   é   NzInvalid script or command ÚmpiÚjacclrí   zKThe backend should be one of {'ring', 'mpi', 'nccl', 'jaccl', 'jaccl-ring'})'ÚargparseÚArgumentParserÚadd_argumentr   Úintr¿   r?   Úparse_known_argsÚprint_pythonr  rM   r~   rÔ   r–   rá   r   Ú	from_fileÚ	from_listrÜ   rç   ró   ÚmxÚcudaÚis_availableÚenvsrP   r   ÚexistsÚis_filer"   ÚresolveÚshutilrú   r.  rå   râ   r%  rë   rø   )rÛ   r    rc   rá   rƒ   rQ   s         r   ÚmainrE  É  sÅ  € Ü×$Ò$Ð1TÔU€FØ
×ÑØØØGð ô ð
 ×ÑØ˜LÐ/Oð ô ð ×ÑØ˜;Ð-Nð ô ð ×ÑØØÜØØ7ð ô ð ×Ñ˜Ð+JÐÔKØ
×ÑØØ2ð ô ð ×ÑØØØØ5ð	 ô ð ×ÑØØØØ3ð	 ô ð ×ÑØØÜØFð	 ô ð ×ÑØØÜØØWð ô ð ×ÑØÐRð ô ð ×ÑØÜØØQð	 ô ð ×ÑØØØØ3ð	 ô ð ×ÑØœCŸN™NÐ1Vð ô ð ×(Ñ(Ó*�J€Dà××ÐÜˆd�k‰kÔÙä
ˆ4ƒy�A„~Ø�‰Ð,Ô-ØˆA…w�$„Ø�‰�Œð ‡}�}Ò Ü×%Ò% d§m¡mÓ4‰ä×%Ò% d§j¡j°$×2CÑ2CÓDˆð ×Ñ˜2Ô $§,¡,Ò"6Ø×'Ñ'ˆŒØ‡|�|ÒÜ!#§¡×!5Ñ!5×!7Ò!7‘v¸VˆŒØ�}‰}˜tŸx™xÕ'€D„Hô �t˜A•w“-Ðˆ×'Ñ'×)Ò)¨f¯n©n×.>Ò.>Ø—[‘[¤# f§n¡nÓ&6Ó"7Ð8ˆˆSŠ	Ü—\’\ $ q¥'Ó*Ð
*ˆ'Ò	7ØˆQŠØ	×	×	Ð	ÜÐ5°d¸1µg°YÐ?Ó@Ð@ð ‡|�|�vÔÜ�FŸN™N¨DÖ7Ø	�‰˜Ô	Ü�6Ÿ>™>¨4Ö6Ø	�‰˜Ô	Ü�FŸN™N¨DÖ7Ø	�‰˜Ô	  D§L¡L°LÔ$@Ü�VŸ^™^¨TÖ8à�‰ØYö	
r   ))r5  Úbase64rØ   r{   rD   rD  r¿   r  r¼   Úcollectionsr   Ú	itertoolsr   Úpathlibr   Úqueuer   rš   r   r   Ú
subprocessr	   r
   r   Úmlx.coreÚcorer=  Úcommonr   r   r   r   r   r9   rÊ   râ   rë   rø   r  r%  rE  r.   r   r   Ú<module>rO     s…   ðó Û Û Û 	Û Û Û 
Û Û Ý Ý Ý Ý %Ý Ý ß 'Ñ 'å ç ?Ó ?÷$ñ $ô,x�Nô xòvwòt òF$òNò<ò,$ôNq
r   