+
    &jaE                     "   ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	H
t
 ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHtHtHt ^ RIHt ^R	IHtHtHtHt  ! R
 R4      t  ! R R] 4      t!R t"R t#R t$R t%R t&R t'R t(R# )    N)Counter)chain)Path)Empty)Queue)select)PIPEPopenrun)Hostfileloglog_warningpositive_numberc                   \   a  ] tR t^t o ]R 4       t]R 4       tRV 3R lR lltR tRt	V t
R# )	CommandProcessc                    \        4       h)z;Return the Popen object that refers to the current command.NotImplementedErrorselfs   &u/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/mlx/_distributed_utils/launch.pyprocessCommandProcess.process        "##    c                    \        4       h)zuReturn a tuple (returncode, killed) for the command. It should be
(None, None) while the command is running normally.r   r   s   &r   exit_statusCommandProcess.exit_status   s     "##r   c                    < V ^8  d   QhRS[ /# )   data)str)format__classdict__s   "r   __annotate__CommandProcess.__annotate__%   s     $ $c $r   c                    \        4       h)zhPreprocess the output of the command so that extra data can be
capture or the format changed on the fly.r   )r   r!   	is_stdouts   &&&r   preprocess_output CommandProcess.preprocess_output%   r   r   c                    \        4       h)z"Terminate or return the exit code.r   r   s   &r   	terminateCommandProcess.terminate*   s    !##r    NF)__name__
__module____qualname____firstlineno__propertyr   r   r)   r,   __static_attributes____classdictcell__r$   s   @r   r   r      s?     $ $ $ $
$ $
$ $r   r   c                   v   a  ] tR t^/t o R t]R 4       t]R 4       tR
R ltR t	]
R 4       t]
R 4       tRtV tR	# )RemoteProcessc           	         VR 8H  p\         P                  WWVWx4      p	V'       g   RV R\        P                  ! V	4       2p	W n        RV n        Wn        \        V	RR\        \        \        R7      V n	        RV n
        R# )	127.0.0.1zssh -tt -o LogLevel=QUIET  NT	/bin/bash)shell
executablestdinstdoutstderrF)r9   make_launch_scriptshlexquote_host_pidfile	_is_localr
   r	   _process_killed)
r   rankhostpythoncwdfilesenvcommandis_localcmds
   &&&&&&&&  r   __init__RemoteProcess.__init__0   sw    ;&..t%gX.tfAekk#6F5GHC
!"
 r   c                    V P                   # N)rI   r   s   &r   r   RemoteProcess.processD   s    }}r   c                N    V P                   P                  4       V P                  3# rW   )rI   pollrJ   r   s   &r   r   RemoteProcess.exit_statusH   s    }}!!#T\\11r   c                |    V P                   f.   VP                  R^R7      vr4W0n         V'       d
   V^ ,          # R# V# )N
maxsplit )rG   split)r   r!   r(   pidfilerests   &&&  r   r)   RemoteProcess.preprocess_outputL   s;    == !ZZqZ9NG#M"47**r   c           	        V P                   '       d   R # V P                  P                  4        V P                  P                  4        \        P                  V P                  4      pV P                  '       g'   RV P                   R\        P                  ! V4       2p\        VRRRRRR7      pVP                  P                  4       R8H  V n         R # )Nzssh r<   Tr=   )checkr>   r?   capture_outputtext1)rJ   rI   r,   waitr9   make_kill_scriptrG   rH   rF   rD   rE   r   rA   strip)r   rS   cs   &  r   r,   RemoteProcess.terminateT   s    <<<! ,,T]];~~~Aekk#&6%78C"
 xx~~'3.r   c                   R pV'       g   RpVR,          pVR,          pVR,          pT;'       g    \         P                  ! 4       pVR\        V4       R2,          pVR\        V4       R2,          pVe    VR	,          pVR
\        V4       R2,          pVR,          pV F  pVP                  R^R7      vr\	        V
4      ^ 8  d   \
        P                  ! V
^ ,          4      MR p
\        ;QJ d    R V	 4       F  '       d   K   RM	  RM! R V	 4       4      '       g   \        RV R24       K  VRV	 RV
 R2,          pK  	  VP                  4        F<  w  rVR,          pVR\
        P                  ! V4       R2,          pVRV R2,          pK>  	  VRV  R2,          pVRRP                  \        \
        P                  V4      4       R2,          pVR,          pV# )r`   zstty -echo; zpidfile=$(mktemp); zecho $$ > $pidfile; zprintf "%s\n" $pidfile; z	if [[ -d z
 ]]; then z  cd z; else z& echo 'Failed to change directory to' z >2; fi; =r^   c              3   Z   "   T F!  qP                  4       ;'       g    VR 8H  x  K#  	  R# 5i)_N)isalnum).0rm   s   & r   	<genexpr>3RemoteProcess.make_launch_script.<locals>.<genexpr>   s#     <1yy{..a3h.s   ++FT'z5' is an invalid environment variable so it is ignoredzexport zfname=$(mktemp); zecho z
 >$fname; z	=$fname; zexport MLX_RANK=zcmd=(r<   ); zexec "${cmd[@]}")osgetcwdreprra   lenrD   rE   allr   itemsjoinmap)rK   rN   rO   rP   rQ   rR   scriptdekeyvalueenv_namecontents   &&&&&&       r   rC    RemoteProcess.make_launch_scriptj   s    #F 	''((-- 299;Id1gYj11E$q'"%%?gF>tAwiuMMF& A''#'2KC-0Z!^EKKa)E3<<333<<<<sOP uAeWB//F  "'H))Fekk'23:>>Fz33F "/ 	$TF"-- 	E#((3u{{G#<=>cBB$$r   c                    R pVRV  R2,          pVR,          pVR,          pVR,          pVR,          pVR,          pVR,          pVR	V  2,          pV# )
r`   z
pid=$(cat rz   zif ps -p $pid >/dev/null; then z    kill $pid; z    echo 1; rp   z    echo 0; rq   zrm r.   )rb   r   s   & r   rk   RemoteProcess.make_kill_script   sm    Jwis++33##. '. &Cy/!r   )rF   rH   rJ   rG   rI   Nr/   )r0   r1   r2   r3   rT   r4   r   r   r)   r,   staticmethodrC   rk   r5   r6   r7   s   @r   r9   r9   /   se     (   2 2/, . .`  r   r9   c                 
  a aaa R oR.\        V4      ,          oV VVV3R lp. p. p. p. p\        V4       F  w  pw  rVP                  \        4       4       VP                  \        4       4       VP                  \        4       4       \        P
                  ! VV	V
RVR,          RVR,          RVR,          /,          R7      pVP                  4        VP                  V4       K  	  \        P                  ! \        P                  P                  4       R 4       \        P                  ! \        P                  P                  4       R4       \        P                  ! \        P                  P                  4       R4       S'       dN   \        ;QJ d%    R \        WV4       4       F  '       g   K   RM	  R M! R \        WV4       4       4      '       Ed   \!        \        P                  P                  4       .. . R	4      w  p pV F4  p\        P"                  ! VR
4      pV F  pVP%                  V4       K  	  K6  	  V FS  p VP'                  4       '       g:   \        P                  P(                  P+                  VP-                  4       4       KO  KU  	  V FS  p VP'                  4       '       g:   \        P                  P(                  P+                  VP-                  4       4       KO  KU  	  \        P                  P(                  P1                  4        \        P                  P(                  P1                  4        \        ;QJ d    R V 4       F  '       g   K   RM	  R M! R V 4       4      '       dK   \        V4       F8  w  rVP3                  4       '       d   K  SV,          ^ ,          ^ 8w  g   K4  Ro EK?  	  EKD   V F  pVP5                  4        K  	  V FR  pVP'                  4       '       d   K  \        P                  P(                  P+                  VP7                  4       4       KQ  	  V FR  pVP'                  4       '       d   K  \        P                  P(                  P+                  VP7                  4       4       KQ  	  \        P                  P(                  P1                  4        \        P                  P(                  P1                  4        R#   \.         d     EK  i ; i  \.         d     EKi  i ; i)FNc                   < VP                  R 4      pVP                  R4      pVP                  R4      pS! V .VO5/ VB pVP                  p\        P                  ! VP                  P                  4       R4       \        P                  ! VP                  P                  4       R4       \        P                  ! VP                  P                  4       R4       VP                  P                  4       VP                  P                  4       .pVP                  P                  4       .p	Rp
VP                  4       Ef     WP                  4       ,          p
\        W. R4      w  rpV F  pWP                  P                  4       8H  p\        P                  ! VR4      P                  RR	7      pVP                  VV4      pV'       d"   VP                  VP!                  4       4       K  VP                  VP!                  4       4       K  	  V F0  p\#        V
4      ^ 8  g   K  \        P$                  ! W4      pV
VR p
K2  	  S'       g   EK!  VP'                  4         VP(                  SV &   SV ,          ^,          '       d   \+        R
V  R24       R# SV ,          ^ ,          ^ 8w  d"   \+        R
V  RSV ,          ^ ,           24       R# \-        SR
V  R24       R#   \         d     ELi ; i)stdin_queuestdout_queuestderr_queueFr   Ng      ?    ignore)errorszNode with rank z was killedz exited with code z
 completed)popr   r{   set_blockingrA   filenorB   r@   rZ   
get_nowait
QueueEmptyr   readdecoder)   putencoder~   writer,   r   r   r   )rK   argskwargsr   r   r   rQ   pto_readto_writestdin_bufferrlistwlistrt   fdr(   msgncommand_class
exit_codesstopverboses   &*,               r   
_thread_fn#_launch_with_io.<locals>._thread_fn   sP   jj/zz.1zz.16t6v6OO
)51
)51
(%088??$ahhoo&78GGNN$%ffh 6 6 88 %WC@OE!((//"33	ggb$'..h.?//Y? $$SZZ\2 $$SZZ\2  |$q(2A#/#3L  t!!#"..
4dA/${;<a A%/$/A*TBRSTBUAVWX?4&
;<3  s   ?K K*)K*r   r   r   )targetr   r   Tc              3   J   "   T F  qP                  4       '       * x  K  	  R # 5irW   )empty)rv   qs   & r   rw   "_launch_with_io.<locals>.<genexpr>   s     U1TAggi--1Ts   !#g?r   c              3   @   "   T F  qP                  4       x  K  	  R # 5irW   )is_alive)rv   ts   & r   rw   r     s     -Wzz||Ws   )NN)r~   	enumerateappendr   	threadingThreadstartr{   r   sysr@   r   rA   rB   anyr   r   r   r   r   bufferr   r   r   flushr   r   get)r   	argumentsr   r   stdin_queuesstdout_queuesstderr_queuesthreadsir   r   r   r   rt   r   r   r   r   r   s   f&f              @@r   _launch_with_ior      s   D#i.0J+= +=Z LMMG&y1>DEG$UW%UW%|B/b 1b 1	
 	
	q 2" OOCII$$&.OOCJJ%%'.OOCJJ%%'.ccU}1TUcccU}1TUUUcii..012r3?q!B772t,L!l# "  A''))JJ%%++ALLN; $  A''))JJ%%++ALLN; $  	

!

! 3-W-333-W---!'*zz||!!}Q'1,#	 +  	  ''))JJ##AEEG,  ''))JJ##AEEG,  JJJJC    s0   S"9S"S559S5"S21S25TTc                 `   \         ;QJ d    R  V 4       F  '       g   K   RM	  RM! R  V 4       4      '       d   V P                  R4       VP                  p. pV Fd  p. pVP                   F>  p\	        VP
                  4       F"  p	VP                  V RV 24       V^,          pK$  	  K@  	  VP                  V4       Kf  	  \        V4      ^8  d   \        P                  ! V4      MRp
RV
/pVP                  pVP                  '       d   VP                  R4       VP                  p\        VP                  R\        P                  ! V4      4       \!        \"        \%        V4       UUu. uF!  w  rWP&                  VP(                  WW3/ 3NK#  	  uppVP                  4       R	# u uppi )
c              3   R   "   T F  p\        VP                  4      ^ 8H  x  K  	  R# 5i)r   N)r~   ipsrv   hs   & r   rw   launch_ring.<locals>.<genexpr>%  s     
*Eq3quu:?Es   %'TFzAThe ring backend requires IPs to be provided instead of hostnames:r`   MLX_HOSTFILEzMLX_RING_VERBOSE=1RunningN)r   errorstarting_portr   rangeconnections_per_ipr   r~   jsondumpsrP   r   rN   r   rD   r   r   r9   r   ssh_hostnamerM   )parserhostsr   rQ   port
ring_hostsr   nodeipr   hostfilerO   rP   rN   rK   s   &&&&           r   launch_ringr   $  s`   
s
*E
*sss
*E
***O	
 DJ%%B4223rd!D6N+	 4  	$  *-Z1)<tzz*%"HX&E
((C|||

'(
((CiG!45 %U+	
+ NNDKKSJBO+	
 		
s   .'F*c                    V^ ,          P                   '       g   \        R4      hV^ ,          P                   ^ ,          pVP                  p\        V4      pVP                  pVP
                  pVP                  '       d   VP                  R4       VP                  RV 24       VP                  RV 24       VP                  RV 24       \        VP                  R\        P                  ! V4      4       \        \        \        V4       U	U
u. uF?  w  rV	V
P                  VP                  V/ VRWP                   ,           2.,           V3/ 3NKA  	  up
p	VP                  4       R# u up
p	i )	r   7Rank 0 should have an IP reachable from all other rankszNCCL_DEBUG=INFOzNCCL_HOST_IP=z
NCCL_PORT=zMLX_WORLD_SIZE=r   zCUDA_VISIBLE_DEVICES=N)r   
ValueError	nccl_portr~   rP   rN   r   r   r   rD   r   r   r9   r   r   rM   repeat_hosts)r   r   r   rQ   master_hostmaster_port
world_sizerP   rN   rK   r   s   &&&&       r   launch_ncclr   G  s<   8<<<RSS(,,q/K..KUJ
((C
((C|||

$%JJ{m,-JJK=)*JJ-.iG!45 %U+	
 , NNKK24:K:K3K2LMNN  ,	
 	#	
s   AE*c                   a S^ ,          P                   '       g   \        R4      hVP                  R8H  p\        ;QJ d    V3R lS 4       F  '       d   K   RM	  RM! V3R lS 4       4      p\        ;QJ d%    R \	        S4       4       F  '       d   K   RM	  RM! R \	        S4       4       4      pV'       d	   V'       g   \        R4      hS^ ,          P                   ^ ,          pVP
                  pVP                  p	VP                  RV R	VP                   24       V'       d   VP                  R
4       R\        P                  ! S U
u. uF  qP                  NK  	  up
4      /p\        VP                  R\        P                  ! V4      4       \!        \"        \	        S4       UU
u. uF!  w  rWP$                  VP&                  WW3/ 3NK#  	  up
pVP                  4       R# u up
i u up
pi )r   r   
jaccl-ringc              3   f   <"   T F&  p\        VP                  4      \        S4      8H  x  K(  	  R # 5irW   )r~   rdma)rv   r   r   s   & r   rw   launch_jaccl.<locals>.<genexpr>s  s"     >1S[CJ.s   .1FTc              3   P   "   T F  w  rVP                   V,          R J x  K  	  R # 5irW   )r   )rv   r   r   s   &  r   rw   r   t  s!     D3C41QVVAY$&3Cs   $&z$Malformed hostfile for jaccl backendzMLX_JACCL_COORDINATOR=r   zMLX_JACCL_RING=1MLX_IBV_DEVICESr   N)r   r   backendr   r   rP   rN   r   r   r   r   r   r   r   rD   r   r   r9   r   rM   )r   r   r   rQ   
jaccl_ring
have_rdmas
have_nullscoordinatorrP   rN   r   rO   rK   s   &f&&         r   launch_jacclr   n  so   8<<<RSS-J>>>>>JD9U3CDD9U3CDDJZ?@@(,,q/K
((C
((CJJ'}Ad6H6H5IJK

%&

E+BEqFFE+B CDEiG!45 %U+	
+ NNDKKSJBO+	
 		 ,C	
s   G,
0'G1c                      \        R R.RRR7      p V P                  P                  4       P                  4       p \        P                  4       R8X  d   \        RRV .RRR7      pM\        RV .RRR7      pVP                  P                  4       p\        \        R VP                  4       4      4      pVP                  4       P                  4       ^ ,          P                  R	4      #     R
# ; i)which	ompi_infoTrf   rg   Darwinotoolz-Llddc                     R V 9   # )libmpir.   )lines   &r   <lambda>!get_mpi_libname.<locals>.<lambda>  s	    D 0r   z@rpath/N)r   rA   rl   r   platformsystemnextfilter
splitlinesra   removeprefix)r   otool_outputlibmpi_lines      r   get_mpi_libnamer    s    +.d4P	$$**,335	??($	*$tL y1dSL#**113 0,2I2I2KL
   "((*1-::9EEs   C$C' 'C,c                 "   \        R R.RRR7      pVP                  P                  4       P                  4       p\	        4       pVeP   \        V4      P                  P                  R,          pR\        V4       2RV 2.VP                  ,           Vn        \        VP                  RV R	24       \        P                  ! R
R7      ;_uu_ 4       p\        R V 4       4      pVP                  4        F  w  r\        V RV	 2VR7       K  	  VP!                  4        VRRRVP"                  .VP$                  '       d   RVP$                  .M. O\'        R VP                   4       . 4      O\'        VP(                   U
u. uF  p
\*        P,                  ! V
4      NK  	  up
. 4      ORNVOp\        VP                  RRP/                  V4      4        \        V4       RRR4       R# u up
i   \0         d     Li ; i  + '       g   i     R# ; i)r   mpirunTr   NlibzDYLD_LIBRARY_PATH=zMLX_MPI_LIBNAME=zUsing 'ry   w)modec              3   8   "   T F  qP                   x  K  	  R # 5irW   )r   r   s   & r   rw   launch_mpi.<locals>.<genexpr>  s     7As   z slots=)filez--outputz:raw
--hostfilez-cwdc              3   *   "   T F	  pR V.x  K  	  R# 5i)z-xNr.   )rv   r   s   & r   rw   r    s     .X4)Xs   --r   r<   )r   rA   rl   r   r  r   parentr"   rP   r   r   tempfileNamedTemporaryFiler   r   printr   namerN   summpi_argrD   ra   r   KeyboardInterrupt)r   r   r   rQ   r  mpi_libnamedyldfr   r   argrS   s   &&&&        r   
launch_mpir%    s   '8$DFF]]  "))+F "#KF|""))E1 T,{m,
 HH
 xq)*		$	$#	.	.!778KKMDAQCwqc"+ "		 FF
 %)HHHvtxx "
 .TXX.3
 dll;ls%++c"l;R@
 
 
 	DLL)SXXc]3	H) 
/	. < ! 		+ 
/	.	.sD   A/G=?A G=?G'3G=G,'G=,G:7G=9G::G==H	c                  d   \         P                  ! R R7      p V P                  RRRR7       V P                  RRRR7       V P                  RR	R
R7       V P                  RR\        ^RR7       V P                  RRR7       V P                  RRR7       V P                  RR. RR7       V P                  RR. RR7       V P                  R^\        RR7       V P                  RR\        R R!R7       V P                  R"R#R7       V P                  R$\        R%R&R7       V P                  R'R(R)R*R+7       V P                  R,\
        P                  R-R7       V P                  4       w  rVP                  '       d   \        VP                  4       R.# \        V4      ^ 8X  d   V P                  R/4       V^ ,          R08X  d   VP                  ^ 4       VP                  e"   \        P                   ! VP                  4      pM+\        P"                  ! VP$                  VP&                  4      pVP(                  R18w  d    VP(                  f   VP(                  Vn        VP(                  f.   \*        P,                  P/                  4       '       d   R2MR3Vn        VP0                  VP2                  ,           Vn        \5        V^ ,          4      ;pP7                  4       '       d@   VP9                  4       '       d*   VP                  \;        VP=                  4       4      .VR4&   MM\>        P@                  ! V^ ,          4      ;pe   WR^ &   M'VPB                  '       d   \E        R5V^ ,           24      hVP(                  R38X  d   \G        WP$                  W4       R.# VP(                  R68X  d   \I        WP$                  W4       R.# VP(                  R28X  d   \K        WP$                  W4       R.# VP(                  R78X  g   VP(                  R88X  d   \M        WP$                  W4       R.# V P                  R94       R.# ):z!Launch an MLX distributed program)descriptionz--print-python
store_truez8Print the path to the current python executable and exit)actionhelpz	--verbosezPrint debug messages in stdoutz--hostsr;   zA comma separated list of hosts)defaultr*  z--repeat-hostsz-nz(Repeat each host a given number of times)typer+  r*  r  zThe file containing the hosts)r*  z	--backendz#Which distributed backend to launchz--envr   z&Set environment variables for the jobs)r)  r+  r*  z	--mpi-argz$Arguments to pass directly to mpirunz--connections-per-ipz7How many connections per ip to use for the ring backend)r+  r,  r*  z--starting-portz-piC~  zHFor the ring backend listen on this port increasing by 1 per rank and IPz--cwdz:Set the working directory on each node to the provided onez--nccl-porti90  zBThe port to use for the NCCL communication (only for nccl backend)z--no-verify-scriptstore_falseverify_scriptz$Do not verify that the script exists)r)  destr*  z--pythonz#Use this python on the remote hostsNzNo script is providedr  r`   ncclring:r      NzInvalid script or command mpijacclr   zKThe backend should be one of {'ring', 'mpi', 'nccl', 'jaccl', 'jaccl-ring'})'argparseArgumentParseradd_argumentr   intr   r?   parse_known_argsprint_pythonr  rM   r~   r   r   r   r   	from_file	from_listr   r   r   mxcudais_availableenvsrP   r   existsis_filer"   resolveshutilr   r.  r   r   r%  r   r   )r   r   rc   r   r   rQ   s         r   mainrE    s   $$1TUF
G  
 L/O   ;-N   7   +JK
2   5	   3	   F	   W   R   Q	   3	   CNN1V   ((*JDdkk
4yA~,-Aw$ }} %%dmm4%%djj$2C2CD 2$,,"6''||!#!5!5!7!7vV}}txx'DH tAw-''))fnn.>.>[[#fnn&6"78S	\\$q'*
*'	7Q				5d1gY?@@ ||vFNND7		6>>46		FNND7		 DLLL$@V^^T8Y	
r   ))r5  base64r   r{   rD   rD  r   r  r   collectionsr   	itertoolsr   pathlibr   queuer   r   r   r   
subprocessr	   r
   r   mlx.corecorer=  commonr   r   r   r   r   r9   r   r   r   r   r  r%  rE  r.   r   r   <module>rO     s       	   
      %   ' '  ? ?$ $,xN xvwt F$N<,$Nq
r   