+
    &jLN                        ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RI	H
t
Ht ^ RIHt ^ RIHt ^RIHtHtHtHtHtHt ] ! R R4      4       t] ! R	 R
4      4       t] ! R R4      4       tR!R ltR tR"R ltR!R lt ! R R4      tR t R t!R t"R t#R t$R#R lt%R#R lt&R!R lt'R t(R t)R t*R t+R t,R  t-R# )$    N)defaultdict)	dataclass)DEVNULLrun)Optional)HostHostfileOptionalBoolActionlog	log_errorlog_warningc                   2   a  ] tR t^t o R tV 3R ltRtV tR# )SSHInfoc                    V P                   # N)can_ssh)selfs   &u/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/mlx/_distributed_utils/config.py__bool__SSHInfo.__bool__   s    ||    c                2   < V ^8  d   Qh/ S[ ;R&   S[ ;R&   # )   r   has_sudo)bool)format__classdict__s   "r   __annotate__SSHInfo.__annotate__   s     M  N r    N)__name__
__module____qualname____firstlineno__r   __annotate_func____static_attributes____classdictcell__r   s   @r   r   r      s     
  r   r   c                   ,   a  ] tR t^"t o V 3R ltRtV tR# )ThunderboltPortc                N   < V ^8  d   Qh/ S[ ;R&   S[ ;R&   S[S[ ,          ;R&   # )r   ifaceuuidconnected_to)strr   )r   r   s   "r   r   ThunderboltPort.__annotate__"   s,     J  I  3-	 r   r    Nr!   r"   r#   r$   r%   r&   r'   r(   s   @r   r*   r*   "         r   r*   c                   ,   a  ] tR t^)t o V 3R ltRtV tR# )ThunderboltHostc                B   < V ^8  d   Qh/ S[ ;R&   S[S[,          ;R&   # )r   nameports)r/   listr*   )r   r   s   "r   r   ThunderboltHost.__annotate__)   s!     
I     r   r    Nr1   r(   s   @r   r4   r4   )   r2   r   r4   c                    V  F  p\        VR VP                  4       \        RVP                  RRR.RRR7      P                  P	                  4       pVR8w  d   VP
                  P                  V4       Ks  \        RVP                  RRR.RRR7      P                  P	                  4       pVR8w  d   VP
                  P                  V4       K  \        R	VP                  4       K  	  R
# )zGetting the ip fromsshipconfig	getifaddren0Tcapture_outputtext en1zCould not extract ip forN)r   ssh_hostnamer   stdoutstripipsappendr   )hostsverbosehips   &&  r   add_ipsrM   /   s    G*ANN;ANNJUC
 &	 	
 8EELLANNJUC
 &	 	
 8EELL.?) r   c                 |   V P                   '       dQ   \        V P                   R 4      ;_uu_ 4       p\        P                  ! VP	                  4       V^R7       RRR4       R# \        R4       \        R4       \        \        P                  ! VP	                  4       ^R7      4       R#   + '       g   i     R# ; i)w)indentNr	   z========)output_hostfileopenjsondumpto_jsonprintdumps)argshostfilefs   && r   save_hostfiler[   H   s~    $&&,,IIh&&(!A6 -, 	jjdjj))+A67 -,,s   (B**B;	c                 B   V'       g   \         M\        pR pV  F  p\        VRVP                  R4       \	        RVP                  R.RRR7      P
                  P                  4       P                  4       pV Uu. uF  qwP                  R4      '       g   K  VNK  	  ppV'       d   K  V! VP                  R4       RpK  	  V'       d'   V! 4        V! R	4       V! 4        V! R
4       V! R4       V'       d   V'       d   \        P                  ! ^4       V'       * # u upi )FzChecking thatzsupports RDMAr;   ibv_devicesTr?   rdma_z"does not seem to have RDMA enabledzESome of the hosts don't have RDMA enabled or they don't support RDMA.zFSee https://ml-explore.github.io/mlx/build/html/usage/distributed.htmlz'for instructions on how to enable RDMA.)r   r   r   rD   r   rE   rF   split
startswithsysexit)rI   rJ   strictlog_ffailedrK   	rdma_devsds   &&&     r   
check_rdmarh   R   s    %K9EFG_annoF6tRVWVEEGUW 	
 !*C	1\\'-BQQ		Cy!.."FGF  UVVW78&: Ds   9DDc                     \         ;QJ d    R  V 4       F  '       d   K   RM	  RM! R  V 4       4      pV'       gR   V'       dJ   \        R4       \        W4       F/  w  rEVP                  '       d   K  \        RVP                  4       K1  	  V# )c              3   8   "   T F  qP                   x  K  	  R # 5ir   )r   ).0infos   & r   	<genexpr>!can_auto_setup.<locals>.<genexpr>p   s     5WT==W   FTzOAutomatic setup requested but the following hosts do not have passwordless sudo - )allr   zipr   rD   )rI   sshinfo
auto_setupr   rK   is   &&&   r   can_auto_setuprv   o   sc    s5W5sss5W55H
]	
 'DA:::E1>>2 ( Or   c                   0   a  ] tR t^{t o R tRR ltRtV tR# )IPConfiguratorc                   \        4       p\        \        4      p^ p^ p\        V4       EF?  w  r\        V	P                  4       EF  w  rVP
                  '       g   K  VP
                  V9  d   K-  W3V9   d   K7  W;P
                  ,          w  rRV RV^,            2pRV RV^,            2pVP                  pW,,          P                  V,          P                  pWXV3,          P                  VV34       W\V3,          P                  VV34       VP                  W34       VP                  W34       V^,          pV^8  d   V^,          p^ pV^8  g   EK  \        R4      h	  EKB  	  WPn
        Wn        W n        R# )r   z192.168..zRan out of available local IPsN)setr   r8   	enumerater7   r.   r,   rH   add
ValueErrorrG   rI   tb_hosts)r   rI   r   uuid_reverse_indexassignedrG   ip0ip1src_noderK   src_portpdst_nodedst_portip_srcip_dst	iface_src	iface_dsts   &&&&              r   __init__IPConfigurator.__init__|   s`   5$$X.KH(1~~~>>);;'83%7%G"#C5#'3#C5#'3GG	$.44X>DD	h&'..	6/BCh&'..	6/BCh12h12q91HCC9$%EFF5  2 /: 
 r   c                n   R p\        \        V P                  V P                  4      4       EF  w  pw  rVRpVR,          p\	        \        V P                  4      4       F~  pWH8X  g   WH3V P                  9  d   K  \        V P                  WH3,          V P                  W3,          4       F-  w  w  rw  rVRV	 RV
 RV R2,          pVRV RV	 R2,          pK/  	  K  	  V'       dt   \        R	VP                   24       VP                  4       P                  RR
4      pRVP                  V.p\        V\        P                  ! V4      4       \        V4       MKRVP                   2p\        V4       \        R\        V4      ,          4       \        V4       \        R4       \        4        EK  	  R# )z255.255.255.252rB   zsudo ifconfig bridge0 down
zsudo ifconfig z inet z	 netmask 
zsudo route change z -interface zRunning auto setup for z ; r;   z
Setup for =zEnter to continueN)r|   rr   rI   r   rangelenrG   rV   rD   rF   replacer   shlexjoinr   input)r   rJ   rt   netmaskru   rK   thcommandjr,   rL   _peermsgs   &&&           r   setupIPConfigurator.setup   sj   ##C

DMM$BCJAwG55G3tzz?+6aV4883.1$((14.$((14..Q*KU!wfRD	'RTUUG!3D6eWBOOG /R , //?@A!--/11$> !..':GUZZ01G"1>>"23c
cCHn%g)*G+ Dr   )rI   rG   r   N)FF)r!   r"   r#   r$   r   r   r&   r'   r(   s   @r   rx   rx   {   s     $!L r   rx   c                 $   / pR pV P                  R4      P                  R4       Fg  pVP                  R4      '       d   VP                  4       R,          pK3  VP                  R4      '       g   KL  VP                  4       R,          W&   R pKi  	  V# )Nzutf-8r   zHardware Port::   NNzDevice::   NN)decoder_   r`   rF   )ports_stringr7   	port_namels   &   r   parse_hardware_portsr      sx    EI  )//5<<())	#I\\)$$ wwy}EI 6 Lr   c                    . pV  Fc  p\        VR VP                  4       VP                  \        P                  ! \        RVP                  RRR.RR7      P                  4      4       Ke  	  . pV  FW  p\        VRVP                  4       VP                  \        \        RVP                  RR	.RR7      P                  4      4       KY  	  . p\        W$4       F  w  rgR
p. p	VR,           F  p
V
P                  R4      pVf   K  V
R,          pV
R,          R,          pV
P                  R. 4      pV Uu. uF  pRV9   g   K  VNK  	  ppV'       d   V^ ,          R,          MRpVRV 2,          pV	P                  \        VVV4      4       K  	  VP                  \        V\        V	R R7      4      4       K  	  / p\        V4       F6  w  pp\        VP                  4       F  w  ppVV3VVP                  &   K  	  K8  	  VV3# u upi )zGetting connectivity fromr;   system_profilerSPThunderboltDataTypez-jsonT)r@   zGetting interface names fromnetworksetupz-listallhardwareportsrB   domain_uuid_keyNdevice_name_keyreceptacle_1_tagreceptacle_id_key_itemszThunderbolt c                     V P                   # r   )r,   xs   &r   <lambda>&extract_connectivity.<locals>.<lambda>   s    !''r   key)r   rD   rH   rS   loadsr   rE   r   rr   getr*   r4   sortedr|   r7   r-   )rI   rJ   thunderbolt_connectionsrK   interface_mapsr   c	iface_mapr6   r7   tr-   tagitemsitemconnected_itemsr.   r,   r   ru   r   r   s   &&                    r   extract_connectivityr      s    G0!..A&&JJ)/ $(	 &	
   NG3Q^^D &/	 $( &
	
 " H3D*++A55*+D|&'D&'(;<CEE(B'E05S9Jd9RttOS9H"#45d  SE23ELLlCD , 	fU@Q.RST! E& (#1agg&DAq*+Qqvv& ' $ ''' Ts   3	HHc                    . p\        V 4       Fw  w  r4^ .\        V 4      ,          pVP                   F>  pVP                  V9   g   K  WP                  ,          w  rxWW;;,          ^,          uu&   K@  	  VP	                  V4       Ky  	  V# r   )r|   r   r7   r.   rH   )	r   r   connectivityru   rK   r   r   r   r   s	   &&       r   make_connectivity_matrixr   	  st    L(#C#h-A~~!33)..9	  	A $ r   c           
      B   . p\        \        V4      4       FL  pR pTp V\        ^aV^,          ,           4      ,          pV^,          pV^ 8X  g   K3   TP                  T4       KN  	  \	        R4       \	        R4       \        V 4       F'  w  rG\	        RW4,           RVP                   R24       K)  	  \        V4       F  w  rGVP                   F  pVP                  '       g   K  VP                  V9  d   K*  W(P                  ,          p	V	^ ,          V8  d   KL  \	        RW4,           RW9^ ,          ,           2R R7       \	        RVP                   RW^ ,          ,          P                  V	^,          ,          P                   R	24       K  	  K  	  \	        R
4       R# )rB   z	graph G {z  node [shape=rectangle];z  z	 [label="z"];z -- )end/z"]}N)
r   r   chrrH   rV   r|   rD   r7   r.   r,   )
rI   r   r   namesru   nr   rK   r   dsts
   &&&       r   tb_connectivity_to_dotr     sS   E3x=!R!b&[!!A"HAAvQ " 
+	
%&% 58*Iann%5S9: !(#A>>>~~%77$^^4C1vzBuxjUq6]O4"=IaggYaQ(8(>(>s1v(F(L(L'MRPQ  $ 
#Jr   c           
        a aaa . p\        4       p\        S 4      oV VV3R  lo\        S4       F  pS! W3. \        4       4       Fk  o\        V V3R l\        \        S4      4       4       4      p\	        \        S4      4      pWR9  g   KG  VP                  SV34       VP                  V4       Km  	  K  	  \        VR R7      # )c              3   @  <"   VP                  V4       VP                  V4       \        S4       FD  pSV,          V,          ^ 8:  d   K  W@8X  d   VR,          x  WC9  g   K3  S! WW#4       Rj  xL
  KF  	  VP                  4        VP	                  V4       R#  L+5i)r   :NNNN)rH   r}   r   popremove)
start_nodenodepathvisitedr   r   dfs	num_nodess   &&&& r   r   extract_rings.<locals>.dfs9  s     DDy!AD!!$)1gzd<<< " 	
t =s   AB$B0B1,Bc              3      <"   T F:  pSSV,          ,          SV^,           \        S4      ,          ,          ,          x  K<  	  R# 5i)   Nr   )rk   ru   r   rs   & r   rm    extract_rings.<locals>.<genexpr>H  s4     U}!l1Q4(AESV+;)<==}s   AAc                 (    \        V ^ ,          4      ) # r   r   r   s   &r   r   extract_rings.<locals>.<lambda>N  s    AaD	zr   r   )r{   r   r   mintupler   rH   r}   )	r   ringsexisting_ringsstartcntrkeyr   r   r   s	   f     @@@r   extract_ringsr   4  s    EUNL!I y!U2su-AUuSQRV}UUC#D)aX&""4( . " %122r   c           	      p   \        V4      p\        V4       F  p\        V4       F  pWE8X  d   K  W,          V,          ^ 8:  g   K!  V'       d_   \        RW,          P                   RW,          P                   24       \        4        \        R4       \        P
                  ! ^4       K    R# 	  K  	  R# )r   zIncomplete mesh, z is not connected to /Try passing --dot to visualize the connectivityFT)r   r   r   rD   ra   rb   )rI   r   rc   r   ru   r   s   &&&   r   check_valid_meshr   Q  s    L!I9y!Avq!Q&+EH,A,A+BBWX]X`XmXmWno KOPHHQK  "  r   c                   a  \        V4      ^ 8  ;'       d&    \        V^ ,          ^ ,          4      \        S 4      8H  pV'       d   V'       g   \        R4       \        4        \        R4       \        V4      ^ 8  d<   \        R4       V F*  p\        RRP                  V 3R lV 4       4       24       K,  	  \        P                  ! ^4       V# )r   zCould not find a full ring.r   zRings found:rp   ,c              3   J   <"   T F  pSV,          P                   x  K  	  R # 5ir   rD   )rk   ru   rI   s   & r   rm   #check_valid_ring.<locals>.<genexpr>m  s     (J1q)>)>s    #)r   r   r   ra   rb   )rI   r   rc   has_ringr   s   f&&  r   check_valid_ringr   d  s    5zA~@@#eAhqk"2c%j"@Hh/0CDu:>n%C(J(J JKLM Or   c           	      <  a R .\        V 4      ,          oV3R lp\        V 4       UUu. uF(  w  r4\        P                  ! W$P                  V3R7      NK*  	  pppV F  pVP                  4        K  	  V F  pVP                  4        K  	  \        S4      '       gz   V'       gr   \        R4       \        V 4       F,  w  r4SV,          '       d   K  \        RVP                  4       K.  	  \        4        \        R4       \        P                  ! ^4       S# u uppi )Nc           
        < \        R R 4      pVSV&   \        RRRRRV RR.\        \        R7      pVP                  ^ 8H  Vn        VP                  '       g   R# \        RRRRRV R	R
.\        \        R7      pVP                  ^ 8H  Vn        R# )Fr;   z-ozBatchMode=yeszConnectTimeout=5echosuccess)rE   stderrNsudols)r   r   r   
returncoder   r   )hostnameru   rl   resultresultss   &&  r   _check%check_ssh_connections.<locals>._checku  s    ue$
 "	 
 ((A-||| "	 
 ))Q.r   )targetrX   z%Could not ssh to the following hosts:z  - z0Maybe they are not set-up for password-less ssh?)r   r|   	threadingThreadrD   r   r   rq   r   ra   rb   )rI   ignore_unreachabler  ru   rK   threadsr   r  s   &&     @r   check_ssh_connectionsr	  r  s    fs5z!G&/T e$$DA 	nna-@A$   		 	  w<< 29:e$DA1::&!..1 % 	DEN%s   .Dc                 $   \        V P                  R 4       \        WP                  4       \        \	        V4       UUu. uF&  w  r#\        W#P                  VP                  . 4      NK(  	  uppRV P                  4      p\        W4       R# u uppi )zPreparing an ethernet hostfilerB   N)
r   rJ   rM   r	   r|   r   rD   rG   envr[   )rX   rI   ru   rK   rY   s   &&   r   prepare_ethernet_hostfiler    sk    68E<< 8A%8HI8Ha	+8HI2txxH $! 	Js   ,B
c                 ,   \        V P                  R 4       Vw  r5. p\        V4       Fz  w  rxW,          p	W7^,
          ,          p
TP                  \	        YyP
                  \        V4       Uu. uF$  qP                  W3,          V,          ^,          NK&  	  up. 4      4       K|  	  \        VRV P                  4      p\        WV P                  4      pTP                  V P                  V P                  ;'       d    TR7       \        W4       R# u upi )zPrepare a ring hostfileringrJ   rt   N)r   rJ   r|   rH   r   rD   r   rG   r	   r  rv   rt   r   r[   )rX   rI   rG   r  rs   count
ring_hostsru   r   rK   r   r   rY   r   s   &&&&&         r   configure_ringr    s    /0KDJT?KE{>>uU|#T|!GGDJ$7$:1$=$=|#TVX	
 # 
FDHH5Hedoo>HIIdllt/K/K8IL$! $Us   ,*Dc           	         \        V P                  R 4       \        WP                  4       . p\        V4       F  w  rV. p\	        \        V4      4       FP  pWX8X  d   VP                  R4       K  VP                  RVP                  WX3,          ^ ,          ^ ,           24       KR  	  VP                  \        WVP                  VP                  V4      4       K  	  \        VRV P                  4      p	\        WV P                  4      p
TP                  V P                  V P                  ;'       d    T
R7       \        W	4       R# )zPrepare a jaccl hostfileNr^   jacclr  )r   rJ   rM   r|   r   r   rH   rG   r   rD   r	   r  rv   rt   r   r[   )rX   rI   rG   rs   jaccl_hostsru   rK   rdmar   rY   r   s   &&&&       r   configure_jacclr    s    01E<< K% s5z"AvD!eCGGADM!$4Q$7#89:	 #
 	4>>155$?@ ! Wdhh7Hedoo>HIIdllt/K/K8IL$!r   c           	      R   \        V P                  R 4       \        WP                  4       . p\        V4      pVw  r7\	        V4       F  w  rW,          p
W8^,
          ,          pW8^,           V,          ,          p. pV F  pWV39  d   VP                  R4       K  . p\        V4       F7  pVP                  RVP                  W3,          V,          ^ ,           24       K9  	  TP                  V^8X  d
   V^ ,          MT4       K  	  VP                  \        WP                  V
P                  V4      4       K  	  \        VRV P                  4      p\        WV P                  4      pTP                  V P                  V P                  ;'       d    TR7       \        V V4       R# )zPrepare a jaccl-ring hostfileNr^   
jaccl-ringr  )r   rJ   rM   r   r|   rH   r   rG   r   rD   r	   r  rv   rt   r   r[   )rX   rI   rG   r  rs   r  r   r  ru   r   rK   	peer_left
peer_rightrdmasotherr  r   rY   r   s   &&&&&              r   configure_jaccl_ringr    sN   56E<< KE
IKDT?KQK	q5I-.
E
33T"uAKK%(<Q(?(B'C DE &
T!W=  	4>>155%@A # \488<Hedoo>HIIdllt/K/K8IL$!r   c                    \        V P                  R 4       \        WP                  4      w  r4V P                  '       d   \	        WV4       R# \        WV4      p\        W44      pV P                  Ef   \        V4      p\        WR4      p\        WR4      p	\        WP                  R4      p
V	'       g7   V'       g/   \        R4       \        R4       \        P                  ! ^4       R# V
'       d   V'       d   \        WWR4       R# V
'       d   V	'       d   \!        WWW^ ,          V4       R# V	'       d   \#        WWW^ ,          V4       R# \        R4       \        R4       \        P                  ! ^4       R# V P                  R8X  d-   \        V4      p\        W4       \#        WWW^ ,          V4       R# V P                  R8X  d/   \        W4       \        WP                  4       \        WWR4       R# V P                  R8X  dB   \        V4      p\        W4       \        WP                  4       \!        WWW^ ,          V4       R# R# )	z,Preparing for communication over thunderboltNFz(Neither thunderbolt mesh nor ring found.z>Perhaps run with --dot to generate a plot of the connectivity.z,RDMA is not available and ring is not found.r  r  r  )r   rJ   r   dotr   rx   r   backendr   r   r   rh   r   ra   rb   r  r  r  )rX   rI   rs   r   r   rG   r   r   has_meshr   has_rdmas   &&&        r   prepare_tb_hostfiler$    s   DF#7||#L Hxxxu0BC
*<
=C+HIL||l+#E?#E%8e\\59@AVWHHQK(D6( c8WE41Xw? DEVWHHQK		l+&tCq7;		 -5,,'S2		%l+&5,,'T#QxA	 
&r   c            	      :   \         P                  ! R R7      p V P                  RRRR7       V P                  RRRR	7       V P                  R
RRR7       V P                  RRR7       V P                  RRR.RRRR7       V P                  RRR7       V P                  RR\        ^ RRR7       V P                  RRRR7       V P                  R. R(ORRR 7       V P                  R!R". R#R$7       V P	                  4       pVP
                  e,   \        P                  ! VP
                  4      P                  pM*\        P                  ! VP                  4      P                  p\        VP                  R%R&P                  R' V 4       4       24       \        W!P                  4      p\        W24       UUu. uF  w  rEV'       g   K  VNK  	  pppV Uu. uF  qD'       g   K  VNK  	  ppVP                   R8X  d   \#        W4       R# \%        WV4       R# u uppi u upi ))z6Configure remote machines for use with MLX distributed)descriptionz	--verbose
store_truezPrint debug messages in stdout)actionhelpz--hostsz	127.0.0.1zA comma separated list of hosts)defaultr)  z--ignore-unreachablez+Ignore hosts that are not reachable via sshz
--hostfilezThe file containing the hosts)r)  z--overthunderboltethernetz&What type of connectivity to configureT)choicesr*  r)  requiredz--output-hostfilez+If provided, save the hostfile to this pathz--auto-setupz--no-auto-setuprt   N)r(  nargsdestr*  z--dotz*Output the topology in DOT format and exitz	--backendz&Which distributed backend to configure)r-  r*  r)  z--envrH   z&Set environment variables for the jobs)r(  r*  r)  zChecking for ssh access for z, c              3   8   "   T F  qP                   x  K  	  R # 5ir   r   )rk   rK   s   & r   rm   main.<locals>.<genexpr>s  s     0OAro   )r  r  r  )argparseArgumentParseradd_argumentr
   
parse_argsrY   r	   	from_filerI   	from_listr   rJ   r   r	  r  rr   overr  r$  )parserrX   rI   rs   r   rK   s         r   mainr;  8  s9   $$LF L/O   ;-N   :  
 +JK

+5   "O   !   +W   /5	   5	   D}} ""4==177""4::.44 
&tyy0O0O'O&PQ $E+B+BCGw.4.41!QQ.E4!''QQqq'G' yyJ!$. 	D1 5's   2HH	HH)F)FT)T).r3  rS   r   ra   r  collectionsr   dataclassesr   
subprocessr   r   typingr   mlx.corecoremxcommonr   r	   r
   r   r   r   r   r*   r4   rM   r[   rh   rv   rx   r   r   r   r   r   r   r   r	  r  r  r  r  r$  r;  r    r   r   <module>rD     s       
  # ! #             ! ! !
@28:	> >B	>(B	>3:&=@""("*":2BjH2r   