+
    &j"?                     v   ^ RI t ^ RIt^ RIt^ RIHt ^ RIHtHt ^ RIH	t	 ^ RI
t
^ RIHu Hu Hu Ht ^ RIHtHt ^ RIHtHt ^ RIHt ^ RIHt ^ R	IHtHtHt ^ R
IH t H!t!H"t" ^ RI#H$t$ ^ RI%H&t& ^ RI'H(t( ^ RI)H*t* ^ RI+H,t, . ROt-]*! ].4      t/] ! R R4      4       t0 ! R R4      t1R R lt2R R lt3R R lt4R# )    N)Callable)	dataclassfield)Any)get_default_numa_optionsjustknobs_check)eventsmetrics)
WorkerSpec)create_healthcheck_server)_AliveCallbackProxyLocalElasticAgentTORCHELASTIC_HEALTH_CHECK_PORT)DefaultLogsSpecs	LogsSpecsSignalException)ChildFailedError)RendezvousParameters)parse_rendezvous_endpoint)
get_logger)NumaOptionsLaunchConfigelastic_launchc                      a  ] tR t^.t o RtRtRtRtRtRt	]
! ]R7      tRt^tRtRtRt]
! ]R7      tRtR	tRtR
tRtRtRtRtR tV 3R ltRtV tR# )r   a  
Creates a rendezvous config.

Args:
    min_nodes: Minimum amount of nodes that the user function will
                    be launched on. Elastic agent ensures that the user
                    function start only when the min_nodes amount enters
                    the rendezvous.
    max_nodes: Maximum amount of nodes that the user function
                    will be launched on.
    nproc_per_node: On each node the elastic agent will launch
                        this amount of workers that will execute user
                        defined function.
    rdzv_backend: rdzv_backend to use in the rendezvous (zeus-adapter, etcd).
    rdzv_endpoint: The endpoint of the rdzv sync. storage.
    rdzv_configs: Key, value pair that specifies rendezvous specific configuration.
    rdzv_timeout: Legacy argument that specifies timeout for the rendezvous. It is going
        to be removed in future versions, see the note below. The default timeout is 900 seconds.
    run_id: The unique run id of the job (if not passed a unique one will be
            deduced from run environment - flow workflow id in flow - or auto generated).
    role: User defined role of the worker (defaults to "trainer").
    max_restarts: The maximum amount of restarts that elastic agent will conduct
                on workers before failure.
    monitor_interval: The interval in seconds that is used by the elastic_agent
                    as a period of monitoring workers.
    start_method: The method is used by the elastic agent to start the
                workers (spawn, fork, forkserver).
    metrics_cfg: configuration to initialize metrics.
    local_addr: address of the local node if any. If not set, a lookup on the local
            machine's FQDN will be performed.
    local_ranks_filter: ranks for which to show logs in console. If not set, show from all.
    event_log_handler: name of the event logging handler as registered in
      `elastic/events/handlers.py <https://docs.pytorch.org/docs/stable/elastic/events.html>`_.
    duplicate_stdout_filters: If non-empty, duplicates stdout to a file containing only lines
                            that match _any_ of the filter strings.
    duplicate_stderr_filters: If non-empty, duplicates stderr to a file containing only lines
                            that match _any_ of the filter strings.
    virtual_local_rank: Enable virtual local rank mode for workers (defaults to False).
                       When enabled, LOCAL_RANK is set to 0 for all workers and
                       CUDA_VISIBLE_DEVICES is adjusted so each worker accesses its
                       assigned GPU at device index 0.
    shutdown_timeout: Time in seconds to wait for graceful shutdown of workers before
                    sending SIGKILL. Can also be set via TORCH_ELASTIC_SHUTDOWN_TIMEOUT
                    environment variable. Defaults to 30 seconds.


.. note::
    `rdzv_timeout` is a legacy argument that will be removed in future.
    Set the timeout via `rdzv_configs['timeout']`

N default_roleetcd)default_factoryg?spawnnullzSIGTERM,SIGINT,SIGHUP,SIGQUITFc                   R pV P                   R8w  d   V P                   V P                  R&   MRV P                  9  d   WP                  R&   V P                  f   \        4       V n        V P                  f   \
        P                  P                  4       '       d]   \
        P                  P                  4       V P                  8X  d0   \        4       V n        \        P                  RV P                  4       V P                  f1   \        \        P                   P#                  RR4      4      V n        R# V P                  ^ 8  d   \%        RV P                   24      hR# )i  timeoutNzUsing default numa options = %rTORCH_ELASTIC_SHUTDOWN_TIMEOUT30z+shutdown_timeout must be non-negative, got )rdzv_timeoutrdzv_configs
logs_specsr   numa_optionstorchcudais_availabledevice_countnproc_per_noder   loggerinfoshutdown_timeoutintosenvironget
ValueError)selfdefault_timeouts   & v/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/torch/distributed/launcher/api.py__post_init__LaunchConfig.__post_init__|   s   "+/+<+<Di(d///+:i( ??".0DO %

''))

'')T-@-@@ 8 :DKK94;L;LM   ($'

?F%D! ""Q&=d>S>S=TU  '    c                  < V ^8  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[R,          ;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[S[S[3,          ;R
&   S[ ;R&   S[ ;R&   S[;R&   S[;R&   S[R,          ;R&   S[S[S[3,          ;R&   S[R,          ;R&   S[;R&   S[R,          ;R&   S[;R&   S[S[,          R,          ;R&   S[S[,          R,          ;R&   S[;R&   S[ R,          ;R&   # )   	min_nodes	max_nodesr.   Nr(   run_idrolerdzv_endpointrdzv_backendr'   r&   max_restartsmonitor_intervalstart_methodlog_line_prefix_templatemetrics_cfg
local_addrevent_log_handlerr)   signals_to_handleduplicate_stdout_filtersduplicate_stderr_filtersvirtual_local_rankr1   )	r2   r   strdictr   floatr   listbool)format__classdict__s   "r9   __annotate__LaunchConfig.__annotate__.   s    l Nm n No p q r D 's t u v w x y z { | sCx.>} ~  @ A B !C D E F "Dj/G H c3h=I J d
!K L #M N $+O P <Q R #3i$.5S T #3i$.5U V $W X Dj'Y r<   )r(   r)   r1   r%   )__name__
__module____qualname____firstlineno____doc__r(   rA   rB   rC   rD   r   rQ   r'   r&   rE   rF   rG   rH   rI   rJ   rK   r)   rL   rM   rN   rO   r1   r:   __annotate_func____static_attributes____classdictcell__rV   s   @r9   r   r   .   s     2n $(JFDML#(#>LLL!L+/"'"=K!J#'+L<1515$#']  r<   c                   <   a  ] tR t^t o RtV 3R lR ltR tRtV tR# )r   a  
Launches an torchelastic agent on the container that invoked the entrypoint.

    1. Pass the ``entrypoint`` arguments as non ``kwargs`` (e.g. no named parameters)/
       ``entrypoint`` can be a function or a command.
    2. The return value is a map of each worker's output mapped
       by their respective global rank.

Usage

::

def worker_fn(foo):
    # ...

def main():
    # entrypoint is a function.
    outputs = elastic_launch(LaunchConfig, worker_fn)(foo)
    # return rank 0's output
    return outputs[0]

    # entrypoint is a command and ``script.py`` is the python module.
    outputs = elastic_launch(LaunchConfig, "script.py")(args)
    outputs = elastic_launch(LaunchConfig, "python")("script.py")
c                D   < V ^8  d   QhRS[ RS[S[,          R,          /# )r>   config
entrypointN)r   r   rP   )rU   rV   s   "r9   rW   elastic_launch.__annotate__   s(     & && sNT)&r<   c                    Wn         W n        R # N_config_entrypoint)r7   rd   re   s   &&&r9   __init__elastic_launch.__init__   s    
 %r<   c                V    \        V P                  V P                  \        V4      4      # rh   )launch_agentrj   rk   rS   )r7   argss   &*r9   __call__elastic_launch.__call__   s    DLL$*:*:DJGGr<   ri   N)	rY   rZ   r[   r\   r]   rl   rq   r_   r`   ra   s   @r9   r   r      s      4& &H Hr<   c                v    V ^8  d   QhR\         \        ,          R,          R\        \        ,          R\        /# )r>   re   Nrp   return)r   rP   rS   r   )rU   s   "r9   rW   rW      s/      X^d%: $s) PS r<   c                    \        V \        4      '       d   V P                  # \        V \        4      '       d+   V \        P
                  8X  d   \        R V 4       R4      # V # R# )a  Retrieve entrypoint name with the rule:
1. If entrypoint is a function, use ``entrypoint.__qualname__``.
2. If entrypoint is a string, check its value:
    2.1 if entrypoint equals to ``sys.executable`` (like "python"), use the first element from ``args``
        which does not start with hifen letter (for example, "-u" will be skipped).
    2.2 otherwise, use ``entrypoint`` value.
3. Otherwise, return empty string.
c              3   D   "   T F  q^ ,          R8w  g   K  Vx  K  	  R# 5i)r   -N ).0args   & r9   	<genexpr>'_get_entrypoint_name.<locals>.<genexpr>   s     >A#s    
 r   )
isinstancer   rY   rP   sys
executablenext)re   rp   s   &&r9   _get_entrypoint_namer      sR     *h''"""	J	$	$'>>CCr<   c                n    V ^8  d   QhR\         R\        \        R,          \        R,          3,          /# )r>   rdzv_parametersrt   N)r   tuplerP   r2   )rU   s   "r9   rW   rW      s.     & &)&
3:sTz!"&r<   c                     V P                   R 8w  d   R# V P                  pVP                  4       pV'       g   \        R4      h\	        VRR7      w  r#VR8X  d   \        RV R24      hW#3# )staticzKEndpoint is missing in endpoint. Try to add --master-addr and --master-port)default_portzport is missing in endpoint: z. Try to specify --master-port)NNr%   )backendendpointstripr6   r   )r   r   master_addrmaster_ports   &   r9   _get_addr_and_portr      s     (*''H~~HY
 	
  9PRSKb+H:5ST
 	
 %%r<   c          
          V ^8  d   QhR\         R\        \        ,          R,          R\        \        ,          R\
        \        \        3,          /# )r>   rd   re   Nrp   rt   )r   r   rP   rS   r   rQ   r2   )rU   s   "r9   rW   rW      sK     ^) ^)^)3%^) s)^) 
#s(^	^)r<   c                 	   V P                   '       gE   \        \        P                  ! 4       P                  4      p\
        P                  R V4       W0n         \        W4      p\
        P                  R/ RVbRV P                  bRV P                  bRV P                  bRV P                   bRV P                  bRV P                  bR	V P                  bR
V P                  bRV P                   bRV P"                  P$                  bRV P&                  bRV P(                  bRV P*                  bRV P,                  bRV P.                  bRV P0                  b4       \3        R#RV P                  RV P                  RV P                   RV P                  RV P                  RV P4                  /V P                  B p\7        V4      w  rgV P,                  \8        P:                  R&   RpRp	\8        P<                  ! \>        4      p
V
e\   \A        RRR7      '       dI    \C        4       p	\E        V	\	        V
4      ^<R7      pVPG                  4        \
        P                  RV
4       \K        V PL                  V P                  V\O        V4      \P        PR                  ! V4      V P                  V P                   VVV P4                  V P(                  V P*                  V P.                  V P0                  V PT                  R 7      p\W        VV P"                  V PX                  V PZ                  V P\                  VR!7      pV	e   V	P_                  VP`                  4       Rp \b        Pd                  ! \b        Pf                  ! V P&                  4      4       VPi                  4       p\j        Pl                  ! VPo                  4       V P(                  4       VPq                  4       '       d   \s        VVPt                  R"7      hVPv                  V'       d   VPx                  P{                  4        # #   \H         d     \
        P                  RRR7       RpRp	 ELi ; i  \r         d    h \|         d4    Rp\j        Pl                  ! TP                  4       T P(                  4       h \H         d2    \j        Pl                  ! TP                  4       T P(                  4       h i ; i  T'       d   TPx                  P{                  4        i i ; i)$z3config has no run_id, generated a random run_id: %saR  Starting elastic_operator with launch configs:
  entrypoint               : %(entrypoint)s
  min_nodes                : %(min_nodes)s
  max_nodes                : %(max_nodes)s
  nproc_per_node           : %(nproc_per_node)s
  run_id                   : %(run_id)s
  rdzv_backend             : %(rdzv_backend)s
  rdzv_endpoint            : %(rdzv_endpoint)s
  rdzv_configs             : %(rdzv_configs)s
  max_restarts             : %(max_restarts)s
  monitor_interval         : %(monitor_interval)s
  log_dir                  : %(log_dir)s
  metrics_cfg              : %(metrics_cfg)s
  event_log_handler        : %(event_log_handler)s
  numa_options             : %(numa_options)s
  signals_to_handle        : %(signals_to_handle)s
  duplicate_stdout_filters : %(duplicate_stdout_filters)s
  duplicate_stderr_filters : %(duplicate_stderr_filters)s
re   r?   r@   r.   rA   rD   rC   r'   rE   rF   log_dirrI   rK   r)   rL   rM   rN   r   r   rJ   TORCHELASTIC_SIGNALS_TO_HANDLENzNai_infra/pytorch_distributed:torchelastic_enable_healthcheck_before_rendezvousF)default)alive_callbackportr"   z>Started early health check server on port %s before rendezvousz)Failed to start early health check serverT)exc_info)rB   local_world_sizere   rp   rdzv_handlerrE   rF   r   r   rJ   rK   r)   rM   rN   rO   )specr(   rG   rH   r1   health_check_server)namefailuresrx   )@rA   rP   uuiduuid4r2   r/   warningr   r0   r?   r@   r.   rD   rC   r'   rE   rF   r(   root_log_dirrI   rK   r)   rL   rM   rN   r   rJ   r   r3   r4   getenvr   r   r   r   start	Exceptionr   rB   r   rdzv_registryget_rendezvous_handlerrO   r   rG   rH   r1   set_delegate_get_alive_timer
   initialize_metricsMetricsConfigrunr	   recordget_event_succeeded	is_failedr   r   return_valuesr   shutdownr   get_event_failed)rd   re   rp   rA   entrypoint_namer   r   r   r   alive_callback_proxyhealthcheck_portr   agentshutdown_rdzvresults   &&&            r9   ro   ro      s   
 ===TZZ\%%&LfU*:<O
KK	F$	
/	
))	
 ))	
 f33		

 fmm	
 F//	
 V11	
 F//	
 F//	
  7 7	
 v((55	
 6--	
  !9!9	
 F//	
  !9!9	
  '(G(G!	
" '(G(G#	
'&P + ##%% }} ""	
 "" $$ 

O  2/BK 4:3K3KBJJ/0 yy!?@#X) )	(#6#8 ";3)*#
  %%'KKP  [[..4["99/J((00$$ 22((!'!@!@!'!@!@!44D$ $$((!'!@!@00/E '))%*?*?@M )""7#8#89K9K#LMe//163K3KL
 #$ 
 ## &&( G  	(NNFQUNV"&#' 	(l    e,,.0H0HI e,,.0H0HI &&( s8   AP B+Q &QQS ;S2SS %S6)r   r   ro   )5r3   r~   r   collections.abcr   dataclassesr   r   typingr   r*   -torch.distributed.elastic.rendezvous.registrydistributedelastic
rendezvousregistryr   torch._utils_internalr   r   torch.distributed.elasticr	   r
   *torch.distributed.elastic.agent.server.apir   :torch.distributed.elastic.agent.server.health_check_serverr   :torch.distributed.elastic.agent.server.local_elastic_agentr   r   r   )torch.distributed.elastic.multiprocessingr   r   r   0torch.distributed.elastic.multiprocessing.errorsr   $torch.distributed.elastic.rendezvousr   *torch.distributed.elastic.rendezvous.utilsr   'torch.distributed.elastic.utils.loggingr   torch.numa.bindingr   __all__rY   r/   r   r   r   r   ro   rx   r<   r9   <module>r      s    
 
  $ (   E E K 5 A 
 
 N E P > * =	H	 i i iX$H $HN(&&^)r<   