+
    &j@                        ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHtHt ^ RI	H
t
 ^ RIHt ^ RIHt ^ RIHtHtHt ^ RIt^ RIHt ^ RIHt ^ RIHt ^R	IHtHtHtHt ^R
I H!t!H"t" RR.t#]! R4      t$]! R4      t%R R lt&R*R R llt'R*R R llt( ! R R4      t)R R lt*R R lt+R R lt,R R lt-R R  lt. ! R! R"]P^                  4      t0R# R$ lt1R% R& lt2R't3]
R( 4       t4R) t5R# )+    N)CallableSequence)contextmanagerwraps)Stats)AnycastTypeVar)ShardedTensor)Shard)_is_wrapped_exception_wrap_exceptionCheckpointExceptionWRAPPED_EXCEPTION)MetadataIndexSTATE_DICT_TYPEfind_tensor_shardfind_state_dict_objectTRc                ~    V ^8  d   QhR\         \        \        ,          ,          R\        \        \        3,          /# )   resultsreturn)listr   r   dictint)formats   "z/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/torch/distributed/checkpoint/utils.py__annotate__r!   "   s2      !''(	# 
 !    c           
          \        \        \        \        3,          \	        V 4       UUu/ uF  w  r\        V4      '       g   K  WbK  	  upp4      # u uppi N)r
   r   r   r   	enumerater   )r   ierrs   &  r    _get_failure_dictr(   "   sI     S##$'0O0FA4I#4N0O Os   AAc                    V ^8  d   QhR\         \        \        3,          R\        P                  R,          R\
        \        ,          /# )r   
local_dictgroupNr   )r   strr	   distProcessGroupset)r   s   "r    r!   r!   +   s;     = =S#X='+'8'84'?=X=r"   c                    \        V P                  4       4      pR.\        P                  ! V4      ,          p\        P                  ! W2VR7       \        \        P                  P                  V4      4      # )z*Gathers all keys, and returns them sorted.Nr+   )	r   keysr-   get_world_sizeall_gather_objectr/   	itertoolschainfrom_iterable)r*   r+   r2   gathered_keyss   &&  r    _all_gather_keysr9   +   sV     
!"D&*Vd.A.A%.H%HM=e<y,,];<<r"   c                x    V ^8  d   QhR\         \        \        3,          R\        P                  R,          RR/# )r   
state_dictprocess_groupNr   )r   r,   r	   r-   r.   )r   s   "r    r!   r!   6   s7     
 
S#X
/3/@/@4/G
	
r"   c                    \         P                  ! V4      ^8X  d   R# \        W4      p\        V P	                  4       4      pW#,
          p\        V4      ^ 8  d   \        RV 24      hR# )z
Asserts that all ranks have the same keys in their state dict.
This is a collective call which requires all ranks in ``process_group`` to
join. It will also induce cross-rank communication and block CPU.
Nz<Key(s) present in other ranks but not this one, difference: )r-   r3   r9   r/   r2   lenAssertionError)r;   r<   all_keysmy_keysdiffs   &&   r    _assert_same_keysrC   6   sd     =)Q.
:H*//#$GD
4y1}J4&Q
 	
 r"   c                      a  ] tR t^Kt o RtV 3R lR ltV 3R lR ltV 3R lR ltV 3R lR	 ltV 3R
 lR lt	V 3R lR lt
V 3R lR ltV 3R lR ltV 3R lR ltV 3R lR ltV 3R lR ltV 3R lR ltRtV tR# )_DistWrappera4  
This is a wrapper around PG that provides a series of features around object collectives.

It works without distributed initialized, where most collectives turns into nops.

All variants that take functions are exception robust, meaning that if one or more
ranks raise errors, all ranks will observe those.
c                N   < V ^8  d   QhRS[ P                  R,          RS[RS[/# )r   r+   Nuse_distcoordinator_rank)r-   r.   boolr   )r   __classdict__s   "r    r!   _DistWrapper.__annotate__U   s4     ' '  4'' ' 	'r"   c                   Wn         W n        W0n        V P                  '       dS   Ve   \        P                  ! W4      MTV n        \        P                  ! V4      V n        V P                  V8H  V n        R # ^ V n        ^ V n        RV n        R # )NT)	r+   rG   rH   r-   get_global_rankglobal_coordinator_rankget_rankrankis_coordinator)selfr+   rG   rH   s   &&&&r    __init___DistWrapper.__init__U   s{     
  0=== $ $$U=% (
 e,DI"&))/?"?D+,D(DI"&Dr"   c                    < V ^8  d   QhRS[ /# r   r   r   )r   rJ   s   "r    r!   rK   k   s      # r"   c                    V P                   # r$   )rP   rR   s   &r    rO   _DistWrapper.get_rankk   s    yyr"   c                    < V ^8  d   QhRS[ /# rV   rW   )r   rJ   s   "r    r!   rK   n   s       r"   c                j    V P                   '       d!   \        P                  ! V P                  4      # ^# )   )rG   r-   r3   r+   rY   s   &r    r3   _DistWrapper.get_world_sizen   s#    ===&&tzz22r"   c                4   < V ^8  d   QhRS[ R,          RS[ /# )r   objectNr   )r   )r   rJ   s   "r    r!   rK   s   s     	' 	'q4x 	'A 	'r"   c                    V.pV P                   '       d.   \        P                  ! VV P                  V P                  R7       \        \        V^ ,          4      # )z_Implement functionality similar to c10d::broadcast_object_list but without distributed enabled.)object_listr+   src)rG   r-   broadcast_object_listr+   rN   r
   r   )rR   r`   rb   s   && r    broadcast_object_DistWrapper.broadcast_objects   sF    h===&&'jj00
 A{1~&&r"   c                D   < V ^8  d   QhRS[ RS[S[ ,          R,          /# )r   r`   r   Nr   r   )r   rJ   s   "r    r!   rK   ~   s#      A $q'D. r"   c                f   V P                   '       d   V P                  '       dB   \        \        \        ,          R.\
        P                  ! V P                  4      ,          4      MRp\
        P                  ! TV P                  '       d   TMRV P                  V P                  R7       TpV# V.pV# )zWImplement functionality similar to c10d::gather_object but without distributed enabled.N)objobject_gather_listdstr+   )
rG   rQ   r
   r   r   r-   r3   r+   gather_objectrN   )rR   r`   gather_objsresults   &&  r    rm   _DistWrapper.gather_object~   s    === &&& T!Wtft':':4::'FFG  262E2E2E;400jj	 !F  XFr"   c                6   < V ^8  d   QhRS[ RS[S[ ,          /# )r   r`   r   rh   )r   rJ   s   "r    r!   rK      s     
 
 
d1g 
r"   c                    V P                   '       df   \        \        \        ,          R.\        P
                  ! V P                  4      ,          4      p\        P                  ! W!V P                  R7       V# V.pV# )z[Implement functionality similar to c10d::all_gather_object but without distributed enabled.N)rb   rj   r+   )rG   r
   r   r   r-   r3   r+   r4   )rR   r`   rn   s   && r    r4   _DistWrapper.all_gather_object   s_    ===tAw1D1DTZZ1P(PQK""'4::
  "(Kr"   c                D   < V ^8  d   QhRS[ S[,          R,          RS[/# )r   rb   Nr   )r   r   )r   rJ   s   "r    r!   rK      s#      $q'D. Q r"   c                ,   V P                   '       dj   \        \        \        ,          R.4      p\        P
                  ! TV P                  '       d   TMRV P                  V P                  R7       V^ ,          pV# Vf   \        R4      hV^ ,          pV# )zXImplement functionality similar to c10d::scatter_object but without distributed enabled.N)scatter_object_output_listscatter_object_input_listrc   r+   zobject_list is None)
rG   r
   r   r   r-   scatter_object_listrQ   rN   r+   r?   )rR   rb   gather_resultlocal_replys   &&  r    scatter_object_DistWrapper.scatter_object   s    === a4&1M$$+89=9L9L9L+RV00jj	 (*K
  "$%:;;%a.Kr"   c          	      ~   < V ^8  d   QhRS[ RS[. S[3,          RS[S[S[,          .S[S[,          3,          RS[/# r   stepmap_fun
reduce_funr   r,   r   r   r   r   )r   rJ   s   "r    r!   rK      sO     . .. "a%. d1gYQ/0	.
 
.r"   c           	         V! 4       pV P                  V4      pRpV P                  '       d   Vf   \	        R4      h\        V4      p\        V4      ^ 8X  dG    \        \        \        \        ,          ,          V! \        \        \        ,          V4      4      4      p\        V4      ^ 8  d"   \        W4      .V P                  4       ,          pV P                  V4      p	\        V	\        4      '       d   V	hV	#   \          d   p\        T4      p Rp?ELRp?ii ; i  \          d"   p\        T4      YP                  &    Rp?LRp?ii ; i)a&  
Compute a value on each rank, then do centralized reduce on a single rank, followed by a scatter.

This method operates in the following way:
    Run ``map_fun`` on all ranks
    Gather results on rank 0
    Call ``reduce_fun`` on all those values
    Scatter to each rank part of the result.
Nall_data is None)BaseExceptionr   rm   rQ   r?   r(   r>   r
   r   r   r   r   rP   r3   r{   
isinstance)
rR   r   r   r   
local_dataeall_dataall_resultsnode_failuresro   s
   &&&&      r    reduce_scatter_DistWrapper.reduce_scatter   s$    	, J %%j1<@$%788-h7M=!Q&B"&Q!445"4Q#:;#K =!A%'<'')* $$[1f122L9  	,(+J	," % B/>q/AM)),Bs0   C: AD :DDDE
)EE
c                n   < V ^8  d   QhRS[ RS[. S[3,          RS[S[S[,          .S[3,          RS[/# r~   r   )r   rJ   s   "r    r!   rK      sJ     )% )%)% "a%)% d1gY\*	)%
 
)%r"   c                @    V! 4       pV P                  V4      pRpV P                  '       dh   Vf   \	        R4      h\        V4      p\        V4      ^ 8X  d#    V! \        \        \        ,          V4      4      p\        V4      ^ 8  d   \        W4      pV P                  V4      p	\        V	\        4      '       d   V	h\        \        V	4      #   \          d   p\        T4      p Rp?LRp?ii ; i  \          d"   p\        T4      YP                  &    Rp?LRp?ii ; i)a)  
Compute a value on each rank, then do centralized reduce on a single rank, followed by a broadcast.

This method operates in the following way:
    Run ``map_fun`` on all ranks
    Gather results on rank 0
    Call ``reduce_fun`` on all those values
    Broadcast the reduced value to all ranks.
Nr   )r   r   rm   rQ   r?   r(   r>   r
   r   r   rP   r   re   r   r   )
rR   r   r   r   r   r   r   ro   r   final_results
   &&&&      r    
all_reduce_DistWrapper.all_reduce   s    	, J %%j115$%788-h7M=!Q&B'T!Wh(?@F =!A%,TA ,,V4l$788A|$$/  	,(+J	, % B/>q/AM)),Bs/   C !C1 C.C))C.1D<DDc                P   < V ^8  d   QhRS[ RS[. S[3,          RS[S[,          /# r   r   r   r   )r,   r   r   r   )r   rJ   s   "r    r!   rK     s5     * ** "a%* 
a	*r"   c                     V! 4       pV P                  V4      p\        V4      p\	        V4      ^ 8  d   \        W4      h\        \        \        ,          V4      #   \          d   p\        T4      p Rp?LmRp?ii ; i)z
Compute a value on each rank, then all_gather them.

This method operates in the following way:
    Run ``map_cp`` on all ranks
    all_gather the values to all ranks
N)	r   r   r4   r(   r>   r   r
   r   r   )rR   r   r   ro   r   r   r   s   &&&    r    
all_gather_DistWrapper.all_gather  so    	(YF ,,V4)+6}!%d::DG[))  	($Q'F	(s   A A;&A66A;c                @   < V ^8  d   QhRS[ RS[. S[3,          RS[/# r   )r,   r   r   )r   rJ   s   "r    r!   rK   $  s1     % %% "a%% 
	%r"   c                   RpV P                   '       d	    V! 4       pV P                  V4      p\        V\        4      '       d   Vh\        \        V4      #   \         d+   p\        YP                  \	        T4      /4      p Rp?LiRp?ii ; i)z
Compute a value on rank 0 and broadcast it.

This method operates in the following way:
    Run ``map_cp`` on rank 0
    broadcast the value
N)	rQ   r   r   rP   r   re   r   r
   r   )rR   r   r   ro   r   r   s   &&&   r    	broadcast_DistWrapper.broadcast$  s     26T  ,,V4l$788A|$$ ! T,TIIq?Q3RSTs   A B! BBc                   < V ^8  d   QhRR/# )r   r   N )r   rJ   s   "r    r!   rK   =  s     ' ' 'r"   c                p    V P                   '       g   R# \        P                  ! V P                  R7       R# )z
Add a synchronization point across all processes when using distributed.
If torch.distributed is initialized, this function will invoke a barrier across the global process group.
If torch.distributed is not initialized, this function is a no-op.
Nr1   )rG   r-   barrierr+   rY   s   &r    r   _DistWrapper.barrier=  s      }}}4::&r"   )rH   rN   r+   rQ   rP   rG   N)__name__
__module____qualname____firstlineno____doc__rS   rO   r3   re   rm   r4   r{   r   r   r   r   r   __static_attributes____classdictcell__)rJ   s   @r    rE   rE   K   s     ' ',  
	' 	' (
 
 $. .`)% )%V* *2% %2' 'r"   rE   c                <    V ^8  d   QhR\         R\        R\        /# r   tensorindexr   )r   r   r   )r   s   "r    r!   r!   H  s&     Y Y Ym Y Yr"   c                 N   VP                   f   \        RVP                   R24      hV P                  4       pVP                  ev   \        V4      VP                  8  d\   \        P                  ! W!P                  ,          P                  P                  4      VP                   8X  d   W!P                  ,          # V F@  p\        P                  ! VP                  P                  4      VP                   8X  g   K>  Vu # 	  \        RVP                    RVP                   R24      h)NzCannot lookup z5 since its a ShardedTensor and no offset was providedzCould not find shard at 'z' for FQN: '')
offset
ValueErrorfqnlocal_shardsr   r>   torchSizemetadatashard_offsets)r   r   shardsshards   &&  r    _find_shardr   H  s    ||UYYK'\]
 	
   "F{{K%++%

6++.77EEF%,,V++&&::enn223u||CL  0l599+UVW
XXr"   c                d    V ^8  d   QhR\         P                  R\        R\         P                  /# r   )r   Tensorr   )r   s   "r    r!   r!   ]  s)      ell = U\\ r"   c                    \        V R 4      '       d   V P                  V4      # \        V \        4      '       d   \	        W4      P
                  # VP                  em   VP                  \        P                  ! ^ .\        V P                  4       4      ,          4      8X  d   V # \        RVP                   RVP                   R24      hV # )__get_tensor_shard__FQN: '1' is not a ShardedTensor, can't find by offset: 'r   )hasattrr   r   r   r   r   r   r   r   r>   sizer   r   )r   r   s   &&r    r   r   ]  s    v-..**511&-((6)000||<<5::qcC,>&>??MUYYKPQVQ]Q]P^^_`
 	
 Mr"   c                <    V ^8  d   QhR\         R\        R\        /# )r   r;   r   r   )r   r   r	   )r   s   "r    r!   r!   m  s"       } QT r"   c                 <   VP                   V 9  d   \        R VP                    R24      hWP                   ,          p\        V\        P                  4      '       d   \        W!4      # VP                  e'   \        RVP                    RVP                   R24      hV# )zCould not find FQN: 'r   r   r   )r   r   r   r   r   r   r   )r;   r   rj   s   && r    r   r   m  s    yy
"01=>>
YY
C#u||$$ ,,		!UYYKPQVQ]Q]P^^_`
 	
 Jr"   c                ~    V ^8  d   QhR\         \        ,          R\         \        ,          R\        \        ,          /# r   abr   r   r   r   )r   s   "r    r!   r!   {  -     1 1# 18C= 1T#Y 1r"   c                 X    \        W4       UUu. uF  w  r#W#,           NK  	  upp# u uppi r$   zipr   r   i_ai_bs   &&  r    _element_wise_addr   {  $    &)!i0i(#CIIi000   &c                ~    V ^8  d   QhR\         \        ,          R\         \        ,          R\        \        ,          /# r   r   )r   s   "r    r!   r!     r   r"   c                 X    \        W4       UUu. uF  w  r#W#,
          NK  	  upp# u uppi r$   r   r   s   &&  r    _element_wise_subr     r   r   c                      a a ] tR tRt oV3R lV 3R llt]P                  3V3R lR lltV3R lR ltV3R lR	 lt	V3R
 lR lt
R tRR ltRtVtV ;t# )_ReaderViewi  c                @   < V ^8  d   QhRS[ P                  RS[RS[/# )r   base_streamr   r>   ioIOBaser   )r   rJ   s   "r    r!   _ReaderView.__annotate__  s&      BII s  r"   c                j   < \         SV `  4        W n        W0n        Wn        V P                  ^ 4       R# r   N)superrS   r   r>   r   seek)rR   r   r   r>   	__class__s   &&&&r    rS   _ReaderView.__init__  s)    &		!r"   c                ,   < V ^8  d   QhRS[ RS[ RS[ /# )r   r   whencer   rW   )r   rJ   s   "r    r!   r     s"     5 53 5 5 5r"   c                  V\         P                  8X  d   V P                  V,           pMIV\         P                  8X  d5   \         P                  pV P                  V P                  ,           V,
          pV P
                  P                  W4      # r$   )osSEEK_SETr   SEEK_ENDr>   r   r   )rR   r   r   s   """r    r   _ReaderView.seek  s_    R[[ [[6)Fr{{"[[FkkDHH,6F$$V44r"   c                    < V ^8  d   QhRS[ /# rV   rW   )r   rJ   s   "r    r!   r     s     5 5c 5r"   c                X    V P                   P                  4       V P                  ,
          # r$   )r   tellr   rY   s   &r    r   _ReaderView.tell  s     $$&44r"   c                    < V ^8  d   QhRS[ /# rV   rI   )r   rJ   s   "r    r!   r          + +$ +r"   c                6    V P                   P                  4       # r$   )r   readablerY   s   &r    r   _ReaderView.readable      ((**r"   c                    < V ^8  d   QhRS[ /# rV   r   )r   rJ   s   "r    r!   r     r   r"   c                6    V P                   P                  4       # r$   )r   seekablerY   s   &r    r   _ReaderView.seekable  r   r"   c                    V P                   V P                  4       ,
          pV^ 8X  d   ^ # \        V4      V8  d   \        V4      RV pV P                  P	                  V4      # r   )r>   r   
memoryviewr   readinto)rR   r   max_sizes   && r    r   _ReaderView.readinto  sS    88diik)q=q6H1ix(A((++r"   c                    V P                   V P                  4       ,
          pVR8X  g   W8  d   TpV P                  P                  V4      # )r]   )r>   r   r   read)rR   r   r   s   && r    r   _ReaderView.read  s=    88diik)2:D$$T**r"   )r   r>   r   )r   )r   r   r   r   rS   r   r   r   r   r   r   r   r   r   r   __classcell__)r   rJ   s   @@r    r   r     sO       /1kk 5 55 5+ ++ +,+ +r"   r   c                p    V ^8  d   QhR\         P                  R\        R\        R\         P                  /# )r   filer   lengthr   r   )r   s   "r    r!   r!     s0     - -BII -s -C -BII -r"   c                     \        WV4      # r$   )r   )r  r   r  s   &&&r    _create_file_viewr    s    tV,,r"   c                <    V ^8  d   QhR\         R\        R\         /# )r   device_type	device_idr   )r,   r   )r   s   "r    r!   r!     s!     ( ( ( ( (r"   c                "    V R8X  d   R# V  RV 2# )zDevice info normalization.cpu:r   )r	  r
  s   &&r    _normalize_device_infor    s    e]!I;''r"   Fc               #     "   \         '       d   \        P                  ! 4       '       d   \        P                  ! 4       ^ 8X  dh   \        P
                  ! 4       p V P                  4         Rx  V P                  4        \        V 4      pVP                  R4      P                  ^
4       R# Rx  R#   T P                  4        \        T 4      pTP                  R4      P                  ^
4       i ; i5i)r   Ntime)ENABLE_PROFILEr-   is_availablerO   cProfileProfileenabledisabler   
sort_statsprint_stats)profilerstatss     r    _profiler    s      ~t0022dmmo6J##%	5(OEV$004	 (OEV$004s#   'C1?C1*B1 .AC11=C..C1c                 6   a  \        S 4      R  V 3R ll4       pV# )c                $    V ^8  d   QhR\         /# rV   )r	   )r   s   "r    r!   #_api_bc_check.<locals>.__annotate__  s     ) )s )r"   c                  8  < \        V 4      ^8X  d   \        P                  ! RSP                   R2^R7       \        P
                  ! S4      pVP                  P                  4        Uu. uF+  q3P                  VP                  8X  g   K  VP                  NK-  	  ppRV9   d$   RV9   d   \        RW3 24      hV ^,          VR&   M8RV9   d$   RV9   d   \        RW3 24      hV ^,          VR&   M\        RV 24      hS! V ^ ,          3/ VB # S! V / VB # u upi )	r   zThe argument order of zG has been changed. Please check the document to avoid future breakages.)
stacklevelstorage_writerzstorage_writer in kwargs: storage_readerzstorage_reader in kwargs: zUnexpected kwonlyargs = )r>   warningswarnr   inspect	signature
parametersvalueskindKEYWORD_ONLYnamer?   RuntimeError)argskwargssigp
kwonlyargsfuncs   *,   r    
inner_func!_api_bc_check.<locals>.inner_func  s+   t9>MM( 8G G
 ##D)C # 5 5 7 7166Q^^;S 7    :-#v-(+EtnEU)VWW+/7'(!Z/#v-(+EtnEU)VWW+/7'("%=j\#JKKQ*6**(((s   *DDr   )r2  r3  s   f r    _api_bc_checkr5    s!    
4[) )2 r"   r$   )6r  r%  r   r5   r   r#  collections.abcr   r   
contextlibr   	functoolsr   pstatsr   typingr	   r
   r   r   torch.distributeddistributedr-   'torch.distributed._shard.sharded_tensorr   -torch.distributed._shard.sharded_tensor.shardr   apir   r   r   r   r   r   r   __all__r   r   r(   r9   rC   rE   r   r   r   r   r   r   r   r  r  r  r  r5  r   r"   r    <module>rA     s      	  	  . %   % %    A ?  5  8
9CLCL=
*z' z'zY* 11%+")) %+P-
(    r"   