+
    &j U                        ^ RI t ^ RIt^ RIHt ^ RIHt ^ RIt^ RIHt ^ RI	H
t
 ^ RIHt ^ RIHt ^ RIHtHtHt . ROt] P(                  ! ]4      t R
t ! R R4      t ! R R]4      t]! ]P4                  ! R4      ]P6                  4      t^ t ! R R	4      t ! R R4      tR R lt R R lt!R t"RR R llt#R R lt$R# )    N)Sequence)Any)DTensor)	local_mapmap_aggregate)	BlockMask)tree_flattentree_maptree_unflattenTensorChunkSpecFc                   *   a  ] tR t^ t o RtR tRtV tR# )_CustomReducera   
Custom reducer class that can be used to specify a custom operation that
reduces losses of multiple microbatches into one value.

Example:
>>> # xdoctest: +SKIP
>>> sum_reducer = _CustomReducer(
>>>     torch.tensor(0.0),
>>>     lambda a, b: a + b
>>> )
c                    Wn         W n        R # N
init_value	reduce_fn)selfr   r   s   &&&/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/torch/distributed/pipelining/microbatch.py__init___CustomReducer.__init__-   s    $"    r   N)__name__
__module____qualname____firstlineno____doc__r   __static_attributes____classdictcell____classdict__s   @r   r   r       s     
# #r   r   c                       ] tR t^2tRtR# )_LossReducer Nr   r   r   r   r   r%   r   r   r$   r$   2       r   r$   g        c                   z   a  ] tR t^=t o RtR tR tR t]V 3R lR l4       t	]V 3R lR l4       t
V 3R	 ltR
tV tR# )r   z*
Class used to specify chunking of inputs
c                    Wn         R # r   	split_dim)r   r+   s   &&r   r   TensorChunkSpec.__init__B   s    "r   c                |    V P                   P                   R V P                   P                   RV P                   R2# ).())	__class__r   r   r+   r   s   &r   __repr__TensorChunkSpec.__repr__G   s9    ~~(()4>>+B+B*C1T^^DTTUV	
r   c                "    R V P                    R2# )zTensorChunkSpec(r0   r*   r2   s   &r   __str__TensorChunkSpec.__str__L   s    !$..!133r   c                4   < V ^8  d   QhRS[ S[R3,          /# )   
chunk_dims.)tupleint)formatr"   s   "r   __annotate__TensorChunkSpec.__annotate__P   s      #s(Or   c                     \        V R 4      pV# )aJ  
A helper for creating a tuple of `TensorChunkSpec` from a tuple of chunk
dimensions (int's).
Example:
    >>> # xdoctest: +SKIP
    >>> # There are three positional arguments to the model, and
    >>> # we are chunking them along dimension 0, 0 and 1, respectively
    >>> args_chunk_spec = TensorChunkSpec.from_tuple((0, 0, 1))
c                     \        V 4      # r   r   dims   &r   <lambda>,TensorChunkSpec.from_tuple.<locals>.<lambda>^   	    ,r   r   )r:   args_chunk_specs   & r   
from_tupleTensorChunkSpec.from_tupleO   s     (,
 r   c                6   < V ^8  d   QhRS[ S[S[3,          /# )r9   r:   )dictstrr<   )r=   r"   s   "r   r>   r?   c   s     ! !cN!r   c                     \        V R 4      pV# )a$  
A helper for creating a dictionary of `TensorChunkSpec` from a
dictionary of chunk dimensions (int's).
Example:
    >>> # xdoctest: +SKIP
    >>> # Chunk dimension 0 for the "id" argument, 1 for the "mask" argument
    >>> kwargs_chunk_spec = TensorChunkSpec.from_dict({"id": 0, "mask": 1})
c                     \        V 4      # r   rB   rC   s   &r   rE   +TensorChunkSpec.from_dict.<locals>.<lambda>p   rG   r   r   )r:   kwargs_chunk_specs   & r   	from_dictTensorChunkSpec.from_dictb   s     *,
 ! r   c                &   < V ^8  d   Qh/ S[ ;R&   # )r9   r+   )r<   )r=   r"   s   "r   r>   r?   =   s      N r   r*   N)r   r   r   r   r   r   r3   r6   staticmethodrI   rR   __annotate_func__r   r    r!   s   @r   r   r   =   sI     #


4  $ ! !M  r   c                       ] tR t^vtRtR# )
_Replicater%   Nr&   r%   r   r   rX   rX   v   r'   r   rX   c                R    V ^8  d   QhR\         R\        R\        \         ,          /# )r9   
block_mask
num_chunksreturn)r	   r<   list)r=   s   "r   r>   r>   z   s*     > >>> 
)_>r   c                N  a  S P                   P                  ^ 4      ^8X  d   S .V,          # S P                   P                  ^ 4      V8  g   \        R4      h^ p\        P                  ! S P                   W4      p\        P                  ! S P
                  W4      pS P                  e"   \        P                  ! S P                  W4      M	R.V,          pS P                  e"   \        P                  ! S P                  W4      M	R.V,          p. p^ p\        V4       F  p	V 3R lp
VP                  \        P                  ! W9,          WI,          WY,          Wi,          S P                  V
! V4      S P                  R7      4       WV	,          P                  ^ 4      ,          pK  	  V# )zGiven a block mask, split the block mask along the batch dimension (dim0).

Args:
    block_mask: Block mask to split
    num_chunks: Number of chunks to split the block mask into

Returns:
    chunk_block_masks: List of chunked block masks
z;Block mask has fewer batch size than the number of chunks. Nc                    <a  VV 3R  lpV# )c                 d   < \         P                  ! V S4      pSP                  W,           WV4      # r   )torch	full_likemask_mod)bhq_idxkv_idxb_offsetrZ   idxs   &&&& r   batch_offset_mask_modI_split_block_mask.<locals>.create_mask_mod.<locals>.batch_offset_mask_mod   s*     ??1c2!**1<6JJr   r%   )ri   rj   rZ   s   f r   create_mask_mod*_split_block_mask.<locals>.create_mask_mod   s    K )(r   )kv_num_blocks
kv_indicesfull_kv_num_blocksfull_kv_indices
BLOCK_SIZErc   seq_lengths)rn   sizeAssertionErrorra   tensor_splitro   rp   rq   rangeappendr	   from_kv_blocksrr   rs   )rZ   r[   	batch_dimkv_num_blocks_chunkskv_indices_chunksfull_kv_num_blocks_chunksfull_kv_indices_chunkschunk_block_masksbatch_offset	chunk_idxrl   s   f&         r   _split_block_maskr   z   s    $$Q'1,|j((##((+z9I
 	
 I --  * **:+@+@*X ((4 	:88*PVj   %%1 	:55zMVj   L:&		) 	  $$2=,7#<#G 6 A%00(6&22
	
 	Y7<<Q??) '* r   c          	          V ^8  d   QhR\         P                  R\        R\        R\        \         P                  ,          /# )r9   tensorspecr[   r\   )ra   Tensorr   r<   r   )r=   s   "r   r>   r>      sC     I< I<LLI<
I< I< ell	I<r   c                  aa V P                  SP                  4      S8  g)   \        RV P                  SP                  4       R24      h\        V \        4      pV'       d2   V P
                  p\        VV3R lV3S,          V3R7      pV! V 4      pM"\        P                  ! V SSP                  4      pV P                  '       d,   V P                  '       d   V F  pVP                  4        K  	  \        '       g   V# R V3R llpV'       dQ   V P
                  p\        V4      p	\        VV3V	,          V3V3V	,          ,           R7      p
\        V
! V .VO5!  4      # \        V! V .VO5!  4      # )zGiven a tensor, and a chunking spec, split the tensor.
Args:

    tensor: Tensor to split
    spec: Chunking spec
    num_chunks: Number of chunks to split the tensor into

Returns:
    chunk_tensors: List of chunked tensors
zTensor size z is smaller than num_chunksc                 H   < \         P                  ! V SSP                  4      # r   )ra   rv   r+   )tr[   r   s   &r   rE   _split_tensor.<locals>.<lambda>   s    e((JGr   out_placementsin_placementsc                    V ^8  d   QhR\         P                  R\         P                  R\        \         P                  R3,          /# )r9   origchunksr\   .)ra   r   r;   )r=   s   "r   r>   #_split_tensor.<locals>.__annotate__   s:      ll%*\\	u||S 	!r   c                 x  < . p^ pV F  p\         P                  ! V 4      pW4P                  SP                  4      ,           p\	        R4      .VP
                  ,          p\	        W64      VSP                  &   WEV&   VP                  V4       W4P                  SP                  4      ,          pK  	  \        V4      # )r   N)ra   
zeros_likert   r+   slicendimrx   r;   )	r   r   expandedri   chunknew_valupperslicesr   s	   &*      r   _expand_chunks%_split_tensor.<locals>._expand_chunks   s     E&&t,G**T^^44E#(;-',,">F%*3%6F4>>"#FOOOG$::dnn--C  Xr   )rt   r+   ru   
isinstancer   
placementsr   ra   rv   requires_gradis_leafretain_grad_debug_mask_minibatcheslenr]   )r   r   r[   _is_dtensorr   split_fnchunk_tensorsr   r   n	expand_fns   &ff        r   _split_tensorr      sR     ;;t~~&*46;;t~~677RS
 	
 VW-K
 &&
G&=:5%-

 190@**6:t~~N
 "E # #"  &&
&=1,%-:-!*;;
	
 If5}566N6:M:;;r   c           	        V '       g   \        V4       Uu. uF  p/ NK  	  up# \        V 4      \        V4      8X  g@   \        R\        V P	                  4       4       R\        VP	                  4       4       24      hVf   \        R4      h\        V R R7      w  rE\        VR R7      w  rc. p\        WFRR7       EF  w  rV	\        J g   \        V	\        4      '       d   VP                  V4       K9  \        V\        P                  4      '       dZ   \        V	\        4      '       g   \        R	\        V	4       24      hVP                  VP                  V	P                  4      4       K  \        V\         4      '       d   \        V	\        4      '       g   \        R	\        V	4       24      hV	P                  ^ 8X  g   \        R
4      hVP"                  P                  ^ 4      ^8X  d   VP                  V4       EKE  VP                  VP"                  P                  ^ 4      4       EKr  \%        RV	 RV R24      h	  \'        . VOVN5!  p
\        V
4       Uu. uF  p. NK  	  pp\        WFRR7       F  w  r. pV	\        J g   \        V	\        4      '       d   V.V
,          pMa\        V\        P                  4      '       d   \)        WV
4      pM4\        V\         4      '       d   \+        W4      pM\%        RV	 RV R24      h\        WRR7       F  w  rVP                  V4       K  	  K  	  V Uu. uF  p\-        W4      NK  	  up# u upi u upi u upi )a3  
Given a dictionary of args, and a dictionary of chunking specs, shard the
args according to the chunking specs.

Args:
    args_dict: Dictionary of args
    args_chunk_spec: Dictionary of chunking specs
    num_chunks: Number of chunks to shard the args into

Returns:
    args_split: List of sharded args
zargs_dict.keys() = z args_chunk_spec.keys() = z.args_chunk_spec should have been set by callerc                 "    \        V \        4      # r   r   r	   xs   &r   rE   %_shard_dict_of_args.<locals>.<lambda>%  s    Z9%=r   r   c                 "    \        V \        4      # r   r   r   s   &r   rE   r   (  s    :a+Cr   TstrictzExpected TensorChunkSpec, got z#BlockMask only supports split_dim=0zUnsupported chunk spec: z and value: z combination.)rw   r   ru   r]   keysr
   ziprX   r   rx   ra   r   r   typert   r+   r	   rn   
ValueErrorminr   r   r   )	args_dictrH   r[   _values	tree_specchunk_specssplit_sizesvr   result_num_chunksflat_split_resultsv_splits_flat_split_result_v_splits   &&&            r   _shard_dict_of_argsr     s   $ !*-.-q-..y>S11!$y~~'7"8!9 :((,_-A-A-C(D'EG
 	
 MNN$=F "!CNK
 Kv488 :D*!=!=z*5<<((dO44$'Ed4j\%RSSqvvdnn569%%dO44$'Ed4j\%RSS>>Q&$%JKK##A&!+"":.""1??#7#7#:;*4&QC}M ) 9. 5[5*5167H1I$J1IAR1I$Jv48"$:D*!=!=s..H5<<(($Q.?@H9%%(>H*4&QC}M  -0-
( %%h/-
 9( #5"4 	)5"4  /X %K&s   MM7Mc                P   V ^8  d   QhR\         \        R3,          R\        \        \        3,          R,          R\        R\         \
        R3,          R,          R\        \        \
        3,          R,          R\         \        \         ,          \        \        ,          3,          /# )	r9   args.kwargsNr   rH   rQ   r\   )r;   r   rL   rM   r<   r   r]   )r=   s   "r   r>   r>   _  s     p$ p$
S/p$cNT!p$ p$ ?C/047	p$
 C01D8p$ 4;T
"#p$r   c                  a	 Vf   / pR pVf   \        WPR R7      pVf   \        WQR R7      p\        \        \        V 4      4      \        \        V4      4      V4      p\	        V4      p\        VVV4      p\	        V4      V8  d=   \	        V4      p\        \        \        V 4      4      \        \        V4      4      V4      p\	        V4      \	        V4      8w  d$   \        R\	        V4       R\	        V4       24      hV U	a	u. uFY  o	\        ;QJ d)    . V	3R l\        \	        S	4      4       4       F  NK  	  5M"! V	3R l\        \	        S	4      4       4       4      NK[  	  p
p	W3# u up	i )a  
Given a sequence of args and kwargs, split them into a number of chunks
according to  their respective chunking specs.

Args:
    args: Tuple of args
    kwargs: Dict of kwargs
    chunks: Number of chunks to split the args and kwargs into
    args_chunk_spec: chunking specs for args, in same shape as args
    kwargs_chunk_spec: chunking specs for kwargs, in same shape as kwargs

Returns:
    args_split: List of sharded args
    kwargs_split: List of sharded kwargs
c                     \        V \        P                  \        ,          4      '       d   \	        \
        4      # \        4       # r   )r   ra   r   r	   r   DEFAULT_CHUNK_DIMrX   r   s   &r   default_spec3split_args_kwargs_into_chunks.<locals>.default_spec  s,    a	122"#455<r   c                 "    \        V \        4      # r   r   r   s   &r   rE   /split_args_kwargs_into_chunks.<locals>.<lambda>  s    *Q	2Jr   r   c                 "    \        V \        4      # r   r   r   s   &r   rE   r     s    Jq)4Lr   z;args and kwargs are split into different number of chunks: z, c              3   6   <"   T F  pSV,          x  K  	  R # 5ir   r%   ).0i
chunk_argss   & r   	<genexpr>0split_args_kwargs_into_chunks.<locals>.<genexpr>  s     <%;jmm%;s   )r   r   rL   	enumerater   RuntimeErrorr;   rw   )r   r   r   rH   rQ   r   args_split_dictreal_num_chunkskwargs_splitr   
args_splits   &&&&&    ` r   split_args_kwargs_into_chunksr   _  sk   p ~  "(J
  $*L
 *Yt_Y'(O
 /*O&L <?* l+-4!?+,
 ?s<00I?#$Bs<'8&9;
 	
 *)J 	<U3z?%;<<U3z?%;<<)  
 ##s   E)AE)c                :    V ^8  d   QhR\         \        ,          /# )r9   r   )r]   r   )r=   s   "r   r>   r>     s     [8 [8I[8r   c                t	  aa Ve   \        V4      w  r#M4\        V ^ ,          4      w  rC\        \        4      .\        V4      ,          p. pV  FK  p\        V4      w  rx\        V4      \        V4      8w  d   \	        RV RV 24      hVP                  V4       KM  	  . p	\        V4       EF  w  p
o\        S\        4      '       Ed   \        \        V4      4       Uu. uF  pW[,          V
,          NK  	  pp\        '       Ed^   V^ ,          P                  pVR,           F/  pVP                  V8X  d   K  \        RV RVP                   24      h	  \        P                  ! \        P                  ! VRR/ \        V4      SP                  R	7      p. p^ p\        V4      \        V4      8X  g$   \        R
\        V4       R\        V4       24      h\!        WRR7       F|  w  ppVVP#                  SP                  4      ,           p\%        RRR4      .VP&                  ,          p\%        VV4      VSP                  &   VV,          pVP                  V4       TpK~  	  MTpV Uu. uF  p\        V\(        4      NK  	  pp\+        V4      '       d   \-        V4      '       g   \        R4      hV^ ,          P.                  o\        VR,          ^4       F5  w  ppVP.                  S8w  g   K  \        RV RS RVP.                   24      h	  \1        V3R lS3\2        ;QJ d)    . V3R l\        \        V4      4       4       F  NK  	  5M"! V3R l\        \        V4      4       4       4      R7      pV	P                  V! V!  4       EK  V	P                  \        P4                  ! VSP                  R7      4       EK  \        S\6        4      '       d[   SP8                  p\        \        V4      4       F"  pSP;                  VW[,          V
,          4      pK$  	  V	P                  V4       EK  V^ ,          V
,          p\        ^\        V4      4       F5  pW[,          V
,          V8X  d   K  \        RV RW[,          V
,           24      h	  V	P                  V4       EK  	  \=        W4      # u upi u upi )z
Given a list of chunks, merge them into a single value according to
the chunk spec.

Args:
    chunks: list of chunks
    chunk_spec: Chunking spec for the chunks

Returns:
    value: Merged value
NzChunk z did not match chunk spec :   NNzExpected shape z, got devicemeta)sectionsrD   z6Expected len(partial_values) == len(meta_chunks), got z != Tr   zRmerge_chunks: expected all values to be DTensors or none to be DTensors, got a mixz*merge_chunks: placement mismatch at chunk z: expected c                  H   < \         P                  ! V SP                  R 7      # )rC   )ra   catr+   )r   args   *r   rE   merge_chunks.<locals>.<lambda>S  s    EIIf#--$Hr   c              3   (   <"   T F  pSx  K	  	  R # 5ir   r%   )r   r   r   s   & r   r   merge_chunks.<locals>.<genexpr>U  s     'V<Uq
<Us   r   rC   z	Expected )r
   r   r   r   r   rx   r   r   rw   r   shaperu   ra   rv   emptyr+   r   rt   r   r   r   anyallr   r   r;   r   r   r   r   r   )r   
chunk_specspec_flattenedflatten_specchunk0_flatchunks_flattenedr   chunk_flattenedr   args_flattenedarg_idxr   partial_valuesoverall_shapevalmeta_chunksvalues_to_catchunk_start_idxpartial_value
meta_chunkchunk_end_idxslice_indicesslicedr   dtensor_flagsr   cat_fnreduced_valvaluer   r   s   &&                           @@r   merge_chunksr    s^   Z '3J'?$ %1$;!)*;<=K@PP )%03~#66veW,FzlSTT0  N!.1c?++ "'s+;'<!=!=I !+G44!=  
 '& .q 1 7 7)"--C995,-m_F399+N  .
 $00KK>v> 0 !#"#>*c+.>>(PQTUcQdPeeijmnyjzi{|  25"2-M: %4joocmm6T$TM%*4t%<$=@R@R$RM38-3XM#--0*=9F!((0&3O2 !/ >KK]Z73]MK=!!=))(9 
 +1-88
%mB&7;DAq||z1,H L((2|6!,,I  < #H$.="'%'VE#mBT<U'V%%'VE#mBT<U'V"V
 %%fm&<=%%eii3==&QR^,,..K"3'7#89	!mm!1!<W!E :
 !!+.$Q'0E"1c*:&;<	'27;uD(#E7&1A1LW1U0VW  =
 !!%(i 2n .77kV Ls   R0'R5)r   r   r  )NN)%loggingoperatorcollections.abcr   typingr   ra   torch.distributed.tensorr   %torch.distributed.tensor.experimentalr   torch.fx.noder   !torch.nn.attention.flex_attentionr	   torch.utils._pytreer
   r   r   __all__	getLoggerr   loggerr   r   r$   r   addsum_reducerr   r   rX   r   r   r   r   r  r%   r   r   <module>r     s      $   , ; ' 7 F F 
		8	$
   # #$	> 	 5<<,hll;  5! 5!r	 	>BI<XUpp$f[8r   