+
    &j                     *   ^ RI Ht ^ RIt^ RIHt ^RIHtHtHtHtH	t	H
t
HtHtHtHtHtHtHtHtHtHtHtHtHtHt RR.t ! R R]4      tRR	] R
]
 R] R] R] R] R2,           ]n        R R ltR R ltR R lt]	! ]R7      RR R ll4       tR# )    )castN)Tensor)_capturable_doc_default_to_fused_or_foreach_device_dtype_check_for_fused_differentiable_doc_disable_dynamo_if_unsupported_foreach_doc
_fused_doc!_get_capturable_supported_devices_get_scalar_dtype
_get_value_maximize_doc_params_doc_stack_if_compiling
_to_scalar_use_grad_for_differentiable_view_as_real
DeviceDictDeviceDtypeDict	OptimizerParamsTAdamadamc                      a a ] tR t^"t oRRRRRRRRRRRRR/V3R	 lV 3R
 lllltV 3R ltR t]RR l4       tRt	Vt
V ;t# )r   FforeachNmaximize
capturabledifferentiablefuseddecoupled_weight_decayc                   < V ^8  d   QhRS[ RS[S[,          RS[S[S[,          S[S[,          3,          RS[RS[RS[RS[R,          R	S[R
S[RS[RS[R,          RS[RR/# )   paramslrbetasepsweight_decayamsgradr   Nr   r   r   r    r!   return)r   floatr   tuplebool)format__classdict__s   "h/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/torch/optim/adam.py__annotate__Adam.__annotate__#   s     NW NWNW FNNW UV^UV^34	NW
 NW NW NW NW NW NW NW d{NW !%NW 
NW    c               D  < \        V\        4      '       d<   V'       d   V	'       g   \        R 4      hVP                  4       ^8w  d   \        R4      hRV8:  g   \        RV 24      hRV8:  g   \        RV 24      hRV^ ,          u;8:  d   R8  g   M \        RV^ ,           24      hRV^,          u;8:  d   R8  g   M \        RV^,           24      hRV8:  g   \        RV 24      h\        V^ ,          \        4      '       d   \        V^,          \        4      '       gF   \        V^ ,          \        4      '       d   \        V^,          \        4      '       g   \        R	4      h\        V^ ,          \        4      '       dC   V	'       g   V'       d   \        R
4      hV^ ,          P                  4       ^8w  d   \        R4      h\        V^,          \        4      '       dC   V	'       g   V'       d   \        R4      hV^,          P                  4       ^8w  d   \        R4      h\        \        \        V4      4      pRVRVRVRVRVRVRVRV	RV
RVRV/p\        SV `%  W4       V'       d0   V
'       d   \        R4      hRV n        V'       d   \        R4      hR# R# )Elr as a Tensor is not supported for capturable=False and foreach=TrueTensor lr must be 1-element        zInvalid learning rate: zInvalid epsilon value:       ?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: z0betas must be either both floats or both TensorszKbetas[0] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[0] must be 1-elementzKbetas[1] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[1] must be 1-elementr%   r&   r'   r(   r)   r   r   r   r   r    r!   z)`fused` does not support `differentiable`Tz0`fused` and `foreach` cannot be `True` together.N)
isinstancer   
ValueErrornumelr+   r,   mapr   super__init__RuntimeError_step_supports_amp_scaling)selfr$   r%   r&   r'   r(   r)   r   r   r   r   r    r!   defaults	__class__s   &&&&&&&$$$$$$ r0   r>   Adam.__init__#   sS     b&!!z [  xxzQ !>??by6rd;<<cz6se<==eAh$$B58*MNNeAh$$B58*MNNl";L>JKKa%((Za%-H-H58V,,E!Hf1M1MOPPeAh''' a  Qx~~1$ !DEEeAh''' a  Qx~~1$ !DEEc*e,- "U3Lww*nU$&<
 	*"#NOO.2D+
 "#UVV  r3   c           	       < \         SV `  V4       V P                   EFe  pVP                  R R4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4      pVR	,           F  pV P                  P                  V. 4      p\        V4      ^ 8w  g   K1  \        P                  ! VR
,          4      '       d   KV  \        VR
,          4      pVR,          '       g   VR,          '       d.   \        P                  ! V\        VR7      VP                  R7      M\        P                  ! V\        4       R7      VR
&   K  	  EKh  	  R# )r)   Fr   r   Nr   r   r!   r    r$   stepis_fuseddtypedevicerJ   )r=   __setstate__param_groups
setdefaultstategetlentorch	is_tensorr+   tensorr   rK   )rA   rP   groupr    pp_statestep_valrC   s   &&     r0   rM   Adam.__setstate__s   s6   U#&&EY.Z/Y-\51-u55u=$$Wd3E8__**..B/w<1$U__WV_-M-M$WV_5H !..%.. $"3U"C#$88 #\\(:K:MN FO	 % 'r3   c                   R pVR,           EF  p	V	P                   f   K  V\        P                  ! V	4      ,          pVP                  V	4       V	P                   P                  '       d   \        R4      hVP                  V	P                   4       V P                  V	,          p
\        V
4      ^ 8X  Ed   VR,          '       d   \        V	4       VR,          '       g   VR,          '       d5   \        P                  ! R\        VR,          R7      V	P                  R7      M\        P                  ! R\        4       R7      V
R	&   \        P                  ! V	\        P                  R
7      V
R&   \        P                  ! V	\        P                  R
7      V
R&   VR,          '       d*   \        P                  ! V	\        P                  R
7      V
R&   VP                  V
R,          4       VP                  V
R,          4       VR,          '       d   VP                  V
R,          4       VR,          '       d%   V
R	,          P                  '       d   \        R4      hVR,          '       d>   \        P                   ! VR,          4      '       d   VR,          '       g   \        R4      hVP                  V
R	,          4       EK  	  V# )Fr$   zJAdam does not support sparse gradients, please consider SparseAdam insteadr    r   rG   rI   r7   rL   rF   )memory_formatexp_avg
exp_avg_sqr)   max_exp_avg_sqr   zB`requires_grad` is not supported for `step` in differentiable moder   r%   r5    )gradrS   
is_complexappend	is_sparser?   rP   rR   r   zerosr   rK   rU   
zeros_likepreserve_formatrequires_gradrT   )rA   rV   params_with_gradgradsexp_avgsexp_avg_sqsmax_exp_avg_sqsstate_stepshas_complexrW   rP   s   &&&&&&&&   r0   _init_groupAdam._init_group   s2    xAvv!u//22 ''*66###&d  QVV$

1u:?W~~5a8 !..%.. "3U7^"L#$88 #\\#5F5HI &M (-'7'7)>)>(E)$ +0*:*:)>)>+E,' Y''272B2BU-B-B3./ i 01""5#67###**51A+BC)**uV}/J/J/J&\  )$$d44!,//&_  ""5=1{ !| r3   c           $     F   V P                  4        RpVe.   \        P                  ! 4       ;_uu_ 4        V! 4       pRRR4       V P                   F  p. p. p. p. p. p. p	VR,          w  rV P	                  VVVVVVV	4      p\        VVVVVV	3RVR,          RVRV
RVRVR,          RVR,          R	VR	,          R
VR
,          RVR,          RVR,          RVR,          RVR,          R\        V RR4      R\        V RR4      RVR,          /  K  	  V#   + '       g   i     L; i)zPerform a single optimization step.

Args:
    closure (Callable, optional): A closure that reevaluates the model
        and returns the loss.
Nr&   r)   ro   beta1beta2r%   r(   r'   r   r   r   r   r    
grad_scale	found_infr!   )'_accelerator_graph_capture_health_checkrS   enable_gradrN   rp   r   getattr)rA   closurelossrV   ri   rj   rk   rl   rm   rn   rs   rt   ro   s   &&           r0   rF   	Adam.step   s    	446""$$y % &&E-/"$E%'H(*K,.O(*K >LE** K   i( (   ; #>2 %L z* i(  !.!"  %%56#$ Gn%& #4t<'( "$T:)* (--E'F+' 'V ] %$s   DD 	)r@   )gMbP?)g?g+?g:0yE>r   FN)__name__
__module____qualname____firstlineno__r>   rM   rp   r   rF   __static_attributes____classdictcell____classcell__)rC   r/   s   @@r0   r   r   "   s     NW  $NW NW !NW  %NW "NW (-NW NW`0IV "9 "9 9r3   af  Implements Adam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}          \\
            &\hspace{13mm}      \lambda \text{ (weight decay)},  \: \textit{amsgrad},
                \:\textit{maximize},  \: \epsilon \text{ (epsilon)}                              \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0\leftarrow 0 \text{ (second moment)},\: v_0^{max}\leftarrow 0          \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{if} \: \lambda \neq 0                                           \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm} v_t^{max} \leftarrow \mathrm{max}(v_{t-1}^{max},v_t)                  \\
            &\hspace{10mm}\widehat{v_t} \leftarrow v_t^{max}/\big(1-\beta_2^t \big)              \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                  \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    z
    Args:
        a;  
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (tuple[float | Tensor, float | Tensor], optional):
            coefficients used for computing running averages of gradient and
            its square. If a tensor is provided, must be 1-element. (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        decoupled_weight_decay (bool, optional): if True, this optimizer is
            equivalent to AdamW and the algorithm will not accumulate weight
            decay in the momentum nor variance. (default: False)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        a=  
    .. Note::
        A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`.
    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    c          (         V ^8  d   QhR\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\        R,          R	\        R,          R
\        R\        R\        \        ,          R\        \        ,          R\        \        ,          R\        R\        R\        R\        R\        R\        RR/# r#   r$   rj   rk   rl   rm   rn   ru   Nrv   r)   ro   rs   rt   r%   r(   r'   r   r   r   r!   r*   listr   r-   r+   )r.   s   "r0   r1   r1   [  s(    KK KKLKK<KK 6lKK f	KK
 &\KK fKK KK }KK KK KK 6>KK 6>KK 	KK KK  
!KK" #KK$ %KK& 'KK( !)KK* 
+KKr3   c          
      \   Vf   Ve   \        R4      h\        P                  P                  4       '       d   \	        V\
        4      '       g   \        R\        V4       24      h\	        V
\
        4      '       g   \        R\        V
4       24      h\	        V\
        4      '       g   \        R\        V4       24      hM!\        V4      p\        V
4      p
\        V4      p\	        V
\        4      '       d   V
P                  V
P                  3V
/pMR p\        V 4       EF  w  ppV'       g
   VV,          M	VV,          ) pVV,          pVV,          pVV,          p\        P                  P                  4       '       gl   V'       dd   \        4       pVP                  P                  VP                  P                  8X  d   VP                  P                  V9   g   \        RV R24      hV^,          pV^ 8w  d   V'       d    VP                  ^W,          ,
          4       MxV'       d^   \	        V\        4      '       dH   VP                   '       d"   VP#                  VP%                  4       V4      pM'VP'                  VVR7      pMVP'                  VVR7      p\        P(                  ! V4      '       d   \        P*                  ! V4      p\        P*                  ! V4      p\        P*                  ! V4      pV'       d!   \        P*                  ! VV,          4      VV&   \        P*                  ! V4      pVP                  pVe9   VP                  pVV3pVV9  d   V
P-                  VVRR	7      VV&   VV,          pMT
pVP/                  V^V,
          4       V'       d   \	        V\        4      '       d{   VP                   '       d0   VP/                  \        P0                  ! V4      ^V,
          R
7       McVP                  V4      P#                  VV\3        \
        ^V,
          4      R7       M*VP                  V4      P#                  VV^V,
          R7       V'       g
   V'       Ed   TpV'       dY   \	        V
\        4      '       dC   V
P                   '       d    ^V
VP%                  4       ,          ,
          p M!^V
V,          ,
          p M^V
V,          ,
          p V'       dY   \	        V\        4      '       dC   VP                   '       d    ^VVP%                  4       ,          ,
          p!M!^VV,          ,
          p!M^VV,          ,
          p!VV ,          p"V"P5                  4       p#V!P7                  4       p$V'       d   V'       d   VV,          P%                  4       p%M	VV,          p%VV,          P9                  \        P:                  ! V%V4      4       VV,          P7                  4       V$V#,          ,          P=                  VV#,          4      p&M4VP7                  4       V$V#,          ,          P=                  VV#,          4      p&V'       d"   VP?                  VP%                  4       V&4       MVP?                  VV&4       M\A        V4      p^V
V,          ,
          p ^VV,          ,
          p!VV ,          p"V!R,          p$V'       dV   \        P:                  ! VV,          VVV,          R7       VV,          P7                  4       V$,          P=                  V4      p&M&VP7                  4       V$,          P=                  V4      p&VP?                  VV&V") R7       V'       g   EKB  \        P(                  ! V V,          4      '       g   EKh  \        PB                  ! VV,          4      VV&   EK  	  R # )N,Expected grad_scale and found_inf to be Nonez#Expected lr to be a float, but got z&Expected beta1 to be a float, but got z&Expected beta2 to be a float, but got IIf capturable=True, params and state_steps must be on supported devices: .alphaT)rK   rJ   non_blocking)weight)value      ?)out)"AssertionErrorrS   jitis_scriptingr9   r+   typer   r   rK   rJ   	enumeratecompileris_compilingr   mul_rh   addcmul_cloneaddrb   view_as_realtolerp_squarer   negsqrtcopy_maximumadd_addcdiv_r   view_as_complex)'r$   rj   rk   rl   rm   rn   ru   rv   r)   ro   rs   rt   r%   r(   r'   r   r   r   r!   
beta1_dictiparamra   r]   r^   step_tcapturable_supported_devicesrK   rJ   keydevice_beta1rF   bias_correction1bias_correction2	step_sizestep_size_negbias_correction2_sqrtr_   denoms'   &&&&&&&&$$$$$$$$$$$                    r0   _single_tensor_adamr   [  s   , !6KLLyy "e$$ #FtBxj!QRR%'' #I$u+!WXX%'' #I$u+!WXX ( ^5!5! %  /4||U[[.I5-Q

f%5'uQxeAhY1+ ^
Q ~~**,,+L+N(!!V]]%7%77LL%%)EE$_`|_}}~ 
 	!1%

1r001 "jv&F&F#111#}}U[[]LI#xx\xB88E8>DE""%%d+D((1G++J7J%*%7%78J%K"&&u-E!KKE 5/C*$"'((!T #+ #
3 ,6c?L L 	dA,- j77"""   d!3AI F&//$d5!e)&< 0  OOE"++D$a%i+HD *UF";";&&&'(5DJJL+@'@$'(5$;$#$ud{?  *UF";";&&&'(5DJJL+@'@$'(5$;$#$ud{? --I%MMOM$4$9$9$;!!%4Q%7%=%=%?N%4Q%7N"((~z)RS $A&++-1F1VW$s]*+ 
 OO%)>)NO$s]*+  w}}6w.f%D 5$; 5$;--I$4c$9!oa0*/RSBTU )+0025JJPPQTU#*-BBHHMNN7E)N< 7u''q	22!&!6!6q7I!JOAu &r3   c          (         V ^8  d   QhR\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\        R,          R	\        R,          R
\        R\        R\        \        ,          R\        \        ,          R\        \        ,          R\        R\        R\        R\        R\        R\        RR/# r   r   )r.   s   "r0   r1   r1   )  s    v vLv<v 6lv f	v
 &\v fv v }v v v 6>v 6>v 	v v  
!v" #v$ %v& 'v( !)v* 
+vr3   c          
         a- \        V 4      ^ 8X  d   R# \        V\        4      '       d4   V'       g   \        R4      hVP	                  4       ^8w  d   \        R4      h\        V
\        4      '       d4   V'       g   \        R4      hV
P	                  4       ^8w  d   \        R4      h\        V\        4      '       d4   V'       g   \        R4      hVP	                  4       ^8w  d   \        R4      h\        P                  P                  4       '       gz   V'       dr   \        RR	7      o-\        ;QJ d*    V-3R
 l\        WRR7       4       F  '       d   K   RM	  RM! V-3R
 l\        WRR7       4       4      '       g   \        RS- R24      hVf   Ve   \        R4      hV'       d   \        R4      h\        V4      p\        V
4      p
\        V4      p\        P                  ! WW#WE.4      p\        V
\        4      '       d)   \!        V
P"                  4      R8w  d   V
P"                  V
/MRpVP%                  4        EF  w  w  ppppppp\'        \(        \        ,          V4      p\'        \(        \        ,          V4      p\'        \(        \        ,          V4      p\'        \(        \        ,          V4      p\'        \(        \        ,          V4      p V^ ,          P"                  p!Ve   V!V9  d   V
P+                  V!RR7      VV!&   V'       d
   VV!,          MT
p"V	'       dB   V'       d,   \'        \(        \        ,          V4      p#\-        VVVVV#4       M\-        VVVV4       V'       d   \        P.                  ! V4      p\        P                  P                  4       '       gJ   V ^ ,          P0                  '       d1   \        P2                  ! V \        P4                  ! RRR7      RR7       M\        P2                  ! V ^4       V^ 8w  di   V'       d&   \        P6                  ! V^W,          ,
          4       M;V'       d   \        P2                  ! VVVR7       M\        P8                  ! VVVR7      p\        P:                  ! VV\'        \<        ^V",
          4      4       \        P6                  ! VV4       \        V\        P                  4      '       d"   \        P>                  ! V^V,
          4      p$Rp%MTp$^V,
          p%\        P@                  ! VV$VV%4       ??$V'       Ed   \        PB                  ! V
V 4      p&\        PB                  ! VV 4      p'\        PD                  ! V&^4       \        PD                  ! V'^4       \        PF                  ! V'4       \        PH                  ! V&V4       \        PJ                  ! V&4       \        PL                  ! V'4       T&p(T'p)V'       dJ   \'        \(        \        ,          V4      p#\        PN                  ! V#V4       \        PP                  ! V#4      p*M\        PP                  ! V4      p*\        PH                  ! V*V)4       \        P2                  ! V*V4       \        PH                  ! V*V(4       \        PR                  ! VVV*4       EKj  V  U+u. uF  p+^V
\U        V+4      ,          ,
          NK  	  p&p+V  U+u. uF  p+^V\U        V+4      ,          ,
          NK  	  p'p+\W        V& U,u. uF  p,VV,,          R,          NK  	  up,4      p(V' U,u. uF  p,V,R,          NK  	  p)p,V'       dJ   \'        \(        \        ,          V4      p#\        PN                  ! V#V4       \        PP                  ! V#4      p*M\        PP                  ! V4      p*\        PH                  ! V*V)4       \        P2                  ! V*V4       \        PR                  ! VVV*V(4       EK  	  R# u up+i u up+i u up,i u up,i )r   Nr5   r6   zHbeta1 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta1 must be 1-elementzHbeta2 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta2 must be 1-elementF)supports_xlac              3      <"   T FU  w  rVP                   P                  VP                   P                  8H  ;'       d    VP                   P                  S9   x  KW  	  R # 5ir}   )rK   r   ).0rW   rF   r   s   &  r0   	<genexpr>%_multi_tensor_adam.<locals>.<genexpr>_  sT      
 A HHMMT[[--- > >!==>@s
   :A "A T)strictr   r   r   z#_foreach ops don't support autogradcpurK   r   r8   )rK   r   r   ),rR   r9   r   r?   r;   r:   rS   r   r   r   allzipr   r   r   "_group_tensors_by_device_and_dtypestrrK   valuesr   r   r   r   _foreach_negis_cpu_foreach_add_rU   _foreach_mul__foreach_add_foreach_lerp_r+   _foreach_mul_foreach_addcmul__foreach_pow_foreach_sub__foreach_neg__foreach_div__foreach_reciprocal__foreach_sqrt__foreach_maximum__foreach_sqrt_foreach_addcdiv_r   r   ).r$   rj   rk   rl   rm   rn   ru   rv   r)   ro   rs   rt   r%   r(   r'   r   r   r   r!   grouped_tensorsr   device_params_device_grads_device_exp_avgs_device_exp_avg_sqs_device_max_exp_avg_sqs_device_state_steps__device_paramsdevice_gradsdevice_exp_avgsdevice_exp_avg_sqsdevice_state_stepsrK   r   device_max_exp_avg_sqsscaled_device_gradsr   r   r   r   r   exp_avg_sq_sqrtrF   bcr   s.   &&&&&&&&$$$$$$$$$$$                          @r0   _multi_tensor_adamr   )  s   , 6{a"fW  88:?:;;%  Z  ;;=A=>>%  Z  ;;=A=>> >>&&((Z'H(
$ s 
 v4@
sss 
 v4@
 
 

 ![\x[yyz{  !6KLLBCC	BBuEuEBB	LO eV$$U\\):e)C 
u  ""$		 	T&\>:DL-8tF|-=>!$v,0CD!$v,0CDq!((!fJ&>!&d!KJv-7z&)U )-d6l<S)T&! #&* !<BT  --l;L ~~**,,1CA1F1M1M1M"ELLU$C3  2A61%##M1r7H3HI ''m<X#(#5#5$m<$L 	\4q<7G+H	
 	.6 eU\\**"'"4"4\1u9"ME".IE 3\5	

  :$11%9KL$11%9KL 0!4 0!4 01  0"5&&'78  !12
 )I$4!)-d6l<S)T&''(>@RS #("5"56L"M"'"5"56H"I1FG5; ##M?OT ;M :L$EZ----:L    ;M :L$EZ----:L    ,FV,WFVb2g^^FV,WXI7G$H7GRWW7G!$H)-d6l<S)T&''(>@RS #("5"56L"M"'"5"56H"I1FG5##	k %|   -X$Hs   *"_,"_1?_6
#_;c          (         V ^8  d   QhR\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\        R,          R	\        R,          R
\        R\        R\        \        ,          R\        \        ,          R\        \        ,          R\        R\        R\        R\        R\        R\        RR/# r   r   )r.   s   "r0   r1   r1   "  s    a aLa<a 6la f	a
 &\a fa a }a a a 6>a 6>a 	a a  
!a" #a$ %a& 'a( !)a* 
+ar3   c                   V '       g   R # V'       d   \        R4      h\        V
4      p
\        V4      pVe   VP                  V/M/ pVe   VP                  V/M/ p\        V\        4      '       d)   \        VP                  4      R8w  d   VP                  V/MR p\        P                  ! WW#WE.4      pVP                  4        EF  w  w  ppw  w  ppppppp\        \        \        ,          V4      p\        \        \        ,          V4      p \        \        \        ,          V4      p!\        \        \        ,          V4      p"\        \        \        ,          V4      p#R R p%p$Ve$   VP                  VVP                  VRR7      4      p$Ve$   VP                  VVP                  VRR7      4      p%Ve'   VV9  d    VP                  VRR7      VV&   VV,          p\        P                  ! V#^4       V'       g   \        P                  M\        P                   p&V&! VV V!V"VV#VVV
VVVVV$V%R7       V%f   EKt  \        P"                  ! V#V%.\%        V#4      ,          4       EK  	  R # )Nz9Adam with fused=True does not support differentiable=Truer   T)r   r   )	r)   r%   rs   rt   r(   r'   r   ru   rv   )r?   r   rK   r9   r   r   r   r   itemsr   r   rO   r   rS   r   _fused_adam__fused_adamw_r   rR   )'r$   rj   rk   rl   rm   rn   ru   rv   r)   ro   rs   rt   r%   r(   r'   r   r   r   r!   grad_scale_dictfound_inf_dictlr_dictr   rK   r   r   r   r   r   r   r   r   r   r   r   r   device_grad_scaledevice_found_inffuncs'   &&&&&&&&$$$$$$$$$$$                    r0   _fused_adamr   "  sV   , VWWuEuE ,6+A		J'r  *3)>		9%B  &b&11c"))n6MBSW   BB	LO 
			 
	 
	
"	T&\>:DL-8tF|-=>!$v,0CD!$v,0CD.2D+! / : :
f4@!  -88	V$?  6#8 ee6eEGFOB.2)?u!!UEXEX"%(&	
" '"%5$6=O9P$PQ 
!r3   )single_tensor_fnc          ,         V ^8  d   QhR\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\        R,          R	\        R
\        R\        R,          R\        R,          R\        R,          R\        R\        R\        R\        \        ,          R\        \        ,          R\        \        ,          R\        R\        R\        RR/# )r#   r$   rj   rk   rl   rm   rn   r   Nr   r   r    ru   rv   ro   r!   r)   rs   rt   r%   r(   r'   r   r*   r   )r.   s   "r0   r1   r1     s0    W WLW<W 6lW f	W
 &\W fW D[W W W $;W W }W W  !!W$ %W& 6>'W( 6>)W* 	+W, -W. 
/W0 1W2 
3Wr3   c               p   V	f=   Vf9   \        WRR7      w  ppV'       d!   \        V\        4      '       d   V'       g   RpV	f   Rp	Vf   Rp\        P                  P                  4       '       gF   \        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   \        R4      hV'       d0   \        P                  P                  4       '       d   \        R4      hV	'       d0   \        P                  P                  4       '       d   \        R4      hV	'       d,   \        P                  P                  4       '       g   \        pM9V'       d,   \        P                  P                  4       '       g   \        pM\        pV! V VVVVV3R	VR
VRVRVRVRVRVRVRVRVRV
RVRV/  R# )zfFunctional API that performs Adam algorithm computation.

See :class:`~torch.optim.Adam` for details.
NF)	use_fusedc              3   V   "   T F  p\        V\        P                  4      x  K!  	  R # 5ir}   )r9   rS   r   )r   ts   & r0   r   adam.<locals>.<genexpr>  s!      5-8
1ell##[s   ')TzPAPI has changed, `state_steps` argument must contain a list of singleton tensorsz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizersr)   ro   rs   rt   r%   r(   r'   r   r   r   ru   rv   r!   )r   r9   r   rS   r   r   r   r?   r   r   r   r   r   )r$   rj   rk   rl   rm   rn   r   r   r   r    ru   rv   ro   r!   r)   rs   rt   r%   r(   r'   r   r   r   s   &&&&&&&&&&&&&&$$$$$$$  r0   r   r     s   F }1e

7 z"f--jG} >>&&(( 5-85 5-85 2 2 ^
 	
 599))++STT''))QRRUYY++--	//11!"       "     &!" #$ %&  6'r3   )NFFNNNFF) typingr   rS   r   	optimizerr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   __all__r   __doc__r   r   r   r   r`   r3   r0   <module>r     s           0 6
n9 nd$J		 	  
 		 		 		 		 -KC PKK\vraH  1DEW FWr3   