
    #Zjg.                        U d dl Z d dlmZ d dlmZmZmZmZmZm	Z	 ddl
mZ ddlmZ ddlmZmZ eeef         Zeeee         ef         Zeeee         ef         Zdd	d
dZeeeeeef         f         ed<   dddded         ed         d	dZeeeeeef         f         ed<    ej        d          ed         fed         ed         ed         ed          e	eed                   ed         d	ddedededededededefd            Z ej        d          ed         fed         ed         ed         ed         ed          e	eed                   d	ddedededededededefd             Z ej        d!          ed         ed          e	eed                   d	d"dededededef
d#            Z G d$ d%e          Z g d&Z!dS )'    N)defaultdict)DictListOptionalTupleUnioncast   )get_array_ops)registry)FloatsXd	Generator        T      ?)L2L2_is_weight_decay	grad_clipSGD_DEFAULTSgMbP?g?g+?g:0yE>r   r   )
learn_ratebeta1beta2epsr   r   r   ADAM_DEFAULTSzRAdam.v1r   r   r   r   r   )r   r   r   r   r   r   use_averagesr   c                2    t          | |||||||d	  	        S )NT)r   r   r   r   r   r   r   	use_radam	Optimizer)r   r   r   r   r   r   r   r   s           \/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/thinc/optimizers.pyRAdamr       s6     -!
 
 
 
    zAdam.v1)r   r   r   r   r   r   r   c                2    t          | |||||||d	  	        S )NF)r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   s           r   Adamr#   8   s6     -!
 
 
 
r!   zSGD.v1)r   r   r   r   c          	      .    t          | |||dd|          S )Nr   )r   r   r   r   r   r   r   )r   r   r   r   r   s        r   SGDr%   Q   s0     -!   r!   c                      e Zd ZU dZeeef         ed<   eeef         ed<   eeeef                  ed<   ee	e
f         ed<   eeef         ed<   eeef         ed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eeee                           ed<   g dZed         ed         ed         ed         ed         ddddd	ededededededededefdZd Zd Zdddeee	f         deded efd!Zd" Zd# Zd$S )%r   zDo various flavours of stochastic gradient descent, with first and
    second order momentum. Currently support 'vanilla' SGD, Adam, and RAdam.
    mom1mom2averages	schedules	nr_update	last_seenr   r   b1b2r   r   r   r   _radam_buffer)r'   r(   r)   r*   r+   r,   r   r   r-   r.   r   r   r   r   r/   r   r   TF)r   r   r   r   r   r   r   r   r   c                   i | _         i | _        |ri | _        nd| _        i | _        t	          t
                    | _        t	          t
                    | _        |                     d|           |                     d|           |                     d|           |                     d|           |                     d|           |                     d|           || _	        |	| _
        d t          d	          D             | _        dS )
al  
        Initialize an optimizer.

        learn_rate (float): The initial learning rate.
        L2 (float): The L2 regularization term.
        beta1 (float): First-order momentum.
        beta2 (float): Second-order momentum.
        eps (float): Epsilon term for Adam etc.
        grad_clip (float): Gradient clipping.
        use_averages (bool): Whether to track moving averages of the parameters.
        use_radam (bool): Whether to use the RAdam optimizer.
        L2_is_weight_decay (bool): Whether to interpret the L2 parameter as a
            weight decay term, in the style of the AdamW optimizer.
        Nr   r   r-   r.   r   r   c                     g | ]}g d S ))NNN ).0_s     r   
<listcomp>z&Optimizer.__init__.<locals>.<listcomp>   s    DDDQ000DDDr!   
   )r'   r(   r)   r*   r   intr+   r,   _set_attr_or_scheduler   r   ranger/   )
selfr   r   r   r   r   r   r   r   r   s
             r   __init__zOptimizer.__init__   s   6 		 	!DMM DM$S))$S))"";	:::""<<<<""4///""4///""5#...""4,,,""4DD%))DDDr!   c                    t          |t          t          t          f          rt	          | ||           d S t          |t
                    rt          |          }|| j        |<   	 t	          | |t          |                     d S # t          t          f$ r,}d| dt          |           d| }t          |          d }~ww xY w)NzInvalid schedule for 'z' (z)
)
isinstancefloatboolr7   setattrlistiterr*   nextStopIteration	TypeErrortype
ValueError)r:   namevalueeerrs        r   r8   zOptimizer._set_attr_or_schedule   s    eeT3/00 
	&D$&&&&&%&& $U#(DN4 &dDKK00000!9- & & &KtKKUKKKK oo%&s   %B C'B==Cc                     | j                                         D ]G\  }}	 t          |          }n # t          $ r t	          | |          }Y nw xY wt          | ||           Hd S )N)r*   itemsrC   rD   getattrr@   )r:   keyschedulerI   s       r   step_scheduleszOptimizer.step_schedules   s    !^1133 	& 	&MC+X  + + +c**+D#u%%%%	& 	&s   0AAr   )lr_scalerO   weightsgradientrR   c                   t          |          dk     r||fS t          |          }| j        |xx         dz  cc<   | j        |         }| j        dk    r| j        s|| j        |z  z  }| j        r|                    || j                  }| j        r|                     ||||||          \  }}nV| j	        dk    r)| j
        dk    r|                     ||||||          \  }}n"| j
        dk    rt          ||| j        z  |z  z  }|dz  }| j        dk    r| j        r||| j        z  | j        z  |z  z  }| j        O|| j        vr$|                    |j        d          | j        |<   |                    | j        |         ||           ||fS )zCall the optimizer with weights and a gradient. The key is the
        identifier for the parameter, usually the node ID and parameter name.
        r
   r   r   Nfloat32)dtype)lenr   r+   r   r   r   clip_gradientr   _radamr-   r.   _adamNotImplementedErrorr   r)   allocshapeupdate_averages)r:   rO   rS   rT   rR   opsnr_upds          r   __call__zOptimizer.__call__   s    x==1H$$G$$sq $7a<< 7<'))H> 	C((4>BBH> 	= $Wh#v! !GXX Ws]]tw}} $

Wh#v! !GXX Ws]]%%x$/1H<<GA7a<<D3<x$/1DG;gEEG=$$-''%(YYw}IY%N%Nc"c 2GVDDD  r!   c                    || j         vr"|                    |j                  | j         |<   || j        vr"|                    |j                  | j        |<   |                    ||j                  }|                    ||j                  }| j        |         | j         |         | j        |         d}	| j        | j        | j        g| j	        d| j
        d}
d}|	d         |	d         }}|
d         \  }}||z  }|d|z
  |d	z  z  z  }||z  }|d|z
  |z  z  }|	d
xx         dz  cc<   |
d         t          |	d
         dz                     }|	d
         |d         k    r|d         |d	         }}n|	d
         |d<   ||	d
         z  }d	d|z
  z  dz
  }|d	|	d
         z  |z  d|z
  z  z
  }||d<   |dk    rEt          j        d|z
  |dz
  z  |dz
  z  |d	z
  z  |z  |z  |d	z
  z            d||	d
         z  z
  z  }n|rdd||	d
         z  z
  z  }nd}||d	<   |dk    r]|
d         dk    r||
d          |
d         z  |z  z  }|j                            |          |
d         z   }|| |
d         z  ||z  z  z  }n<|dk    r6|
d         dk    r||
d          |
d         z  |z  z  }|| |
d         z  |z  z  }|                    ||j                  |                    ||j                  fS )N)stepexp_avg
exp_avg_sqr   )lrbetasr   weight_decaybufferTre   rf   rh   r
      rd   rj   r6   r         r   ri   rg   r   )r'   alloc1fsizer(   	reshape1fr+   r   r-   r.   r   r/   r7   mathsqrtxp	reshape_fr^   )r:   r`   rS   gradrR   rO   ra   
weights_1Dgradient_1Dstategroupdegenerated_to_sgdre   rf   r   r   bufferedN_sma	step_sizebeta2_t	N_sma_maxdenoms                         r   rZ   zOptimizer._radam   s   di [[66DIcNdi [[66DIcN]]7GL99
mmD$)44
 N3'y~)C.
 
 /gtw'8(
 
 "#I.l0CW~u 	e
q5y[!^44
5AI,,f?3uV}r'9#:#:;=HQK'''{HQK9EE-HQKuV},GQY!+IE&M 1G ;q7{ KKEHQK zz I[qy" 1}& qy" 	
    !1}&  %-//1		 $ 1uf'=#=>			#HQK A::^$))u^44uT{BZOO
FKK
++eEl:E9*uT{2goFFJJ]]^$))u^44uT{BZOO
9*uT{2W<<JMM*gm44MM+tz22
 	
r!   c                    |                     ||j                  }|                     ||j                  }|| j        vr"|                    |j                  | j        |<   || j        vr"|                    |j                  | j        |<   | j        |         }	| j        |         }
| j        }| j        }d||z  z
  }d||z  z
  }| j        |dz  z  |z  }| j        }|	                    |||	|
|||||z            \  }}}	}
|	| j        |<   |
| j        |<   |
                    ||j                  |
                    ||j                  fS )Nr   g      ?)rq   rp   r'   ro   r(   r-   r.   r   r   adamru   r^   )r:   r`   rS   rT   rR   rO   ra   rw   rx   r'   r(   r-   r.   fix1fix2rg   r   s                    r   r[   zOptimizer._adamH  sM   ]]7GL99
mmHhm<<di [[66DIcNdi [[66DIcNy~y~WWb&j!b&j!_tSy(4/h.1hhT4Rb8m/
 /
+
Kt 	#	#MM*gm44MM+x~66
 	
r!   N)__name__
__module____qualname____doc__r   KeyTr   __annotations__r   strr   r7   r>   r?   r   	__slots__r   
FloatOrSeqr;   r8   rQ   r   rb   rZ   r[   r2   r!   r   r   r   e   sk          tX~

tX~
tD(N+,,,,CN####D#ID#IIIIIII	JJJIIIOOOXh/01111  I, 't,)'2)'2'. -k :!#',E ,E ,E,E 	,E
 ,E ,E ,E ,E ,E ,E !,E ,E ,E ,E\& & && & & '! '! '!38_'! '! 	'! '! '! '! '!RL
 L
 L
\
 
 
 
 
r!   r   )r#   r    r%   r   r   r   )"rr   collectionsr   typingr   r   r   r   r   r	   backendsr   configr   typesr   r   r7   r   r   r>   r   IntOrSeqr   r?   r   r   
optimizersr    r#   r%   objectr   __all__r2   r!   r   <module>r      s    # # # # # # ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; # # # # # #       & & & & & & & &S#X5$u+y01
d3i*+ 4 4d3eT3.//0    
t
k*5 5tCudC/001    Z  *<8 &g.%g.#E*"4(#tD-8L*MNN)+6    	
 
 	      ! 0 Y*<8 #4(%g.%g.#E*)+6#tD-8L*MNN   	 	
  
       0 X "$'(5#tD,7K*LMM   	 	
     &{
 {
 {
 {
 {
 {
 {
 {
| Q
P
Pr!   