+
    &j/                         ^ RI t ^ RIHtHtHt ^ RIHt ^ RIH	t	 ^ RI
Ht ^ RIHt ^ RIHt ^ RIHt ^ RIHt  ! R	 R
]4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      tR# )    N)AnyCallableOptional)relu)Module)Dropout)Linear)	LayerNorm)
checkpointc                      a a ] tR t^t oRtRV3R lV 3R llltR	R lt]]P                  3V3R lR ll4       t
RtVtV ;t# )
MultiHeadAttentiona  Implements the scaled dot product attention with multiple heads.

Given inputs for queries, keys and values the ``MultiHeadAttention``
produces new values by aggregating information from the input values
according to the similarities of the input queries and keys.

All inputs as well as the output are linearly projected without biases by
default.

``MultiHeadAttention`` also takes an optional additive attention mask that
should be broadcastable with ``(batch, num_heads, # queries, # keys)``. The
mask should have ``-inf`` or very large negative numbers at the positions
that should *not* be attended to.

Args:
    dims (int): The model dimensions. This is also the default
        value for the queries, keys, values, and the output.
    num_heads (int): The number of attention heads to use.
    query_input_dims (int, optional): The input dimensions of the queries.
        Default: ``dims``.
    key_input_dims (int, optional): The input dimensions of the keys.
        Default: ``dims``.
    value_input_dims (int, optional): The input dimensions of the values.
        Default: ``key_input_dims``.
    value_dims (int, optional): The dimensions of the values after the
        projection. Default: ``dims``.
    value_output_dims (int, optional): The dimensions the new values will
        be projected to. Default: ``dims``.
    bias (bool, optional): Whether or not to use a bias in the projections.
        Default: ``False``.
c                   < V ^8  d   QhRS[ RS[ RS[S[ ,          RS[S[ ,          RS[S[ ,          RS[S[ ,          RS[S[ ,          RS[/# )	   dims	num_headsquery_input_dimskey_input_dimsvalue_input_dims
value_dimsvalue_output_dimsbias)intr   bool)format__classdict__s   "q/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/mlx/nn/layers/transformer.py__annotate__MultiHeadAttention.__annotate__0   sy     I II I #3-	I
 !I #3-I SMI $C=I I    c	                v  < \         S	V `  4        W,          ^ 8w  d   \        RV RV R24      hT;'       g    TpT;'       g    TpT;'       g    TpT;'       g    TpT;'       g    TpW n        \	        W1VR7      V n        \	        WAVR7      V n        \	        WVVR7      V n        \	        WgVR7      V n        R# )r   zIThe input feature dimensions should be divisible by the number of heads (z % z) != 0)r   N)	super__init__
ValueErrorr   r	   
query_projkey_proj
value_projout_proj)
selfr   r   r   r   r   r   r   r   	__class__s
   &&&&&&&&&r   r"   MultiHeadAttention.__init__0   s     	"$$(6YKv? 
 ,33t'//4+==~''4
-55" !1dC~$? !1DIz4Hr   c                   V P                  V4      pV P                  V4      pV P                  V4      pV P                  p\        P
                  ! VRVR34      P                  ^ ^^^4      p\        P
                  ! VRVR34      P                  ^ ^^^4      p\        P
                  ! VRVR34      P                  ^ ^^^4      p\        P                  ! ^VP                  R,          ,          4      p\        P                  P                  WW6VR7      pVP                  ^ ^^^4      P                  RR4      pV P                  V4      # )   )scalemask)r$   r%   r&   r   mx	unflatten	transposemathsqrtshapefastscaled_dot_product_attentionflattenr'   )r(   querieskeysvaluesr.   r   r-   outputs   &&&&&   r   __call__MultiHeadAttention.__call__O   s   //'*}}T"(NN	,,wYO<FFq!QPQR||D"y"o6@@Aq!Lfb9b/:DDQ1aP		!gmmB//0556T 6 
 !!!Q1-55b"=}}V$$r   c                :   < V ^8  d   QhRS[ RS[P                  /# )r   Ndtype)r   r1   Dtype)r   r   s   "r   r   r   `   s      s 288 r   c                    \         P                  ! V 4      pVR,          VR,          8  pVP                  V4      \         P                  ! V4      P                  ,          pV# )NNNN)rE   N)r1   arangeastypefinfomin)rA   rB   indicesr.   s   &&  r   create_additive_causal_mask.MultiHeadAttention.create_additive_causal_mask_   sH    ))A,w'$-/{{5!BHHUO$7$77r   )r%   r   r'   r$   r&   )NNNNNFN)__name__
__module____qualname____firstlineno____doc__r"   r>   staticmethodr1   float32rK   __static_attributes____classdictcell____classcell__r)   r   s   @@r   r   r      sA     @I I>%  >@jj    r   r   c                   P   a a ] tR t^gt oRR]R3V3R lV 3R llltR tRtVtV ;t	# )TransformerEncoderLayerN        Tc                f   < V ^8  d   QhRS[ RS[ RS[S[ ,          RS[RS[S[.S[3,          RS[/# r   r   r   mlp_dimsdropout
activation
norm_firstr   r   floatr   r   r   )r   r   s   "r   r   $TransformerEncoderLayer.__annotate__h   sV     % %% % 3-	%
 % cUCZ(% %r   c                B  < \         SV `  4        T;'       g
    V^,          p\        W4      V n        \	        V4      V n        \	        V4      V n        \        W4      V n        \        W14      V n	        \        V4      V n        \        V4      V n        WPn        W`n        R#    N)r!   r"   r   	attentionr
   ln1ln2r	   linear1linear2r   dropout1dropout2r`   ra   r(   r   r   r^   r_   r`   ra   r)   s   &&&&&&&r   r"    TransformerEncoderLayer.__init__h   s}     	''tax+D<T?T?d-h-(($$r   c                   V P                   '       d   V P                  V4      pV P                  W3W24      pV P                  V4      pW,           pV P	                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pW,           pV# V P                  WW4      pV P                  V4      pV P                  W,           4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P	                  W,           4      pV# rM   )	ra   ri   rh   rm   rj   rk   r`   rn   rl   )r(   xr.   ys   &&& r   r>    TransformerEncoderLayer.__call__}   s   ???AqQ-Aa AAAQA"Aa AQAA  qQ-Aa AAQA"Aa AQAAr   )	r`   rh   rm   rn   rk   rl   ri   rj   ra   
rN   rO   rP   rQ   r   r"   r>   rU   rV   rW   rX   s   @@r   rZ   rZ   g   s'     
 #'+/% %* r   rZ   c                   R   a a ] tR t^t oRR]RR3V3R lV 3R llltR tRtVtV ;t	# )	TransformerEncoderNr[   TFc                T   < V ^8  d   QhRS[ RS[ RS[ RS[S[ ,          RS[RS[RS[/# r   
num_layersr   r   r^   r_   ra   r   r   r   rc   r   )r   r   s   "r   r   TransformerEncoder.__annotate__   U     % %% % 	%
 3-% % % %r   c	                   < \         S
V `  4        \        V4       U	u. uF  p	\        W#WEWg4      NK  	  up	V n        \        V4      V n        Wn        R # u up	i rM   )r!   r"   rangerZ   layersr
   lnr   r(   rz   r   r   r^   r_   r`   ra   r   ir)   s   &&&&&&&&& r   r"   TransformerEncoder.__init__   ]     	
 :&	
 ' $J '	
 D/$
   Ac                    V P                    F*  pV P                  '       d   \        V4      MTpV! W4      pK,  	  V P                  V4      # rM   r   r   r   )r(   rr   r.   ls   &&& r   r>   TransformerEncoder.__call__   s<    A!%
1aA!
A  wwqzr   r   r   r   ru   rX   s   @@r   rw   rw      *      #' % %* r   rw   c                   P   a a ] tR t^t oRR]R3V3R lV 3R llltR tRtVtV ;t	# )TransformerDecoderLayerNr[   Tc                f   < V ^8  d   QhRS[ RS[ RS[S[ ,          RS[RS[S[.S[3,          RS[/# r]   rb   )r   r   s   "r   r   $TransformerDecoderLayer.__annotate__   sV     % %% % 3-	%
 % cUCZ(% %r   c                  < \         SV `  4        T;'       g
    V^,          p\        W4      V n        \        W4      V n        \        V4      V n        \        V4      V n        \        V4      V n        \        W4      V n
        \        W14      V n        \        V4      V n        \        V4      V n        \        V4      V n        WPn        W`n        R# rf   )r!   r"   r   self_attentioncross_attentionr
   ri   rj   ln3r	   rk   rl   r   rm   rn   dropout3r`   ra   ro   s   &&&&&&&r   r"    TransformerDecoderLayer.__init__   s     	''tax0A1$BT?T?T?d-h-((($$r   c                v   V P                   '       d   V P                  V4      pV P                  WUWS4      pV P                  V4      pW,           pV P	                  V4      pV P                  WRW$4      pV P                  V4      pW,           pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pW,           pV# V P                  WW4      pV P                  V4      pV P                  W,           4      pV P                  WW$4      pV P                  V4      pV P	                  W,           4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  W,           4      pV# rM   )ra   ri   r   rm   rj   r   rn   r   rk   r`   r   rl   )r(   rr   memoryx_maskmemory_maskrs   s   &&&&& r   r>    TransformerDecoderLayer.__call__   sn   ???A##A!4Aa AAA$$QDAa AAAQA"Aa AQAA"  ##A!4Aa AA$$QDAa AAQA"Aa AQAAr   )r`   r   rm   rn   r   rk   rl   ri   rj   r   ra   r   ru   rX   s   @@r   r   r      s'     
 #'+/% %0" "r   r   c                   R   a a ] tR t^t oRR]RR3V3R lV 3R llltR tRtVtV ;t	# )	TransformerDecoderNr[   TFc                T   < V ^8  d   QhRS[ RS[ RS[ RS[S[ ,          RS[RS[RS[/# ry   r{   )r   r   s   "r   r   TransformerDecoder.__annotate__   r}   r   c	                   < \         S
V `  4        \        V4       U	u. uF  p	\        W#WEWg4      NK  	  up	V n        \        V4      V n        Wn        R # u up	i rM   )r!   r"   r   r   r   r
   r   r   r   s   &&&&&&&&& r   r"   TransformerDecoder.__init__   r   r   c                    V P                    F+  pV P                  '       d   \        V4      MTpV! WW44      pK-  	  V P                  V4      # rM   r   )r(   rr   r   r   r   r   s   &&&&& r   r>   TransformerDecoder.__call__
  s?    A!%
1aA!V1A  wwqzr   r   ru   rX   s   @@r   r   r      r   r   r   c                   b   a a ] tR tRt oRtR^^^RR]RRRR3V3R lV 3R	 llltR
 tRtVt	V ;t
# )Transformeri  a  
Implements a standard Transformer model.

The implementation is based on `Attention Is All You Need
<https://arxiv.org/abs/1706.03762>`_.

The Transformer model contains an encoder and a decoder. The encoder
processes the input sequence and the decoder generates the output sequence.
The interaction between encoder and decoder happens through the attention
mechanism.

Args:
    dims (int, optional): The number of expected features in the
        encoder/decoder inputs. Default: ``512``.
    num_heads (int, optional): The number of attention heads. Default:
        ``8``.
    num_encoder_layers (int, optional): The number of encoder layers in the
        Transformer encoder. Default: ``6``.
    num_decoder_layers (int, optional): The number of decoder layers in the
        Transformer decoder. Default: ``6``.
    mlp_dims (int, optional): The hidden dimension of the MLP block in each
        Transformer layer. Defaults to ``4*dims`` if not provided. Default:
        ``None``.
    dropout (float, optional): The dropout value for the Transformer
        encoder and decoder. Dropout is used after each attention layer and
        the activation in the MLP layer. Default: ``0.0``.
    activation (function, optional): the activation function for the MLP
        hidden layer. Default: :func:`mlx.nn.relu`.
    custom_encoder (nn.Module, optional): A custom encoder to replace the
        standard Transformer encoder. Default: ``None``.
    custom_decoder (nn.Module, optional): A custom decoder to replace the
        standard Transformer decoder. Default: ``None``.
    norm_first (bool, optional): if ``True``, encoder and decoder layers
        will perform layer normalization before attention and MLP
        operations, otherwise after. Default: ``True``.
    checkpoint (bool, optional): if ``True`` perform gradient checkpointing
        to reduce the memory usage at the expense of more computation.
        Default: ``False``.
i   Nr[   TFc                   < V ^8  d   QhRS[ RS[ RS[ RS[ RS[S[ ,          RS[RS[S[.S[3,          RS[S[,          R	S[S[,          R
S[RS[/# )r   r   r   num_encoder_layersnum_decoder_layersr^   r_   r`   custom_encodercustom_decoderra   r   rb   )r   r   s   "r   r   Transformer.__annotate__:  s     $
 $
$
 $
  	$

  $
 3-$
 $
 cUCZ($
 !$
 !$
 $
 $
r   c           
        < \         SV `  4        T;'       g    \        VVVVVVV
V4      V n        T	;'       g    \	        VVVVVVV
V4      V n        R # rM   )r!   r"   rw   encoderr   decoder)r(   r   r   r   r   r^   r_   r`   r   r   ra   r   r)   s   &&&&&&&&&&&&r   r"   Transformer.__init__:  st     	% 	
 	
);	*
 & 	
 	
);	*
r   c                H    V P                  W4      pV P                  W&WE4      # rM   )r   r   )r(   srctgtsrc_masktgt_maskr   r   s   &&&&&& r   r>   Transformer.__call__`  s!    c,||C??r   )r   r   )rN   rO   rP   rQ   rR   r   r"   r>   rU   rV   rW   rX   s   @@r   r   r     sE     &T "#"#"&+/(,(, $
 $
L@ @r   r   )r4   typingr   r   r   mlx.corecorer1   mlx.nn.layers.activationsr   mlx.nn.layers.baser   mlx.nn.layers.dropoutr   mlx.nn.layers.linearr	   mlx.nn.layers.normalizationr
   mlx.nn.utilsr   r   rZ   rw   r   r   r    r   r   <module>r      sv     * *  * % ) ' 1 #U Up/f /d :;f ;| :Q@& Q@r   