+
    &je                     t    ^ RI t ^ RIHt ^ RIHt ^ RIHt  ! R R]4      t ! R R]4      t	 ! R R	]4      t
R# )
    N)Optional)Modulec                   b   a a ] tR t^
t oRtRV3R lV 3R llltR tR	V3R lR lltRtVt	V ;t
# )
RoPEa}  Implements the rotary positional encoding.

The traditional implementation rotates consecutive pairs of elements in the
feature dimension while the default implementation rotates pairs with
stride half the feature dimensions for efficiency.

For more details see `RoFormer: Enhanced Transformer with Rotary Position
Embedding <https://arxiv.org/abs/2104.09864>`_.

Args:
    dims (int): The feature dimensions to be rotated. If the input feature
        is larger than dims then the rest is left unchanged.
    traditional (bool, optional): If set to ``True`` choose the traditional
        implementation which is slightly less efficient. Default: ``False``.
    base (float, optional): The base used to compute angular frequency for
        each dimension in the positional encodings. Default: ``10000``.
    scale (float, optional): The scale used to scale the positions. Default: ``1.0``.
c                2   < V ^8  d   QhRS[ RS[RS[RS[/# )   dimstraditionalbasescale)intboolfloat)format__classdict__s   "y/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/mlx/nn/layers/positional_encoding.py__annotate__RoPE.__annotate__   s3        	
     c                T   < \         SV `  4        Wn        W n        W0n        W@n        R # )N)super__init__r	   r
   r   r   )selfr	   r
   r   r   	__class__s   &&&&&r   r   RoPE.__init__   s%     		&	
r   c                8    V P                    R V P                   2# )z, traditional=)r	   r
   )r   s   &r   _extra_reprRoPE._extra_repr+   s    ))N4+;+;*<==r   c                    < V ^8  d   QhRS[ /# )r   offsetr   )r   r   s   "r   r   r   .   s     
 
# 
r   c           	         \         P                  P                  VV P                  V P                  V P
                  V P                  VR 7      # ))r
   r   r   r    )mxfastroper	   r
   r   r   )r   xr    s   &&&r   __call__RoPE.__call__.   s@    ww||II((**  
 	
r   )r   r	   r   r
   )Fi'  g      ?)r   )__name__
__module____qualname____firstlineno____doc__r   r   r'   __static_attributes____classdictcell____classcell__r   r   s   @@r   r   r   
   s(     & >
 
 
r   r   c                   L   a a ] tR t^9t oRtRV3R lV 3R llltR tRtVtV ;t	# )SinusoidalPositionalEncodinga  Implements sinusoidal positional encoding.

For more details see the paper `Attention Is All You Need
<https://arxiv.org/abs/1706.03762>`_.

Args:
    dims (int): The dimensionality of the resulting positional embeddings.
    min_freq (float, optional): The minimum frequency expected. Default:
        ``0.0001``.
    max_freq (float, optional): The maximum frequency expected. Default:
        ``1``.
    scale (float, optional): A multiplicative scale for the embeddings.
        Default: ``sqrt(2/dims)``.
    cos_first (bool, optional): If ``True`` embed using ``[cos(x); sin(x)]``
        instead of the reverse. Default: ``False``.
    full_turns (bool, optional): If ``True`` multiply the frequencies with
        :math:`2\pi`. Default: ``False``.
c                N   < V ^8  d   QhRS[ RS[RS[RS[S[,          RS[RS[/# )r   r	   min_freqmax_freqr   	cos_first
full_turns)r   r   r   r   )r   r   s   "r   r   )SinusoidalPositionalEncoding.__annotate__M   sK     # ## # 	#
 # # #r   c                  < \         SV `  4        ^\        P                  ! ^ V^,          4      V^,          ^,
          ,          ,
          p\        P
                  ! V4      p\        P
                  ! V4      p\        P                  ! WsV,
          ,          V,           4      V n        V'       d.   V P                  ^\        P                  ,          ,          V n        T;'       g    ^V,          R,          V n	        WPn
        R# )   g      ?N)r   r   r#   arangemathlogexp_sigmaspir   r7   )	r   r	   r5   r6   r   r7   r8   one_zeror   s	   &&&&&&& r   r   %SinusoidalPositionalEncoding.__init__M   s     	ryyDAI.$!)a-@@88H%88H% vvhX*=>IJ<<1tww;7DL //q4xC/
"r   c                b   VR,          V P                   ,          p\        P                  ! V4      p\        P                  ! V4      pV P                  '       d   \        P
                  ! W4.RR7      pM\        P
                  ! WC.RR7      pV P                  ^8w  d   W P                  ,          pV# ).axis).N)r@   r#   cossinr7   concatenater   )r   r&   ycosysinys   &&   r   r'   %SinusoidalPositionalEncoding.__call__e   sv    iL4<<'vvayvvay>>>|"5A|"5A::?JJAr   )r@   r7   r   )g-C6?r;   NFF)
r)   r*   r+   r,   r-   r   r'   r.   r/   r0   r1   s   @@r   r3   r3   9   s     &# #0 r   r3   c                   p   a  ] tR t^ut o ]]P                  3V 3R lR ll4       t]R 4       tRR lt	Rt
V tR# )ALiBic                2   < V ^8  d   QhRS[ RS[ RS[ RS[ /# )r   q_sequence_lengthk_sequence_length	num_headsr    r!   )r   r   s   "r   r   ALiBi.__annotate__w   s1        	r   c                >   \         P                  ! W04      p\         P                  ! ^ V4      p\         P                  ! \         P                  ! VR,          VR,          ,
          RR7      4      ) p\        P                  W$R7      pWx,          P                  V4      p	V	# )r   rE   )rT   dtype)NNNN)NrX   )r   r;   )r#   r<   absexpand_dimsrP   create_alibi_slopeastype)
rR   rS   rT   r    rW   x1x2distance_matrixalibi_slope
alibi_masks
   &&&&&     r   create_alibi_matrixALiBi.create_alibi_matrixv   s     YYv1YYq+,66NN2g;G46B
 
 ...P%3;;EB
r   c                   a R  V3R lloS! V 4      p\         P                  ! W!R7      p\         P                  ! VRR7      # )c                $    V ^8  d   QhR\         /# )r   nr!   )r   s   "r   r   .ALiBi.create_alibi_slope.<locals>.__annotate__   s     		 		# 		r   c                   < \         P                  ! V 4      P                  4       '       dU   ^^\         P                  ! V 4      ^,
          ) ,          ) ,          p\        V 4       Uu. uF  q!W,          ,          NK  	  up# ^\         P                  ! \         P                  ! V 4      4      ,          pS! V4      S! ^V,          4      R,          RW,
           ,           # u upi )r   :r   Nr   N)r=   log2
is_integerrangefloor)rf   starticlosest_power_of_2
get_slopess   &   r   rp   ,ALiBi.create_alibi_slope.<locals>.get_slopes   s    yy|&&((tyy|a'7%8 89:27(;(Q(((;;%&$**TYYq\*B%B"12 %7!78>?WAWXY <s   &C)rW   rE   )rG   )r#   arrayrZ   )rT   rW   slopesoutrp   s   &&  @r   r[   ALiBi.create_alibi_slope   s8    		 		 I&hhv+~~c11r   Nc                    \         P                  VP                  R,          V,           VP                  R,          VP                  ^,          VVP                  R7      pVe	   WC,           pW,           # )r   )rR   rS   rT   r    rW   rr   rG   )rP   rb   shaperW   )r   attention_scoresr    maskra   s   &&&& r   r'   ALiBi.__call__   sk    ...44R86A.44R8&,,Q/"(( / 

 #*J,,r    )r   N)r)   r*   r+   r,   staticmethodr#   float32rb   r[   r'   r.   r/   )r   s   @r   rP   rP   u   sC      jj    2 2 
- 
-r   rP   )r=   typingr   mlx.corecorer#   mlx.nn.layers.baser   r   r3   rP   r|   r   r   <module>r      s9       %,
6 ,
^96 9x--F --r   