
    IZjC                    ~   d Z ddlmZ ddlZddlZddlZddlmZmZ ddl	m
Z
m	Z	 ddlmZ ddlmZmZmZmZmZmZmZmZmZmZ ddlZddlZddlZddlmZ  eej                  Z 	 dd	l!m"Z"m#Z# n# e$$ r e j%        d
k    r Y nw xY werddl&m'Z' ddl(m)Z)  G d de          Z* ej+                    fd@dZ, ej+                    dfdAdZ- ej+                    dfdBdZ.dCd!Z/e j%        d
k     rdDd%Z0ndDd&Z0dEd)Z1dFd+Z2dGd.Z3dHd/Z4dId0Z5dJd4Z6dKd7Z7 G d8 d9ej8                  Z9dLd<Z:e j%        d
k     rdMd>Z;dS dMd?Z;dS )Nz&Pydantic (v1 / v2) adapter for LanceDB    )annotationsN)ABCabstractmethod)datedatetime)Enum)
TYPE_CHECKINGAnyCallableDict	GeneratorListTypeUnion_GenericAliasGenericAlias)Version)
CoreSchemacore_schema   )	FieldInfo   EmbeddingFunctionConfigc                  ^    e Zd Zeedd                        Zeedd                        ZdS )	FixedSizeListMixinreturnintc                     t           NNotImplementedError     \/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/lancedb/pydantic.pydimzFixedSizeListMixin.dim.   
     "!r$   pa.DataTypec                     t           r    r!   r#   r$   r%   value_arrow_typez#FixedSizeListMixin.value_arrow_type3   r'   r$   Nr   r   r   r(   )__name__
__module____qualname__staticmethodr   r&   r*   r#   r$   r%   r   r   -   sf        " " " ^ \" " " " ^ \" " "r$   r   r&   r   
value_typer(   c                P    ddl m}  |dt                     t          | |          S )Nr   )warnz|lancedb.pydantic.vector() is deprecated, use lancedb.pydantic.Vector instead.This function will be removed in future release)warningsr3   DeprecationWarningVector)r&   r1   r3   s      r%   vectorr7   9   sB    D	:  
 #z"""r$   Tnullableboolr   Type[FixedSizeListMixin]c                F      G  fddt           t                    }|S )a=  Pydantic Vector Type.

    !!! warning
        Experimental feature.

    Parameters
    ----------
    dim : int
        The dimension of the vector.
    value_type : pyarrow.DataType, optional
        The value type of the vector, by default pa.float32()
    nullable : bool, optional
        Whether the vector is nullable, by default it is True.

    Examples
    --------

    >>> import pydantic
    >>> from lancedb.pydantic import Vector
    ...
    >>> class MyModel(pydantic.BaseModel):
    ...     id: int
    ...     url: str
    ...     embeddings: Vector(768)
    >>> schema = pydantic_to_schema(MyModel)
    >>> assert schema == pa.schema([
    ...     pa.field("id", pa.int64(), False),
    ...     pa.field("url", pa.utf8(), False),
    ...     pa.field("embeddings", pa.list_(pa.float32(), 768))
    ... ])
    c                      e Zd Z fdZedfd            Zed fd            Zedfd            Zed fd            Z	edd            Z
e fd            Zej        dk     red fd            ZdS dS )Vector.<locals>.FixedSizeListc                    d dS )NzFixedSizeList(dim=)r#   selfr&   s    r%   __repr__z&Vector.<locals>.FixedSizeList.__repr__j   s    .....r$   r   r9   c                      S r    r#   r8   s   r%   r8   z&Vector.<locals>.FixedSizeList.nullablem       Or$   r   c                      S r    r#   r&   s   r%   r&   z!Vector.<locals>.FixedSizeList.dimq       Jr$   r(   c                      S r    r#   r1   s   r%   r*   z.Vector.<locals>.FixedSizeList.value_arrow_typeu   	    r$   _source_typer
   _handlerpydantic.GetCoreSchemaHandlerr   c           	     z    t          j        | t          j        t          j                                        S )N
min_length
max_lengthitems_schemar    no_info_after_validator_functionlist_schemafloat_schemaclsrL   rM   r&   s      r%   __get_pydantic_core_schema__z:Vector.<locals>.FixedSizeList.__get_pydantic_core_schema__y   sD     ?'""!,!9!;!;    r$   Generator[Callable, None, None]c              3     K   | j         V  d S r    validaterY   s    r%   __get_validators__z0Vector.<locals>.FixedSizeList.__get_validators__         ,r$   c                    t          |t          t          t          j        f          rt          |          k    rt          d           | |          S )Nz,A list of numbers or numpy.ndarray is needed)
isinstancelistrangenpndarraylen	TypeError)rY   vr&   s     r%   r^   z&Vector.<locals>.FixedSizeList.validate   sI    a$rz!:;; Ps1vv}} NOOO3q66Mr$   r   field_schemaDict[str, Any]c                *    ddi|d<   |d<   |d<   d S )NtypenumberitemsmaxItemsminItemsr#   rY   rk   r&   s     r%   __modify_schema__z/Vector.<locals>.FixedSizeList.__modify_schema__   s,    )/(:W%+.Z(+.Z(((r$   Nr   r9   r+   r,   rL   r
   rM   rN   r   r   r   r[   rk   rl   )r-   r.   r/   rB   r0   r8   r&   r*   classmethodrZ   r`   r^   PYDANTIC_VERSIONmajorrt   r&   r8   r1   s   r%   FixedSizeListr=   i   s[       	/ 	/ 	/ 	/ 	/ 
	 	 	 	 	 
	 
	 	 	 	 	 
	 
	 	 	 	 	 
	 

	 
	 
	 
	 
	 

	 
	 	 	 
	 
	 	 	 	 
	
 !A%%/ / / / / [/ / / &%r$   r}   rd   r   )r&   r1   r8   r}   s   ``` r%   r6   r6   E   sP    H./ ./ ./ ./ ./ ./ ./ ./ ./0 ./ ./ ./` r$   r   c                F      G  fddt           t                    }|S )a  Pydantic MultiVector Type for multi-vector embeddings.

    This type represents a list of vectors, each with the same dimension.
    Useful for models that produce multiple embeddings per input, like ColPali.

    Parameters
    ----------
    dim : int
        The dimension of each vector in the multi-vector.
    value_type : pyarrow.DataType, optional
        The value type of the vectors, by default pa.float32()
    nullable : bool, optional
        Whether the multi-vector is nullable, by default it is True.

    Examples
    --------

    >>> import pydantic
    >>> from lancedb.pydantic import MultiVector
    ...
    >>> class MyModel(pydantic.BaseModel):
    ...     id: int
    ...     text: str
    ...     embeddings: MultiVector(128)  # List of 128-dimensional vectors
    >>> schema = pydantic_to_schema(MyModel)
    >>> assert schema == pa.schema([
    ...     pa.field("id", pa.int64(), False),
    ...     pa.field("text", pa.utf8(), False),
    ...     pa.field("embeddings", pa.list_(pa.list_(pa.float32(), 128)))
    ... ])
    c                  
   e Zd Z fdZedfd            Zed fd            Zedfd            Zedd	            Ze	d fd            Z
e	dd            Ze	 fd            Zej        dk     re	d fd            ZdS dS )$MultiVector.<locals>.MultiVectorListc                    d dS )NzMultiVector(dim=r?   r#   r@   s    r%   rB   z-MultiVector.<locals>.MultiVectorList.__repr__   s    ,c,,,,r$   r   r9   c                      S r    r#   rD   s   r%   r8   z-MultiVector.<locals>.MultiVectorList.nullable   rE   r$   r   c                      S r    r#   rG   s   r%   r&   z(MultiVector.<locals>.MultiVectorList.dim   rH   r$   r(   c                      S r    r#   rJ   s   r%   r*   z5MultiVector.<locals>.MultiVectorList.value_arrow_type   rK   r$   c                     dS )NTr#   r#   r$   r%   is_multi_vectorz4MultiVector.<locals>.MultiVectorList.is_multi_vector   s    4r$   rL   r
   rM   rN   r   c                    t          j        | t          j        t          j        t          j                                                  S )NrP   )rS   rT   rX   s      r%   rZ   zAMultiVector.<locals>.MultiVectorList.__get_pydantic_core_schema__   sY     ?'!,!8#&#&%0%=%?%?" " "  	 	 	r$   r[   c              3     K   | j         V  d S r    r]   r_   s    r%   r`   z7MultiVector.<locals>.MultiVectorList.__get_validators__   ra   r$   c                   t          |t          t          f          st          d          |D ]O}t          |t          t          t          j        f          rt          |          k    rt          d d          P | |          S )NzA list of vectors is neededzEach vector must be a list of z numbers)rc   rd   re   ri   rf   rg   rh   )rY   rj   vecr&   s      r%   r^   z-MultiVector.<locals>.MultiVectorList.validate   s    a$// ? =>>> T T!#eRZ'@AA TSXXQT__#$RS$R$R$RSSS FU3q66Mr$   r   rk   rl   c                     dddid|d<   d S )Narrayrn   ro   )rn   rp   rr   rq   rp   r#   rs   s     r%   rt   z6MultiVector.<locals>.MultiVectorList.__modify_schema__   s-     $$h/ # #	) )W%%%r$   Nru   r+   r,   rv   rw   rx   )r-   r.   r/   rB   r0   r8   r&   r*   r   ry   rZ   r`   r^   rz   r{   rt   r|   s   r%   MultiVectorListr      s{       	- 	- 	- 	- 	- 
	 	 	 	 	 
	 
	 	 	 	 	 
	 
	 	 	 	 	 
	 
	 	 	 
	 
	 	 	 	 	 
	 
	 	 	 
	 
	 	 	 	 
	 !A%%     [   &%r$   r   r~   )r&   r1   r8   r   s   ``` r%   MultiVectorr      sP    F: : : : : : : : :$ 2 : : :x r$   py_type	Type[Any]fieldr   c                N   | t           u rt          j                    S | t          u rt          j                    S | t
          u rt          j                    S | t          u rt          j                    S | t          u rt          j
                    S | t          u rt          j                    S | t          u r&t          |d          }t          j        d|          S t!          | dd          t"          t$          fv r| j        d         }t)          ||          S t+          d|  d          )	zConvert a field with native Python type to Arrow data type.

    Raises
    ------
    TypeError
        If the type is not supported.
    tzus)r   
__origin__Nr   z9Converting Pydantic type to Arrow Type: unsupported type .)r   paint64floatfloat64strutf8r9   bool_bytesbinaryr   date32r   
get_extras	timestampgetattrrd   tuple__args___pydantic_list_child_to_arrowri   )r   r   r   childs       r%   _py_type_to_arrow_typer      s    #~~xzz	E		z||	Cwyy	Dxzz	E		y{{	Dy{{	H		t$$|DR((((	,	-	-$	>	> #,UE:::
NGNNN  r$   modelpydantic.BaseModelList[pa.Field]c                H    d | j                                         D             S )Nc                4    g | ]\  }}t          ||          S r#   _pydantic_to_field.0namer   s      r%   
<listcomp>z-_pydantic_model_to_fields.<locals>.<listcomp>   s4     
 
 
0;etU++
 
 
r$   )
__fields__rp   r   s    r%   _pydantic_model_to_fieldsr     s2    
 
?D?O?U?U?W?W
 
 
 	
r$   c                H    d | j                                         D             S )Nc                4    g | ]\  }}t          ||          S r#   r   r   s      r%   r   z-_pydantic_model_to_fields.<locals>.<listcomp>'  s6     
 
 
e tU++
 
 
r$   )model_fieldsrp   r   s    r%   r   r   &  s2    
 
$17799
 
 
 	
r$   tpr
   c                r   dd}t          j        |           r || t          j                  r#t	          |           }t          j        |          S  || t                    r t          | dd	                       rKt          j	        t          j	        | 
                                |                                                     S t          j	        | 
                                |                                           S  || t                    rd
 | D             }t          |          dk    rd|                                }|t          u r7t          j        t          j                    t          j                              S t'          ||          S t          j                    S t'          | |          S )N	candidater
   basern   r   r9   c                F    	 t          | |          S # t          $ r Y dS w xY wNF)
issubclassri   )r   r   s     r%   _safe_issubclassz6_pydantic_type_to_arrow_type.<locals>._safe_issubclass.  s9    	i... 	 	 	55	s    
  r   c                     dS r   r#   r#   r$   r%   <lambda>z._pydantic_type_to_arrow_type.<locals>.<lambda>:  s    e r$   c                6    h | ]}t          |j                  S r#   )rn   value)r   ms     r%   	<setcomp>z/_pydantic_type_to_arrow_type.<locals>.<setcomp>C  s     555Q4==555r$   r   )r   r
   r   rn   r   r9   )inspectisclasspydantic	BaseModelr   r   structr   r   list_r*   r&   r   rh   popr   
dictionaryint32r   r   )r   r   r   fieldsvalue_typesr1   s         r%   _pydantic_type_to_arrow_typer   -  s       r B 233 	%.r22F9V$$$B 233 	=<wr,mm<<>> Kx)<)<)>)> I IJJJ8B//1126688<<<B%% 	
 65"555K;1$$(__..
$$=RWYY???-j%@@@799!"e,,,r$   r   c           	         t          |           }|6t          j        t          j        dt	          ||          d                    S t          j        t	          | |                    S )NitemT)_unwrap_optional_annotationr   r   r   r   )r   r   	unwrappeds      r%   r   r   N  sb    +E22IxHV9)UKKTRR
 
 	
 80>>???r$   
annotation
Any | Nonec                   t          | t          t          f          ra| j        }| j        }|t
          k    rGd |D             }t          |          dk    r(t          |          t          |          k    r|d         S nxt          j        dk    rht          | t          j
                  rN| j        }d |D             }t          |          dk    r(t          |          t          |          k    r|d         S d S )Nc                4    g | ]}|t          d           u|S r    rn   r   args     r%   r   z/_unwrap_optional_annotation.<locals>.<listcomp>\  s(    EEEs$t**/D/D/D/D/Dr$   r   r      
   c                4    g | ]}|t          d           u|S r    r   r   s     r%   r   z/_unwrap_optional_annotation.<locals>.<listcomp>a  s(    AAAC3d4jj+@+@C+@+@+@r$   )rc   r   r   r   r   r   rh   sysversion_infotypes	UnionType)r   originargsnon_nones       r%   r   r   W  s    *}l;<< &"U??EEtEEEH8}}!!c(mms4yy&@&@{"		W	$	$J)P)P	$"AA4AAAx==A#h--3t99"<"<A;4r$   c                ,   t          | j                  }|t          ||           S t          | j        t          t
          f          r9| j        j        }| j        j        }|t          u r|d         }t          ||           S t          | j        |           S )z.Convert a Pydantic FieldInfo to Arrow DataTypeNr   )
r   r   r   rc   r   r   r   r   rd   r   )r   r   r   r   r   s        r%   _pydantic_to_arrow_typer   g  s    +E,<==I+Iu===%"]L$ABB ?!,(T>>GE0>>>'(8%@@@r$   c                Z   t          | j                  dS t          | j        t          t          f          r?| j        j        }| j        j        }|t          k    rt          d |D                       rdS nt          j
        dk    rEt          | j        t          j                  r&| j        j        }|D ]}|t          d          u r dS n_t          j        | j                  rF	 t!          | j        t"                    r| j                                        S n# t&          $ r Y dS w xY wdS )z*Check if a Pydantic FieldInfo is nullable.NTc              3  8   K   | ]}|t          d           u V  d S r    r   )r   typs     r%   	<genexpr>zis_nullable.<locals>.<genexpr>~  s-      553$t**$555555r$   r   F)r   r   rc   r   r   r   r   r   anyr   r   r   r   rn   r   r   r   r   r8   ri   )r   r   r   r   s       r%   is_nullabler   v  sG   "5#344@t%"]L$ABB !,(U??5555555 t		W	$	$E4Deo)V)V	$( 	 	Cd4jj  tt !	 
)	*	* 	%*,>?? 3'002223 	 	 	55	5s   &2D 
D('D(r   r   pa.Fieldc                f    t          |          }t          j        | |t          |                    S )z,Convert a Pydantic field to a PyArrow Field.)r   r   r   r   )r   r   dts      r%   r   r     s+    	 	'	'B8D"k%00111r$   Type[pydantic.BaseModel]	pa.Schemac                H    t          |           }t          j        |          S )a  Convert a [Pydantic Model][pydantic.BaseModel] to a
       [PyArrow Schema][pyarrow.Schema].

    Parameters
    ----------
    model : Type[pydantic.BaseModel]
        The Pydantic BaseModel to convert to Arrow Schema.

    Returns
    -------
    pyarrow.Schema
        The Arrow Schema

    Examples
    --------

    >>> from typing import List, Optional
    >>> import pydantic
    >>> from lancedb.pydantic import pydantic_to_schema, Vector
    >>> class FooModel(pydantic.BaseModel):
    ...     id: int
    ...     s: str
    ...     vec: Vector(1536)  # fixed_size_list<item: float32>[1536]
    ...     li: List[int]
    ...
    >>> schema = pydantic_to_schema(FooModel)
    >>> assert schema == pa.schema([
    ...     pa.field("id", pa.int64(), False),
    ...     pa.field("s", pa.utf8(), False),
    ...     pa.field("vec", pa.list_(pa.float32(), 1536)),
    ...     pa.field("li", pa.list_(pa.int64()), False),
    ... ])
    )r   r   schema)r   r   s     r%   pydantic_to_schemar     s"    D 'u--F9Vr$   c                  n    e Zd ZdZed             Zed
d            Zed             Zedd            Zd	S )
LanceModelam  
    A Pydantic Model base class that can be converted to a LanceDB Table.

    Examples
    --------
    >>> import lancedb
    >>> from lancedb.pydantic import LanceModel, Vector
    >>>
    >>> class TestModel(LanceModel):
    ...     name: str
    ...     vector: Vector(2)
    ...
    >>> db = lancedb.connect("./example")
    >>> table = db.create_table("test", schema=TestModel)
    >>> table.add([
    ...     TestModel(name="test", vector=[1.0, 2.0])
    ... ])
    AddResult(version=2)
    >>> table.search([0., 0.]).limit(1).to_pydantic(TestModel)
    [TestModel(name='test', vector=FixedSizeList(dim=2))]
    c                    t          |           }|                                 }t          |          dk    rBddlm} |                                                    |          }|                    |          }|S )z6
        Get the Arrow Schema for this model.
        r   r   )EmbeddingFunctionRegistry)r   parse_embedding_functionsrh   
embeddingsr   get_instanceget_table_metadatawith_metadata)rY   r   	functionsr   metadatas        r%   to_arrow_schemazLanceModel.to_arrow_schema  s    
 $C((1133	y>>A======0==??RR H ))(33Fr$   r   	List[str]c                h    t          |                                                                           S )z4
        Get the field names of this model.
        )rd   safe_get_fieldskeysr_   s    r%   field_nameszLanceModel.field_names  s*    
 C''))..00111r$   c                >    t           j        dk     r| j        S | j        S )Nr   )rz   r{   r   r   r_   s    r%   r	  zLanceModel.safe_get_fields  s!    !A%%>!r$   List['EmbeddingFunctionConfig']c           	        ddl m} g }|                                                                 D ].\  }}t	          |d          }||                    ||g           /g }|D ]f\  }}|                                                                 D ]:\  }}t	          |d          }	|	|u r!|                     ||||                     ;g|S )z@
        Parse the embedding functions from this model.
        r   r   vector_column_forNsource_column_for)source_columnvector_columnfunction)r   r   r	  rp   r   append)
rY   r   vec_and_functionr   
field_infofuncconfigsr   sourcesrc_funcs
             r%   r   z$LanceModel.parse_embedding_functions  s   
 	877777 # 3 3 5 5 ; ; = = 	6 	6D*j*=>>D ''t555) 	 	IC&)&9&9&;&;&A&A&C&C  "
%j2EFFt## NN//*0d     r$   N)r   r  )r   r  )	r-   r.   r/   __doc__ry   r  r  r	  r   r#   r$   r%   r   r     s         ,   [  2 2 2 [2     [ 
    [  r$   r   r  keyc                    t           j        dk    r| j        pi                     |          S | j        j        pi                     di                               |          S )z;
    Get the extra metadata from a Pydantic FieldInfo.
    r   json_schema_extra)rz   r{   r  getr  extra)r  r  s     r%   r   r     s_     "",277<<<!'-2223FKKOOPSTTTr$   rl   c                *    |                                  S z;
        Convert a Pydantic model to a dictionary.
        )dictr   s    r%   model_to_dictr$    s     zz||r$   c                *    |                                  S r"  )
model_dumpr   s    r%   r$  r$  !  s     !!!r$   )r&   r   r1   r(   )r&   r   r1   r(   r8   r9   r   r:   )r&   r   r1   r(   r8   r9   r   r   )r   r   r   r   r   r(   )r   r   r   r   )r   r
   r   r   r   r(   )r   r
   r   r   r   r(   )r   r
   r   r   )r   r   r   r(   )r   r   r   r9   )r   r   r   r   r   r   )r   r   r   r   )r  r   r  r   r   r
   )r   r   r   rl   )<r  
__future__r   r   r   r   abcr   r   r   r   enumr   typingr	   r
   r   r   r   r   r   r   r   r   numpyrf   pyarrowr   r   packaging.versionr   __version__rz   pydantic_corer   r   ImportErrorr{   pydantic.fieldsr   r   r   r   float32r7   r6   r   r   r   r   r   r   r   r   r   r   r   r   r   r$  r#   r$   r%   <module>r3     s   - , " " " " " "  



  # # # # # # # # # # # # # # # #                                       % % % % % %78/00 555555555   "" #"  4))))))333333	" 	" 	" 	" 	" 	" 	" 	" 0:rz|| 	# 	# 	# 	# 	# )3
tT T T T Tp )3
t_ _ _ _ _D   > A
 
 
 
 

 
 
 
- - - -B@ @ @ @    A A A A   02 2 2 2# # # #LQ Q Q Q Q# Q Q QhU U U U A     " " " " " "s   &A/ /BB