
    IZj$7              
       6   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
mZmZmZmZ d dlmZ d dlZd dlZd dlmZ ddlmZ d Z e            Zd	ed
efdZd	ed
efdZd	ed
e
ej         ef         fdZ!deeej"        f         ded
efdZ#d<defdZ$d<dej%        deee&e'f                  fdZ(dee         d
efdZ)	 d<dej*        dee         dee&         d
ee         fdZ+d<dej,        dee&         d
efdZ-dej.        d
e'fdZ/dej.        d
ee&         fdZ0d ee         d
ee&         fd!Z1dej,        d"ed ee         d#ee         fd$Z2ed%             Z3e34                    e          d&efd'            Z5e34                    e6          d&e6fd(            Z5e34                    e&          d&e&fd)            Z5e34                    e7          d&e7fd*            Z5e34                    e'          d&e'fd+            Z5e34                     e8d                    d& e8d          fd,            Z5e34                    e          d&efd-            Z5e34                    e          d&efd.            Z5e34                    e9          d&e9fd/            Z5e34                    e:          d&e:fd0            Z5e34                    ej;                  d&ej;        fd1            Z5d2 Z<d3efd4Zd5e=d6efd7Z>dej%        fd8Z?d9ej@        fd:ZAd9ej@        fd;ZBdS )=    N)datedatetime)singledispatch)TupleUnionOptionalAnyList)urlparse   )validate_table_namec                  2    	 dd l } | S # t          $ r Y d S w xY w)Nr   )adlfsImportError)r   s    X/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/lancedb/util.pysafe_import_adlfsr      s7       tts    
urireturnc                 t    t          |           }|j        }|sd}n|dv rd}nt          |          dk    rd}|S )z
    Get the scheme of a URI. If the URI does not have a scheme, assume it is a file URI.

    Parameters
    ----------
    uri : str
        The URI to parse.

    Returns
    -------
    str: The scheme of the URI.
    file)s3as3ns3r   )r   schemelen)r   parsedr   s      r   get_uri_schemer   #   sT     c]]F]F 	>	!	!	V		 M    c                     t          |           }t          |j                  dk    r| S |j        s|j        S |j        |j        z   S )z
    Get the location of a URI. If the parameter is not a url, assumes it is just a path

    Parameters
    ----------
    uri : str
        The URI to parse.

    Returns
    -------
    str: Location part of the URL, without scheme
    r   )r   r   r   netlocpath)r   r   s     r   get_uri_locationr"   >   sL     c]]F
6=Q 
= +{}v{**r   c                 F   t          |           dk    rGt          j        t          j                            d          dd          }t          |           }||fS t          |           dk    rt          t                              t          j                            d          t          j                            d          	          }t          j	        t          j
        |                    }t          |           }||fS t          j                            |           S )
zT
    Get a PyArrow FileSystem from a URI, handling extra environment variables.
    r   AWS_ENDPOINT   )endpoint_overriderequest_timeoutconnect_timeoutazNAZURE_STORAGE_ACCOUNT_NAMEAZURE_STORAGE_ACCOUNT_KEY)account_nameaccount_key)r   pa_fsS3FileSystemosenvirongetr"   r   AzureBlobFileSystemPyFileSystemFSSpecHandler
FileSystemfrom_uri)r   fsr!   
az_blob_fss       r   fs_from_urir:   X   s     cd"" jnn^<<
 
 

  $$4x				$	$):..(DEE
'BCC / 
 


  3J ? ?@@$$4x$$S)))r   basepartsc                    t          | t          j                  r
 | j        | S t	          |           } t          |           dk    rt	          t          j        | g|R            S t          |           }d                    d |j        g|D                       }|	                    |          
                                S )a  
    Join a URI with multiple parts, handles both local and remote paths

    Parameters
    ----------
    base : str
        The base URI
    parts : str
        The parts to join to the base URI, each separated by the
        appropriate path separator for the URI scheme and OS
    r   /c                 8    g | ]}|                     d           S )r>   )rstrip).0ps     r   
<listcomp>zjoin_uri.<locals>.<listcomp>   s"    GGGqQXXc]]GGGr   )r!   )
isinstancepathlibPathjoinpathstrr   r   joinr!   _replacegeturl)r;   r<   urlnew_paths       r   join_urirN   s   s     $%% %t}e$$t99Ddv%% 7<-u---... tnn88GGSX4F4FGGGHH|||**11333r   modulec                 p    	 t          j        |           S # t          $ r t          d|p|            w xY w)ac  
    Import the specified module. If the module is not installed,
    raise an ImportError with a helpful message.

    Parameters
    ----------
    module : str
        The name of the module to import
    mitigation : Optional[str]
        The package(s) to install to mitigate the error.
        If not provided then the module name will be used.
    zPlease install )	importlibimport_moduler   )rO   
mitigations     r   attempt_import_or_raiserT      sT    D&v... D D DBJ,@&BBCCCDs    5tblflattenc                    |du r5	 |                                  } t          d | j        D                       r4nOt          |t                    r:|dk    rt          d          |dk    r|                                  } |dz  }|dk    | S )aJ  
    Flatten all struct columns in a table.

    Parameters
    ----------
    flatten: Optional[Union[int, bool]]
        If flatten is True, flatten all nested columns.
        If flatten is an integer, flatten the nested columns up to the
        specified depth.
        If unspecified, do not flatten the nested columns.
    Tc              3   ^   K   | ](}t           j                            |j                  V  )d S N)patypes	is_structtyperA   cols     r   	<genexpr>z"flatten_columns.<locals>.<genexpr>   s4      FFC28%%ch//FFFFFFr   r   zIPlease specify a positive integer for flatten or the boolean value `True`r   )rV   anyschemarD   int
ValueError)rU   rV   s     r   flatten_columnsre      s     $	++--CFF3:FFFFF 	GS	!	! a<<   kk++--CqLG kk Jr   r!   c                 j    dt           dt           fdd                    fd| D                       S )Nsegmentr   c                 l    t          d | D                       r| S d|                     dd           dS )Nc              3   J   K   | ]}|                                 p|d k    V  dS )_N)isalnum)rA   chars     r   r`   z=_format_field_path.<locals>.format_segment.<locals>.<genexpr>   s3      AAt||~~,AAAAAAr   `z``)allreplace)rg   s    r   format_segmentz*_format_field_path.<locals>.format_segment   sE    AAAAAAA 	N07??3--0000r   .c              3   .   K   | ]} |          V  d S rY    )rA   rg   rp   s     r   r`   z%_format_field_path.<locals>.<genexpr>   s-      @@NN7++@@@@@@r   )rH   rI   )r!   rp   s    @r   _format_field_pathrt      sN    1 1 1 1 1 1
 88@@@@4@@@@@@r   fielddimc                    g || j         }t          | j                  r.t          | j                  }|||k    rt	          |          gS g S t
          j                            | j                  r\g }t          | j        j	                  D ]>}|
                    t          | j                            |          ||                     ?|S g S rY   )nameis_vector_columnr]   infer_vector_column_dimrt   rZ   r[   r\   range
num_fieldsextend_iter_vector_columnsru   )ru   r!   rv   
field_path
vector_dimcolumnsidxs          r   r~   r~      s     %4$$J
## ,UZ88
;*++&z2233		x%*%% .// 	Y 	YCNN/
0@0@0E0EzSVWWXXXXIr   rb   c                     g }| D ]&}|                     t          |g |                     't          |          dk    rt          d|           t          |          dk    rt          d          |d         S )z
    Get the vector column name

    Parameters
    ----------
    schema : pa.Schema
        The schema of the vector column.

    Returns
    -------
    str: the vector column name.
    r   zmSchema has more than one vector column. Please specify the vector column name for vector search. Candidates: r   z^There is no vector column in the data. Please specify the vector column name for vector search)r}   r~   r   rd   )rb   rv   vector_col_namesru   s       r   inf_vector_column_queryr      s      F F 4UB D DEEEE
q  A.>A A
 
 	

 !!F
 
 	
 Ar   	data_typec                 >   t           j                            |           rJt           j                            | j                  s$t           j                            | j                  rdS t           j                            |           rt          | j                  S dS )z
    Check if the column is a vector column.

    Parameters
    ----------
    data_type : pa.DataType
        The data type of the column.

    Returns
    -------
    bool: True if the column is a vector column.
    TF)rZ   r[   is_fixed_size_listis_floating
value_typeis_uint8is_listry   r   s    r   ry   ry      s     
x""9-- 6
Y12268Y1226 t			)	$	$ 6	 45555r   c                     t           j                            |           r| j        S t           j                            |           rt          | j                  S d S rY   )rZ   r[   r   	list_sizer   rz   r   r   s    r   rz   rz     sQ    	x""9-- #""	x	"" =&y';<<<4r   queryc                 >   | d S t          | t          j                  r| j        dk    rd S | j        d         S t          | t
                    rO| rM| d         }t          |t
          t          t          j        f          rt          |          S t          |           S d S )Nr   )rD   npndarrayndimshapelisttupler   )r   firsts     r   _query_vector_dimr     s    }t%$$ :??4{2% 5 aedE2:677 	u::5zz4r   
query_typevector_column_namec                     ||S |dk    rd S ||dk    r4	 t          | t          |                    }n# t          $ r}|d }~ww xY w|S )Nftshybridrv   )r   r   	Exception)rb   r   r   r   es        r   infer_vector_column_namer   &  s     %!!UtJ(22	!8-e44" " "  	 	 	G	 s   5 
AAAc                      t          d          )Nz/SQL conversion is not implemented for this type)NotImplementedErrorvalues    r   value_to_sqlr   >  s    
O
P
PPr   r   c                 :    |                      dd          } d|  dS )N'z'')ro   r   s    r   rj   rj   C  s#    MM#t$$Eu<<<r   c                 V    dt          j        |                                            dS )z~Convert bytes to a hex string literal.

    See https://datafusion.apache.org/user-guide/sql/data_types.html#binary-types
    zX'r   )binasciihexlifydecoder   s    r   rj   rj   I  s,     4 ''..003333r   c                      t          |           S rY   rH   r   s    r   rj   rj   R      u::r   c                      t          |           S rY   r   r   s    r   rj   rj   W  r   r   c                 D    t          |                                           S rY   )rH   upperr   s    r   rj   rj   \  s    u::r   c                     dS )NNULLrs   r   s    r   rj   rj   a  s    6r   c                 2    d|                                   dS Nr   	isoformatr   s    r   rj   rj   f      #u  ####r   c                 2    d|                                   dS r   r   r   s    r   rj   rj   k  r   r   c                 ^    dd                     t          t          |                     z   dz   S )N[, ])rI   mapr   r   s    r   rj   rj   p  s(    3|U33444s::r   c                 p    dd                     d |                                 D                       z   dz   S )Nznamed_struct(r   c              3   F   K   | ]\  }}d | dt          |           V  dS )r   z', N)r   )rA   kvs      r   r`   z_.<locals>.<genexpr>z  s=      JJDAq///l1oo//JJJJJJr   ))rI   itemsr   s    r   rj   rj   u  s@     	
))JJEKKMMJJJ
J
J	K
	r   c                 D    t          |                                           S rY   )r   tolistr   s    r   rj   rj     s    '''r   c                 F     t          j                    fd            }|S )zThis is a decorator which can be used to mark functions
    as deprecated. It will result in a warning being emitted
    when the function is used.c                      t          j        dt                     t          j        dj         dt          d           t          j        dt                      | i |S )Nalwaysz	Function z6 is deprecated and will be removed in a future version   )category
stackleveldefault)warningssimplefilterDeprecationWarningwarn__name__)argskwargsfuncs     r   new_funczdeprecated.<locals>.new_func  s{    h(:;;;.DM . . . (	
 	
 	
 	
 	i);<<<tT$V$$$r   )	functoolswraps)r   r   s   ` r   
deprecatedr     s:    
 _T% % % % % Or   rx   c                 $    t          |            dS )zVerify the table name is valid.N)native_validate_table_name)rx   s    r   r   r     s    t$$$$$r   base_exceptionnotec                    t          | d          r|                     |           d S t          | j        d         t                    r+| j        d         dz   |z   g| j        dd          R | _        d S t          d          )Nadd_noter   
r   zCannot add note to exception)hasattrr   rD   r   rH   rd   )r   r   s     r   r   r     s    ~z** 9%%%%%	N'*C	0	0 9"T)D0
 $
 

 7888r   c                      t          dd          dt          j        ffd                     fdt	           j                  D                       S )a&  
    Convert a PyArrow Table to a PyTorch Tensor.

    Each column is converted to a tensor (using zero-copy via DLPack)
    and the columns are then stacked into a single tensor.

    Fails if torch is not installed.
    Fails if any column is more than one chunk.
    Fails if a column's data type is not supported by PyTorch.

    Parameters
    ----------
    tbl : pa.Table or pa.RecordBatch
        The table or record batch to convert to a tensor.

    Returns
    -------
    torch.Tensor: The tensor containing the columns of the table.
    torchr_   c                     | j         dk    rt          d                              |                     d                    S )Nr   z8Single batch was too large to fit into a one-chunk tabler   )
num_chunksr   from_dlpackchunk)r_   r   s    r   	to_tensorz tbl_to_tensor.<locals>.to_tensor  s=    >AVWWW  1...r   c                 L    g | ] }                      |                    !S rs   )column)rA   irU   r   s     r   rC   z!tbl_to_tensor.<locals>.<listcomp>  s-    QQQQ		#**Q--00QQQr   )rT   rZ   ChunkedArraystackr{   num_columns)rU   r   r   s   `@@r   tbl_to_tensorr     ss    ( $GW55E/r / / / / / /
 ;;QQQQQ%:P:PQQQRRRr   batchc                 p    t          dd                              fd| j        D                       S )a  
    Convert a PyArrow RecordBatch to a PyTorch Tensor.

    Each column is converted to a tensor (using zero-copy via DLPack)
    and the columns are then stacked into a single tensor.

    Fails if torch is not installed.
    Fails if a column's data type is not supported by PyTorch.

    Parameters
    ----------
    batch : pa.RecordBatch
        The record batch to convert to a tensor.

    Returns
    -------
    torch.Tensor: The tensor containing the columns of the record batch.
    r   c                 :    g | ]}                     |          S rs   )r   )rA   r_   r   s     r   rC   z#batch_to_tensor.<locals>.<listcomp>  s'    HHH3))#..HHHr   )rT   r   r   )r   r   s    @r   batch_to_tensorr     s=    & $GW55E;;HHHH%-HHHIIIr   c                     t          dd          }t          dd          }d | j        D             }|                    |                    |                    }t	          |                    d                    }|S )a  
    Convert a PyArrow RecordBatch to a list of PyTorch Tensor, one per row

    Each column is converted to a tensor (using zero-copy via DLPack)
    and the columns are then stacked into a single tensor.  The 2D tensor
    is then converted to a list of tensors, one per row

    Fails if torch or numpy is not installed.
    Fails if a column's data type is not supported by PyTorch.
    r   numpyc                 :    g | ]}|                     d           S )F)zero_copy_only)to_numpyr^   s     r   rC   z(batch_to_tensor_rows.<locals>.<listcomp>  s&    KKKcs||5|11KKKr   r   r   )rT   r   tensorcolumn_stackr   unbind)r   r   r   r   stackedrowss         r   batch_to_tensor_rowsr    sv     $GW55E#GW55EKKU]KKKGll5--g6677G1%%&&DKr   rY   )Cr   r   rQ   r0   rE   r   r   r   r   typingr   r   r   r	   r
   urllib.parser   r   r   pyarrowrZ   
pyarrow.fsr8   r.   _lancedbr   r   r   r   rH   r   r"   r6   r:   rF   rN   rT   Tablerc   boolre   rt   Fieldr~   Schemar   DataTypery   rz   r   r   r   registerrj   bytesfloatr]   r   dictr   r   BaseExceptionr   r   RecordBatchr   r  rs   r   r   <module>r     s  
          				   # # # # # # # # $ $ $ $ $ $ 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ! ! ! ! ! !               G G G G G G   	     6+# +# + + + +4*S *U5#3S#89 * * * *645gl*+ 4S 4S 4 4 4 44D DC D D D D&  HU395E,F    @AT#Y A3 A A A A <@ 89+3C=	#Y   " BI HSM S    <     .r{ x}    Xc] x}    I C= !	   0 Q Q Q sS    
 u4U 4 4 4 4 sS     uU     tT     ttDzz""TT$ZZ    #" x  $X $ $ $ ! $ t$T $ $ $ $ t;T ; ; ; ; tT     rz""(RZ ( ( ( #"(  ,%c % % % %
	9] 	9# 	9 	9 	9 	9Srx S S S S<J2> J J J J.      r   