
    IZj                        U d dl mZ d dlmZ d dlZd dlmZmZmZ d dl	m
Z
 d dlZd dlmZ ddlmZ e G d d	                      Zed
efd            Ze                    ej                  dej        d
efd            Ze                    ej                  dej        d
efd            Ze                    ej                  dej        d
efd            Ze                    ej                  dej        d
efd            Ze                    ej                  dej        d
efd            Ze                    e          ded
efd            Z e                    e!          de!d
efd            Z"e                    e          ded
efd            Z#ded
efdZ$ e%            Z&e%e'         e(d<   d Z) e)             dS )    )	dataclass)singledispatchN)CallableIteratorOptionalto_arrow   )
LanceModelc                   l    e Zd ZU ej        ed<   ee         ed<   eg ej	        f         ed<   dZ
eed<   dS )	Scannableschemanum_rowsreaderTrescannableN)__name__
__module____qualname__paSchema__annotations__r   intr   RecordBatchReaderr   bool     ]/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/lancedb/scannable.pyr   r      s[         IsmR--.... Kr   r   returnc                     t          | d          rt          t          |                     S t          dt	          |                      )N__iter__z&to_scannable not implemented for type )hasattr_from_iterableiterNotImplementedErrortypedatas    r   to_scannabler(      sG     tZ   *d4jj)))
StDzzSS
T
TTr   r'   c                 :     t           j        d  fdd          S )Nc                       S Nr   r&   s   r   <lambda>z_from_reader.<locals>.<lambda>'   s    $ r   Fr   r   r   r   )r   r   r&   s   `r   _from_readerr.   #   s-     {T,,,,E   r   c                 B     t           j         j         fd          S )Nc                  P    t           j                             j         g          S r+   )r   r   from_batchesr   r&   s   r   r,   z_from_batch.<locals>.<lambda>0   s    r+88tfMM r   r   r   r   )r   r   r   r&   s   `r   _from_batchr3   +   s0    {MMMM   r   c                 D    t          | j        | j        | j                  S Nr2   )r   r   r   	to_readerr&   s    r   _from_tabler7   4   s    DK$-WWWWr   c                 \     t           j                                          fd          S )Nc                  P                                                                      S r+   scannerr6   r&   s   r   r,   z_from_dataset.<locals>.<lambda>>   s    t||~~//11 r   r2   r   r   
count_rowsr&   s   `r   _from_datasetr>   9   s8    {""1111   r   c                 <    t          | j        d | j        d          S )NFr-   )r   projected_schemar6   r&   s    r   _from_scannerrA   B   s,     $~	   r   c                     | st          d          t          |           }t          |j        |j        |j                  S )Nz4Cannot create table from empty list without a schemar2   )
ValueErrorr	   r   r   r   r6   r'   tables     r   
_from_listrF   M   sI     QOPPPTNNE|enU_   r   c                      t          d          )Nz6Cannot add a single dictionary to a table. Use a list.rC   r&   s    r   
_from_dictrI   W       
M
N
NNr   c                      t          d          )Nz6Cannot add a single LanceModel to a table. Use a list.rH   r&   s    r   _from_lance_modelrL   \   rJ   r   c                      t           d           }|t          d          t          |          j         fd}t          j                             |                      }t          |          S )Nz'Cannot create table from empty iteratorc               3   D  K                                    E d {V  D ]} t          |           }|j        k    rI	 |                              }n2# t          j        j        $ r t          d d|j                   w xY w|                                 E d {V  d S )NzfInput iterator yielded a batch with schema that does not match the schema of other batches.
Expected:
z
Got:
)
to_batchesr	   r   castr   libArrowInvalidrC   )itembatchr'   firstr   s     r   r#   z_from_iterable.<locals>.iterh   s      ##%%%%%%%%% 	* 	*DTNNE|v%%!JJv..EEv*   $E&,E E6;lE E   ''))))))))))	* 	*s   A/B)nextrC   r	   r   r   r   r1   r(   )r'   
first_itemr#   r   rU   r   s   `   @@r   r"   r"   a   s    dD!!JBCCCZ  E\F* * * * * * * !..vttvv>>Fr   _registered_modulesc                     dt           j        v rdt          vrt                              d           ddl} t          j        | j                  d| j        dt          j	        fd            t                              | j                  d| j        dt          ffd            }dt           j        v rdt          vrt                              d           ddl}t          j        |j                  d|j        dt          j	        fd	            }t                              |j                  d|j        dt          fd
            }t                              |j                  d|j        dt          fd            }dt           j        v rdt          vrt                              d           ddlm} ddlm} t                              |          d|dt          fd            }t                              |          d|dt          fd            }	dt           j        v r`dt          vrYt                              d           ddl}
t                              |
j                  d|
j        dt          fd            }dS dS dS )zHRegister converters for optional dependencies that are already imported.pandasr   Nr'   r   c                 n    t           j                            | d          }|                    d           S )NF)preserve_index)r   Tablefrom_pandasreplace_schema_metadatarD   s     r   _arrow_from_pandasz9_register_optional_converters.<locals>._arrow_from_pandas   s0    H((e(DDE00666r   c                 4    t           |                     S r+   )r(   )r'   r`   s    r   _from_pandasz3_register_optional_converters.<locals>._from_pandas   s     2 24 8 8999r   polarsc                 *    |                                  S r+   r   r&   s    r   _arrow_from_polarsz9_register_optional_converters.<locals>._arrow_from_polars   s    ==??"r   c                 |    |                                  }t          |j        t          |           |j                  S r5   )r	   r   r   lenr6   r'   arrows     r   _from_polarsz3_register_optional_converters.<locals>._from_polars   s7    MMOOE|c$ii   r   c                     |                                                                  }t          |j        |j        |j                  S r5   )collectr	   r   r   r   r6   rh   s     r   _from_polars_lazyz8_register_optional_converters.<locals>._from_polars_lazy   s?    LLNN++--E|enU_   r   datasets)Dataset)DatasetDictc                 l    | j         j        }t          |j        t	          |           |j                  S r5   )r'   rE   r   r   rg   r6   rD   s     r   _from_hf_datasetz7_register_optional_converters.<locals>._from_hf_dataset   s3    IOE|c$ii   r   c                      t                                                     d                  j        j        dj        vr9                    t          j        dt          j                                         fdt          d  
                                D                       }t          |fd          S )Nr   splitc               3   Z  K                                    D ]\  } }|j                                        D ]s}t          j        | gt          |          z  t          j                              }t          j                            t          |j
                  |gz             V  td S )N)r%   )r   )itemsr'   rO   r   arrayrg   stringRecordBatchfrom_arrayslistcolumns)
split_namedatasetrT   	split_arrr'   r   s       r   genzI_register_optional_converters.<locals>._from_hf_dataset_dict.<locals>.gen   s      +/::<<  'J!(!8!8!:!:  $&H'L3u::5BIKK% % %	 !n88 //9+=f 9      	 r   c              3   4   K   | ]}t          |          V  d S r+   )rg   ).0r~   s     r   	<genexpr>zO_register_optional_converters.<locals>._from_hf_dataset_dict.<locals>.<genexpr>   s(      GGgS\\GGGGGGr   c                  T    t           j                                                    S r+   )r   r   r1   )r   r   s   r   r,   zN_register_optional_converters.<locals>._from_hf_dataset_dict.<locals>.<lambda>   s     r3@@OO r   r2   )r{   keysfeaturesarrow_schemanamesappendr   fieldrx   sumvaluesr   )r'   
total_rowsr   r   s   ` @@r   _from_hf_dataset_dictz<_register_optional_converters.<locals>._from_hf_dataset_dict   s     $tyy{{++A./8EFfl**rx'E'EFF      GGGGGGGJ#OOOOO   r   lancec                 \     t           j                                          fd          S )Nc                  P                                                                      S r+   r:   r&   s   r   r,   zD_register_optional_converters.<locals>._from_lance.<locals>.<lambda>   s    t||~~7799 r   r2   r<   r&   s   `r   _from_lancez2_register_optional_converters.<locals>._from_lance   s8    {**9999   r   )sysmodulesrX   addrZ   r	   register	DataFramer   r]   r(   r   rc   	LazyFramern   ro   rp   r   LanceDataset)pdrb   plre   rj   rm   	HFDatasetHFDatasetDictrr   r   r   r   r`   s               @r   _register_optional_convertersr   ~   s    3;83F#F#F)))		2<	(	(	7R\ 	7bh 	7 	7 	7 
)	(	7 
		r|	,	,	:r| 	:	 	: 	: 	: 	: 	: 
-	,	: 3;83F#F#F)))		2<	(	(	#R\ 	#bh 	# 	# 	# 
)	(	# 
		r|	,	,	r| 		 	 	 	 
-	,	 
		r|	,	,	BL 	Y 	 	 	 
-	,	 S[  Z7J%J%J
+++111111999999			y	)	)	9 	 	 	 	 
*	)	 
		}	-	-	 	) 	 	 	 
.	-	. #+'1D"D"D(((			u1	2	2	e0 	Y 	 	 	 
3	2	 	 	 "D"Dr   )*dataclassesr   	functoolsr   r   typingr   r   r   lancedb.arrowr	   pyarrowr   pyarrow.datasetr~   dspydanticr   r   r(   r   r   r.   ry   r3   r]   r7   ro   r>   ScannerrA   r{   rF   dictrI   rL   r"   setrX   strr   r   r   r   r   <module>r      sp   " ! ! ! ! ! ! $ $ $ $ $ $ 



 / / / / / / / / / / " " " " " "                               U) U U U U r+,,r+ 	    -, r~&&bn     '& rx  Xbh X9 X X X ! X rz""
 y    #" rz""
 y    #" tT i     tOT Oi O O O O z""OJ O9 O O O #"O   i        4 !$ SX % % %T T Tp       r   