IZj<ddlmZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z m Z ddl mZddlmZmZmZmZmZmZmZmZmZmZmZddlmZdd lmZmZd d l m!Z!dd l"m#Z#dd l$m%Z%d dl&m'Z'm(Z(m)Z)m*Z*m+Z,m-Z.ddl/Z0ddl1Z/ddl2m3Z4ddl5m6Z7ddl8Z9d dl:m;Z;mm?Z?m@Z@d dlAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLd dlMmNZNd dlOmPZPmQZQd dlRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_d dl`maZambZbmcZcmdZdmeZemfZfd dlAmgZgedZhdZidZjdd Zkdd#Zldd$dd'ZmerFd d(lnmoZod d)lpmqZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~d d*lAmZddl+Z+ddlZd d+lmZmZmZmZmZmZmZmZ ddd/Zdd2Z dd5d6dd@Z dddCZddGZddIZ dddKZddLZdMZ dddNddOZddRZddSZdTZGdUdVeZqGdWdXeqZ dddZZdd]Zdd_ZddeZ dddlZddpZddrZddtZddvZddxZddzZdd|Zdd}Zdd~ZGddZe GddZe GddZe GddZe GddZGddZGddZdS)) annotationsN)ABCabstractmethod) dataclass)datetime timedelta)cached_property) TYPE_CHECKINGAnyCallableDictIterableListLiteralOptionalTupleUnionoverload)urlparse)_register_optional_converters to_scannable) __version__) peek_reader)LOOP)_check_for_hugging_face_check_for_lance_check_for_pandaslancepandaspolars)DATAVECVECTOR_COLUMN_NAME)EmbeddingFunctionConfigEmbeddingFunctionRegistry) BTreeIvfFlatIvfPqIvfSqBitmapIvfRq LabelListHnswPqHnswSqHnswFlatFTS)LanceMergeInsertBuilder) LanceModel model_to_dict) AsyncFTSQueryAsyncHybridQuery AsyncQueryAsyncTakeQueryAsyncVectorQuery FullTextQueryLanceEmptyQueryBuilderLanceFtsQueryBuilderLanceHybridQueryBuilderLanceQueryBuilderLanceVectorQueryBuilderLanceTakeQueryBuilderQuery)add_note fs_from_uriget_uri_schemeinfer_vector_column_namejoin_uri value_to_sql) lang_mapping)lazybytes descriptions)jiebalindera)zunknown base tokenizerzInvalid directory path:zFailed to load JiebazFailed to load tokenizer configz&Failed to initialize default tokenizer exception BaseExceptionnotestrreturnNonect|ddpd}|jr-t|jdtr |jdnd}||vr||vrt ||dSdSdS)N __notes__r)getattrargs isinstancerQrB)rNrPexisting_notesmessages Y/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/lancedb/table.py_add_unique_noter^fsY R88>BN > ():C@@  q   >!!d'&9&9D!!!!!"!&9&9base_tokenizerboolcDtfdtDS)Nc3TK|]"}|kp|dV#dS)/N) startswith).0prefixr`s r] z-_is_model_backed_tokenizer..rsU  & KN$=$=lll$K$Kr_)any _MODEL_BACKED_TOKENIZER_PREFIXES)r`s`r]_is_model_backed_tokenizerrkqs; 6  r_)languagerl Optional[str]c.t||?dvr;dtj}t |d|dSt |sdSt fdtDsdSt |ddS)Nz"not support the requested languagez, zSupported languages: c3 K|]}|vV dSNrV)rfmarkerr\s r]rhz,_maybe_add_fts_error_note..s(NNVv NNNNNNr_aKModel-backed tokenizers such as 'jieba/default' and 'lindera/ipadic' require tokenizer models in Lance's language model home. Set LANCE_LANGUAGE_MODEL_HOME to override the default platform data directory under 'lance/language_models'. Expected layouts include '/jieba/default/...' and '/lindera/ipadic/...'.)rQjoinrHvaluesr^rkri_MODEL_BACKED_TOKENIZER_ERRORS)rNr`rlsupported_langsr\s @r]_maybe_add_fts_error_notervxs)nnG D O O))L$7$9$9::$MO$M$MNNN %n 5 5 NNNN/MNNN N N -r_LanceDBConnection) Table OptimizeStats CleanupStatsCompactionStatsTagAddColumnsResult AddResultAlterColumnsResult DeleteResultDropColumnsResult LsmWriteSpec MergeResult UpdateResult) IndexConfig) QueryTypeOnBadVectorsTypeAddMode CreateModeVectorIndexTypeScalarIndexTypeBaseTokenizerType DistanceTypeschemaOptional[pa.Schema]pa.RecordBatchReaderc` ddlm}t|rt||jrC|jj}tj ||j St||j j rt|d}d|jvr9|tjdtj}tj |t'|St|t(rt+dt|t,rt+dt|t.rA|sD|t+dtj||St|dt(r^|dj}d|D}tj||St|dtjr1tj |Stj|St=|rt|t>j rtj!|d }|j"j# |j"j#ni}d |$D}|%|St|tjr|St|tjr&tj |j"|gStM|r@t|tNj(r&|)St|tj*jr&|)St|tj*j+r|St|tjr|StY|j-.d r6|jj/d kr&|0StY|j-.d rH|jj/dkr8|10St|tdrtg|StidtY|d)Nr)datasetssplitz6Cannot add a single LanceModel to a table. Use a list.z6Cannot add a single dictionary to a table. Use a list.z4Cannot create table from empty list without a schemarc,g|]}t|SrV)r4)rfds r] z(_into_pyarrow_reader..s 333M!$$333r_F)preserve_indexc&i|]\}}|dk ||S)spandasrVrfkvs r] z(_into_pyarrow_reader..s#@@@Ai1r_r! DataFrame LazyFramezUnknown data type z. Supported types: list of dicts, pandas DataFrame, polars DataFrame, pyarrow Table/RecordBatch, or Pydantic models. See https://docs.lancedb.com/tables/ for examples.)5lancedb.dependenciesrrrZDatasetfeatures arrow_schemapaRecordBatchReader from_batchesdata to_batches dataset_dict DatasetDict_schema_from_hfnamesappendfieldstring_to_batches_with_splitr3 ValueErrordictlistry from_pylist to_reader __class__to_arrow_schema RecordBatchrpdr from_pandasrmetadataitemsreplace_schema_metadatarr LanceDatasetscannerdatasetScannertype __module__re__name__to_arrowcollectr_iterator_to_reader TypeError)rrrtablemetas r]_into_pyarrow_readerrs.-----t$$  dH, - - ]/F'44VTY=Q=Q=S=STT T h3? @ @ $T400Ffl**rx'E'EFF'44.t44 $ ##SQRRR$SQRRR$4  I~ !WXXX8''V'<<FFHH H d1gz * * :!W&6688F33d333D8''V'<<FFHH H Q 0 0 :8((..88:: :8''--7799 9 4 $ Zbl%C%C$ $$T%$@@(- (=(Iu|$$r@@@@@,,T22<<>>> D"( # # ~~ D". ) ) #00tfEEE $   JtU5G$H$H ||~~''))) D"*, - - ||~~''))) D"*, - - ~~ D". / /   T ((22 N #{ 2 2}}((*** T ((22  N #{ 2 2||~~&&((22444 D( # # "4((( Ad A A A   r_rrcttjfd}tj|S)Nc3DKEd{VD]}t|}|jkrI |}n2#tjj$rtdd|jwxYw|Ed{VdS)NzfInput iterator yielded a batch with schema that does not match the schema of other batches. Expected: z Got: ) rread_allrcastrlib ArrowInvalidrr)batchrrfirstrs r]genz _iterator_to_reader..gens * *E2599BBDDE|v%%!JJv..EEv*$E&,EE6;lEE '')) ) ) ) ) ) ) ) ) * *s A/B)rnextrrrr)rrrrs` @@r]rrse !d , ,E \F * * * * * * *   , ,VSSUU ; ;;r_errorF)allow_subschema'DATA' target_schemarOptional[dict]on_bad_vectorsr fill_valuefloatrc.t||}t|||}t|||||}|t|\}}|r(|t |j|}t|t|||}|S)a Handle input data, applying all standard transformations. This includes: * Converting the data to a PyArrow Table * Adding vector columns defined in the metadata * Adding embedding metadata into the schema * Casting the table to the target schema * Handling bad vectors Parameters ---------- target_schema : Optional[pa.Schema], default None The schema to cast the table to. This is typically the schema of the table if it already exists. Otherwise it might be a user-requested schema. allow_subschema : bool, default False If True, the input table is allowed to omit columns from the target schema. The target schema will be filtered to only include columns that are present in the input table before casting. metadata : Optional[dict], default None The embedding metadata to add to the schema. on_bad_vectors : Literal["error", "drop", "fill", "null"], default "error" What to do if any of the vectors are not the same size or contains NaNs. fill_value : float, default 0.0 The value to use when filling vectors. Only used if on_bad_vectors="fill". All entries in the vector will be set to this value. r)rrrr) r_append_vector_columns_handle_bad_vectors_infer_target_schema with_metadata_merge_metadatar_validate_schema_cast_to_target_schema)rrrrrrreaders r]_sanitize_datarsR"$ 6 6F #FMH M M MF!%# F 4V < < v %33 M2H = =  ]### #FM? K KF Mr_r pa.Schemacj|drSttt jtt |}t j||j|s/tt|krtd|rtt|krdttt jtt }t j||jfd}t j |S)NTcheck_metadatarz>Input table has different number of columns than target schemac3KD]w}tj|g}|r.tj|djVxdS)Nrr)rryrrrr from_arrayscolumns)r cast_batchesrreordered_schemas r]rz#_cast_to_target_schema..genus  E%%ug..334DEEPPRR  n00 O+4D1   r_) requals_align_field_typesriterrrlenr_infer_subschemarr)rrrfieldsrrs` @r]rr\s`}M$?? T&-%8%8 9 94]@S@S;T;T U UFy-2HIII  s#344M8J8JJJ L   N3/00C 4F4FFF! fm$$ % %tD1A,B,B'C'C  9Vm6LMMM         , ,-=ssuu E EEr_rList[pa.Field] target_fieldsc ,g}|D]tfd|Dd}|tdjdtj|jretjjr8tjtjj |jj }n|j}ntj |jr]tjr@tj tjj g|jj gd}n|j}ntj|jr[tjr?tjtjj g|jj gd}n|j}ntj|jrftjrJtj tjj g|jj gd|jj}n|j}n|j}|tjj|j|j|S)zD Apply the data types from the target_fields to the fields. c3<K|]}|jjk|VdSrpname)rffrs r]rhz%_align_field_types..s1NN15:9M9MQ9M9M9M9MNNr_NzField 'z' not found in target schemar)rrrrtypes is_structrstructrris_list _is_list_likelist_ value_field is_large_list large_listis_fixed_size_list list_sizerrnullabler)rr new_fields target_fieldnew_typers @r]rrsJ1 1 NNNN NNNPTUU  OuzOOOPP P 8  l/ 0 0* )x!!%*-- -9& )$)0(, X  l/ 0 0 )UZ(( -8&/0%*67(, X # #L$5 6 6 )UZ(( -=&/0%*67(, X ( (): ; ; )UZ(( -8&/0%*67!%/ (,#(H HUZ5><;P Q Q     r_reference_fieldscg}d|D}|D]}||j}|"td|tj|jrWt jt|jj |jj }t j |j||j }n|}| ||S)z Transform the list of fields so the types match the reference_fields. The order of the fields is preserved. ``schema`` may have fewer fields than `reference_fields`, but it may not have more fields. ci|] }|j| SrVrrfrs r]rz$_infer_subschema..s 2 2 2Aafa 2 2 2r_NzUnexpected field in schema: {})getrrformatrrrrrrrrr r)rrrlookupr referencer new_fields r]rrsF 2 2!1 2 2 2F!!JJuz**  =DDUKKLL L 8  in - - "y J%N)H  "II "I i    Mr_Union[pa.Schema, LanceModel]ctj|r)t|tr|}|'| ||j}t |||||}|j}n"| tj g|}|(|tdt|dr|j}|rt|j|}||}t|tj r||}n:t|tjr tj||}||fS)N)rrrz&Either data or schema must be providedr)inspectisclass issubclassr3rrrrrryrrhasattrrrrZrrr)rrrrrs r]sanitize_create_tablers^v5:fj#A#A5#2244    2H  )!      8''F33D ~ <EFF F T8 $ $ ![FC"6?H==%%h// dBH % % C//99DD b2 3 3 C'44VTBBD <r_c|D]2}| |jj}||jjkrd}t|3|S)z Extract pyarrow schema from HuggingFace DatasetDict and validate that they're all the same schema between splits NzCAll datasets in a HuggingFace DatasetDict must have the same schema)rsrrr)rrrmsgs r]rrs[ ;;==!! >%2FF w'4 4 4WCC.. 5 Mr_c #K|D]\}}|jD]}tj|g}d|jvrC|dtj|g|j ztj }|D]}|VdS)zm Return a generator of RecordBatches from a HuggingFace DatasetDict with an extra `split` column rN) rrrrryr column_names append_columnarraynum_rowsr)rkeyrrrbs r]rr$s   W\,,..  EH))5'22Ee000++RXseen&tjtj||r|||j}|rV|  |tj | |j }||j| |tj | |j }>|V[dS)Nr)rfunctionr#pcallis_nullas_py$compute_source_embeddings_with_retry source_columnr$rrr%r set_columnindex) r vector_columnconffuncno_vector_columncol_data functionsrrs r]rz#_append_vector_columns..genRsa  E'0'8'8  # t}#08J#J #rvbj}9M.N.N'O'O'U'U'W'W#HHd01  H(  % 3 3"LL77HXFLL4O4O4TUUU!! !& 0 0!.44]CC"LL77HXFLL4O4O4TUUU!! KKKK'  r_)rrr& get_instanceparse_functionsrrrrrrrfloat32r,ndimsrrr) rrrrr5r6rdtyperr:s `` @r]rr4sV~"8,,"6?H==)688HHRRI  &- F(00!! t  3 3 3! ]33t}/B/B/D/DEEUTJJJ MM% Yv (> ? ? ?F,   , ,VSSUU ; ;;r_base table_namect||}t|}|jdkr|d}|dS)z Get a table path that can be used in PyArrow FS. Removes any weird schemes (such as "s3+ddb") and drops any query params. zs3+ddbs3)schemeNquery) _table_urirrD_replacegeturl)r@rAuriparseds r] _table_pathrLks^ T: & &C c]]F }  -- ??? & & - - / //r_c(t||dS)Nz.lance)rF)r@rAs r]rGrG{s DZ/// 0 00r_cJ|durddlm}|ddfS|dus|dS|dfS) zNormalize a ``progress`` parameter for :meth:`Table.add`. Returns ``(progress_obj, owns)`` where *owns* is True when we created a tqdm bar that the caller must close. Tr)tqdmz rows)unitFN)NF) tqdm.autorO)progressrOs r]_normalize_progressrSsX 4""""""t!!!4''5H,{ U?r_ceZdZdZeeddZeeddZeeddZeedd Z dd Z eedd Z edddZ dddZ eddZddZddZddded ddfd!dd"d#dd$d%dd dd& dd9Zdd;Zed%<fddAZeddCZed dDdddEddHZddIdJdIddIdKdLdMd d d d dNdNdIdddOddcZe dddrZddvZe dddZedIdddZddZedIdddZedddddZedddZ eddZ!eddZ"eddZ#eddZ$e dddddZ%e ddIdddZ&edZ'eddIdIdddZ(eddZ)eddZ*eddZ+eddZ,eddZ-eddZ.edZ/edddZ0eddZ1e2dd„Z3ddĄZ4dńZ5eddƄZ6edDŽZ7dS)rya& A Table is a collection of Records in a LanceDB Database. Examples -------- Create using [DBConnection.create_table][lancedb.DBConnection.create_table] (more examples in that method's documentation). >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2}]) >>> table.head() pyarrow.Table vector: fixed_size_list[2] child 0, item: float b: int64 ---- vector: [[[1.1,1.2]]] b: [[2]] Can append new data with [Table.add()][lancedb.table.Table.add]. >>> table.add([{"vector": [0.5, 1.3], "b": 4}]) AddResult(version=2) Can query the table with [Table.search][lancedb.table.Table.search]. >>> table.search([0.4, 0.4]).select(["b", "vector"]).to_pandas() b vector _distance 0 4 [0.5, 1.3] 0.82 1 2 [1.1, 1.2] 1.13 Search queries are much faster when an index is created. See [Table.create_index][lancedb.table.Table.create_index]. rRrQct)zThe name of this TableNotImplementedErrorselfs r]rz Table.name "!r_intct)zThe version of this TablerVrXs r]versionz Table.versionrZr_rct)lThe [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#) of this Table rVrXs r]rz Table.schema "!r_Tagsct)aWTag management for the table. Similar to Git, tags are a way to add metadata to a specific version of the table. .. warning:: Tagged versions are exempted from the :py:meth:`cleanup_old_versions()` process. To remove a version that has been tagged, you must first :py:meth:`~Tags.delete` the associated tag. Examples -------- .. code-block:: python table = db.open_table("my_table") table.tags.create("v2-prod-20250203", 10) tags = table.tags.list() rVrXs r]tagsz Table.tagss 6"!r_c,|dS)z The number of rows in this TableN) count_rowsrXs r]__len__z Table.__len__st$$$r_"Dict[str, EmbeddingFunctionConfig]cdS)z^ Get a mapping from vector column name to it's configured embedding function. NrVrXs r]embedding_functionszTable.embedding_functionsr_Nfilterrmct) Count the number of rows in the table. Parameters ---------- filter: str, optional A SQL where clause to filter the rows to count. rVrYrks r]rezTable.count_rowss "!r_rI blob_modeBlobMode'pandas.DataFrame'c @|jdi|S)a~Return the table as a pandas DataFrame. Parameters ---------- blob_mode: str, default "lazy" Controls how blob columns are returned for backends that support Lance blob-aware pandas conversion. **kwargs Forwarded to PyArrow / Lance pandas conversion. Returns ------- pd.DataFrame rV)r to_pandasrYrokwargss r]rszTable.to_pandass%)t}}(226222r_pa.Tablect)_Return the table as a pyarrow Table. Returns ------- pa.Table rVrXs r]rzTable.to_arrows "!r_lance.LanceDatasetc t)znReturn the table as a lance.LanceDataset. Returns ------- lance.LanceDataset rVrYrus r]to_lancezTable.to_lancer`r_'pl.DataFrame'c t)zjReturn the table as a polars.DataFrame. Returns ------- polars.DataFrame rVr{s r] to_polarszTable.to_polars!r`r_l2`TIVF_PQ2,) index_type wait_timeoutnum_bitsmax_iterations sample_ratemef_constructionrtraintarget_partition_sizevector_column_namereplacera acceleratorindex_cache_size Optional[int]rrrOptional[timedelta]rrrrrrrrc t)aCreate an index on the table. Parameters ---------- metric: str, default "l2" The distance metric to use when creating the index. Valid values are "l2", "cosine", "dot", or "hamming". l2 is euclidean distance. Hamming is available only for binary vectors. num_partitions: int, default 256 The number of IVF partitions to use when creating the index. Default is 256. num_sub_vectors: int, default 96 The number of PQ sub-vectors to use when creating the index. Default is 96. vector_column_name: str, default "vector" The vector column name to create the index. replace: bool, default True - If True, replace the existing index if it exists. - If False, raise an error if duplicate index exists. accelerator: str, default None If set, use the given accelerator to create the index. Only support "cuda" for now. index_cache_size : int, optional The size of the index cache in number of entries. Default value is 256. num_bits: int The number of bits to encode sub-vectors. Only used with the IVF_PQ index. Only 4 and 8 are supported. wait_timeout: timedelta, optional The timeout to wait if indexing is asynchronous. name: str, optional The name of the index. If not provided, a default name will be generated. train: bool, default True Whether to train the index with existing data. Vector indices always train with existing data. rV)rYmetricnum_partitionsnum_sub_vectorsrrrrrrrrrrrrrrs r] create_indexzTable.create_index* t"!r_rSct)a Drop an index from the table. Parameters ---------- name: str The name of the index to drop. Notes ----- This does not delete the index from disk, it just removes it from the table. To delete the index, run [optimize][lancedb.table.Table.optimize] after dropping the index. Use [list_indices][lancedb.table.Table.list_indices] to find the names of the indices. rVrYrs r] drop_indexzTable.drop_indexfs $"!r_seconds index_names Iterable[str]timeoutrct) Wait for indexing to complete for the given index names. This will poll the table until all the indices are fully indexed, or raise a timeout exception if the timeout is reached. Parameters ---------- index_names: str The name of the indices to poll timeout: timedelta Timeout to wait for asynchronous indexing. The default is 5 minutes. rVrYrrs r]wait_for_indexzTable.wait_for_indexzs "!r_TableStatisticsct)9 Retrieve table and fragment statistics. rVrXs r]statsz Table.statss "!r_BTREE)rrrrcolumnrct)a Create a scalar index on a column. Parameters ---------- column : str The column to be indexed. Must be a boolean, integer, float, or string column. replace : bool, default True Replace the existing index if it exists. index_type: Literal["BTREE", "BITMAP", "LABEL_LIST"], default "BTREE" The type of index to create. wait_timeout: timedelta, optional The timeout to wait if indexing is asynchronous. name: str, optional The name of the index. If not provided, a default name will be generated. Examples -------- Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column ``my_col`` has a scalar index: >>> import lancedb # doctest: +SKIP >>> db = lancedb.connect("/data/lance") # doctest: +SKIP >>> img_table = db.open_table("images") # doctest: +SKIP >>> my_df = img_table.search().where("my_col = 7", # doctest: +SKIP ... prefilter=True).to_pandas() Scalar indices can also speed up scans containing a vector search and a prefilter: >>> import lancedb # doctest: +SKIP >>> db = lancedb.connect("/data/lance") # doctest: +SKIP >>> img_table = db.open_table("images") # doctest: +SKIP >>> img_table.search([1, 2, 3, 4], vector_column_name="vector") # doctest: +SKIP ... .where("my_col != 7", prefilter=True) ... .to_pandas() Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. ``my_col BETWEEN 0 AND 100``), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. ``my_col < 0 AND other_col> 100``) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column ``not_indexed`` does not have a scalar index then the filter ``my_col = 0 OR not_indexed = 1`` will not be able to use any scalar index on ``my_col``. rV)rYrrrrrs r]create_scalar_indexzTable.create_scalar_indexs ~"!r_F@simpleEnglish()ordering_field_namesrwriter_heap_size use_tantivytokenizer_name with_positionr`rlmax_token_length lower_casestemremove_stop_words ascii_foldingngram_min_lengthngram_max_length prefix_onlyrr field_namesUnion[str, List[str]]rOptional[Union[str, List[str]]]rrrrr`rrlrrrrrrrrct)uCreate a full-text search index on the table. Warning - this API is highly experimental and is highly likely to change in the future. Parameters ---------- field_names: str or list of str The name of the field to index. Native FTS indexes can only be created on a single field at a time. To search over multiple text fields, create a separate FTS index for each field. replace: bool, default False If True, replace the existing index if it exists. Note that this is not yet an atomic operation; the index will be temporarily unavailable while the new index is being created. writer_heap_size: int, default 1GB Deprecated legacy Tantivy parameter. Any value other than the default raises an error. ordering_field_names: Deprecated legacy Tantivy parameter. Setting this raises an error. tokenizer_name: str, default "default" A compatibility alias for native tokenizer configs. Can be "raw", "default" or the 2 letter language code followed by "_stem". So for english it would be "en_stem". For new native FTS indexes, use ``base_tokenizer`` directly; ``tokenizer_name`` is a legacy compatibility alias and does not expose model-backed tokenizer names such as ``jieba/default`` or ``lindera/ipadic``. use_tantivy: bool, default False Deprecated legacy Tantivy parameter. Setting this to True raises an error. with_position: bool, default False If False, do not store the positions of the terms in the text. This can reduce the size of the index and improve indexing speed. But it will raise an exception for phrase queries. base_tokenizer : str, default "simple" The base tokenizer to use for tokenization. Options are: - "simple": Splits text by whitespace and punctuation. - "whitespace": Split text by whitespace, but not punctuation. - "raw": No tokenization. The entire text is treated as a single token. - "ngram": N-Gram tokenizer. - "jieba/*": Jieba tokenizer loaded from Lance's language model home. - "lindera/*": Lindera tokenizer loaded from Lance's language model home. language : str, default "English" The language to use for stemming and stop-word removal. This is not the primary way to enable CJK tokenization. max_token_length : int, default 40 The maximum token length to index. Tokens longer than this length will be ignored. lower_case : bool, default True Whether to convert the token to lower case. This makes queries case-insensitive. stem : bool, default True Whether to stem the token. Stemming reduces words to their root form. For example, in English "running" and "runs" would both be reduced to "run". remove_stop_words : bool, default True Whether to remove stop words. Stop words are common words that are often removed from text before indexing. For example, in English "the" and "and". ascii_folding : bool, default True Whether to fold ASCII characters. This converts accented characters to their ASCII equivalent. For example, "café" would be converted to "cafe". ngram_min_length: int, default 3 The minimum length of an n-gram. ngram_max_length: int, default 3 The maximum length of an n-gram. prefix_only: bool, default False Whether to only index the prefix of the token for ngram tokenizer. wait_timeout: timedelta, optional The timeout to wait if indexing is asynchronous. name: str, optional The name of the index. If not provided, a default name will be generated. Notes ----- Model-backed tokenizers such as ``jieba/default`` and ``lindera/ipadic`` require tokenizer models in Lance's language model home. Set ``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data directory under ``lance/language_models``. rV)rYrrrrrrrr`rlrrrrrrrrrrs r]create_fts_indexzTable.create_fts_indexs L"!r_rrrrr"moderrrrrrR$Optional[Union[bool, Callable, Any]]rct)ajAdd more data to the [Table](Table). Parameters ---------- data: DATA The data to insert into the table. Acceptable types are: - list-of-dict - pandas.DataFrame - pyarrow.Table or pyarrow.RecordBatch mode: str The mode to use when writing the data. Valid values are "append" and "overwrite". on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill". fill_value: float, default 0. The value to use when filling vectors. Only used if on_bad_vectors="fill". progress: bool, callable, or tqdm-like, optional Progress reporting during the add operation. Can be: - ``True`` to automatically create and display a tqdm progress bar (requires ``tqdm`` to be installed):: table.add(data, progress=True) - A **callable** that receives a dict with keys ``output_rows``, ``output_bytes``, ``total_rows``, ``elapsed_seconds``, ``active_tasks``, ``total_tasks``, and ``done``:: def on_progress(p): print(f"{p['output_rows']}/{p['total_rows']} rows, " f"{p['active_tasks']}/{p['total_tasks']} workers") table.add(data, progress=on_progress) - A **tqdm-compatible** progress bar whose ``total`` and ``update()`` will be called automatically. The postfix shows write throughput (MB/s) and active worker count:: with tqdm() as pbar: table.add(data, progress=pbar) Returns ------- AddResult An object containing the new version number of the table after adding data. rV)rYrrrrrRs r]addz Table.add;rr_onUnion[str, Iterable[str]]r2ct|tr|gntt|}t ||Sa Returns a [`LanceMergeInsertBuilder`][lancedb.merge.LanceMergeInsertBuilder] that can be used to create a "merge insert" operation This operation can add rows, update rows, and remove rows all in a single transaction. It is a very generic tool that can be used to create behaviors like "insert if not exists", "update or insert (i.e. upsert)", or even replace a portion of existing data with new data (e.g. replace all data where month="january") The merge insert operation works by combining new data from a **source table** with existing data in a **target table** by using a join. There are three categories of records. "Matched" records are records that exist in both the source table and the target table. "Not matched" records exist only in the source table (e.g. these are new data) "Not matched by source" records exist only in the target table (this is old data) The builder returned by this method can be used to customize what should happen for each category of data. Please note that the data may appear to be reordered as part of this operation. This is because updated rows will be deleted from the dataset and then reinserted at the end with the new values. Parameters ---------- on: Union[str, Iterable[str]] A column (or columns) to join on. This is how records from the source table and target table are matched. Typically this is some kind of key or id column. Examples -------- >>> import lancedb >>> data = pa.table({"a": [2, 1, 3], "b": ["a", "b", "c"]}) >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", data) >>> new_data = pa.table({"a": [2, 3, 4], "b": ["x", "y", "z"]}) >>> # Perform a "upsert" operation >>> res = table.merge_insert("a") \ ... .when_matched_update_all() \ ... .when_not_matched_insert_all() \ ... .execute(new_data) >>> res MergeResult(version=2, num_updated_rows=2, num_inserted_rows=1, num_deleted_rows=0, num_attempts=1) >>> # The order of new rows is non-deterministic since we use >>> # a hash-join as part of this operation and so we sort here >>> table.to_arrow().sort_by("a").to_pandas() a b 0 1 b 1 2 x 2 3 y 3 4 z rZrQrrr2rYrs r] merge_insertzTable.merge_insertw;t C(( ct)a~ Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search][search] and [full-text search][experimental-full-text-search]. All query options are defined in [LanceQueryBuilder][lancedb.query.LanceQueryBuilder]. Examples -------- >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> data = [ ... {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]}, ... {"original_width": 2000, "caption": "foo", "vector": [0.5, 3.4, 1.3]}, ... {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]} ... ] >>> table = db.create_table("my_table", data) >>> query = [0.4, 1.4, 2.4] >>> (table.search(query) ... .where("original_width > 1000", prefilter=True) ... .select(["caption", "original_width", "vector"]) ... .limit(2) ... .to_pandas()) caption original_width vector _distance 0 foo 2000 [0.5, 3.4, 1.3] 5.220000 1 test 3000 [0.3, 6.2, 2.6] 23.089996 Parameters ---------- query: list/np.ndarray/str/PIL.Image.Image, default None The targetted vector to search for. - *default None*. Acceptable types are: list, np.ndarray, PIL.Image.Image - If None then the select/where/limit clauses are applied to filter the table vector_column_name: str, optional The name of the vector column to search. The vector column needs to be a pyarrow fixed size list type - If not specified then the vector column is inferred from the table schema - If the table has multiple vector columns then the *vector_column_name* needs to be specified. Otherwise, an error is raised. query_type: str *default "auto"*. Acceptable types are: "vector", "fts", "hybrid", or "auto" - If "auto" then the query type is inferred from the query; - If `query` is a list/np.ndarray then the query type is "vector"; - If `query` is a PIL.Image.Image then either do vector search, or raise an error if no corresponding embedding function is found. - If `query` is a string, then the query type is "vector" if the table has embedding functions else the query type is "fts" Returns ------- LanceQueryBuilder A query builder object representing the query. Once executed, the query returns - selected columns - the vector - and also the "_distance" column which is the distance between the query vector and the returned vector. rVrYrFrrrrs r]searchz Table.searchs l"!r_) with_row_idoffsets list[int]rr@cdS)a Take a list of offsets from the table. Offsets are 0-indexed and relative to the current version of the table. Offsets are not stable. A row with an offset of N may have a different offset in a different version of the table (e.g. if an earlier row is deleted). Offsets are mostly useful for sampling as the set of all valid offsets is easily known in advance to be [0, len(table)). No guarantees are made regarding the order in which results are returned. If you desire an output order that matches the order of the given offsets, you will need to add the row offset column to the output and align it yourself. Parameters ---------- offsets: list[int] The offsets to take. Returns ------- pa.RecordBatch A record batch containing the rows at the given offsets. NrV)rYrrs r] take_offsetszTable.take_offsets rjr_pa.RecordBatchct}tt|}t|fd}dg|z}t|D] }||||<|jj}|d|| d | dg}|S)a Take a list of offsets from the table and return as a record batch. This method uses the `take_offsets` method to take the rows. However, it aligns the offsets to the passed in offsets. This means the return type is a record batch (and so users should take care not to pass in too many offsets) Note: this method is primarily intended to fulfill the Dataset contract for pytorch. Parameters ---------- offsets: list[int] The offsets to take. Returns ------- pa.RecordBatch A record batch containing the rows at the given offsets. c|SrprV)idxrs r]z$Table.__getitems__..Is gclr_)r'r _rowoffset)rrrangesortedrrrrselectrsort_bytakecombine_chunks drop_columns) rYr num_offsetsindices permutationpermutation_invirtbls ` r] __getitems__zTable.__getitems__*s:'ll u[))**W*B*B*B*BCCC # +{## 0 0A./OKN + ++#|$$$   g & & VG__ XZZ W\ " " T/ " " ^   \<. ) )  r_row_idscdS)ap Take a list of row ids from the table. Row ids are not stable and are relative to the current version of the table. They can change due to compaction and updates. No guarantees are made regarding the order in which results are returned. If you desire an output order that matches the order of the given ids, you will need to add the row id column to the output and align it yourself. Unlike offsets, row ids are not 0-indexed and no assumptions should be made about the possible range of row ids. In order to use this method you must first obtain the row ids by scanning or searching the table. Even so, row ids are more stable than offsets and can be useful in some situations. There is an ongoing effort to make row ids stable which is tracked at https://github.com/lancedb/lancedb/issues/1120 Parameters ---------- row_ids: list[int] The row ids to take. Returns ------- AsyncTakeQuery A query object that can be executed to get the rows. NrV)rYrrs r] take_row_idszTable.take_row_ids\rjr_ batch_sizerrArrcdSrprV)rYrFrrs r]_execute_queryzTable._execute_querys  #sr_verboseOptional[bool]cdSrprVrYrFrs r] _explain_planzTable._explain_plansSVSVr_cdSrprVrYrFs r] _analyze_planzTable._analyze_plans25#r_cdSrprVrs r]_output_schemazTable._output_schemas9<r_mergenew_datarcdSrprVrYrrrrs r] _do_mergezTable._do_merges cr_whererct)ajDelete rows from the table. This can be used to delete a single row, many rows, all rows, or sometimes no rows (if your predicate matches nothing). Parameters ---------- where: str The SQL where clause to use when deleting rows. - For example, 'x = 2' or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error. Returns ------- DeleteResult An object containing the new version number of the table after deletion. Examples -------- >>> import lancedb >>> data = [ ... {"x": 1, "vector": [1.0, 2]}, ... {"x": 2, "vector": [3.0, 4]}, ... {"x": 3, "vector": [5.0, 6]} ... ] >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", data) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 2 [3.0, 4.0] 2 3 [5.0, 6.0] >>> table.delete("x = 2") DeleteResult(num_deleted_rows=1, version=2) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 3 [5.0, 6.0] If you have a list of values to delete, you can combine them into a stringified list and use the `IN` operator: >>> to_remove = [1, 5] >>> to_remove = ", ".join([str(v) for v in to_remove]) >>> to_remove '1, 5' >>> table.delete(f"x IN ({to_remove})") DeleteResult(num_deleted_rows=1, version=3) >>> table.to_pandas() x vector 0 3 [5.0, 6.0] rVrYr s r]deletez Table.deletes p"!r_ values_sqlrsrrOptional[Dict[str, str]]rct)a This can be used to update zero to all rows depending on how many rows match the where clause. If no where clause is provided, then all rows will be updated. Either `values` or `values_sql` must be provided. You cannot provide both. Parameters ---------- where: str, optional The SQL where clause to use when updating rows. For example, 'x = 2' or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error. values: dict, optional The values to update. The keys are the column names and the values are the values to set. values_sql: dict, optional The values to update, expressed as SQL expression strings. These can reference existing columns. For example, {"x": "x + 1"} will increment the x column by 1. Returns ------- UpdateResult - rows_updated: The number of rows that were updated - version: The new version number of the table after the update Examples -------- >>> import lancedb >>> import pandas as pd >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]}) >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", data) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 2 [3.0, 4.0] 2 3 [5.0, 6.0] >>> table.update(where="x = 2", values={"vector": [10.0, 10]}) UpdateResult(rows_updated=1, version=2) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 3 [5.0, 6.0] 2 2 [10.0, 10.0] >>> table.update(values_sql={"x": "x + 1"}) UpdateResult(rows_updated=3, version=3) >>> table.to_pandas() x vector 0 2 [1.0, 2.0] 1 4 [5.0, 6.0] 2 3 [10.0, 10.0] rVrYr rsrs r]updatez Table.updates |"!r_delete_unverified older_thanr'CleanupStats'cdS)aT Clean up old versions of the table, freeing disk space. Parameters ---------- older_than: timedelta, default None The minimum age of the version to delete. If None, then this defaults to two weeks. delete_unverified: bool, default False Because they may be part of an in-progress transaction, files newer than 7 days old are not deleted by default. If you are sure that there are no in-progress transactions, then you can set this to True to delete all files older than `older_than`. Returns ------- CleanupStats The stats of the cleanup operation, including how many bytes were freed. See Also -------- [Table.optimize][lancedb.table.Table.optimize]: A more comprehensive optimization operation that includes cleanup as well as other operations. Notes ----- This function is not available in LanceDb Cloud (since LanceDB Cloud manages cleanup for you automatically) NrVrYrrs r]cleanup_old_versionszTable.cleanup_old_versionsrjr_cdS)a Run the compaction process on the table. This can be run after making several small appends to optimize the table for faster reads. Arguments are passed onto Lance's [compact_files][lance.dataset.DatasetOptimizer.compact_files]. For most cases, the default should be fine. See Also -------- [Table.optimize][lancedb.table.Table.optimize]: A more comprehensive optimization operation that includes cleanup as well as other operations. Notes ----- This function is not available in LanceDB Cloud (since LanceDB Cloud manages compaction for you automatically) NrV)rYrYrus r] compact_fileszTable.compact_files:rjr_cleanup_older_thanrretrainrr cdS) Optimize the on-disk data and indices for better performance. Modeled after ``VACUUM`` in PostgreSQL. Optimization covers three operations: * Compaction: Merges small files into larger ones * Prune: Removes old versions of the dataset * Index: Optimizes the indices, adding new data to existing indices Parameters ---------- cleanup_older_than: timedelta, optional default 7 days All files belonging to versions older than this will be removed. Set to 0 days to remove all versions except the latest. The latest version is never removed. delete_unverified: bool, default False Files leftover from a failed transaction may appear to be part of an in-progress operation (e.g. appending new data) and these files will not be deleted unless they are at least 7 days old. If delete_unverified is True then these files will be deleted regardless of their age. .. warning:: This should only be set to True if you can guarantee that no other process is currently working on this dataset. Otherwise the dataset could be put into a corrupted state. retrain: bool, default False This parameter is no longer used and is deprecated. The frequency an application should call optimize is based on the frequency of data modifications. If data is frequently added, deleted, or updated then optimize should be run frequently. A good rule of thumb is to run optimize if you have added or modified 100,000 or more records or run more than 20 data modification operations. NrVrYrrr s r]optimizezTable.optimizePrjr_Iterable[IndexConfig]cdS)z} List all indices that have been created with [Table.create_index][lancedb.table.Table.create_index] NrVrXs r] list_indiceszTable.list_indicesrjr_ index_nameOptional[IndexStatistics]cdS)G Retrieve statistics about an index Parameters ---------- index_name: str The name of the index to retrieve statistics for Returns ------- IndexStatistics or None The statistics about the index. Returns None if the index does not exist. NrVrYr(s r] index_statszTable.index_statsrjr_ transforms6Dict[str, str] | pa.Field | List[pa.Field] | pa.SchemacdS)a Add new columns with defined values. Parameters ---------- transforms: Dict[str, str], pa.Field, List[pa.Field], pa.Schema A map of column name to a SQL expression to use to calculate the value of the new column. These expressions will be evaluated for each row in the table, and can reference existing columns. Alternatively, a pyarrow Field or Schema can be provided to add new columns with the specified data types. The new columns will be initialized with null values. Returns ------- AddColumnsResult version: the new version number of the table after adding columns. NrVrYr.s r] add_columnszTable.add_columnsrjr_ alterationsIterable[Dict[str, str]]cdS)a Alter column names and nullability. Parameters ---------- alterations : Iterable[Dict[str, Any]] A sequence of dictionaries, each with the following keys: - "path": str The column path to alter. For a top-level column, this is the name. For a nested column, this is the dot-separated path, e.g. "a.b.c". - "rename": str, optional The new name of the column. If not specified, the column name is not changed. - "data_type": pyarrow.DataType, optional The new data type of the column. Existing values will be casted to this type. If not specified, the column data type is not changed. - "nullable": bool, optional Whether the column should be nullable. If not specified, the column nullability is not changed. Only non-nullable columns can be changed to nullable. Currently, you cannot change a nullable column to non-nullable. Returns ------- AlterColumnsResult version: the new version number of the table after the alteration. NrVrYr3s r] alter_columnszTable.alter_columnsrjr_rrcdS)a- Drop columns from the table. Parameters ---------- columns : Iterable[str] The names of the columns to drop. Returns ------- DropColumnsResult version: the new version number of the table dropping the columns. NrVrYrs r]rzTable.drop_columnsrjr_r]Union[int, str]cdS)h Checks out a specific version of the Table Any read operation on the table will now access the data at the checked out version. As a consequence, calling this method will disable any read consistency interval that was previously set. This is a read-only operation that turns the table into a sort of "view" or "detached head". Other table instances will not be affected. To make the change permanent you can use the `[Self::restore]` method. Any operation that modifies the table will fail while the table is in a checked out state. Parameters ---------- version: int | str, The version to check out. A version number (`int`) or a tag (`str`) can be provided. To return the table to a normal state use `[Self::checkout_latest]` NrVrYr]s r]checkoutzTable.checkoutrjr_cdSz Ensures the table is pointing at the latest version This can be used to manually update a table when the read_consistency_interval is None It can also be used to undo a `[Self::checkout]` operation NrVrXs r]checkout_latestzTable.checkout_latestrjr_Optional[Union[int, str]]cdS)aRestore a version of the table. This is an in-place operation. This creates a new version where the data is equivalent to the specified previous version. Data is not copied (as of python-v0.2.1). Parameters ---------- version : int or str, default None The version number or version tag to restore. If unspecified then restores the currently checked out version. If the currently checked out version is the latest version then this is a no-op. NrVr=s r]restorez Table.restorerjr_List[Dict[str, Any]]cdS)List all versions of the tableNrVrXs r] list_versionszTable.list_versionsrjr_c@t|jj|jSrp)rG_connrJrrXs r] _dataset_urizTable._dataset_uris$*.$)444r_"Tuple[str, pa_fs.FileSystem, bool]cddlm}t||st|jdkrdSt |jdd}t |\}}||jtj j k}|||fS)Nr) RemoteTablefile)rWNF_indicesfts) remote.tablerNrZrDrKrFrC get_file_inforpa_fsFileTypeNotFound)rYrNpathfs index_existss r]_get_fts_index_pathzTable._get_fts_index_paths------ dK ( ( %N4;L,M,MQW,W,W$$):u==t$$D''--2en6MM b,''r_c`|\}}}|rtd|ddS)Nz%Legacy Tantivy FTS index detected at zo. Tantivy-based FTS has been removed. Delete the legacy index and recreate it with table.create_fts_index(...).)rZr)rYrW_existss r]_ensure_no_legacy_fts_indexz!Table._ensure_no_legacy_fts_index sT2244a  ////   r_cdS Check if the table is using the new v2 manifest paths. Returns ------- bool True if the table is using the new v2 manifest paths, False otherwise. NrVrXs r]uses_v2_manifest_pathszTable.uses_v2_manifest_paths*rjr_cdS)ap Migrate the manifest paths to the new format. This will update the manifest to use the new v2 format for paths. This function is idempotent, and can be run multiple times without changing the state of the object store. !!! danger This should not be run while other concurrent operations are happening. And it should also run until completion before resuming other operations. You can use [Table.uses_v2_manifest_paths][lancedb.table.Table.uses_v2_manifest_paths] to check if the table is already using the new path style. NrVrXs r]migrate_v2_manifest_pathszTable.migrate_v2_manifest_paths5rjr_rRrQrRr[rRrrRrarRrgrprkrmrRr[rI)rorprRrqrRrvrRry)rRr})rrQrrarrmrrrrrrrr[rr[rr[rr[rr[rrmrrarrrrQrRrSrrrrrRrSrRr) rrQrrarrrrrrm)&rrrrrrarrrrarrmrrar`rrlrQrrrrarrarrarrarr[rr[rrarrrrmrrrN rr"rrrrrrrRrrRrrrrRr2NNrNN rFrrrmrrrrmrrrRr>)rrrrarRr@)rrrRr)rrrrarRr@rFrArrrrrRrFrFrArrrRrQrFrArRrQrFrArRr rr2rr"rrrrrRrr rQrRrNNr rmrsrrrrRrrrrrarRrrrrrar rarRr%r(rQrRr))r.r/)r3r4rrrRrr]r:r]rBrRrE)rRrLrRra)8rr __qualname____doc__propertyrrr]rrcrfrirersrr|rr$rrrrrrrrrrrrrrrrrr rrrrr$r'r-r2r7rr>rArDrHr rKrZr^rbrdrVr_r]ryrys##J"""^X""""^X""""^X""""^X"6%%%%   ^X  " " " "^ "33333""""^""""""""""4%)*.:"'/,0 ""/3':":":":":":"x""""*@IyQT?U?U?U"""""""""^"  &-,0">">">">">"^>"HAE*<!(,#,4!*,"&" ! !!,0"-f"f"f"f"f"f"P!+29= 9"9"9"9"^9"v<1<1<1<1| ,0 &-17;U"U"U"U"^U"n9>     ^ 80000d9>      ^  D %)'+ #####^#VVVV^V555^5<<<^<^7"7"7"^7"r $!%=" 04 ="="="="="^="~+/# #( # # # # # ^# J  ^ *37"' , , , , , ^, \   ^     ^     ^ ,   ^ :    ^     ^ 0  ^      ^  ---^-555_5((((   ^   ^   r_ryceZdZdZdddddddddddZeddZeddZeddZe dd Z e dddddddd!dd"Z e dd#Z dd%Zedd'Zdd)Zedd+Zdd/Zdd1Zedd3Zd d6Zd7Zd!d"d9Zd!d#d;Zdd<Zdd=Zd$d%d@Zd&d'dEZd%dFZd!d(dHZdIddedJdddKdLdMdNdOdPf ddJddQd)daZ d*dcZ!d*ddZ"d!d+dfZ#e$dPgfd,dlZ%d-dnZ&eddoZ'd.dpZ(d.dqZ)dJdrddsd/dvZ*ddwdxdwddwdydzd{dJdJdJdJd|d|dwdd}d0dZ+e,d1dZ- d2d3dZ. d4d5dZ/e d6dZ0e1 d7d8dZ2e1 d9d:dZ2e1 d;ddZ2 d=d?dÄZ2e d@ddddddddŜdAdфZ3dBdԄZ4 d4dd՜dCdڄZ5dddۜdDdZ6dEdFdZ7dGdZ8dHdZ9dIdZ:edJdZ;ed d!dwddKdZ?eddLdZ@ddwdwddMdZAdNdZBdOdZCdPdZDdQdZEdRdZFdSd ZGdTd ZHdUd ZIdVdZJdZKdWdZLdS(X LanceTablea A table in a LanceDB database. This can be opened in two modes: standard and time-travel. Standard mode is the default. In this mode, the table is mutable and tracks the latest version of the table. The level of read consistency is controlled by the `read_consistency_interval` parameter on the connection. Time-travel mode is activated by specifying a version number. In this mode, the table is immutable and fixed to a specific version. This is useful for querying historical versions of the table. N)namespace_pathstorage_optionsrlocationnamespace_clientmanaged_versioningpushdown_operations_async connection'LanceDBConnection'rrQrOptional[List[str]]rrrrrrmr Optional[Any]rrr Optional[set]r AsyncTablec |g}||_||_||_||_| p t |_| | |_dStj|j ||||||||_dS)N)rrrrrr) rJ_namespace_path _location_namespace_clientset_pushdown_operations_tablerrun open_table) rYrrrrrrrrrrs r]__init__zLanceTable.__init__Ys  !N -!!1$7$@355!   DKKK( ++#1$3%5%%5'9,  DKKKr_rRc|jjSrp)rrrXs r]rzLanceTable.name}s {r_ List[str]c|jS)z'Return the namespace path of the table.)rrXs r] namespacezLanceTable.namespaces ##r_cd|jr#d|j|jgzS|jS)z9Return the full identifier of the table (namespace$name).$)rrrrrXs r]idz LanceTable.ids6   @88D0DI;>?? ?yr_r LanceDBTablecddlm}t|}|j|}|||j|S)Nrrw)r)dbrxr from_innerdatabaser)clsrrx async_tblconns r]rzLanceTable.from_inners^))))))sOO + +CLLNN;;s  N    r_rrrrrrrc |g}||||||||||  } | jn8#t$r+} dt| vrtd|d| d} ~ wwxYw| S)Nrz Not found:zTable z does not exist)r]rrQFileNotFoundError) rrrrrrrrrrres r]openzLanceTable.opens  !Nc  )+--1 3      KKK   s1vv%%'(F(F(F(FGGGG   s" A&AAc\|j|jSt|jj|jSrp)rrLrJrJrrXs r] _dataset_pathzLanceTable._dataset_paths* > %> !4:>49555r_ryc   ddl}n#t$rtdwxYw|j6|j|jgz}|jd|j|jj|j|d|S|j|j f|j|jjd|S)z+Return the LanceDataset backing this table.rN^The lance library is required to use this function. Please install with `pip install pylance`.)r]rrtable_idr]rrV) r ImportErrorrrrrr]rJrr)rYrurrs r]r|zLanceTable.to_lances  LLLL   =    ! -+tyk9H 5= $ :!%!7!    u}   L J6      s!rcXtj|jS)zwReturn the schema of the table. Returns ------- pa.Schema A PyArrow schema object.)rrrrrXs r]rzLanceTable.schemas"x **,,---r_rEcXtj|jS)rG)rrrrHrXs r]rHzLanceTable.list_versionss x 1133444r_r[cXtj|jS)z$Get the current version of the table)rrrr]rXs r]r]zLanceTable.versions"x ++--...r_rrr@cPt|j|Srp)r@rrrYrs r]rzLanceTable.take_offsets $T[%=%=g%F%FGGGr_rcPt|j|Srp)r@rrrYrs r]rzLanceTable.take_row_idsrr_rac*t|jS)a0Tag management for the table. Similar to Git, tags are a way to add metadata to a specific version of the table. .. warning:: Tagged versions are exempted from the :py:meth:`cleanup_old_versions()` process. To remove a version that has been tagged, you must first :py:meth:`~Tags.delete` the associated tag. Returns ------- Tags The tag manager for managing tags for the table. Examples -------- >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", ... [{"vector": [1.1, 0.9], "type": "vector"}]) >>> table.tags.create("v1", table.version) >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}]) AddResult(version=2) >>> tags = table.tags.list() >>> print(tags["v1"]["version"]) 1 >>> table.checkout("v1") >>> table.to_pandas() vector type 0 [1.1, 0.9] vector )rarrXs r]rczLanceTable.tagssJDK   r_r]r:c^tj|j|dS)aCheckout a version of the table. This is an in-place operation. This allows viewing previous versions of the table. If you wish to keep writing to the dataset starting from an old version, then use the `restore` function. Calling this method will set the table into time-travel mode. If you wish to return to standard mode, call `checkout_latest`. Parameters ---------- version: int | str, The version to check out. A version number (`int`) or a tag (`str`) can be provided. Examples -------- >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", ... [{"vector": [1.1, 0.9], "type": "vector"}]) >>> table.version 1 >>> table.to_pandas() vector type 0 [1.1, 0.9] vector >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}]) AddResult(version=2) >>> table.version 2 >>> table.checkout(1) >>> table.to_pandas() vector type 0 [1.1, 0.9] vector N)rrrr>r=s r]r>zLanceTable.checkout#s+H %%g../////r_c\tj|jdS)zCheckout the latest version of the table. This is an in-place operation. The table will be set back into standard mode, and will track the latest version of the table. N)rrrrArXs r]rAzLanceTable.checkout_latestIs( ,,../////r_rBc|,tj|j|tj|jdS)aRestore a version of the table. This is an in-place operation. This creates a new version where the data is equivalent to the specified previous version. Data is not copied (as of python-v0.2.1). Parameters ---------- version : int or str, default None The version number or version tag to restore. If unspecified then restores the currently checked out version. If the currently checked out version is the latest version then this is a no-op. Examples -------- >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", [ ... {"vector": [1.1, 0.9], "type": "vector"}]) >>> table.version 1 >>> table.to_pandas() vector type 0 [1.1, 0.9] vector >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}]) AddResult(version=2) >>> table.version 2 >>> table.tags.create("v2", 2) >>> table.restore(1) >>> table.to_pandas() vector type 0 [1.1, 0.9] vector >>> len(table.list_versions()) 3 >>> table.restore("v2") >>> table.to_pandas() vector type 0 [1.1, 0.9] vector 1 [0.5, 0.2] vector >>> len(table.list_versions()) 4 N)rrrr>rDr=s r]rDzLanceTable.restoreQsOX   HT[))'22 3 3 3 $$&&'''''r_rkcZtj|j|Srp)rrrrerns r]rezLanceTable.count_rowss"x ..v66777r_c|jjd|j}|jj"|d|jjz }|d|jdz }|S)Nz(name=z , read_consistency_interval={!r}z, _conn=))rrrrJread_consistency_intervalr)rYvals r]__repr__zLanceTable.__repr__sk(== == : / ; 5<< 4 C )$*)))) r_c*|Srp)rrXs r]__str__zLanceTable.__str__s}}r_rvcZtj|j|S)z%Return the first n rows of the table.)rrrheadrYns r]rzLanceTable.heads"x ((++,,,r_rIrorp'pd.DataFrame'c |dkr>|jt|jdkr|jdi|S|jdd|i|S)a1Return the table as a pandas DataFrame. Parameters ---------- blob_mode: str, default "lazy" Controls how Lance blob columns are returned. **kwargs Forwarded to Lance pandas conversion. Returns ------- pd.DataFrame rINmemoryrorV)rrDrrrsr|rts r]rszLanceTable.to_pandassu     " .d011X==,4==??,66v66 6(t}}(GG9GGGGr_cXtj|jS)zVReturn the table as a pyarrow Table. Returns ------- pa.Table)rrrrrXs r]rzLanceTable.to_arrows" x ,,..///r_'pl.LazyFrame'cRddlm}||}tj|d|S)aFReturn the table as a polars LazyFrame. Parameters ---------- batch_size: int, optional Passed to polars. This is the maximum row count for scanned pyarrow record batches Note ---- 1. This requires polars to be installed separately 2. Currently we've disabled push-down of the filters from polars because polars pushdown into pyarrow uses pyarrow compute expressions rather than SQl strings (which LanceDB supports) Returns ------- pl.LazyFrame r)PyarrowDatasetAdapterF)allow_pyarrow_filterr)lancedb.integrations.pyarrowrplscan_pyarrow_dataset)rYrrrs r]rzLanceTable.to_polarssH( GFFFFF''--& %J    r_rTrrrrrr)rrrrrrrrarrr`Literal['IVF_FLAT', 'IVF_SQ', 'IVF_PQ', 'IVF_RQ', 'IVF_HNSW_SQ', 'IVF_HNSW_PQ', 'IVF_HNSW_FLAT']rrrrrrc|I||| |||||||| | | |dS| dkrt||| | |}n| dkrt ||| | |}n| dkrt ||||| | |}n| dkrt |||| | | }nk| d krt||||| | | | | }nL| d krt||| | | | | }n/| dkrt||| | | | | }ntd| tj |j |||||S)zCreate an index on the table.N) rrrrrrrrrrrrIVF_FLAT) distance_typerrrrIVF_SQr)rrrrrrrIVF_RQ)rrrrrr IVF_HNSW_PQ) rrrrrrrrr IVF_HNSW_SQ)rrrrrrr IVF_HNSW_FLATUnknown index type )rconfigrr)r|rrAr(r*r)r,r.r/r0rrrr)rYrrrrrrrrrrrrrrrrrs r]rzLanceTable.create_indexs?:  " MMOO ( ()%- /'!1! /&; )     " " " F : % %$--'&; FF8 # #$--'&; FF8 # #$- /!-'&;FF8 # #$-!-'&; FF= ( ($- /!-' /&;   FF= ( ($--' /&;FF? * *$--' /&;FF?:??@@ @x K $ $" %     r_rScZtj|j|S)z Drops an index from the table Parameters ---------- name: str The name of the index to drop )rrrrrs r]rzLanceTable.drop_indexL s$x ..t44555r_cZtj|j|S) Prewarm an index in the table. This is a hint to the database that the index will be accessed in the future and should be loaded into memory if possible. This can reduce cold-start latency for subsequent queries. This call initiates prewarming and returns once the request is accepted. It is idempotent and safe to call from multiple clients concurrently. It is generally wasteful to call this if the index does not fit into the available cache. Not all index types support prewarming; unsupported indices will silently ignore the request. Parameters ---------- name: str The name of the index to prewarm )rrr prewarm_indexrs r]rzLanceTable.prewarm_indexW s$(x 11$77888r_rcZtj|j|S)n Prewarm data for the table. This is a hint to the database that the given columns will be accessed in the future and the database should prefetch the data if possible. Currently only supported on remote tables. This call initiates prewarming and returns once the request is accepted. It is idempotent and safe to call from multiple clients concurrently. This operation has a large upfront cost but can speed up future queries that need to fetch the given columns. Large columns such as embeddings or binary data may not be practical to prewarm. This feature is intended for workloads that issue many queries against the same columns. Parameters ---------- columns: list of str, optional The columns to prewarm. If None, all columns are prewarmed. )rrr prewarm_datar9s r]rzLanceTable.prewarm_datam s$*x 0099:::r_rrrrrc\tj|j||Srp)rrrrrs r]rzLanceTable.wait_for_index s&x 22;HHIIIr_rcXtj|jSrp)rrrrrXs r]rzLanceTable.stats s x ))++,,,r_cXtj|jSrp)rrrrJrXs r]rJzLanceTable.uri sx ))***r_cXtj|jS)Get the initial storage options that were passed in when opening this table. For dynamically refreshed options (e.g., credential vending), use :meth:`latest_storage_options`. Warning: This is an internal API and the return value is subject to change. Returns ------- Optional[Dict[str, str]] The storage options, or None if no storage options were configured. )rrrinitial_storage_optionsrXs r]rz"LanceTable.initial_storage_options s"x ;;==>>>r_cXtj|jS) Get the latest storage options, refreshing from provider if configured. This method is useful for credential vending scenarios where storage options may be refreshed dynamically. If no dynamic provider is configured, this returns the initial static options. Warning: This is an internal API and the return value is subject to change. Returns ------- Optional[Dict[str, str]] The storage options, or None if no storage options were configured. )rrrlatest_storage_optionsrXs r]rz!LanceTable.latest_storage_options s"x ::<<===r_r)rrrrrc|dkrt}n<|dkrt}n'|dkrt}ntd|t j|j||||S)NrBITMAP LABEL_LISTrrrr)r'r+r-rrrrr)rYrrrrrs r]rzLanceTable.create_scalar_index s  WWFF 8 # #XXFF < ' '[[FF?:??@@ @x K $ $VWVRV $ W W   r_Frrrrr)rrrrrrr`rlrrrrrrrrrrrrrrrrrr`rrlrrrrrrrrc ||rtd|td|dkrtdt|tstd||| || | | | ||||d }n||}t d i|} t j|j ||||dS#ttf$r#}t||j |j |d}~wwxYw) Nz^Tantivy-based FTS has been removed. Remove use_tantivy and recreate the index with native FTS.zXordering_field_names was only supported by the removed Tantivy-based FTS implementation.rzTwriter_heap_size was only supported by the removed Tantivy-based FTS implementation.zNative FTS indexes can only be created on a single field at a time. To search over multiple text fields, create a separate FTS index for each field.) r`rlrrrrrrrrrrr`rlrV)r^rrZrQinfer_tokenizer_configsr1rrrr RuntimeErrorrvr`rl)rYrrrrrrrr`rlrrrrrrrrrtokenizer_configsrrs r]rzLanceTable.create_fts_index s. ((***  M  +4  1 1 14 +s++ 5   !"0$!.$4(%6!.$4$4* ! !  !% < <^ L L       H ((#! )     L)    %%4     G  s#0CD &DD rc x|dkr ddddddddddd S|d kr d dddddddddd S|d kr d dddddddddd St|d krtd ||dd }|dddkrtd ||tvrtd|dt|ddddddddd S)NdefaultrrrTFr) r`rlrrrrrrrrraw whitespacezInvalid tokenizer name _stemzInvalid language code )rrrH)rlangs r]rz"LanceTable.infer_tokenizer_configs sg Y & &"*%$&"%*!&$%$%$   u $ $"'%$(#%*!&$%$%$   | + +".%$(#%*!&$%$%$    ~  ! # #G~GGHH Hbqb! "## ' ) )G~GGHH H | # #>> import lancedb >>> import pyarrow as pa >>> df = pa.table({'x': [1, 2, 3], 'y': ['a', 'b', 'c']}) >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("dataset", df) >>> table.to_pandas() x y 0 1 a 1 2 b 2 3 c >>> new_df = pa.table({'x': [1, 2, 3], 'z': ['d', 'e', 'f']}) >>> table.merge(new_df, 'x') >>> table.to_pandas() x y z 0 1 a d 1 2 b e 2 3 c f )rrrN)rZrr|rrrA)rYrrrrs r]rzLanceTable.merge sh k: . . %..00KK(K  8F     r_rgcbtj|jjS) Get the embedding functions for the table Returns ------- funcs: Dict[str, EmbeddingFunctionConfig] A mapping of the vector column to the embedding function or empty dict if not configured. )r&r;r<rrrXs r]rizLanceTable.embedding_functions s-)577GG K    r_vectorrF3Optional[Union[VEC, str, 'PIL.Image.Image', Tuple]]rLiteral['vector']rrr?cdSrprVrs r]rzLanceTable.search s #&#r_rQLiteral['fts']r<cdSrprVrs r]rzLanceTable.search s  #sr_hybridrLiteral['hybrid']r=cdSrprVrs r]rzLanceTable.search s #&#r_rrr;cdSrprVrs r]rzLanceTable.search s "%r_r>ct|trd}t|j|||}t j||||||pgS)a Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search][search] and [full-text search][search]. Examples -------- >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> data = [ ... {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]}, ... {"original_width": 2000, "caption": "foo", "vector": [0.5, 3.4, 1.3]}, ... {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]} ... ] >>> table = db.create_table("my_table", data) >>> query = [0.4, 1.4, 2.4] >>> (table.search(query) ... .where("original_width > 1000", prefilter=True) ... .select(["caption", "original_width", "vector"]) ... .limit(2) ... .to_pandas()) caption original_width vector _distance 0 foo 2000 [0.5, 3.4, 1.3] 5.220000 1 test 3000 [0.3, 6.2, 2.6] 23.089996 Parameters ---------- query: list/np.ndarray/str/PIL.Image.Image, default None The targetted vector to search for. - *default None*. Acceptable types are: list, np.ndarray, PIL.Image.Image - If None then the select/[where][sql]/limit clauses are applied to filter the table vector_column_name: str, optional The name of the vector column to search. The vector column needs to be a pyarrow fixed size list type *default "vector"* - If not specified then the vector column is inferred from the table schema - If the table has multiple vector columns then the *vector_column_name* needs to be specified. Otherwise, an error is raised. query_type: str, default "auto" "vector", "fts", or "auto" If "auto" then the query type is inferred from the query; If `query` is a list/np.ndarray then the query type is "vector"; If `query` is a PIL.Image.Image then either do vector search or raise an error if no corresponding embedding function is found. If the `query` is a string, then the query type is "vector" if the table has embedding functions, else the query type is "fts" fts_columns: str or list of str, default None The column(s) to search in for full-text search. If None then the search is performed on all indexed columns. For now, only one column can be searched at a time. Returns ------- LanceQueryBuilder A query builder object representing the query. Once executed, the query returns selected columns, the vector, and also the "_distance" column which is the distance between the query vector and the returned vector. rQrrrFr)rrr)rZr:rErr>creaters r]rzLanceTable.search srX e] + + J5;!1    !'   1 3#)r     r_r()rrdata_storage_versionenable_v2_manifest_pathsrrrrrxOptional[DATA]rrexist_okri'Optional[List[EmbeddingFunctionConfig]]Optional[Dict[str, str | bool]]r)r*c | g} ||}||_| |_||_||_|p t |_| $tjddt| i} | | d<| $tjddt| i} | | d<tj |jj |||||||| | | || |_ |S) a. Create a new table. Examples -------- >>> import lancedb >>> data = [ ... {"x": 1, "vector": [1.0, 2]}, ... {"x": 2, "vector": [3.0, 4]}, ... {"x": 3, "vector": [5.0, 6]} ... ] >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", data) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 2 [3.0, 4.0] 2 3 [5.0, 6.0] Parameters ---------- db: LanceDB The LanceDB instance to create the table in. name: str The name of the table to create. data: list-of-dict, dict, pd.DataFrame, default None The data to insert into the table. At least one of `data` or `schema` must be provided. schema: pa.Schema or LanceModel, optional The schema of the table. If not provided, the schema is inferred from the data. At least one of `data` or `schema` must be provided. mode: str, default "create" The mode to use when writing the data. Valid values are "create", "overwrite", and "append". exist_ok: bool, default False If the table already exists then raise an error if False, otherwise just open the table, it will not add the provided data but will validate against any schema that's specified. on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill", "null". fill_value: float, default 0. The value to use when filling vectors. Only used if on_bad_vectors="fill". embedding_functions: list of EmbeddingFunctionModel, default None The embedding functions to use when creating the table. data_storage_version: optional, str, default "stable" Deprecated. Set `storage_options` when connecting to the database and set `new_table_data_storage_version` in the options. enable_v2_manifest_paths: optional, bool, default False Deprecated. Set `storage_options` when connecting to the database and set `new_table_enable_v2_manifest_paths` in the options. NzEsetting data_storage_version directly on create_table is deprecated. zUse database_options instead.new_table_data_storage_versionz=setting enable_v2_manifest_paths directly on create_table is z)deprecated. Use database_options instead."new_table_enable_v2_manifest_paths) rrr,rrrirrrr)__new__rJrrrrrwarningswarnDeprecationWarningrr create_tabler)rrrrrrr,rrrirrr)r*rrrrYs r]r(zLanceTable.create_ s&T  !N{{3 -!!1$7$@355!  + MW/"    &"$@TO< = # / MO;"    &"$( @ Ah J  ) )!-%$7- /!!1 *      r_r rcZtj|j|Srp)rrrrr s r]rzLanceTable.delete s"x **511222r_rrsrrrc`tj|j|||S)a7 This can be used to update zero to all rows depending on how many rows match the where clause. Parameters ---------- where: str, optional The SQL where clause to use when updating rows. For example, 'x = 2' or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error. values: dict, optional The values to update. The keys are the column names and the values are the values to set. values_sql: dict, optional The values to update, expressed as SQL expression strings. These can reference existing columns. For example, {"x": "x + 1"} will increment the x column by 1. Returns ------- UpdateResult - rows_updated: The number of rows that were updated - version: The new version number of the table after the update Examples -------- >>> import lancedb >>> import pandas as pd >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]}) >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", data) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 2 [3.0, 4.0] 2 3 [5.0, 6.0] >>> table.update(where="x = 2", values={"vector": [10.0, 10]}) UpdateResult(rows_updated=1, version=2) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 3 [5.0, 6.0] 2 2 [10.0, 10.0] r  updates_sql)rrrrrs r]rzLanceTable.update s,fx **6J*WWXXXr_rrArrrc6d|jvr/|j(ddlm}|j|jgz}||j||St j|j |||fd}tj j |S)N QueryTabler)_execute_server_side_queryrc3K tjV)#t$rYdSwxYwrp)rr __anext__StopAsyncIteration) async_itersr] iter_syncz,LanceTable._execute_query..iter_sync% sZ ;(:#7#7#9#9:::::;%    s */ ==)rrlancedb.namespacer=rrrrrrrrrr)rYrFrrr=rrBrAs @r]rzLanceTable._execute_query s D5 5 5&2 D D D D D D+tyk9H--d.DhPUVV VX K & &uW & U U       #001BIIKKPPPr_rc\tj|j||Srp)rrrrrs r]rzLanceTable._explain_plan. s$x 11%AABBBr_cZtj|j|Srp)rrrrrs r]rzLanceTable._analyze_plan1 s"x 11%88999r_cZtj|j|Srp)rrrrrs r]rzLanceTable._output_schema4 s"x 22599:::r_rr2rrc`tj|j||||Srp)rrrr r s r]r zLanceTable._do_merge7 s1x K ! !%>: N N   r_c|jjSrp)r_innerrXs r]rIzLanceTable._innerB s {!!r_z0.21.0zUse `Table.optimize` instead.) deprecated_incurrent_versiondetailsrrrrcT|||S)a Clean up old versions of the table, freeing disk space. Parameters ---------- older_than: timedelta, default None The minimum age of the version to delete. If None, then this defaults to two weeks. delete_unverified: bool, default False Because they may be part of an in-progress transaction, files newer than 7 days old are not deleted by default. If you are sure that there are no in-progress transactions, then you can set this to True to delete all files older than `older_than`. Returns ------- CleanupStats The stats of the cleanup operation, including how many bytes were freed. r)r|rrs r]rzLanceTable.cleanup_old_versionsF s/>}}33 *;4   r_r|cv|jj|i|}||S)aa Run the compaction process on the table. This can be run after making several small appends to optimize the table for faster reads. Arguments are passed onto `lance.dataset.DatasetOptimizer.compact_files`. (see Lance documentation for more details) For most cases, the default should be fine. )r|r$rrA)rYrYrurs r]rzLanceTable.compact_filesi s= 7 (6GGG  r_rrr cdtj|j|||dS)r"rN)rrrr$r#s r]r$zLanceTable.optimize} sFZ  K #5"3 !       r_r%cXtj|jS)Q List all indices that have been created with Self::create_index )rrrr'rXs r]r'zLanceTable.list_indices s"x 0022333r_r(r)cZtj|j|S)r+)rrrr-r,s r]r-zLanceTable.index_stats s$x // ;;<<>wGGHHHr_spec'LsmWriteSpec'cZtj|j|S)znInstall an LsmWriteSpec. See [`AsyncTable.set_lsm_write_spec`][lancedb.AsyncTable.set_lsm_write_spec].)rrrset_lsm_write_specrYrYs r]r\zLanceTable.set_lsm_write_spec s$x 66t<<===r_cXtj|jS)zrRemove the LsmWriteSpec. See [`AsyncTable.unset_lsm_write_spec`][lancedb.AsyncTable.unset_lsm_write_spec].)rrrunset_lsm_write_specrXs r]r_zLanceTable.unset_lsm_write_spec s"x 88::;;;r_cXtj|jS)ra)rrrrbrXs r]rbz!LanceTable.uses_v2_manifest_paths s"x ::<<===r_c\tj|jdS)az Migrate the manifest paths to the new format. This will update the manifest to use the new v2 format for paths. This function is idempotent, and can be run multiple times without changing the state of the object store. !!! danger This should not be run while other concurrent operations are happening. And it should also run until completion before resuming other operations. You can use [LanceTable.uses_v2_manifest_paths][lancedb.table.LanceTable.uses_v2_manifest_paths] to check if the table is already using the new path style. N)rrrrdrXs r]rdz$LanceTable.migrate_v2_manifest_paths s($ 668899999r_ field_name new_metadataDict[str, str]c`tj|j||dSz Replace the metadata of a field in the schema Parameters ---------- field_name: str The name of the field to replace the metadata for new_metadata: dict The new metadata to set N)rrrreplace_field_metadatarYrbrcs r]rgz!LanceTable.replace_field_metadata s, 33J MMNNNNNr_)rrrrQrrrrrrrrmrrrrrrrrre)rRr)rr)rrrrrrrrmrrrrrrrmrgrrf)rrrRr@)rrrRr@rhrrprrjrrlrkrorprRr)rRr)rrrrQrrarrmrrrr[rrrr[rr[rr[rr[rrmrrarrrnrrrRrSrorprRr)rrQrrarrrrm)$rrrrrrarrrrarrmrrar`rrlrQrrrrarrarrarrarr[rr[rrarrm)rrQrRrrqrrr})rrrrQrrmrrri)NNrNN) rFrrrmrrrrmrrrRr?)NNrQNN) rFrrrmrr rrmrrrRr<)NNr"NN) rFrrrmrr#rrmrrrRr=rt) rFrSrrmrrrrmrrrRr;ru)NNr(FrrN) rrxrrQrr+rrrrr,rarrrrrir-rrrr.r)rmr*rrrmrrrrr|r~rvrwrxryrzr{)rRrr)rRr|rrr)r.rSrRr~)r3r4rRrrrrrRrSrYrZrRrSrRrSr)rbrQrcrd)Mrrrrrrrrr classmethodrrr rr|rrHr]rrrcr>rArDrerrrrsrrr$rrrrrrrrJrrrr staticmethodrrrrirrr(rrrrrrr rI deprecation deprecatedrrrr$r'r-r2r7rrXr\r_rbrdrgrVr_r]rrJs!   &/348*."&*.-1-1!""""""H   X $$$X$X     [   /348*."&*.-1-1#####[#J666_6    6...X.5555///X/HHHHHHHH$!$!$!X$!L$0$0$0$0L000.(.(.(.(.(`88888-----HHHHH,0000     : $"4%)*.  "-} 0#/35} } } } } } ~ 6 6 6 69999,;;;;;0@IyQT?U?U?UJJJJJ ----+++X+ ? ? ? ?>>>>(&-"       0AE*<!(,#,4!*,"&" ! !!"+SSSSSSj< < < \< B!+29= /!/!/!/!/!j#'9= >>>>>@    _  FJ,0(0-17; &&&&X&FJ,0%*-17; ####X# ,0(0-17; & & & &X &,0 &-17; %%%%X% ,0 &-17;\ \ \ \ \ | $&*#+2GKw/3;?.237"&*.-1%wwwww[wr3333 $!%3Y 04 3Y3Y3Y3Y3Y3Yr%)'+ QQQQQQ:CCCCC::::;;;;     """X"[#/+/ #(        <[#/     $37"' 3 3 3 3 3 3 j4444 ==== ==== AAAA ;;;;IIII >>>> <<<< > > > >:::( O O O O O Or_r(Literal['error', 'drop', 'fill', 'null']ctj|sStjfd}tj|S)Nc 3dKD](}g} D]_}|d} 2|0t||d}||t||d||d}`|D](}|dt||d|d<)|jdr|Vt j|g }|r.t j |dj V*dS) N expected_dimrexpected_value_type)rrrrwrxTrrr) _infer_vector_dimr_handle_bad_vector_columnrrrryrrrrrr) r pending_dimsr5dimrrr output_schemarrvector_columnss r]rz _handle_bad_vectors..gen s  EL!/   #N3 ,+E-2G,HIIC '' 6661'4V'<#1)!$(56K(L ".    084EmF3455M.1|""="FF  %%ug..33MBBMMOO  n00 O+(19  r_)_find_vector_columnsr_vector_output_schemarrr)rrrrrrr}r~s```` @@r]rr s*&-QQN  )&-HHM          D   , ,]CCEE B BBr_ reader_schema List[dict]ch|g}|D]}t|jo8tj|jjo|jtk}tj|jo8tj|jjo|jj dk}|s|r| |jddd|St|j }t|j|}ttj|} g}|D]f}|j|vr t|jr)tj|jjsK||j} |j| vp>|jtkp.|jdko#tj|j}tj| jo8tj| jjo| jj dk} |s| rX| |jtj|jr |jj nd|jjdh|S)N )rrwrx embedding)rrrr is_floating value_typerr$r r rrrrrr&r;r<keysr) rrrr~rnamed_vector_collikely_vector_colreader_column_namesactive_metadataembedding_function_columns reader_fieldtyped_fixed_vector_cols r]rrC s "  Eej))5H(()>??5J"44  ++EJ771H(()>??1Z)R/    #4 %% % (,/3m122%m&. sKKK&vf~vKKKr_r) rrr_vector_output_typerrrr rr)rr~columns_by_namerrr output_types r]rr sLKNKKKO FYY $$UZ00 >*KK-eV< >8 x &;<=== n%1 H ' ' 3 3 2 J M.$A A Ax -... :r_rrrwrrxOptional[pa.DataType]c|j|}||}t|js|S|tj|jrk|jj|kr[t j| t j |jj }| |||}|tj |jj s)tj|jj rQt j| t j |}| |||}tj|jj rt!|}n%t jdgt#|z}||} nDtj|jr |jj} nt%|} | |St'jt'j|| } t'j|p%t'j| } | r)t'j|| } |dkrLt'j| rt3d|dt3d|d|dkr)t'j| t jd|}n|d kr0|t'j| }||}n]|d krE|t3d t'j| t j|g| z|j|}nt3d || |||S) a Ensure that the vector column exists and has type fixed_size_list(float) Parameters ---------- data: pa.Table The table to sanitize. vector_column_name: str The name of the vector column. on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill", "null". fill_value: float, default 0.0 The value to use when filling vectors. Only used if on_bad_vectors="fill". Nr+FrzVector column 'z' has variable length vectors. Set on_bad_vectors='drop' to remove them, set on_bad_vectors='fill' and fill_value= to replace them, or set on_bad_vectors='null' to replace them with null.z' has NaNs. Set on_bad_vectors='drop' to remove them, set on_bad_vectors='fill' and fill_value= to replace them, or set on_bad_vectors='null' to replace them with null.nulldropfillz;`fill_value` must not be None if `on_bad_vectors` is 'fill'z"Invalid value for on_bad_vectors: )r#r4rrrrr r r% to_pylistrrr3rrrhas_nan_valuesrryr- not_equallist_value_lengthrir0or_rif_elsescalarrkinvert) rrrrrwrxpositionvec_arrhas_nanr| has_wrong_dimhas_bad_vectorsis_bads r]rzrz s. &&'9::H%&G  & &   H ' ' 5 5 ! L "l 2 2(7,,..RXgl>U5V5VWWWx);WEE& GL344' 8 ' ' (? @ @'(7,,..RX>Q5R5RSSSx);WEE xGL3443 ))(E7S\\122  $ $W\ 2 2l$(( ;KL!5g!>!>DDMfWoo++--N 1F1F1L1L1N1NO%T// W $ $vm$$**,,  N&8NNN!N&8NNN v % %j $GG v % %;;ry0011D-.GG v % %! Qj :,,7<@@@GG R.RRSS S ??8%7 A AAr_arr$Union[pa.ListArray, pa.ChunkedArray]pa.BooleanArrayct|tjr$tjd|jD}n|}tj|jr9tj | tj }ntj |}tj |}tjtj||}tjt#t%|tj}tj||S)Nc6g|]}|SrV)flatten)rfchunks r]rz"has_nan_values..s "K"K"Ku5==??"K"K"Kr_r+)rZr ChunkedArray chunked_arraychunksrr is_float16rr-is_nanrr=list_parent_indicesuniquerkr%rruint32is_in)rrsvalues_has_nanvalues_indiceshas_nan_indicesrs r]rrs#r''!"K"K "K"K"KLL x6;''+6;;rz||#<#<==6**+C00Ni .. I IJJOhuSXXRY[[999G 8G_ - --r_ data_typectj|p=tj|ptj|Srp)rrrrr )rs r]rr%sJ ## 2 8 ! !) , , 2 8 & &y 1 1r_metadata_dictsci}|D]x}||D]^\}}t|tr|d}t|tr|d}|||<_y|S)Nzutf-8)rrZrQencode)rmergedrr'values r]rr-s F"     "..**  JC#s## *jj))%%% . W--F3KK  Mr_rbc:|}d|vpd|vS)z0Check if a field name indicates a vector column.rr)lower)rb name_lowers r]_name_suggests_vector_columnr;s(!!##J z ! >[J%>>r_&Tuple[pa.Schema, pa.RecordBatchReader]c|j}d}t|D]\}}tj|jp#tj|j}t|jr?|r<|t|\}}t| |}tj |jj r(tjtj|}n~tj|jj rR| |}t#|tjr|}|} t+j| d} | dkr'tjtj|}nt+j| } | d} | d} | | dks| -| dkr'tjtj|}n)tjtj|}nktj|j||j}|||}||fS)N only_valid)rrminmax)r )r enumeraterrrrrrrr_modal_list_sizerrrrr=rrZrrrr-countr0uint8min_maxrr r)rrpeekedrr is_list_typer| target_typers flattened valid_countr min_value max_valuers r]rrAsU]F Ff%%-.-.5x'' 33 rx7M7M J8 8  ( 3 3( . ( .~!,V!4!4"6==#3#344Cx##EJ$9::  hrz||S99 $$UZ%:;; q))fbo665#2244F"NN,,  hy|DDDJJLL !##"$(28::s";";KK j33G ' 4 4 6 6I ' 4 4 6 6I!-)a--!-)c//&(hrz||S&A&A &(hrxzz3&?&?  I ZZ9--F 6>r_r[ctjtj|ddSNrr)r-rrr0)rs r]rrys3 72',, - -a 0 6 6 8 8 @@r_ Union[pa.Array, pa.ChunkedArray]c4t|jsdStj|}tj|tj|d}t |dkrdStj|ddSr) rrr-rrkgreaterrrr0)rlengthss r]ryry~s  " "t"3''GiGQ!7!788G 7||qt 77  A  $ $ & &v ..r_c@|jt|jdSdS)z. Make sure the metadata is valid utf8 N)r_validate_metadatars r]rrs*"6?+++++#"r_c|D]v\}}t|tr8 |d1#t$rt d|dwxYwt|t rt|wdS)zo Make sure the metadata values are valid utf8 (can be nested) Raises ValueError if not valid utf8 utf8z Metadata key zG is not valid utf8. Consider base64 encode for generic binary metadata.N)rrZrJdecodeUnicodeDecodeErrorrrr)rrrs r]rrs   " "1 a   "     %    JAJJJ  4  " q ! ! ! " "s AA$ceZdZdZddZdZdZdZdd Zd Z ddZ ddZ ddZ e ddZddZddZdddZddd!Zdd#Zdd%Zddd*Zdd+Zddddd,d-dd7Zdd8Zdd9Zddd;Zed<=fddBZddDZddEZddGZddHZ dIddddJddVZ!ddYZ"e# dddcZ$e# dddfZ$e# dddiZ$e# dddlZ$e# dddoZ$ dddrZ$dduZ%ddxZ&dddydd}Z'ddZ(ddZ)ddZ*ddZ+ddZ, ddddddZ-ddZ.ddZ/ddZ0ddZ1dZ2ddZ3dZ4dddZ5ddZ6ddZ7e ddZ8ddddddZ9ddZ:ddZ;ddZdS)ra8 An AsyncTable is a collection of Records in a LanceDB Database. An AsyncTable can be obtained from the [AsyncConnection.create_table][lancedb.AsyncConnection.create_table] and [AsyncConnection.open_table][lancedb.AsyncConnection.open_table] methods. An AsyncTable object is expected to be long lived and reused for multiple operations. AsyncTable objects will cache a certain amount of index data in memory. This cache will be freed when the Table is garbage collected. To eagerly free the cache you can call the [close][lancedb.AsyncTable.close] method. Once the AsyncTable is closed, it cannot be used for any further operations. An AsyncTable can also be used as a context manager, and will automatically close when the context is exited. Closing a table is optional. If you do not close the table, it will be closed when the AsyncTable object is garbage collected. Examples -------- Create using [AsyncConnection.create_table][lancedb.AsyncConnection.create_table] (more examples in that method's documentation). >>> import lancedb >>> async def create_a_table(): ... db = await lancedb.connect_async("./.lancedb") ... data = [{"vector": [1.1, 1.2], "b": 2}] ... table = await db.create_table("my_table", data=data) ... print(await table.query().limit(5).to_arrow()) >>> import asyncio >>> asyncio.run(create_a_table()) pyarrow.Table vector: fixed_size_list[2] child 0, item: float b: int64 ---- vector: [[[1.1,1.2]]] b: [[2]] Can append new data with [AsyncTable.add()][lancedb.table.AsyncTable.add]. >>> async def add_to_table(): ... db = await lancedb.connect_async("./.lancedb") ... table = await db.open_table("my_table") ... await table.add([{"vector": [0.5, 1.3], "b": 4}]) >>> asyncio.run(add_to_table()) Can query the table with [AsyncTable.vector_search][lancedb.table.AsyncTable.vector_search]. >>> async def search_table_for_vector(): ... db = await lancedb.connect_async("./.lancedb") ... table = await db.open_table("my_table") ... results = ( ... await table.vector_search([0.4, 0.4]).select(["b", "vector"]).to_pandas() ... ) ... print(results) >>> asyncio.run(search_table_for_vector()) b vector _distance 0 4 [0.5, 1.3] 0.82 1 2 [1.1, 1.2] 1.13 Search queries are much faster when an index is created. See [AsyncTable.create_index][lancedb.table.AsyncTable.create_index]. rrc||_dS)aCreate a new AsyncTable object. You should not create AsyncTable objects directly. Use [AsyncConnection.create_table][lancedb.AsyncConnection.create_table] and [AsyncConnection.open_table][lancedb.AsyncConnection.open_table] to obtain Table objects.N)rIrYrs r]rzAsyncTable.__init__s r_c4|jSrp)rIrrXs r]rzAsyncTable.__repr__s{##%%%r_c|SrprVrXs r] __enter__zAsyncTable.__enter__s r_c.|dSrp)r)rYr\s r]__exit__zAsyncTable.__exit__s r_rRrac4|jS)z!Return True if the table is open.)rIis_openrXs r]rzAsyncTable.is_opens{""$$$r_c4|jS)zClose the table and free any resources associated with it. It is safe to call this method multiple times. Any attempt to use the table after it has been closed will raise an error.)rIrrXs r]rzAsyncTable.closes {  """r_rrrScKt|tr|g}nt|}|j|d{VdS)aSet the unenforced primary key for this table to the given ordered list of columns. "Unenforced" means LanceDB does not check uniqueness on writes; the columns are recorded in the schema as the primary key so that features such as `merge_insert` can use them. Calling this again replaces any previously-set primary key. Parameters ---------- columns : str or Iterable[str] Either a single column name (single-column key) or an ordered iterable of column names (composite key). Each column dtype must be one of: int32, int64, utf8, large_utf8, binary, large_binary, fixed_size_binary. N)rZrQrrIrXr9s r]rXz%AsyncTable.set_unenforced_primary_keys\& gs # # $iGG7mmGk44W===========r_rYrZcJK|j|d{VdS)u(Install an LsmWriteSpec on this table. The spec selects Lance's MemWAL LSM-style write path for future `merge_insert` calls. ``LsmWriteSpec`` chooses one of three sharding strategies: - ``LsmWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by the single-column unenforced primary key. - ``LsmWriteSpec.identity(column)`` — shard by the raw value of a scalar column. - ``LsmWriteSpec.unsharded()`` — route every write to a single shard. All variants require the table to have an unenforced primary key set via [`set_unenforced_primary_key`]; bucket sharding additionally requires it to be the single column being bucketed. Parameters ---------- spec : LsmWriteSpec The sharding spec to install. Examples -------- >>> from lancedb._lancedb import LsmWriteSpec >>> # table.set_unenforced_primary_key("id") >>> # table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 16)) N)rIr\r]s r]r\zAsyncTable.set_lsm_write_specs68k,,T22222222222r_cHK|jd{VdS)zRemove the LsmWriteSpec from this table. Reverts to the standard `merge_insert` write path. Errors if no spec is currently set. N)rIr_rXs r]r_zAsyncTable.unset_lsm_write_spec<s4 k..00000000000r_rQc4|jS)zThe name of the table.)rIrrXs r]rzAsyncTable.nameDs{!!!r_rcDK|jd{VS)r_N)rIrrXs r]rzAsyncTable.schemaIs. ['')))))))))r_rgcK|d{V}tj|jS)rN)rr&r;r<r)rYrs r]rizAsyncTable.embedding_functionsPsF{{}}$$$$$$(577GGXXXr_Nrkrmr[cFK|j|d{VS)rmN)rIrerns r]rezAsyncTable.count_rows]s0[++F333333333r_rrvcK||d{VS)z Return the first `n` rows of the table. Parameters ---------- n: int, default 5 The number of rows to return. N)rFlimitrrs r]rzAsyncTable.headhsBZZ\\''**33555555555r_r7cNt|jS)a] Returns an [AsyncQuery][lancedb.query.AsyncQuery] that can be used to search the table. Use methods on the returned query to control query behavior. The query can be executed with methods like [to_arrow][lancedb.query.AsyncQuery.to_arrow], [to_pandas][lancedb.query.AsyncQuery.to_pandas] and more. )r7rIrFrXs r]rFzAsyncTable.queryss $+++--...r_rycK ddl}n#t$rtdwxYw|j|d{Vf|d{V|d{Vd|S)Nrrr)rrrrJr]r)rYrurs r] _to_lancezAsyncTable._to_lance~s  LLLL   =   u}((**       ,,..(((((("&"="="?"???????      s #rIrorprcK|dkr%|d{Vjdi|S|d{Vjdd|i|S)a;Return the table as a pandas DataFrame. Parameters ---------- blob_mode: str, default "lazy" Controls how Lance blob columns are returned. **kwargs Forwarded to PyArrow / Lance pandas conversion. Returns ------- pd.DataFrame rINrorV)rrsrrts r]rszAsyncTable.to_pandass   4$--//))))))4>>v>> >1dnn&&&&&&&&1PPIPPPPr_c^K|d{VS)rxN)rFrrXs r]rzAsyncTable.to_arrows4ZZ\\**,,,,,,,,,r_T)rrrrrrrrr_Optional[Union[IvfFlat, IvfPq, IvfRq, HnswPq, HnswSq, HnswFlat, BTree, Bitmap, LabelList, FTS]]rrrrcK|~t|ttttt t tttttf s,tdtt|z |j||||||d{VdS#t"t$f$r8}t|trt'||j|j|d}~wwxYw)aCreate an index to speed up queries Indices can be created on vector columns or scalar columns. Indices on vector columns will speed up vector searches. Indices on scalar columns will speed up filtering (in both vector and non-vector searches) Parameters ---------- column: str The column to index. replace: bool, default True Whether to replace the existing index If this is false, and another index already exists on the same columns and the same name, then an error will be returned. This is true even if that index is out of date. The default is True config: default None For advanced configuration you can specify the type of index you would like to create. You can also specify index-specific parameters when creating an index object. wait_timeout: timedelta, optional The timeout to wait if indexing is asynchronous. name: str, optional The name of the index. If not provided, a default name will be generated. train: bool, default True Whether to train the index with existing data. Vector indices always train with existing data. Nzmconfig must be an instance of IvfSq, IvfPq, IvfRq, HnswPq, HnswSq, BTree, Bitmap, LabelList, or FTS, but got )r4rrrrr)rZr(r*r)r,r.r/r0r'r+r-r1rrQrrIrrrrvr`rl)rYrrrrrrrs r]rzAsyncTable.create_indexs=l      BDGV DUDUV +**) +         L)   &#&& )#)#8#_ G s&B,,C5=3C00C5cJK|j|d{VdS)a Drop an index from the table. Parameters ---------- name: str The name of the index to drop. Notes ----- This does not delete the index from disk, it just removes it from the table. To delete the index, run [optimize][lancedb.table.AsyncTable.optimize] after dropping the index. Use [list_indices][lancedb.table.AsyncTable.list_indices] to find the names of the indices. N)rIrrs r]rzAsyncTable.drop_indexs6$k$$T***********r_cJK|j|d{VdS)rN)rIrrs r]rzAsyncTable.prewarm_indexs6(k''-----------r_rcJK|j|d{VdS)rN)rIrr9s r]rzAsyncTable.prewarm_data0s6*k&&w///////////r_rrrrrrcLK|j||d{VdS)rN)rIrrs r]rzAsyncTable.wait_for_indexGs8k((g>>>>>>>>>>>r_rcDK|jd{VS)rN)rIrrXs r]rzAsyncTable.statsXs.[&&(((((((((r_cDK|jd{VS)a8 Get the table URI (storage location). For remote tables, this fetches the location from the server via describe. For local tables, this returns the dataset URI. Returns ------- str The full storage location of the table (e.g., S3/GCS path). N)rIrJrXs r]rJzAsyncTable.uri^s,[__&&&&&&&&&r_rcDK|jd{VS)rN)rIrrXs r]rz"AsyncTable.initial_storage_optionsls.[88:::::::::r_cDK|jd{VS)rN)rIrrXs r]rz!AsyncTable.latest_storage_options{s.[77999999999r_rrrr"r(Optional[Literal['append', 'overwrite']]rOptional[OnBadVectorsType]rOptional[float]rRrrcK|d{V}|d}|d}|dkrt|d||\}}n0|dks|j#d|jvrt|||j||d}t t |}t |\}} |j||pd | d{V |r| SS#t$rF} d t| vrt| d t| vrt| d} ~ wwxYw#|r| wwxYw) aeAdd more data to the [Table](Table). Parameters ---------- data: DATA The data to insert into the table. Acceptable types are: - list-of-dict - pandas.DataFrame - pyarrow.Table or pyarrow.RecordBatch mode: str The mode to use when writing the data. Valid values are "append" and "overwrite". on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill", "null". fill_value: float, default 0. The value to use when filling vectors. Only used if on_bad_vectors="fill". progress: callable or tqdm-like, optional A callback or tqdm-compatible progress bar. See :meth:`Table.add` for details. Nrr overwrite)rrsembedding_functionsTrrrrr)rRz Cast errorzVector column contains NaN) rrrrrrrSrIrrrrQr) rYrrrrrRrr\rrs r]rzAsyncTable.addsD{{}}$$$$$$  !$N  J ;  ,d>jGD!!w & & O ',Bfo,U,U!-% $ D &'''D!!,X66$ !t/?x(SSSSSSSS S !     !   s1vv%% mm#-Q77 mm#   !     !s%"$C D/)AD**D//D22E rr2ct|tr|gntt|}t ||Srrrs r]rzAsyncTable.merge_insertrr_.rFrrLiteral['auto']rrr8Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]c KdSrprVrs r]rzAsyncTable.searchsDG3r_r#r6c KdSrprVrs r]rzAsyncTable.search 3r_.Optional[Union[VEC, 'PIL.Image.Image', Tuple]]r9c KdSrprVrs r]rzAsyncTable.search(rr_r r5c KdSrprVrs r]rzAsyncTable.search2s r_rrc KdSrprVrs r]rzAsyncTable.search<s 3r_rrc Kd}dfd }d }|d kr||r|} d }nkt|trd}nRt|tr tj||d |d{V\} \} 3| |d{V} t fd| Drd}nd }nd}n#t$r1} dt| vrdt| vrd}n| Yd} ~ nd} ~ wwxYwd }n|d kr7||r|} nq||||d{V\} | |d{V} nH|dkrB||rtd||||d{V\} | |d{V} |d kr@ | } |r| |} | S|dkr) ||S|dkrU | } |r| |} | ||Std|d)aCreate a search query to find the nearest neighbors of the given query vector. We currently support [vector search][search] and [full-text search][experimental-full-text-search]. All query options are defined in [AsyncQuery][lancedb.query.AsyncQuery]. Parameters ---------- query: list/np.ndarray/str/PIL.Image.Image, default None The targetted vector to search for. - *default None*. Acceptable types are: list, np.ndarray, PIL.Image.Image - If None then the select/where/limit clauses are applied to filter the table vector_column_name: str, optional The name of the vector column to search. The vector column needs to be a pyarrow fixed size list type - If not specified then the vector column is inferred from the table schema - If the table has multiple vector columns then the *vector_column_name* needs to be specified. Otherwise, an error is raised. query_type: str *default "auto"*. Acceptable types are: "vector", "fts", "hybrid", or "auto" - If "auto" then the query type is inferred from the query; - If `query` is a list/np.ndarray then the query type is "vector"; - If `query` is a PIL.Image.Image then either do vector search, or raise an error if no corresponding embedding function is found. - If `query` is a string, then the query type is "vector" if the table has embedding functions else the query type is "fts" Returns ------- LanceQueryBuilder A query builder object representing the query. cpt|ttjtjtjfSrp)rZrnpndarrayrArrayrrEs r] is_embeddingz'AsyncTable.search..is_embeddings!edBJ"/%RSS Sr_rrmrrrFrrR#Tuple[str, EmbeddingFunctionConfig]cKt|trd}d{V}t||||}t j|j}||}|ktd|d}t|dkr3t|dt| nt|d|||fS)NrQr'zColumn 'z'' has no registered embedding function.rz0Embedding functions are registered for columns: z6No embedding functions are registered for any columns.)rZr:rrEr&r;r<rrrrrBrr)rrrFrfuncsr7rrYs r]get_embedding_funcz-AsyncTable.search..get_embedding_funcs= %// #" ;;==((((((F!9%#5 """  .:<<LLE99/00D|"*1***u::>>0 --00 W %t+ +r_cK|@tj}|d|jj|d{VdSdS)Nr)asyncioget_running_looprun_in_executorr,#compute_query_embeddings_with_retry)rrFloops r]make_embeddingz)AsyncTable.search..make_embeddingsr$/11..!*N  tr_rrrQNc3ZK|]%}|jdjko |jdkV&dS)rr1N)rr2r)rfrembedding_confs r]rhz$AsyncTable.search..sV!"IaLN,HH6 !  5r_r"Columnz$has no registered embedding functionz#Hybrid search requires a text query)rzUnknown query type: '')rrmrrrFrrRr") rZr:rQr'gatherr'rirrF nearest_tornearest_to_text)rYrFrrrrr!r%r, vector_queryrrbuilderr.s` @r]rzAsyncTable.searchHsr T T T" ," ," ," ," ," ,H      |E""& &$ % E=11# &" E3''! &+&n))++**+=vuMM<+^&1-;^NE-R-R'R'R'R'R'R'R &-2 *2JJ)1JJ%* +"   3$$@CFFJJ&+ #  0& 8 # #|E"" K$ ;M;M& E<<6666662"N&4^NE%J%JJJJJJJ 8 # #|E"" K !FGGG;M;M& E<<6666662"N&4^NE%J%JJJJJJJ  ! !jjll--l;;G! =!..);<<N 5 ::<<//{/KK K 8 # #jjll--l;;G! =!..);<<**5+*FF FBZBBBCC Cs>C D'DD query_vectorUnion[VEC, Tuple]cP||S)a2 Search the table with a given query vector. This is a convenience method for preparing a vector query and is the same thing as calling `nearestTo` on the builder returned by `query`. Seer [nearest_to][lancedb.query.AsyncQuery.nearest_to] for more details. )rFr2)rYr6s r] vector_searchzAsyncTable.vector_searchs zz||&&|444r_rA@AsyncHybridQuery | AsyncFTSQuery | AsyncVectorQuery | AsyncQuerycB|}|j||j}|j||j}|jr||j}|j||j}|jr|}|jr|}|j r| |j }|j r[| |j  |j |j}|j||j}|j:|j3||j|j}nC|j||j}n!|j||j}|j||j}|jr||j}|jr||j}|jr|}|jr|}|jr*||jj|jj}|Srp)rFroffsetrrrkr  fast_searchrorder_byrr2distance_range lower_bound upper_boundrminimum_nprobesmaximum_nprobesnprobes refine_factorr5refbypass_vector_index postfilterfull_text_queryr3rYrF async_querys r]_sync_query_to_asynczAsyncTable._sync_query_to_async sjjll ; "%++EK88K < #%,,U\::K = <%,,U];;K < #%++EL99K   4%1133K   4%1133K > ?%..u~>>K < @%00>>MM!5#4K".)778KLL $0U5J5V)11)!/%"788 &2)99%:OPP &2)99%:OPP ".)778KLL " F)001DEE x 7)nnUX66 ( @)==??   3%0022K   %55%+U-B-JKr_rrrrcjK||}|||d{VS)N)max_batch_lengthr)rLr)rYrFrrrKs r]rzAsyncTable._execute_queryBs[//66  ++',         r_rcfK||}||d{VSrp)rL explain_plan)rYrFrrKs r]rzAsyncTable._explain_planSs?//66  --g666666666r_cdK||}|d{VSrp)rL analyze_planrJs r]rzAsyncTable._analyze_planXs=//66  --/////////r_cdK||}|d{VSrp)rLr}rJs r]rzAsyncTable._output_schema]s=//66  ..000000000r_rrrrrc K|d{V}|d}|d}t|||j||d}t|tjr7tj|j|}|j |t|j |j |j|j|j|j|j|jd{VS)NrrTr)rwhen_matched_update_all!when_matched_update_all_conditionwhen_not_matched_insert_all!when_not_matched_by_source_delete$when_not_matched_by_source_conditionr use_index)rrrrZrryrrrrIexecute_merge_insertr_on_when_matched_update_all"_when_matched_update_all_condition_when_not_matched_insert_all"_when_not_matched_by_source_delete%_when_not_matched_by_source_condition_timeout _use_index)rYrrrrrrs r]r zAsyncTable._do_mergeas{{}}$$$$$$  !$N  J  _)!     dBH % % U'44T[$//BSBSTTD[55  9(-(F272Z,1,N272Z5:5`*            r_r rcFK|j|d{VS)aDelete rows from the table. This can be used to delete a single row, many rows, all rows, or sometimes no rows (if your predicate matches nothing). Parameters ---------- where: str The SQL where clause to use when deleting rows. - For example, 'x = 2' or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error. Examples -------- >>> import lancedb >>> data = [ ... {"x": 1, "vector": [1.0, 2]}, ... {"x": 2, "vector": [3.0, 4]}, ... {"x": 3, "vector": [5.0, 6]} ... ] >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("my_table", data) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 2 [3.0, 4.0] 2 3 [5.0, 6.0] >>> table.delete("x = 2") DeleteResult(num_deleted_rows=1, version=2) >>> table.to_pandas() x vector 0 1 [1.0, 2.0] 1 3 [5.0, 6.0] If you have a list of values to delete, you can combine them into a stringified list and use the `IN` operator: >>> to_remove = [1, 5] >>> to_remove = ", ".join([str(v) for v in to_remove]) >>> to_remove '1, 5' >>> table.delete(f"x IN ({to_remove})") DeleteResult(num_deleted_rows=1, version=3) >>> table.to_pandas() x vector 0 3 [5.0, 6.0] N)rIrr s r]rzAsyncTable.deletes1d[''.........r_r9updatesOptional[Dict[str, Any]]r:rcK||td||td|d|D}|j||d{VS)a This can be used to update zero to all rows in the table. If a filter is provided with `where` then only rows matching the filter will be updated. Otherwise all rows will be updated. Parameters ---------- updates: dict, optional The updates to apply. The keys should be the name of the column to update. The values should be the new values to assign. This is required unless updates_sql is supplied. where: str, optional An SQL filter that controls which rows are updated. For example, 'x = 2' or 'x IN (1, 2, 3)'. Only rows that satisfy this filter will be udpated. updates_sql: dict, optional The updates to apply, expressed as SQL expression strings. The keys should be column names. The values should be SQL expressions. These can be SQL literals (e.g. "7" or "'foo'") or they can be expressions based on the previous value of the row (e.g. "x + 1" to increment the x column by 1) Returns ------- UpdateResult An object containing: - rows_updated: The number of rows that were updated - version: The new version number of the table after the update Examples -------- >>> import asyncio >>> import lancedb >>> import pandas as pd >>> async def demo_update(): ... data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]}) ... db = await lancedb.connect_async("./.lancedb") ... table = await db.create_table("my_table", data) ... # x is [1, 2], vector is [[1, 2], [3, 4]] ... await table.update({"vector": [10, 10]}, where="x = 2") ... # x is [1, 2], vector is [[1, 2], [10, 10]] ... await table.update(updates_sql={"x": "x + 1"}) ... # x is [2, 3], vector is [[1, 2], [10, 10]] >>> asyncio.run(demo_update()) Nz2Only one of updates or updates_sql can be providedz.Either updates or updates_sql must be providedc4i|]\}}|t|SrV)rGrs r]rz%AsyncTable.update..s$JJJ$!Q1l1ooJJJr_)rrrIr)rYrer r:s r]rzAsyncTable.updatesf  ;#:QRR R ?{2MNN N  JJ'--//JJJK['' U;;;;;;;;;r_r.6dict[str, str] | pa.field | List[pa.field] | pa.Schemar~cKt|tjr|g}t|tr-t d|Drtj|}t|tjr |j|d{VS|j t| d{VS)aX Add new columns with defined values. Parameters ---------- transforms: Dict[str, str] A map of column name to a SQL expression to use to calculate the value of the new column. These expressions will be evaluated for each row in the table, and can reference existing columns. Alternatively, you can pass a pyarrow field or schema to add new columns with NULLs. Returns ------- AddColumnsResult version: the new version number of the table after adding columns. cBh|]}t|tjSrV)rZrFieldrs r] z)AsyncTable.add_columns..s$ 9 9 9Z28 $ $ 9 9 9r_N) rZrrlrr.rSchemarIadd_columns_with_schemar2rr1s r]r2zAsyncTable.add_columnss* j"( + + &$J j$ ' ' /C 9 9j 9 9 9- -  /:..J j") , , K<rrQr)rYr]rs r]r>zAsyncTable.checkoutXs. +&&w// / / / / / / / / /   c!ff$$ MwMMM  s & A%AAcHK|jd{VdSr@)rIrArXs r]rAzAsyncTable.checkout_latestys4k))+++++++++++r_Optional[int | str]cJK|j|d{VdS)a Restore the table to the currently checked out version This operation will fail if checkout has not been called previously This operation will overwrite the latest version of the table with a previous version. Any changes made since the checked out version will no longer be visible. Once the operation concludes the table will no longer be in a checked out state and the read_consistency_interval, if any, will apply. N)rIrDr=s r]rDzAsyncTable.restores6k!!'***********r_rrr8cPt|j|S)a Take a list of offsets from the table. Offsets are 0-indexed and relative to the current version of the table. Offsets are not stable. A row with an offset of N may have a different offset in a different version of the table (e.g. if an earlier row is deleted). Offsets are mostly useful for sampling as the set of all valid offsets is easily known in advance to be [0, len(table)). Parameters ---------- offsets: list[int] The offsets to take. Returns ------- pa.RecordBatch A record batch containing the rows at the given offsets. )r8rIrrs r]rzAsyncTable.take_offsetss"*dk66w??@@@r_rcPt|j|S)ay Take a list of row ids from the table. Row ids are not stable and are relative to the current version of the table. They can change due to compaction and updates. Unlike offsets, row ids are not 0-indexed and no assumptions should be made about the possible range of row ids. In order to use this method you must first obtain the row ids by scanning or searching the table. Even so, row ids are more stable than offsets and can be useful in some situations. There is an ongoing effort to make row ids stable which is tracked at https://github.com/lancedb/lancedb/issues/1120 Parameters ---------- row_ids: list[int] The row ids to take. Returns ------- AsyncTakeQuery A query object that can be executed to get the rows. )r8rIrrs r]rzAsyncTable.take_row_idss"6dk66w??@@@r_ AsyncTagsc*t|jS)aTag management for the dataset. Similar to Git, tags are a way to add metadata to a specific version of the dataset. .. warning:: Tagged versions are exempted from the :py:meth:`optimize(cleanup_older_than)` process. To remove a version that has been tagged, you must first :py:meth:`~Tags.delete` the associated tag. )rrIrXs r]rczAsyncTable.tagss %%%r_FrrrrzcKd}|$t|dz}|rddl}|jdt|j||d{VS)r"NirzNThe 'retrain' parameter is deprecated and will be removed in a future version.)cleanup_since_msr)round total_secondsr3r4r5rIr$)rYrrr rr3s r]r$zAsyncTable.optimizesZ+/  )$%7%E%E%G%G$%NOO    OOO HM""    [))-/*         r_r%cDK|jd{VS)rQN)rIr'rXs r]r'zAsyncTable.list_indicess.[--/////////r_r(r)cfK|j|d{V}|dStdi|S)r+NrV)rIr-IndexStatistics)rYr(rs r]r-zAsyncTable.index_statssMk--j99999999 =4"++U++ +r_cDK|jd{VSr`)rIrbrXs r]rbz!AsyncTable.uses_v2_manifest_paths1s.[77999999999r_cHK|jd{VdS)az Migrate the manifest paths to the new format. This will update the manifest to use the new v2 format for paths. This function is idempotent, and can be run multiple times without changing the state of the object store. !!! danger This should not be run while other concurrent operations are happening. And it should also run until completion before resuming other operations. You can use [AsyncTable.uses_v2_manifest_paths][lancedb.table.AsyncTable.uses_v2_manifest_paths] to check if the table is already using the new path style. N)rImigrate_manifest_paths_v2rXs r]rz$AsyncTable.migrate_manifest_paths_v2<s4$k3355555555555r_rbrcdict[str, str]cLK|j||d{VdSrf)rIrgrhs r]rgz!AsyncTable.replace_field_metadataPs8k00\JJJJJJJJJJJr_)rrrrmrnrorergrirprjrirl)rRr7rmrkrj) rrQrrrrrrrrmrrarnrkrorprl) rr"rr rr rrrRrrRrrs)NN.NN) rFrmrrmrrrrmrrrRr) rFrmrrmrr#rrmrrrRr6) rFrrrmrrrrmrrrRr9) rFrmrrmrr rrmrrrRr5) rFrrrmrrrrmrrrRr9rt) rFrrrmrrrrmrrrRr)r6r7rRr9)rFrArRr:rvrxryrzr{r|)rerfr rmr:rrRr)r.rirRr~)r3rprRr)rrrf)r]rz)r]r})rrrRr8)rrrRr8)rRr)rrrrarRrzrr)rbrQrcr)?rrrrrrrrrrrXr\r_rrrrirerrFrrsrrrrrrrrrJrrrrrrr9rLrrrrr rrr2r7rr]rHr>rArDrrrcr$r'r-rbrrgrVr_r]rrs@@D&&&%%%%###>>>>23333<1111"""X"**** Y Y Y Y 4 4 4 4 4 6 6 6 6 6 / / / /    QQQQQ$----#' ,0"+[[[[[[z++++(....,000000@IyQT?U?U?U?????")))) ' ' ' ' ; ; ; ;::::(:B59&*9=I!I!I!I!I!I!V<1<1<1<1| $,0&)-17; GGGGXG $,0(+-17; XAE,0&)-17; X $,0%(-17; X ,0(+-17;    X  ,0 &-17;vDvDvDvDvDp 5 5 5 53333r%)'+       "7777 0000 1111" " " " H2/2/2/2/l-1;< $04 ;<;<;<;<;<;r?r@rAutilrBrCrDrErFrGrHrprjrtr^rkrvrrx_lancedbryrrzr{r|r}r~rrrrrrrrPILrrrrrrrrrrrrrrrrrrrrLrGrSrrrrrrzrrrrrrryrrrrrrrrarrVr_r]rsC #"""""########!!!!!!((((((((%%%%%%                          "!!!!!IIIIIIII%%%%%%((((((1111111111JJJJJJJJ                          +*****////////  2 3#7 """""QU4%%%%%%#"""""MMMJJJ                    )-J J J J J Z<<<<4*.#'. B"BBBBBBP"$F$F$F$F$FN::::z%%%%V'. *****Z       $#'4< $ 4<4<4<4<4<4 > > > 0""&15 cBcBcBcBcBL...."    ???? 5555pAAAA ////,,,,""""&zKzKzKzKzKzKzKzKz- *"*"*"*"*"*"*" *"Z ''''''' ', """"""" "(          >B8B8B8B8B8B8B8B8JB4B4B4B4B4B4B4B4B4B4r_