diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index c2ce277..bbc8b53 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -79,6 +79,13 @@ jobs: with: name: assembled-site path: build/site + # Without this the upload silently drops dot-prefixed paths, so the + # published branch would be missing files the assembler produced — + # docs/.cursor/ went missing from the first published build that way. + # The assembler must publish what it assembled, editor config included: + # deciding a file is "not really content" is how a byte-comparability + # guarantee turns into a list of exceptions. + include-hidden-files: true retention-days: 7 publish: diff --git a/docs/agent-branch-experiments.mdx b/docs/agent-branch-experiments.mdx index f2e0a4f..59bcf68 100644 --- a/docs/agent-branch-experiments.mdx +++ b/docs/agent-branch-experiments.mdx @@ -58,7 +58,7 @@ Use a different column name for each model. Within an experiment, use the same model for the stored text and the search queries. -## Apply the winning experiment to `main` +## Apply the winning experiment to `main` {#apply-the-winning-experiment-to-main} The branch experiments leave you with the results side by side, and they deliberately never touch `main`. Once you've picked a winner, apply it to @@ -76,7 +76,7 @@ the losing branch around until you're confident in the result. See [Branches](/tables/branching) for more on how branches, versions, and tags relate. -## More experiments you can run +## More experiments you can run {#more-experiments-you-can-run} Swapping embedding models on a branch while working with agents is only one example of what you can do with the LanceDB plugin. The table below shows other experiments diff --git a/docs/api-reference/index.mdx b/docs/api-reference/index.mdx index 7ead054..5075602 100644 --- a/docs/api-reference/index.mdx +++ b/docs/api-reference/index.mdx @@ -13,7 +13,7 @@ If you're looking for a REST API reference, visit the [REST API](/api-reference/ If you're looking for conceptual and practical namespace guidance before diving into method signatures, see [Namespaces and Catalog Model](/namespaces) and [Using Namespaces in SDKs](/namespaces/usage). -## Supported SDKs +## Supported SDKs {#supported-sdks} Python, Typescript and Rust SDKs are officially supported by LanceDB. You can use these SDKs to interact with both LanceDB OSS and Enterprise deployments. @@ -23,7 +23,7 @@ Python, Typescript and Rust SDKs are officially supported by LanceDB. You can us | [Typescript](https://lancedb.github.io/lancedb/js/) | A TypeScript wrapper around the Rust library, built with `napi-rs` | [Rust](https://docs.rs/lancedb/latest/lancedb/index.html) | Native Rust library with persistent-storage and high performance | -## REST API SDKs +## REST API SDKs {#rest-api-sdks} Enterprise @@ -33,7 +33,7 @@ REST API-based SDKs provide a convenient way to interact with LanceDB Enterprise |:--------------|-------------------| | [Java](https://lancedb.github.io/lancedb/java/java/)| REST API Enterprise SDK in Java | -## Community-driven SDKs +## Community-driven SDKs {#community-driven-sdks} In addition to the officially supported SDKs, the LanceDB community may contribute SDKs in other languages. These SDKs may not have the same level of support or feature parity as the official ones supported by LanceDB, but they can be an option diff --git a/docs/api-reference/rest/index.mdx b/docs/api-reference/rest/index.mdx index 6793f57..1442225 100644 --- a/docs/api-reference/rest/index.mdx +++ b/docs/api-reference/rest/index.mdx @@ -15,7 +15,7 @@ additional endpoints for managing tables and data. If you have specific needs or questions about the Enterprise REST API Namespace, please [contact us](mailto:support@lancedb.com). -## Authentication +## Authentication {#authentication} Enterprise @@ -24,7 +24,7 @@ must be encoded as JSON or Arrow RPC. To authenticate to the Enterprise REST API, you need the endpoint for your deployment and a valid API key for that deployment. -### Get your Enterprise credentials +### Get your Enterprise credentials {#get-your-enterprise-credentials} 1. Obtain the following values from your LanceDB administrator or the LanceDB team that provisioned your Enterprise deployment: - an API key @@ -41,7 +41,7 @@ export LANCEDB_DATABASE="your-database-name" 3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint. -### Verify authentication +### Verify authentication {#verify-authentication} 4. Check that you can reach the deployment and list tables: diff --git a/docs/build-with-ai-agents.mdx b/docs/build-with-ai-agents.mdx index 384c002..8296400 100644 --- a/docs/build-with-ai-agents.mdx +++ b/docs/build-with-ai-agents.mdx @@ -77,14 +77,14 @@ Not every agent supports project scope; `npx plugins targets` lists what it found and what each target supports. -## Get started with the LanceDB agent plugin +## Get started with the LanceDB agent plugin {#get-started-with-the-lancedb-agent-plugin} This tutorial uses the Camelot dataset from the [quickstart](/quickstart), with a portrait added for each character. Each LanceDB row contains validated metadata and raw JPEG bytes. Text, images, and any embeddings you add later remain in the same table. -### 1. Download the multimodal dataset +### 1\. Download the multimodal dataset {#1-download-the-multimodal-dataset} From a new project directory, download the JSON file and portraits: @@ -133,7 +133,7 @@ Pydantic models before writing it. After the agent writes the pipeline, inspect the schema, batching, and write path rather than assuming it followed the plugin's guidance correctly. -### 2. Prompt your agent to build the pipeline +### 2\. Prompt your agent to build the pipeline {#2-prompt-your-agent-to-build-the-pipeline} Install the Python packages used by the example: @@ -173,7 +173,7 @@ simply ask it to use the `lancedb` plugin in the prompt, as shown above. That should be enough! The agent will create `ingest_multimodal.py`, or similar. The following sections inspect the script to verify that it follows the plugin's guidance. -#### Data validation +#### Data validation {#data-validation} The plugin encourages the agent to validate each record with Pydantic before writing it. The agent should ideally define a schema for the table and a nested schema for the @@ -186,7 +186,7 @@ The agent should ideally define a schema for the table and a nested schema for t In this case, our agent correctly defined `Character` and `Stats` Pydantic models and validated the JSON before adding it to the table. -#### Batched ingestion +#### Batched ingestion {#batched-ingestion} Naively calling `table.add()` once per row is slow, and is considered an anti-pattern in LanceDB. The plugin encourages the agent to collect incoming rows into batches and @@ -202,7 +202,7 @@ batch. If validation fails, that batch is never written. The function yields up to `batch_size` rows at a time, providing an iterable of batches for the ingestion step, shown next. -#### Table maintenance +#### Table maintenance {#table-maintenance} For LanceDB OSS, the plugin instructs the agent to call `table.optimize()` after the ingestion loop. This compacts small fragments, cleans up old versions according to the retention policy, and incorporates new data into indexes. @@ -220,7 +220,7 @@ This example dataset has only eight rows, so the default batch size writes it in one call. Larger inputs should still avoid single-row write commits. -### 3. Run the OSS pipeline +### 3\. Run the OSS pipeline {#3-run-the-oss-pipeline} ```bash uv run python ingest_multimodal.py @@ -238,7 +238,7 @@ Here are the first three rows: Once your pipeline works, you can [run experiments on branches](/agent-branch-experiments) to try new embedding models, parsers, or search settings without touching `main`. -## Takeaways +## Takeaways {#takeaways} The example in this tutorial was small, but similar ideas apply to other workflows, too. Give the agent the data source, the constraints it must respect, and the @@ -247,7 +247,7 @@ artifacts it should return. The plugin supplies LanceDB-specific guidance, but it's the user's responsibility to ensure the output makes sense for the application. -### Try the plugin with your own dataset +### Try the plugin with your own dataset {#try-the-plugin-with-your-own-dataset} The plugin shown in this tutorial should generalize reasonably well to other use cases. If you find any issues, open [an issue](https://github.com/lancedb/lancedb-agent-plugins/issues) diff --git a/docs/demos/index.mdx b/docs/demos/index.mdx index 28e905f..95f1ab3 100644 --- a/docs/demos/index.mdx +++ b/docs/demos/index.mdx @@ -13,7 +13,7 @@ App | Description [Video Search](#video-search) | A video search application that allows searching through a library of videos using natural language queries. -## Semantic.art +## Semantic.art {#semantic-art} multimodal hybrid-search @@ -37,7 +37,7 @@ Read in detail about how Semantic.art is built in this blog post. -## Wikipedia 41M Hybrid Search +## Wikipedia 41M Hybrid Search {#wikipedia-41m-hybrid-search} multimodal hybrid-search @@ -61,7 +61,7 @@ Read in detail about how the Wikipedia 41M hybrid search demo is built in this b -## Video Search +## Video Search {#video-search} multimodal video-search diff --git a/docs/embedding/index.mdx b/docs/embedding/index.mdx index 1a238c1..30ad260 100644 --- a/docs/embedding/index.mdx +++ b/docs/embedding/index.mdx @@ -34,7 +34,7 @@ that automatically generates vector embeddings during data ingestion. Automatic generation is available in LanceDB OSS, with SDK-specific query ergonomics. The API abstracts embedding generation, allowing you to focus on your application logic. -## Embedding Registry +## Embedding Registry {#embedding-registry} You can get a supported embedding function from the registry, and then use it in your table schema. Once configured, the embedding function will automatically generate embeddings when you insert data @@ -55,7 +55,7 @@ while Rust examples typically compute query embeddings explicitly before vector -### Using an embedding function +### Using an embedding function {#using-an-embedding-function} Create an embedding function before you attach it to table or schema metadata. Python and TypeScript fetch provider implementations from the embedding registry, while Rust constructs the provider embedding function @@ -123,7 +123,7 @@ For non-sensitive settings such as inference device selection, you can also use Find the full list of arguments for each provider in the [integrations](/integrations/embedding) section. -## Multiple embedding columns +## Multiple embedding columns {#multiple-embedding-columns} A single table can include more than one embedding definition when you want to store multiple semantic views of the same data, or generate embeddings from different source columns. In practice, each embedding definition @@ -136,11 +136,11 @@ In TypeScript, automatic query embedding currently uses the first embedding func table metadata. If a table has multiple embedding definitions and you need to query a specific vector column, compute the query embedding explicitly and pass the vector to the search builder. -## Embedding model providers +## Embedding model providers {#embedding-model-providers} LanceDB supports most popular embedding providers. -### Text embeddings +### Text embeddings {#text-embeddings} | Provider | Model ID | Default Model | |----------|----------|---------------| @@ -150,7 +150,7 @@ LanceDB supports most popular embedding providers. | Cohere | `cohere` | `embed-english-v3.0` | | ... | ... | ... | -### Multimodal embedding +### Multimodal embedding {#multimodal-embedding} | Provider | Model ID | Supported Inputs | |----------|----------|------------------| @@ -160,7 +160,7 @@ LanceDB supports most popular embedding providers. You can find all supported embedding models in the [integrations](/integrations/embedding) section. -## Embeddings in LanceDB Enterprise +## Embeddings in LanceDB Enterprise {#embeddings-in-lancedb-enterprise} Enterprise In LanceDB Enterprise, embedding generation during data ingestion is client-side and the resulting vectors are @@ -171,7 +171,7 @@ The Enterprise server does not currently generate embeddings from query text on query-time embedding happens on the client side. -### How string queries are interpreted +### How string queries are interpreted {#how-string-queries-are-interpreted} For the Python remote client, `table.search("hello")` can take two different paths: @@ -218,7 +218,7 @@ want full control over query-time behavior. -## Custom Embedding Functions +## Custom Embedding Functions {#custom-embedding-functions} You can always implement your own embedding function: - Python/TypeScript: subclass `TextEmbeddingFunction` (text) or `EmbeddingFunction` (multimodal). diff --git a/docs/embedding/quickstart.mdx b/docs/embedding/quickstart.mdx index 08ff9ce..bcaf90f 100644 --- a/docs/embedding/quickstart.mdx +++ b/docs/embedding/quickstart.mdx @@ -17,7 +17,7 @@ import { LanceDB will automatically vectorize the data both at ingestion and query time. All you need to do is specify which model to use. Popular embedding models like OpenAI, Hugging Face, Sentence Transformers, CLIP, and more, are supported. -## Step 1: Import Required Libraries +## Step 1: Import Required Libraries {#step-1-import-required-libraries} First, import the necessary LanceDB components: @@ -40,7 +40,7 @@ from lancedb.embeddings import get_registry - TypeScript uses `lancedb.embedding.getRegistry()` and `lancedb.embedding.LanceSchema()` for the same registry/schema workflow - In TypeScript, import the provider module before calling `getRegistry().get(...)`; the provider import is what registers names such as `"huggingface"` or `"openai"` -## Step 2: Connect to LanceDB +## Step 2: Connect to LanceDB {#step-2-connect-to-lancedb} Establish a connection to your LanceDB OSS directory or Enterprise cluster: @@ -55,7 +55,7 @@ db = lancedb.connect(...) -## Step 3: Initialize the Embedding Function +## Step 3: Initialize the Embedding Function {#step-3-initialize-the-embedding-function} Choose and configure your embedding model: @@ -76,7 +76,7 @@ This creates an embedding function from the local embedding registry. The Python - Modify the model name for different embedding models - Set `device="cuda"` for GPU acceleration if available -## Step 4: Define Your Schema +## Step 4: Define Your Schema {#step-4-define-your-schema} Create a Pydantic model that defines your table structure: @@ -97,7 +97,7 @@ class Words(LanceModel): - `model.ndims()`: Sets vector dimensions for your model - In TypeScript, use `model.sourceField(...)` and `model.vectorField()` inside `LanceSchema(...)` -## Step 5: Create Table and Ingest Data +## Step 5: Create Table and Ingest Data {#step-5-create-table-and-ingest-data} Create a table with your schema and add data: @@ -124,7 +124,7 @@ If your input already includes the vector column, automatic embedding only runs absent or entirely null for the batch. Partially supplied vectors are treated as manual data, so LanceDB preserves them instead of filling only the missing rows. -## Step 6: Query with Automatic Embedding +## Step 6: Query with Automatic Embedding {#step-6-query-with-automatic-embedding} Note: On LanceDB Enterprise, the server does not generate embeddings from query text. In the Python remote client, `table.search("greetings")` can still work when the table schema includes embedding metadata, because diff --git a/docs/enterprise/architecture.mdx b/docs/enterprise/architecture.mdx index f8d58ac..4f78f3b 100644 --- a/docs/enterprise/architecture.mdx +++ b/docs/enterprise/architecture.mdx @@ -30,13 +30,13 @@ flowchart TB CP -.->|Govern and configure| IX ``` -## Compute-storage separation +## Compute-storage separation {#compute-storage-separation} In LanceDB Enterprise, storage and compute are deliberately decoupled. Table data and index artifacts live in object storage, while query-serving and background workers read from and write to that shared durable layer. This means compute can be replaced, scaled, or specialized without making any individual node the owner of the dataset. This design has practical consequences. Query fleets can scale for interactive traffic without also scaling background indexing capacity. Heavy indexing and compaction work can run on dedicated workers instead of stealing resources from user-facing queries. Caches can accelerate hot reads without becoming the source of truth. And because the data remains in object storage, durability does not depend on the lifecycle of a particular server or local disk. -## Architecture +## Architecture {#architecture} At a high level, the control plane governs the system and the data plane executes the work. The control plane is responsible for configuration, service discovery, identity integration, policy, and cluster lifecycle. It determines how the system should behave, but it is not the layer serving table data or executing user queries -- that's the role of the data plane. @@ -80,7 +80,7 @@ flowchart TB WAL -->|Persist| OS ``` -## Remote tables +## Remote tables {#remote-tables} A [remote table](/tables-and-namespaces#understanding-tables) is the user-facing abstraction over this architecture. From the client side, you connect to a logical storage layer and table over the network by providing a `db://...` connection identifier. The system then resolves that logical name to the underlying storage-backed table and executes the operation inside the cluster. @@ -88,7 +88,7 @@ This is why Enterprise feels familiar at the API level while operationally behav This design makes LanceDB Enterprise suitable for catalog-backed layouts, see [Namespaces and the Catalog Model](/namespaces) for more details. For the basic application flow, see the shared [quickstart](/quickstart). -## Read path +## Read path {#read-path} When a client issues a query against a remote table, the path is straightforward: @@ -100,7 +100,7 @@ When a client issues a query against a remote table, the path is straightforward This separation is what lets Enterprise combine a clean remote API with a serving layer that can scale horizontally and keep hot data close to execution. -## Write path +## Write path {#write-path} Writes follow a different path because durability comes first: @@ -110,7 +110,7 @@ Writes follow a different path because durability comes first: Keeping the commit path centered on object storage ensures that the durable record of the table lives outside any single query node. Regardless of whether you're using Lance namespaces or an external catalog, the catalog's role is mainly to resolve table names and provide access details -- the table’s actual data and index artifacts remain in object storage. -## Background work +## Background work {#background-work} Indexing, compaction, and cleanup are intentionally moved off the user request path. After table changes are committed, the system can determine that additional work is needed and assign it to background workers built for heavyweight processing. @@ -122,7 +122,7 @@ In practice, that usually looks like this: This separation is one of the clearest architectural reasons to use LanceDB Enterprise: the same query-serving infrastructure does not have to handle every expensive indexing or compaction task itself. -## What this means for users +## What this means for users {#what-this-means-for-users} For teams using LanceDB Enterprise, the architecture changes the _operational model_ more than the programming model. You still work with tables and queries, but the cluster now takes responsibility for distributed execution, cache-aware reads, and long-running background jobs. diff --git a/docs/enterprise/authentication.mdx b/docs/enterprise/authentication.mdx index 75c970a..62b756b 100644 --- a/docs/enterprise/authentication.mdx +++ b/docs/enterprise/authentication.mdx @@ -13,7 +13,7 @@ LanceDB Enterprise supports two ways for clients to authenticate against a `db:/ OAuth is the recommended option when you want to rotate credentials centrally, plug into an existing identity provider, or run on Azure with managed identities so no secret material lives on the client. -## API key +## API key {#api-key} Pass the API key from your Enterprise tenant on `connect`. This works with both the synchronous and asynchronous Python clients, as well as TypeScript and Rust. @@ -28,7 +28,7 @@ db = lancedb.connect( ) ``` -## OAuth +## OAuth {#oauth} The async Python client and the TypeScript client can obtain bearer tokens from an OIDC issuer and attach them to every request. Token acquisition, caching, and refresh are handled inside the client — your application code only provides the configuration. @@ -36,7 +36,7 @@ The async Python client and the TypeScript client can obtain bearer tokens from In Python, OAuth is supported through `lancedb.connect_async`. The synchronous `connect` entry point continues to use API key authentication for `db://` URIs. In TypeScript, `lancedb.connect` accepts `oauthConfig` directly. -### Supported flows +### Supported flows {#supported-flows} `OAuthFlowType` selects how the client acquires tokens: @@ -45,7 +45,7 @@ In Python, OAuth is supported through `lancedb.connect_async`. The synchronous ` | Client Credentials | `OAuthFlowType.CLIENT_CREDENTIALS` | `OAuthFlowType.ClientCredentials` | Service-to-service / machine-to-machine. Requires a client ID and client secret registered with your identity provider. | | Azure Managed Identity | `OAuthFlowType.AZURE_MANAGED_IDENTITY` | `OAuthFlowType.AzureManagedIdentity` | Workloads running on Azure compute (VMs, AKS, App Service, Container Apps). Tokens are fetched from the Azure IMDS endpoint, so no client secret is stored on the client. | -### Configure OAuth +### Configure OAuth {#configure-oauth} Build an OAuth config and pass it on connect. Use `oauth_config` in Python and `oauthConfig` in TypeScript. @@ -122,7 +122,7 @@ const oauthConfig: OAuthConfig = { ``` -### Configuration reference +### Configuration reference {#configuration-reference} The same configuration is available in both SDKs. Python uses `snake_case` field names; TypeScript uses `camelCase`. diff --git a/docs/enterprise/benchmarks.mdx b/docs/enterprise/benchmarks.mdx index 1f05e12..05a31ae 100644 --- a/docs/enterprise/benchmarks.mdx +++ b/docs/enterprise/benchmarks.mdx @@ -21,7 +21,7 @@ If you want performance guidance for your own workload, reach out to [contact@la Depending on workload and tuning, Enterprise clusters can also be configured for high concurrency, including thousands of QPS in some deployments, but the right configuration varies by use case. Training, search, and analytics workloads often benefit from different cluster shapes and resource allocation strategies. To understand which parts of the system influence these results, see the [Enterprise architecture](/enterprise/architecture) guide. -## Dataset +## Dataset {#dataset} We used two datasets for this benchmark: the [dbpedia-entities-openai-1M](https://huggingface.co/datasets/KShivendu/dbpedia-entities-openai-1M) for vector search, and a synthetic dataset for vector search with metadata filtering. @@ -33,7 +33,7 @@ for vector search, and a synthetic dataset for vector search with metadata filte These benchmark results are most useful as a directional baseline. Different data distributions, index choices, cache behavior, and cluster settings can materially change the latency profile. -## Vector Search +## Vector Search {#vector-search} We ran vector queries against `dbpedia-entities-openai-1M` with a warmed-up cache. In that benchmark setup, we observed the following latency profile: @@ -44,7 +44,7 @@ We ran vector queries against `dbpedia-entities-openai-1M` with a warmed-up cach | P99 | 35ms | | Max | 49ms | -## Full-Text Search +## Full-Text Search {#full-text-search} With the same dataset and a warmed-up cache, full-text search fell into the following range: @@ -55,7 +55,7 @@ With the same dataset and a warmed-up cache, full-text search fell into the foll | P99 | 42ms | | Max | 98ms | -## Vector Search with Metadata Filtering +## Vector Search with Metadata Filtering {#vector-search-with-metadata-filtering} We created a 15M-vector dataset to evaluate metadata-aware search under more complex filtering conditions. These filters can span a wide range of scalar columns, for example, "find Sci-fi movies since 1900". diff --git a/docs/enterprise/deployment/azure.mdx b/docs/enterprise/deployment/azure.mdx index e846989..30781ac 100644 --- a/docs/enterprise/deployment/azure.mdx +++ b/docs/enterprise/deployment/azure.mdx @@ -8,7 +8,7 @@ icon: "cloud" LanceDB Enterprise can be deployed on Azure using Azure Kubernetes Service (AKS) with Azure Blob Storage for data persistence and Azure Private Link for secure connectivity. -## General Architecture Overview +## General Architecture Overview {#general-architecture-overview} ```mermaid graph TB @@ -47,14 +47,14 @@ graph TB style WI fill:#e6f4ea,stroke:#66bb6a,stroke-width:2px,color:#1d3a1f ``` -### Key Components +### Key Components {#key-components} - **LanceDB architecture** is deployed in an AKS cluster within its own VPC - **Client applications** connect to the cluster securely using Azure Private Link - **AKS cluster** is granted Azure Blob Storage read/write permissions using Azure Workload Identity - **Azure EventHub** can be used as the message queue by LanceDB Enterprise for internal message communication (alternative: self-hosted Kafka cluster in AKS) -## Read Path Architecture +## Read Path Architecture {#read-path-architecture} ```mermaid graph LR @@ -84,14 +84,14 @@ graph LR style BS fill:#e0f2f1,stroke:#26a69a,color:#09312d ``` -### Read Path Flow +### Read Path Flow {#read-path-flow} 1. **Client Application** sends query request through Private Link 2. **Query Nodes** receive and process the request 3. **Plan Executors** optimize and execute the query using distributed data cache to speed up read queries 4. **Azure Blob Storage** stores data and indices in Lance, while Plan Executors maintain distributed cache for performance -## Write Path Architecture +## Write Path Architecture {#write-path-architecture} ```mermaid graph LR @@ -131,13 +131,13 @@ graph LR style BS fill:#e0f2f1,stroke:#26a69a,color:#09312d ``` -### Write Path Flow +### Write Path Flow {#write-path-flow} Query nodes write data and indices synchronously to Azure Blob Storage in Lance data format while asynchronously sending data modification events to Azure EventHub (or self-hosted Kafka cluster). These write events are processed by the Lance Agent, which launches indexing pods or data optimization pods to optimize data for better read performance. -## Deployment Options +## Deployment Options {#deployment-options} -### Storage Architecture Support +### Storage Architecture Support {#storage-architecture-support} ```mermaid graph TB @@ -162,21 +162,21 @@ graph TB style SA3 fill:#e0f2f1,stroke:#26a69a,color:#09312d ``` -### Deployment Models +### Deployment Models {#deployment-models} LanceDB Enterprise supports three deployment models on Azure: -#### 1. Fully Managed Service +#### 1\. Fully Managed Service {#1-fully-managed-service} - **Infrastructure and storage** in LanceDB's Azure account - **Complete management** by LanceDB team - **Simplest setup** for customers -#### 2. BYOC (Bring Your Own Cloud) +#### 2\. BYOC (Bring Your Own Cloud) {#2-byoc-bring-your-own-cloud} - **Infrastructure and storage** in customer's Azure account - **Fully Managed by LanceDB** - **Full control** over data residency -#### 3. Hybrid - Bring Your Own Container +#### 3\. Hybrid - Bring Your Own Container {#3-hybrid-bring-your-own-container} - **Infrastructure** in LanceDB's account - **Storage containers** in customer's account diff --git a/docs/enterprise/deployment/index.mdx b/docs/enterprise/deployment/index.mdx index 643b199..cf5806a 100644 --- a/docs/enterprise/deployment/index.mdx +++ b/docs/enterprise/deployment/index.mdx @@ -8,7 +8,7 @@ icon: "list" There are two deployment models available for LanceDB Enterprise: **Managed** and **BYOC**. Both models support AWS, GCP, and Azure cloud platforms. -## Managed deployment +## Managed deployment {#managed-deployment} This is a private deployment of LanceDB Enterprise. All applications run in cloud accounts managed by LanceDB in the same location as your client applications. @@ -16,7 +16,7 @@ This hands-off approach is recommended for users who do not wish to manage the i To access your deployment, LanceDB can provision either a public or private load balancer. -## Bring-your-own-cloud (BYOC) deployment +## Bring-your-own-cloud (BYOC) deployment {#bring-your-own-cloud-byoc-deployment} With this deployment model, LanceDB Enterprise is installed into your own cloud account. This approach is recommended when: @@ -25,7 +25,7 @@ This approach is recommended when: To deploy, an identity will be provisioned in your account with permissions to manage the infrastructure. -## Custom deployments +## Custom deployments {#custom-deployments} LanceDB Enterprise installation is highly configurable and customizable to your needs. If you have any other specific deployment requirements, please reach out to our support team diff --git a/docs/enterprise/index.mdx b/docs/enterprise/index.mdx index 69ae326..e8c595d 100644 --- a/docs/enterprise/index.mdx +++ b/docs/enterprise/index.mdx @@ -18,13 +18,13 @@ If you need private deployments, high performance at extreme scale, or if you ha [reach out to our team](mailto:contact@lancedb.com) to set up a LanceDB Enterprise cluster in your environment. -## Why use LanceDB Enterprise? +## Why use LanceDB Enterprise? {#why-use-lancedb-enterprise} If you are evaluating LanceDB for a production AI system, Enterprise is built around three practical needs: handling very large vector workloads, running feature engineering close to the data, and operating the platform with production visibility. -### 1. 100B+ row scale +### 1\. Scale to hundreds of billions of rows {#1-scale-to-hundreds-of-billions-of-rows} LanceDB Enterprise is built for demanding workloads that exceed the capabilities of a single machine, whether from extremely large data volumes or a high number of concurrent queries. Instead of asking your application to own caching, query scaling, and maintenance, Enterprise turns those into **platform** capabilities. @@ -44,7 +44,7 @@ more concurrent requests. - **Enterprise training cache**: Coming soon. Enterprise is extending the same storage-aware caching model to training and feature engineering pipelines so large jobs can use GPU capacity more efficiently. -### 2. Feature engineering with Geneva +### 2\. Feature engineering with Geneva {#2-feature-engineering-with-geneva} For many teams, retrieval is only part of the problem. They also need a reliable way to derive new columns, run backfills, and keep feature pipelines close to the data they already store in LanceDB. This is what [Geneva](/geneva/) enables. @@ -55,7 +55,7 @@ backfills, and keep feature pipelines close to the data they already store in La batch system around OSS tables. - **Shared workflows**: Use Geneva clusters, manifests, and jobs to manage feature engineering work in one place. -### 3. Enterprise-grade monitoring +### 3\. Enterprise-grade monitoring {#3-enterprise-grade-monitoring} Production retrieval systems need more than search or training performance. Teams also need to observe the system, choose how it is deployed, and satisfy security and compliance requirements. @@ -67,7 +67,7 @@ is deployed, and satisfy security and compliance requirements. - **Private networking and compliance**: Designed for production environments that need encryption at rest, private connectivity options, and compliance coverage such as SOC 2 Type II and HIPAA. -## How is LanceDB Enterprise different from OSS? +## How is LanceDB Enterprise different from OSS? {#how-is-lancedb-enterprise-different-from-oss} LanceDB OSS runs inside your application process. LanceDB Enterprise runs as a distributed cluster across many machines. Both are built on the same Lance columnar file format, so moving data from one edition to the other does @@ -83,7 +83,7 @@ not require a data conversion step. | **Data format** | Supports multiple available standards | Supports multiple available standards | No vendor lock-in; data moves freely between editions. | | **Deployment** | Embedded in your code | BYOC or Managed | Enterprise meets uptime, compliance, and support goals that OSS cannot. | -### Architecture and scale +### Architecture and scale {#architecture-and-scale} LanceDB OSS is directly embedded into your service. The process owns all CPU, memory, and storage, so scale is limited to what one host can provide. @@ -92,7 +92,7 @@ adding nodes, and the platform can keep serving traffic even when individual nod Read More: [LanceDB Enterprise Architecture](/enterprise/architecture/) -### Latency of data retrieval +### Latency of data retrieval {#latency-of-data-retrieval} With LanceDB OSS, read latency depends heavily on where the data lives. If you use local disk or shared file storage, reads can be quite fast. But if you point an embedded deployment at S3, GCS, or Azure Blob, every read still takes a full round trip to remote object storage, especially when the data is cold. @@ -100,20 +100,20 @@ LanceDB Enterprise is designed for the object-storage-backed case. It uses NVMe Read More: [LanceDB Enterprise Benchmarks](/enterprise/benchmarks/) -### Throughput of search queries +### Throughput of search queries {#throughput-of-search-queries} A single LanceDB OSS process shares one CPU pool with the rest of the application. When concurrent queries hit that CPU, retrieval and similarity processes compete for cores. The server cannot process more work in parallel and any extra traffic waits in the queue, raising latency without increasing queries per second. LanceDB Enterprise distributes queries across many execution nodes. A load balancer assigns queries to the least-loaded node, so throughput grows as more nodes join the cluster instead of stalling at a single-process ceiling. -### Caching of commonly retrieved data +### Caching of commonly retrieved data {#caching-of-commonly-retrieved-data} LanceDB OSS has no built-in cache. Every read repeats the same object-store round trip and pays the same latency penalty. LanceDB Enterprise shards a cache across the fleet with consistent hashing. Popular vectors remain on local NVMe drives until they age out under a least-recently-used policy. Cache misses fall back to the object store, fill the local shard, and serve future reads faster. This design slashes both latency and egress cost for workloads with temporal locality. -### Maintenance of vector indexes +### Maintenance of vector indexes {#maintenance-of-vector-indexes} Vector indexes fragment when data is inserted, updated, or deleted. Fragmentation slows queries because the engine must scan more blocks. LanceDB OSS offers a CLI call to compact or rebuild the index, but you must schedule it yourself. @@ -124,7 +124,7 @@ largest workloads. Read More: [Indexing in LanceDB](/indexing/) -### Deployment and governance +### Deployment and governance {#deployment-and-governance} When you work with LanceDB OSS, it is included as part of your binary, Docker, or serverless function. The footprint is small, and no extra services run beside it. @@ -134,26 +134,26 @@ monitoring. Both enterprise modes are designed for private networking, complianc Read More: [LanceDB Enterprise Deployment](/enterprise/deployment/) -## Usage differences between Enterprise and OSS +## Usage differences between Enterprise and OSS {#usage-differences-between-enterprise-and-oss} The [quickstart](/quickstart) guide shows both local embedded connections and Enterprise `db://...` connections. Once connected to LanceDB, the table API is largely the same: create a table, search, filter, evolve the schema, and store multimodal records. However, there are some semantic differences worth understanding when your code is talking to LanceDB Enterprise. -### 1. Connection model +### 1\. Connection model {#1-connection-model} In LanceDB Enterprise, your app connects via a `db://...` URI and sends requests to the cluster API. The cluster executes table operations on your behalf. Your code is coupled to a **managed service endpoint**, whereas embedded LanceDB is directly coupled to local or object-storage paths. -### 2. Returned table type +### 2\. Returned table type {#2-returned-table-type} Connecting to an Enterprise table via `open_table(...)` returns a `RemoteTable`, unlike embedded LanceDB, which returns a `LanceTable`. `RemoteTable` is a catalog-backed table accessed through a server/cluster, and does not support all the same methods as `LanceTable` (see below). -### 3. Materialization APIs +### 3\. Materialization APIs {#3-materialization-apis} For Python users working with LanceDB Enterprise, `RemoteTable` does not support table-level materialization methods like `table.to_arrow()` or `table.to_pandas()`. This protects users from @@ -163,7 +163,7 @@ Instead, materialize results through query/search builders, for example `table.search(...).limit(...).to_pandas()` or `table.query(...).to_arrow()`. For quick previews, use `table.head()`. -### 4. Maintenance lifecycle +### 4\. Maintenance lifecycle {#4-maintenance-lifecycle} In Enterprise, maintenance operations like `optimize` and `compact_files` are handled by the cluster as background work. You can trigger them manually, but they are not required for performance or @@ -172,7 +172,7 @@ correctness in the same way they are in embedded LanceDB. That means maintenance is managed by platform behavior and cluster configuration, not by explicit per-table maintenance calls in your application code. -### 5. Guardrails and limits +### 5\. Guardrails and limits {#5-guardrails-and-limits} Enterprise can enforce platform-level guardrails, such as index/table limits and safety checks around operations like `merge_insert` when too many rows are unindexed. Embedded LanceDB mostly exposes @@ -181,7 +181,7 @@ storage/format-level behavior, and you tune many lifecycle tasks yourself. This means an operation in LanceDB Enterprise can fail due to service-level policy, not just because of local table shape or schema mismatch. -### 6. Cluster-managed background work +### 6\. Cluster-managed background work {#6-cluster-managed-background-work} In Enterprise, async writes and reindexing workflows are handled by cluster background systems. In embedded LanceDB, if you want ongoing upkeep, you usually schedule and run it yourself in your @@ -196,7 +196,7 @@ use query builders to fetch results, and otherwise interact with your tables as LanceDB. -## Which one should I use? +## Which one should I use? {#which-one-should-i-use} [It's very simple to get started with OSS](/quickstart/): Get started with `pip install lancedb` and begin ingesting your data and vectors into LanceDB. LanceDB OSS makes sense when your dataset fits on one machine, traffic is still diff --git a/docs/enterprise/security.mdx b/docs/enterprise/security.mdx index 019219a..b004274 100644 --- a/docs/enterprise/security.mdx +++ b/docs/enterprise/security.mdx @@ -7,14 +7,14 @@ icon: "shield-alt" LanceDB Enterprise maintains high security standards with SOC 2 Type II, HIPAA, and GDPR compliance. Our security framework is designed to provide enterprise-grade protection for your data and workloads across deployment models. -## Security Certifications +## Security Certifications {#security-certifications} - **SOC 2 Type II**: Independent audit confirming our security controls and operational effectiveness - **HIPAA Compliance**: Certified to handle protected health information (PHI) in healthcare applications - **GDPR Compliance**: Supports organizations with data privacy requirements under the General Data Protection Regulation - **Regular Audits**: Ongoing security assessments to maintain compliance standards -### Ongoing Compliance +### Ongoing Compliance {#ongoing-compliance} LanceDB maintains SOC 2 Type II, HIPAA, and GDPR compliance through ongoing audits and continuous improvement of our security practices as standards and risks evolve. @@ -22,16 +22,16 @@ LanceDB maintains SOC 2 Type II, HIPAA, and GDPR compliance through ongoing audi Visit the [LanceDB Trust Center](https://trust.lancedb.com/) to learn more about LanceDB's security posture, data privacy practices, and to request access to security documentation. -## LanceDB Enterprise +## LanceDB Enterprise {#lancedb-enterprise} -### Data Security +### Data Security {#data-security} Customer data is strictly protected and remains within the confines of your account. We maintain rigorous data isolation and encryption protocols to ensure confidentiality. LanceDB Enterprise only receives telemetry data for monitoring system health. At LanceDB, customer data security is paramount. -### Encryption +### Encryption {#encryption} LanceDB Enterprise safeguards your data through encryption at rest, preventing unauthorized access. This comprehensive encryption covers all data stored within the diff --git a/docs/faq/faq-enterprise.mdx b/docs/faq/faq-enterprise.mdx index 8210edb..7c209cd 100644 --- a/docs/faq/faq-enterprise.mdx +++ b/docs/faq/faq-enterprise.mdx @@ -9,9 +9,9 @@ mode: wide This section provides answers to the most common questions asked about LanceDB Enterprise. For assistance with LanceDB Enterprise, please [contact us](mailto:support@lancedb.com) via email and one of our support staff will get back to you. -### Architecture and Fault Tolerance +### Architecture and Fault Tolerance {#architecture-and-fault-tolerance} -#### What's the impact of losing each component (query node, indexer, etc.) in the LanceDB stack? +#### What is the impact of losing each component (query node, indexer, etc.) in the LanceDB stack? {#what-is-the-impact-of-losing-each-component-query-node-indexer-etc-in-the-lancedb-stack} LanceDB Enterprise employs component-level replication to ensure fault tolerance and continuous operations. While the system remains fully functional during replica failures, transient performance impacts (e.g., elevated latency or reduced throughput) @@ -19,14 +19,14 @@ may occur until automated recovery completes. For architectural deep dives, including redundancy configurations, please contact the LanceDB team. -#### What does plan executor cache versus not cache? +#### What does plan executor cache versus not cache? {#what-does-plan-executor-cache-versus-not-cache} The plan executor caches the table data, not the table indices. -#### Should I use disk cache or memory cache for the plan executor? +#### Should I use disk cache or memory cache for the plan executor? {#should-i-use-disk-cache-or-memory-cache-for-the-plan-executor} LanceDB implements highly performant consistent hashing for our plan executors. NVMe SSD caching is enabled by default for all deployments. -#### How is the PE (Plan Executor) fleet shared? What fault tolerance exists (how many nodes can be lost)? +#### How is the PE (Plan Executor) fleet shared? What fault tolerance exists (how many nodes can be lost)? {#how-is-the-pe-plan-executor-fleet-shared-what-fault-tolerance-exists-how-many-nodes-can-be-lost-} LanceDB's plan executor is typically deployed with 2+ replicas for fault tolerance: - Mirrored Caches: Each query replica maintains synchronized copies of data subsets, @@ -37,9 +37,9 @@ With a single replica failure, there is no downtime - the system remains operational with degraded performance, as the remaining replicas will handle all the traffic until the failed replica comes back online. -### Consistency +### Consistency {#consistency} -#### How is strong/weak consistency configured in the enterprise stack? +#### How is strong and weak consistency configured in the enterprise stack? {#how-is-strong-and-weak-consistency-configured-in-the-enterprise-stack} By default, LanceDB Enterprise operates in strong consistency mode. Once a write is successfully acknowledged, a new Lance dataset version manifest file is created. Subsequent reads always load the latest manifest file to @@ -60,24 +60,24 @@ keeping data reasonably fresh for most applications. Note that **this setting only affects read operations**. Write operations always remain strongly consistent. -### Indexing +### Indexing {#indexing} -#### Can I use GPU for indexing? +#### Can I use GPU for indexing? {#can-i-use-gpu-for-indexing} Yes! Please [contact](mailto:support@lancedb.com) the LanceDB team to enable GPU-based indexing for your deployment. Then you just need to call `create_index`, and the backend will use GPU for indexing. LanceDB is able to index a few billion vectors under 4 hours. -### Cluster Configuration +### Cluster Configuration {#cluster-configuration} -#### What are the parameters that can be configured for my LanceDB cluster? +#### What are the parameters that can be configured for my LanceDB cluster? {#what-are-the-parameters-that-can-be-configured-for-my-lancedb-cluster} LanceDB Enterprise offers granular control over performance, resilience, and operational behavior through a comprehensive set of parameters: replication factors for each component, consistency level, graceful shutdown time intervals, etc. Please contact the LanceDB team for detailed documentation on such parameter configurations. -### Monitoring and Alerts +### Monitoring and Alerts {#monitoring-and-alerts} -#### What are the metrics that LanceDB exposes for monitoring? +#### What are the metrics that LanceDB exposes for monitoring? {#what-are-the-metrics-that-lancedb-exposes-for-monitoring} We have various metrics set up for monitoring each component in the LanceDB stack: - Query node: RPS, query latency, error codes, slow take count, CPU/memory utilization, etc. @@ -85,7 +85,7 @@ We have various metrics set up for monitoring each component in the LanceDB stac Please contact the LanceDB team for the comprehensive list of monitoring metrics. -#### How do I integrate LanceDB's monitoring metrics with my monitoring dashboard? +#### How do I integrate LanceDB monitoring metrics with my monitoring dashboard? {#how-do-i-integrate-lancedb-monitoring-metrics-with-my-monitoring-dashboard} LanceDB uses Prometheus for metrics collection and OpenTelemetry (OTel) to export such metrics with data enrichment. The LanceDB team will work with you to integrate the monitoring metrics with your preferred dashboard. \ No newline at end of file diff --git a/docs/faq/faq-oss.mdx b/docs/faq/faq-oss.mdx index ce1b1be..7c1b4ce 100644 --- a/docs/faq/faq-oss.mdx +++ b/docs/faq/faq-oss.mdx @@ -8,43 +8,43 @@ mode: wide This section covers some common questions and issues that you may encounter when using LanceDB. -### Is LanceDB open source? +### Is LanceDB open source? {#is-lancedb-open-source} LanceDB OSS is a permissively licensed embedded retrieval library available under an Apache 2.0 license. We also have a LanceDB Enterprise, a commercial product that can be deployed on a private cloud or a bring-your-own-cloud (BYOC) solution. LanceDB Enterprise transforms your data lake into a high-performance multimodal lakehouse. -### What is the difference between Lance and LanceDB? +### What is the difference between Lance and LanceDB? {#what-is-the-difference-between-lance-and-lancedb} [Lance](https://github.com/lancedb/lance) is a modern lakehouse format for multimodal AI. It's perfect for building search engines, feature stores and being the foundation of large-scale ML training jobs requiring high performance IO and shuffles. It also has native support for storing, querying, and inspecting deeply nested data for robotics or large blobs like images, point clouds, and more. LanceDB is the multimodal lakehouse that's built on top of Lance, and utilizes the underlying optimized storage format to build efficient disk-based indexes that power semantic search & retrieval applications, from RAGs to QA bots to recommender systems. -### Why invent another data format instead of using Parquet? +### Why invent another data format instead of using Parquet? {#why-invent-another-data-format-instead-of-using-parquet} As we mention in our talk titled "[Lance, a modern columnar data format](https://www.youtube.com/watch?v=ixpbVyrsuL8)", Parquet and other tabular formats that derive from it are rather dated (Parquet is over 10 years old), especially when it comes to random access on vectors. We needed a format that's able to handle the complex trade-offs involved in shuffling, scanning, OLAP and filtering large datasets involving vectors, and our extensive experiments with Parquet didn't yield sufficient levels of performance for modern ML. [Our benchmarks](https://lancedb.com/blog/benchmarking-random-access-in-lance/) show that Lance is up to 1000x faster than Parquet for random access, which we believe justifies our decision to create a new data format for AI. -### Why build in Rust? +### Why build in Rust? {#why-build-in-rust} We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust's safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database. -### What makes LanceDB different? +### What makes LanceDB different? {#what-makes-lancedb-different} LanceDB is among the few embedded vector DBs out there that we believe can unlock a whole new class of LLM-powered applications in the browser or via edge functions. Lance's multimodal nature allows you to store the raw data, metadata and the embeddings all at once, unlike other solutions that typically store just the embeddings and metadata. The Lance data format that powers our storage system also provides true zero-copy access and seamless interoperability with numerous other data formats (like Pandas, Polars, Pydantic) via Apache Arrow, as well as automatic data versioning and data management without needing extra infrastructure. -### How large of a dataset can LanceDB handle? +### How large of a dataset can LanceDB handle? {#how-large-of-a-dataset-can-lancedb-handle} LanceDB and its underlying data format, Lance, are built to scale to really large amounts of data. LanceDB OSS can comfortably handle millions of vectors on a single node, making it a great fit for most applications. Its disk-based indexes keep performance strong without requiring expensive infrastructure. If you need to scale to hundreds of millions of vectors or work with terabytes of data, we recommend [LanceDB Enterprise](/enterprise). Enterprise customers regularly operate on billions of rows, backed by distributed infrastructure designed for large-scale production workloads. -### Do I need to build a vector index to run vector search? +### Do I need to build a vector index to run vector search? {#do-i-need-to-build-a-vector-index-to-run-vector-search} No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, a vector index is usually not necessary. For large-scale (>1M) or higher dimension vectors, it is beneficial to create a vector index. See the [Vector Indexes](/indexing/vector-index/) section for more details. -### How can I speed up data inserts? +### How can I speed up data inserts? {#how-can-i-speed-up-data-inserts} LanceDB auto-parallelizes large writes when you call `table.add()` with materialized data such as `pa.Table`, `pd.DataFrame`, or `pa.dataset()`. No extra configuration @@ -61,24 +61,24 @@ For best results: See [Loading Large Datasets](/tables/create#loading-large-datasets) for full examples. -### Do I need to set a refine factor when using an index? +### Do I need to set a refine factor when using an index? {#do-i-need-to-set-a-refine-factor-when-using-an-index} Yes. LanceDB uses PQ, or Product Quantization, to compress vectors and speed up search when using an ANN index. However, because PQ is a lossy compression algorithm, it tends to reduce recall while also reducing the index size. To address this trade-off, we introduce a process called **refinement**. The normal process computes distances by operating on the compressed PQ vectors. The refinement factor (*rf*) is a multiplier that takes the top-k similar PQ vectors to a given query, fetches `rf * k` *full* vectors and computes the raw vector distances between them and the query vector, reordering the top-k results based on these scores instead. For example, if you're retrieving the top 10 results and set `refine_factor` to 25, LanceDB will fetch the 250 most similar vectors (according to PQ), compute the distances again based on the full vectors for those 250 and then re-rank based on their scores. This can significantly improve recall, with a small added latency cost (typically a few milliseconds), so it's recommended you set a `refine_factor` of anywhere between 5-50 and measure its impact on latency prior to deploying your solution. -### How can I improve IVF-PQ recall while keeping latency low? +### How can I improve IVF-PQ recall while keeping latency low? {#how-can-i-improve-ivf-pq-recall-while-keeping-latency-low} When using an IVF-PQ index, there's a trade-off between recall and latency at query time. You can improve recall by increasing the number of probes and the `refine_factor`. In our benchmark on the GIST-1M dataset, we show that it's possible to achieve >0.95 recall with a latency of under 10 ms on most systems, using ~50 probes and a `refine_factor` of 50. This is, of course, subject to the dataset at hand and a quick sensitivity study can be performed on your own data. You can find more details on the benchmark in a past [blog post](https://medium.com/etoai/benchmarking-lancedb-92b01032874a). ![](/static/assets/images/faq/recall-vs-latency.webp) -### How much data can LanceDB practically manage without affecting performance? +### How much data can LanceDB practically manage without affecting performance? {#how-much-data-can-lancedb-practically-manage-without-affecting-performance} We target good performance on ~10-50 billion rows and ~10-30 TB of data. For the best performance and scalability guarantees, check out [LanceDB Enterprise](/enterprise). -### Does LanceDB support concurrent operations? +### Does LanceDB support concurrent operations? {#does-lancedb-support-concurrent-operations} LanceDB can handle concurrent reads very well, and can scale horizontally. The main constraint is how well the storage layer you've chosen, scales. For writes, we support concurrent writing, though too many concurrent writers can lead to failing writes as there is a limited number of times a writer retries a commit. diff --git a/docs/index.mdx b/docs/index.mdx index 2dcd73d..e59402a 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -13,7 +13,7 @@ It is built on top of [Lance](/lance), an open-source lakehouse format designed Move from data exploration to model training on one, unified platform without needing to manage a fragmented stack of storage, feature, retrieval, and training systems. -## Build better models, faster +## Build better models, faster {#build-better-models-faster} Training data and experimentation slow down when raw data, metadata, embeddings, features, and governance artifacts live in separate systems. LanceDB keeps them together in one versioned multimodal table, so AI teams spend less @@ -27,7 +27,7 @@ GPU-ready batches from a tagged dataset version. For a deeper look at how this works in training pipelines, start with [Why LanceDB for training](/training/why-lancedb). -## LanceDB suite +## LanceDB suite {#lancedb-suite} The LanceDB suite includes LanceDB OSS, an open-source embedded retrieval library, and LanceDB Enterprise, a multimodal lakehouse platform for the full AI data lifecycle. @@ -37,7 +37,7 @@ feature engineering, search and retrieval, and efficient training data access. ![LanceDB suite: OSS search and Enterprise multimodal lakehouse on Lance format](/static/assets/images/overview/lancedb-suite.svg) -## Why teams use LanceDB +## Why teams use LanceDB {#why-teams-use-lancedb} @@ -55,7 +55,7 @@ feature engineering, search and retrieval, and efficient training data access. -## Start with your workload +## Start with your workload {#start-with-your-workload} @@ -72,13 +72,13 @@ feature engineering, search and retrieval, and efficient training data access. -## From local development to production scale +## From local development to production scale {#from-local-development-to-production-scale} LanceDB OSS and LanceDB Enterprise share the same Lance format and table model. Start locally with the embedded OSS library, then move to Enterprise when your team needs distributed scale, managed infrastructure, private deployment, or higher-throughput curation, feature engineering, search and retrieval, and training workflows. -### 1. LanceDB OSS +### 1\. LanceDB OSS {#1-lancedb-oss} The fastest way to get started is the open-source embedded library, with client SDKs in Python, TypeScript and Rust. Run it locally in just a few steps, which lets you explore datasets, curate data, and run search and retrieval workloads for agents. Start here: @@ -100,7 +100,7 @@ for agents. Start here: -### 2. LanceDB Enterprise +### 2\. LanceDB Enterprise {#2-lancedb-enterprise} [LanceDB Enterprise](/enterprise) is a petabyte-scale (and beyond), distributed **multimodal lakehouse** platform built for search, curation, feature engineering, and high-throughput training data access workflows on top of the same core table diff --git a/docs/indexing/fts-index.mdx b/docs/indexing/fts-index.mdx index 400e07f..2d62ad8 100644 --- a/docs/indexing/fts-index.mdx +++ b/docs/indexing/fts-index.mdx @@ -13,9 +13,9 @@ examples on how to create and configure FTS indexes in LanceDB OSS and Enterpris In LanceDB Enterprise, `create_fts_index` API returns immediately, but index building happens asynchronously. -## Creating FTS Indexes +## Creating FTS Indexes {#creating-fts-indexes} -### Synchronous API +### Synchronous API {#synchronous-api} Use `create_fts_index` with synchronous LanceDB connections: @@ -35,7 +35,7 @@ Check FTS index status using the API: `wait_for_index(...)` waits until the named FTS index exists and `index_stats(...)` reports `num_unindexed_rows == 0`. It can time out if writes keep adding rows faster than the index catches up. If a table has multiple FTS indexes, specify the target text column when querying instead of relying on implicit selection. -### Asynchronous API +### Asynchronous API {#asynchronous-api} When using async connections (`connect_async`), use `create_index` with the `FTS` configuration: @@ -52,7 +52,7 @@ The `create_fts_index` method is not available on `AsyncTable`. Use `create_inde The current FTS implementation is Lance-native. Legacy Tantivy-only options, including `use_tantivy`, are no longer accepted by the index creation APIs. -## Nested field paths +## Nested field paths {#nested-field-paths} FTS indexes can target text leaves inside struct columns by passing a dotted path (for example, `payload.text`). The same path works for [`MatchQuery`](/search/full-text-search) and [`PhraseQuery`](/search/full-text-search), and for the `columns` argument on async `nearest_to_text` queries. @@ -78,9 +78,9 @@ from lancedb.index import FTS await async_table.create_index("payload.text", config=FTS(with_position=True)) ``` -## Configuration Options +## Configuration Options {#configuration-options} -### FTS Parameters +### FTS Parameters {#fts-parameters} | Parameter | Type | Default | Description | |:----------|:-----|:--------|:------------| @@ -104,7 +104,7 @@ await async_table.create_index("payload.text", config=FTS(with_position=True)) - `ascii_folding` helps with international text (e.g., “café” → “cafe”). -### Tokenizer choices +### Tokenizer choices {#tokenizer-choices} `base_tokenizer` controls segmentation before token filters run: @@ -118,7 +118,7 @@ Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ `language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`. -### Posting block size +### Posting block size {#posting-block-size} `block_size` controls the number of documents packed into each compressed posting block on disk. The default of `128` matches the current FTS layout and is the right choice for most workloads. Setting it to `256` opts in to the experimental FTS V3 format, which changes how postings are encoded and may introduce breaking changes in future releases. Any other value is rejected at index creation time. @@ -136,7 +136,7 @@ await table.createIndex("text", { }); ``` -### Phrase Query Configuration +### Phrase Query Configuration {#phrase-query-configuration} Enable phrase queries by setting: @@ -145,7 +145,7 @@ Enable phrase queries by setting: | `with_position` | `True` | Track token positions for phrase matching | | `remove_stop_words` | `False` | Preserve stop words for exact phrase matching | -## Indexing nested string fields +## Indexing nested string fields {#indexing-nested-string-fields} You can build an FTS index on a string field inside a struct by passing its full dotted path, like `nested.text`. The same path is used when you query the index through `fts_columns`, and the indexed column is reported back as the full path from `list_indices()`. diff --git a/docs/indexing/gpu-indexing.mdx b/docs/indexing/gpu-indexing.mdx index 6702510..60b5536 100644 --- a/docs/indexing/gpu-indexing.mdx +++ b/docs/indexing/gpu-indexing.mdx @@ -32,7 +32,7 @@ GPU acceleration changes how the vector index is built, not the lifecycle after appended later still need `optimize()` before they are part of the index, and `fast_search()` only searches indexed rows. -## Manual GPU indexing in LanceDB OSS +## Manual GPU indexing in LanceDB OSS {#manual-gpu-indexing-in-lancedb-oss} You can use the Python SDK to manually create the `IVF_PQ` index on a GPU. You'll need [PyTorch>2.0](https://pytorch.org/). Note that GPU-based indexing is currently only @@ -41,7 +41,7 @@ supported by the synchronous SDK in LanceDB OSS. Specify the values `cuda` or `mps` (on Apple Silicon) for the `accelerator` parameter to enable GPU training on your device. -### GPU indexing on Linux +### GPU indexing on Linux {#gpu-indexing-on-linux} @@ -49,7 +49,7 @@ to enable GPU training on your device. -### GPU indexing on macOS (Apple Silicon) +### GPU indexing on macOS (Apple Silicon) {#gpu-indexing-on-macos-apple-silicon} @@ -57,14 +57,14 @@ to enable GPU training on your device. -## Performance considerations +## Performance considerations {#performance-considerations} - GPU memory usage scales with `num_partitions` and vector dimensions - For optimal performance, ensure GPU memory exceeds dataset size - Batch size is automatically tuned based on available GPU memory - Indexing speed improves with larger batch sizes -## Troubleshooting +## Troubleshooting {#troubleshooting} If you encounter the error `AssertionError: Torch not compiled with CUDA enabled`, you need to [install PyTorch with CUDA support](https://pytorch.org/get-started/locally/). diff --git a/docs/indexing/index.mdx b/docs/indexing/index.mdx index 7977b8e..d3b123f 100644 --- a/docs/indexing/index.mdx +++ b/docs/indexing/index.mdx @@ -20,7 +20,7 @@ Scalar indices serve as a foundational optimization layer, accelerating filterin - Key-value lookups (enabling rapid primary key-based retrievals) -## Supported Index Types +## Supported Index Types {#supported-index-types} LanceDB provides a comprehensive suite of indexing strategies for different data types and use cases: @@ -48,7 +48,7 @@ By default, automatic vector indexing creates `IVF_PQ`, and scalar index creatio columns, not list columns; use `LabelList` for list containment filters. -### Quantization Types +### Quantization Types {#quantization-types} Vector indexes can use different quantization methods to compress vectors and improve search performance: @@ -59,19 +59,19 @@ Vector indexes can use different quantization methods to compress vectors and im | `RQ` (RabitQ Quantization) | Use when you need maximum compression or have specific per-dimension requirements. | Per-dimension quantization using a RabitQ codebook. Provides fine-grained control over compression per dimension. For `IVF_RQ`, vector dimensions must be divisible by `8`. | | `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization—stores raw vectors. Provides the highest accuracy but requires more storage and memory. | -## Understanding the IVF-PQ Index +## Understanding the IVF-PQ Index {#understanding-the-ivf-pq-index} An ANN (Approximate Nearest Neighbors) index is a data structure that quickly produces an approximate solution to the **k-nearest neighbors (kNN)** problem. It greatly improves upon the runtime of a brute-force kNN search, while admitting a slight decrease in accuracy. LanceDB uses the disk-based indexing technique IVF-PQ, discussed below. LanceDB differs from other vector databases in that it is built on top of [Lance](https://github.com/lancedb/lance), an open-source columnar data format designed for performant ML workloads and fast random access. Due to the design of Lance, LanceDB's indexing philosophy adopts a primarily *disk-based* indexing philosophy. -## IVF-PQ +## IVF-PQ {#ivf-pq} LanceDB uses **IVF-PQ** indexing, which combines the clustering-based **Inverted File Index (IVF)** with **Product Quantization (PQ)** to efficiently compress embeddings. The implementation provides several parameters to fine-tune the index's size, query throughput, latency, and recall. -### Product Quantization +### Product Quantization {#product-quantization} Quantization is a compression technique used to reduce the dimensionality of an embedding to speed up search. @@ -90,7 +90,7 @@ Quantized: `4 × 8 = 32` bits Quantization results in a **128x** reduction in memory requirements for each vector in the index, which is substantial. -### Inverted File Index (IVF) Implementation +### Inverted File Index (IVF) Implementation {#inverted-file-index-ivf-implementation} While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space. @@ -101,11 +101,11 @@ In IVF, the PQ vector space is divided into *Voronoi cells*, which are essential During query time, depending on where the query lands in vector space, it may be close to the border of multiple Voronoi cells, which could make the top-k results ambiguous and span across multiple cells. To address this, the IVF-PQ introduces the `nprobe` parameter, which controls the number of Voronoi cells to search during a query. The higher the `nprobe`, the more accurate the results, but the slower the query. ![](/static/assets/images/indexing/ivfpq_query_vector.webp) -## HNSW Index Implementation +## HNSW Index Implementation {#hnsw-index-implementation} Approximate Nearest Neighbor (ANN) search is a method for finding data points near a given point in a dataset, though not always the exact nearest one. HNSW is one of the most accurate and fastest Approximate Nearest Neighbour search algorithms, It's beneficial in high-dimensional spaces where finding the same nearest neighbor would be too slow and costly. -### Types of ANN Search Algorithms +### Types of ANN Search Algorithms {#types-of-ann-search-algorithms} Approximate Nearest Neighbor (ANN) search is a method for finding data points near a given point in a dataset, though not always the exact nearest one. For example, HNSW is an ANN index that performs well in high-dimensional spaces where other techniques prove too slow and costly. @@ -119,7 +119,7 @@ There are three main types of ANN search algorithms: HNSW is a graph-based algorithm. All graph-based search algorithms rely on the idea of a k-nearest neighbor (or k-approximate nearest neighbor) graph, which we outline below. HNSW also combines this with the ideas behind a classic 1-dimensional search data structure: the skip list. -### Understanding k-Nearest Neighbor Graphs +### Understanding k-Nearest Neighbor Graphs {#understanding-k-nearest-neighbor-graphs} The k-nearest neighbor graph actually predates its use for ANN search. Its construction is quite simple: @@ -141,7 +141,7 @@ That is, if you start with a k-ANN graph for n-1 vertices, you can extend it to One downside of k-NN and k-ANN graphs alone is that one must typically build them with a large value of k to get decent results, resulting in a large index. -### Hierarchical Navigable Small Worlds (HNSW) +### Hierarchical Navigable Small Worlds (HNSW) {#hierarchical-navigable-small-worlds-hnsw} HNSW builds on k-ANN in two main ways: diff --git a/docs/indexing/quantization.mdx b/docs/indexing/quantization.mdx index c0d5156..8acc1fc 100644 --- a/docs/indexing/quantization.mdx +++ b/docs/indexing/quantization.mdx @@ -24,11 +24,11 @@ Two axes are being combined here: whether partitions are searched flatly or via Use the same distance metric when training the index and running queries against it. For IVF-based indexes, `num_partitions` controls the number of groups and `sample_rate` controls how many training vectors are sampled per partition, so the training sample is roughly `sample_rate * num_partitions`. -## RaBitQ quantization +## RaBitQ quantization {#rabitq-quantization} RaBitQ is a binary quantization method that represents each normalized embedding using **1 bit per dimension**, plus a couple of small corrective scalars. In practice, a 1,024-dimensional `float32` vector that would normally take 4 KB can be compressed to roughly a few hundred bytes with RaBitQ, while still maintaining reasonable recall. -### How RaBitQ works +### How RaBitQ works {#how-rabitq-works} - Embeddings are grouped around centroids (as in other IVF indexes). - Each residual vector is normalized and mapped to the nearest vertex of a randomly rotated hypercube on the unit sphere. @@ -44,7 +44,7 @@ Compared to `IVF_PQ`, RaBitQ: For a deeper dive into the theory and some benchmark results, see the blog post: [LanceDB's RaBitQ Quantization for Blazing Fast Vector Search](https://lancedb.com/blog/feature-rabitq-quantization/). -### Using RaBitQ +### Using RaBitQ {#using-rabitq} You can create an RaBitQ-backed vector index by setting `index_type="IVF_RQ"` when calling `create_index`. @@ -61,7 +61,7 @@ It's also possible to tune the number of IVF partitions in `IVF_RQ`, similar to Indexes built with `num_bits >= 2` use an updated on-disk layout. Older LanceDB versions cannot read them and will fail with a clear missing-column error rather than returning incorrect results. Existing indexes keep working and upgrade automatically when they are rewritten (for example, during compaction, optimize, or remap). `num_bits=1` indexes are unaffected in both directions. -## API Reference +## API Reference {#api-reference} The full list of parameters to the algorithm are listed below. diff --git a/docs/indexing/reindexing.mdx b/docs/indexing/reindexing.mdx index b90516a..2a1cabc 100644 --- a/docs/indexing/reindexing.mdx +++ b/docs/indexing/reindexing.mdx @@ -14,7 +14,7 @@ As data is being added and a reindex operation is running, LanceDB will combine Rather than dropping an existing index entirely and reindexing from scratch, LanceDB supports **incremental indexing**. -## Incremental Reindexing +## Incremental Reindexing {#incremental-reindexing} You can manually trigger an incremental indexing operation on updated data using the `optimize()` method on a table. @@ -42,7 +42,7 @@ The benefit of using LanceDB Enterprise is that it automates the reindexing proc and operates continuously in the background, minimizing the impact on latency under high loads. In OSS, you must manually manage the reindexing cadence based on your data growth and performance needs. -## Disk utilization +## Disk utilization {#disk-utilization} Compaction by itself does not immediately free disk space, and can temporarily increase it because new compacted files are written before old-version files are deleted. Disk space is reclaimed when old versions diff --git a/docs/indexing/scalar-index.mdx b/docs/indexing/scalar-index.mdx index 7e71459..68bffdd 100644 --- a/docs/indexing/scalar-index.mdx +++ b/docs/indexing/scalar-index.mdx @@ -27,7 +27,7 @@ LanceDB supports four types of scalar indexes: - `LABEL_LIST`: Special index for `List` and `LargeList` columns of primitive values supporting `array_contains_all` and `array_contains_any` queries. - `FM`: FM-Index over string or binary columns that accelerates substring search via `contains(col, 'needle')`. -## Choosing the Right Index Type +## Choosing the Right Index Type {#choosing-the-right-index-type} | Data Type | Filter | Index Type | |:----------------------------------------------------------------|:------------------------------------------|:-------------| @@ -36,9 +36,9 @@ LanceDB supports four types of scalar indexes: | List of low cardinality of numbers or strings | `array_has_any`, `array_has_all` | `LABEL_LIST` | | String or binary (`Utf8`, `LargeUtf8`, `Binary`, `LargeBinary`) | `contains(col, 'needle')` | `FM` | -## Scalar Index Operations +## Scalar Index Operations {#scalar-index-operations} -### 1. Build the Index +### 1\. Build the Index {#1-build-the-index} You can create multiple scalar indexes within a table. By default, the index will be `BTREE`, but you can always configure another type like `BITMAP` @@ -52,7 +52,7 @@ You can create multiple scalar indexes within a table. By default, the index wil If you are using LanceDB Enterprise, the `create_scalar_index` API returns immediately, but the building of the scalar index is asynchronous. To wait until all data is fully indexed, you can specify the `wait_timeout` parameter on `create_scalar_index()` or call `wait_for_index()` on the table. -### 2. Check Index Status +### 2\. Check Index Status {#2-check-index-status} @@ -62,7 +62,7 @@ If you are using LanceDB Enterprise, the `create_scalar_index` API returns immed `wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. -### 3. Update the Index +### 3\. Update the Index {#3-update-the-index} Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index. @@ -76,7 +76,7 @@ Updating the table data (adding, deleting, or modifying records) requires that y New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise automates the optimize process, minimizing the impact on search speed. -### 4. Run Indexed Searches +### 4\. Run Indexed Searches {#4-run-indexed-searches} The following scan will be faster if the column `book_id` has a scalar index: @@ -94,7 +94,7 @@ Scalar indexes can also speed up scans containing a vector search or full text s -## Indexing nested fields +## Indexing nested fields {#indexing-nested-fields} Scalar indexes can target a scalar field inside a struct by passing its full dotted path. The path is preserved end to end: it's the value you pass to `create_scalar_index`, it's what `list_indices()` reports under `columns`, and it's the column reference you use in filter predicates. @@ -110,7 +110,7 @@ table.search().where("metadata.user_id = 42").limit(1).to_list() Nested paths follow Lance field-path semantics: dot-separate each struct field from root to leaf (for example, `metadata.author.name`). The same convention applies to FTS and vector indexes. -## FM-Index for substring search +## FM-Index for substring search {#fm-index-for-substring-search} The `FM` index is a scalar index built over string or binary columns that accelerates substring lookups expressed as `contains(col, 'needle')`. Unlike the @@ -129,7 +129,7 @@ Use the FM-Index when: Pick `FTS` instead when you need word-level relevance ranking, phrase queries, or language-aware tokenization. -### Create an FM-Index +### Create an FM-Index {#create-an-fm-index} Build an FM-Index with the async `create_index` API by passing the `Fm` config in Python or `Index.fm()` in TypeScript. In Rust, use `Index::Fm(FmIndexBuilder::default())`. @@ -166,7 +166,7 @@ table.search().where("contains(text, 'needle')").limit(10).to_pandas() `list_indices()` reports the index type as `"Fm"`. -## Index UUID Columns +## Index UUID Columns {#index-uuid-columns} LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)`), enabling efficient lookups and filtering on UUID-based primary keys. @@ -177,7 +177,7 @@ LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)` - TypeScript SDK version `0.19.0` or later -### 1. Define UUID Type +### 1\. Define UUID Type {#1-define-uuid-type} @@ -185,7 +185,7 @@ LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)` -### 2. Generate UUID Data +### 2\. Generate UUID Data {#2-generate-uuid-data} @@ -193,7 +193,7 @@ LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)` -### 3. Create Table with UUID Column +### 3\. Create Table with UUID Column {#3-create-table-with-uuid-column} @@ -201,7 +201,7 @@ LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)` -### 4. Create and Wait for the Index +### 4\. Create and Wait for the Index {#4-create-and-wait-for-the-index} @@ -209,7 +209,7 @@ LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)` -### 5. Perform Operations with the UUID Index +### 5\. Perform Operations with the UUID Index {#5-perform-operations-with-the-uuid-index} @@ -217,7 +217,7 @@ LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)` -## Index nested fields +## Index nested fields {#index-nested-fields} You can build a scalar index on a field inside a struct column by passing the canonical dot-separated path to `create_index`. This is useful when filters diff --git a/docs/indexing/vector-index.mdx b/docs/indexing/vector-index.mdx index 359a6d2..febe764 100644 --- a/docs/indexing/vector-index.mdx +++ b/docs/indexing/vector-index.mdx @@ -32,11 +32,11 @@ You can create and manage multiple vector indexes on any Lance dataset. LanceDB In LanceDB, HNSW is not exposed as a top-level vector index. Instead, it's available as a sub-index inside IVF partitions. What this means in practice is that vectors are first partitioned by IVF, then each selected partition is searched using an HNSW graph. LanceDB supports the unquantized variant `IVF_HNSW_FLAT`, along with quantized variants such as `IVF_HNSW_PQ` and `IVF_HNSW_SQ`. This combines IVF's scalability with HNSW's higher-recall ANN search within partitions. -### Manual Indexing +### Manual Indexing {#manual-indexing} If using LanceDB OSS, you will have to create the vector index manually, by calling `table.create_index()`, and updating the index as new data arrives and tuning its parameters is also a manual process. -### Automatic Indexing +### Automatic Indexing {#automatic-indexing} Enterprise-only Vector indexing is managed **automatically** in LanceDB Enterprise. As soon as data is updated, the system updates the index and optimizates it. *This is done asynchronously as a background process*. @@ -60,7 +60,7 @@ Rows appended after an index build remain outside that index until optimization search still checks those unindexed rows with a slower fallback path; `fast_search()` skips that fallback and searches only indexed rows. -## Choose the Right Index +## Choose the Right Index {#choose-the-right-index} Use this table as a quick starting point for choosing the right index type and quantization method for your use case: @@ -78,7 +78,7 @@ If your vector search frequently includes metadata filters (`where(...)`), prefe Compression ratios are practical rules of thumb and can vary with vector distribution, metric, and configuration. For small dimensions, choose `IVF_PQ` for accuracy, not for guaranteed higher compression than `IVF_RQ`. -### Index Tuning +### Index Tuning {#index-tuning} Start with these values, then tune for your workload: @@ -93,13 +93,13 @@ Start with these values, then tune for your workload: - `num_sub_vectors`: start at `dimension // 8`. Increase for better recall, decrease for faster search and smaller indexes. - For small dimensions (`dimension <= 256`), `IVF_PQ` is often preferred over `IVF_RQ` for better accuracy at similar query performance. -## Example: Construct an IVF Index +## Example: Construct an IVF Index {#example-construct-an-ivf-index} In this example, we will create an index for a table containing 1536-dimensional vectors. The index will use IVF_PQ with L2 distance, which is well-suited for high-dimensional vector search. Make sure you have enough data in your table (at least a few thousand rows) for effective index training. -### Index Configuration +### Index Configuration {#index-configuration} Sometimes you need to configure the index beyond default parameters: @@ -127,7 +127,7 @@ Let's take a look at a sample request for an IVF index: -### 1. Setup +### 1\. Setup {#1-setup} Connect to LanceDB and open the table you want to index. @@ -137,7 +137,7 @@ Connect to LanceDB and open the table you want to index. -### 2. Construct an IVF Index +### 2\. Construct an IVF Index {#2-construct-an-ivf-index} Create an `IVF_PQ` index with `cosine` similarity. Specify `vector_column_name` if you use multiple vector columns or non-default names. For a vector field nested inside a struct, use dot notation (e.g. `image.embedding`); see [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full syntax. You can switch `index_type` to `IVF_RQ`, `IVF_HNSW_SQ`, or `IVF_HNSW_FLAT` depending on your recall/latency/compression target. @@ -147,7 +147,7 @@ Create an `IVF_PQ` index with `cosine` similarity. Specify `vector_column_name` -#### Indexing nested vector fields +#### Indexing nested vector fields {#indexing-nested-vector-fields} If your vector column lives inside a struct, pass its full dotted path as `vector_column_name`. The same path is used at query time and is what `list_indices()` reports under `columns`: @@ -161,7 +161,7 @@ If your vector column lives inside a struct, pass its full dotted path as `vecto Nested paths follow Lance field-path semantics: dot-separate each struct field from root to leaf (for example, `image.thumbnail.embedding`). The same convention applies to FTS and scalar indexes. -### Async API and Config Objects +### Async API and Config Objects {#async-api-and-config-objects} With asynchronous Python connections, create vector indexes with `await table.create_index("vector", config=...)`. The `config` object carries the same index choices you configure in the synchronous API, such as distance metric, partition count, and quantization settings: @@ -183,7 +183,7 @@ Use these Python config classes for the index types shown on this page: | `IVF_HNSW_PQ` | `IvfHnswPq` | | `IVF_HNSW_SQ` | `IvfHnswSq` | -### 3. Query the IVF Index +### 3\. Query the IVF Index {#3-query-the-ivf-index} Search using a random 1,536-dimensional embedding. @@ -193,7 +193,7 @@ Search using a random 1,536-dimensional embedding. -#### Search Configuration +#### Search Configuration {#search-configuration} Core knobs available on a vector search call: @@ -224,7 +224,7 @@ Recommended `nprobes` behavior by index type: | `IVF_RQ` | Keep auto-tuned `nprobes`; raise only when recall is insufficient. | | `IVF_PQ` | Keep auto-tuned `nprobes`; raise when recall is insufficient. Often preferred over `IVF_RQ` when `dimension <= 256`. | -#### Advanced Search Controls +#### Advanced Search Controls {#advanced-search-controls} These controls are useful for thresholded retrieval, recall measurement, and working around index-level metric constraints. @@ -259,9 +259,9 @@ Flat search is $O(n)$ — reserve `bypass_vector_index()` for sampled recall mea Multivector indexing currently requires `distance_type="cosine"` — `l2` is rejected at index-creation time. That restriction is why `bypass_vector_index()` is the escape hatch for non-cosine queries on a multivector column: the metric you want at query time cannot be served by the index, so you fall back to a flat scan. See [Multivector Search](/search/multivector-search) for the full rules. -## Example: Construct an HNSW Index +## Example: Construct an HNSW Index {#example-construct-an-hnsw-index} -### Index Configuration +### Index Configuration {#index-configuration-2} There are four key parameters to set when constructing an HNSW index: @@ -270,7 +270,7 @@ There are four key parameters to set when constructing an HNSW index: - `m`: The number of neighbors to select for each vector in the HNSW graph. - `ef_construction`: The number of candidates to evaluate during the construction of the HNSW graph. -### 1. Construct an HNSW Index +### 1\. Construct an HNSW Index {#1-construct-an-hnsw-index} The snippet below uses `IVF_HNSW_SQ`. If you want the unquantized variant, change `index_type` to `IVF_HNSW_FLAT`. @@ -280,7 +280,7 @@ The snippet below uses `IVF_HNSW_SQ`. If you want the unquantized variant, chang -### 2. Query the HNSW Index +### 2\. Query the HNSW Index {#2-query-the-hnsw-index} @@ -288,11 +288,11 @@ The snippet below uses `IVF_HNSW_SQ`. If you want the unquantized variant, chang -## Example: Construct a Binary Vector Index +## Example: Construct a Binary Vector Index {#example-construct-a-binary-vector-index} Binary vectors are useful for hash-based retrieval, fingerprinting, or any scenario where data can be represented as bits. -### Index Configuration +### Index Configuration {#index-configuration-3} - Store binary vectors as fixed-size binary data (uint8 arrays, with 8 bits per byte). For storage, pack binary vectors into bytes to save space. - Index Type: `IVF_FLAT` is used for indexing binary vectors @@ -306,7 +306,7 @@ Binary vectors are useful for hash-based retrieval, fingerprinting, or any scena - Quantized index types (`IVF_PQ`, `IVF_RQ`, `IVF_SQ`, `IVF_HNSW_PQ`, `IVF_HNSW_SQ`) do not accept binary inputs — their `distance_type` is restricted to `l2`, `cosine`, or `dot`. -### 1. Create Table and Schema +### 1\. Create Table and Schema {#1-create-table-and-schema} @@ -314,7 +314,7 @@ Binary vectors are useful for hash-based retrieval, fingerprinting, or any scena -### 2. Generate and Add Data +### 2\. Generate and Add Data {#2-generate-and-add-data} @@ -322,7 +322,7 @@ Binary vectors are useful for hash-based retrieval, fingerprinting, or any scena -### 3. Construct the Binary Index +### 3\. Construct the Binary Index {#3-construct-the-binary-index} @@ -330,7 +330,7 @@ Binary vectors are useful for hash-based retrieval, fingerprinting, or any scena -### 4. Vector Search +### 4\. Vector Search {#4-vector-search} @@ -338,7 +338,7 @@ Binary vectors are useful for hash-based retrieval, fingerprinting, or any scena -## Check Index Status +## Check Index Status {#check-index-status} Vector index creation runs in the background and may take some time to complete. While it is ongoing, you can check its status either programmatically through the API or from the **LanceDB Enterprise UI**. @@ -368,7 +368,7 @@ These fields are populated for local and embedded tables. On LanceDB Enterprise -## Custom Index Names +## Custom Index Names {#custom-index-names} The `{column}_idx` suffix is a default convention, not the only supported naming path. Pass `name=...` to `create_index()` to override it — useful when you want to manage multiple indexes on the same column (for example, side-by-side `IVF_PQ` and `IVF_HNSW_SQ` builds) or when you script index replacement by name. Once set, `list_indices()`, `index_stats(name)`, and `wait_for_index([name])` all reference the custom name. diff --git a/docs/integrations/ai/agno.mdx b/docs/integrations/ai/agno.mdx index 3537dbc..b5edef0 100644 --- a/docs/integrations/ai/agno.mdx +++ b/docs/integrations/ai/agno.mdx @@ -21,7 +21,7 @@ We'll walk through the steps below to build a YouTube transcript-aware Agno assi - Retrieve context during responses with hybrid search - Ask questions about the video content in a CLI chat loop -## Prerequisites +## Prerequisites {#prerequisites} Install dependencies: @@ -35,7 +35,7 @@ uv add agno openai lancedb youtube-transcript-api beautifulsoup4 ``` -## Step 1: Configure LanceDB-backed knowledge +## Step 1: Configure LanceDB-backed knowledge {#step-1-configure-lancedb-backed-knowledge} First, you can initialize the core `Knowledge` object that your agent will use for retrieval. It configures LanceDB as the vector store, enables hybrid search with native LanceDB FTS, and sets the embedding model. @@ -44,7 +44,7 @@ It configures LanceDB as the vector store, enables hybrid search with native Lan {PyFrameworksAgnoSetup} -## Step 2: Fetch and ingest the YouTube transcript +## Step 2: Fetch and ingest the YouTube transcript {#step-2-fetch-and-ingest-the-youtube-transcript} Next, extract a YouTube video ID, fetch the full transcript, and flatten it into text for indexing. The snippet shown below then inserts that transcript text into the Agno knowledge base, which writes vectors and metadata to LanceDB. @@ -57,7 +57,7 @@ The snippet shown below then inserts that transcript text into the Agno knowledg This path explicitly fetches the transcript first, then inserts transcript text into LanceDB through Agno. -## Step 3: Create an Agno agent with knowledge search +## Step 3: Create an Agno agent with knowledge search {#step-3-create-an-agno-agent-with-knowledge-search} The next step is to construct an Agno `Agent` and attach the knowledge base you just populated. With `search_knowledge=True`, the agent performs retrieval before answering, so responses stay grounded in transcript context. @@ -69,7 +69,7 @@ When `search_knowledge=True`, Agno makes a knowledge-search tool (shown in outpu {PyFrameworksAgnoAgent} -## Step 4: Start a CLI chat loop +## Step 4: Start a CLI chat loop {#step-4-start-a-cli-chat-loop} You can now ask an initial question and then start an interactive loop for follow-up queries. Each prompt runs through the same retrieval pipeline, so you can iteratively inspect what the transcript contains. @@ -82,7 +82,7 @@ Each prompt runs through the same retrieval pipeline, so you can iteratively ins Want local-first inference? Replace OpenAI model/embedder classes with Agno's Ollama providers. See Agno's Ollama knowledge examples: [docs.agno.com/examples/models/ollama/chat/knowledge](https://docs.agno.com/examples/models/ollama/chat/knowledge). -### Question 1 +### Question 1 {#question-1} The following question is asked in the CLI chat loop: ``` @@ -109,7 +109,7 @@ The following question is asked in the CLI chat loop: We get the response based on the transcript's contents as expected. -### Question 2 +### Question 2 {#question-2} Let's ask a more specific question about the CEO of LanceDB, which is also in the transcript: @@ -137,7 +137,7 @@ INFO Found 10 documents We get the response based on the transcript's contents and title as expected. -## Why this works well +## Why this works well {#why-this-works-well} To start, LanceDB OSS can run from a local directory, so transcript data can stay on your machine when you are using the OSS stack. diff --git a/docs/integrations/ai/genkit.mdx b/docs/integrations/ai/genkit.mdx index d72c4bc..6314842 100644 --- a/docs/integrations/ai/genkit.mdx +++ b/docs/integrations/ai/genkit.mdx @@ -10,18 +10,18 @@ import { TsFrameworksGenkitUsage, } from '/snippets/integrations.mdx'; -### genkitx-lancedb +### genkitx-lancedb {#genkitx-lancedb} Genkit is an open-source framework for building end-to-end AI and RAG pipelines with a clean, TypeScript-first developer experience. The genkitx-lancedb plugin lets you use LanceDB as a high-performance vector store inside your Genkit flows, so you can index, search, and retrieve data efficiently as part of your AI applications. -### Installation +### Installation {#installation} ```bash pnpm install genkitx-lancedb ``` -### Usage +### Usage {#usage} Adding LanceDB plugin to your genkit instance. @@ -46,11 +46,11 @@ On running this query, you'll get 5 results fetched from the lancedb table, wher -## Creating a custom RAG flow +## Creating a custom RAG flow {#creating-a-custom-rag-flow} Now that we've seen how you can use LanceDB in a Genkit pipeline, let's refine the flow and create a RAG. A RAG flow will consist of an index and a retriever with its outputs postprocessed and fed into an LLM for final response -### Creating custom indexer flows +### Creating custom indexer flows {#creating-custom-indexer-flows} You can also create custom indexer flows, utilizing more options and features provided by LanceDB. @@ -63,7 +63,7 @@ In your console, you can see the logs Screenshot 2025-05-11 at 7 19 14 PM -### Creating custom retriever flows +### Creating custom retriever flows {#creating-custom-retriever-flows} You can also create custom retriever flows, utilizing more options and features provided by LanceDB. {TsFrameworksGenkitCustomRetriever} diff --git a/docs/integrations/ai/hermes-agent.mdx b/docs/integrations/ai/hermes-agent.mdx index 3e6b4c0..a5521a9 100644 --- a/docs/integrations/ai/hermes-agent.mdx +++ b/docs/integrations/ai/hermes-agent.mdx @@ -22,7 +22,7 @@ process, storing a single LanceDB table on local disk. There's no memory server - LanceDB manages the durable long-term memory and offers semantic recall. -## Why LanceDB fits agent memory +## Why LanceDB fits agent memory {#why-lancedb-fits-agent-memory} Out of the box, Hermes remembers with a small curated notes file frozen into the system prompt, plus lexical (keyword) search over past sessions. Both are useful, but keyword search @@ -42,7 +42,7 @@ LanceDB is an embedded retrieval library, which makes it a natural fit here: - **It scales up** — the same table abstraction carries over to larger LanceDB deployments later, so the local setup is never a dead end. -## Install and activate +## Install and activate {#install-and-activate} Want to try this without touching your existing Hermes setup? Run everything in an isolated @@ -116,7 +116,7 @@ You want to see `Provider: lancedb` with both `installed ✓` and `available ✓ -## The memory tools +## The memory tools {#the-memory-tools} Once activated, the agent has four tools for working with long-term memory: @@ -131,7 +131,7 @@ Beyond these tools, the plugin also captures durable facts from your conversatio automatically — an auxiliary model distills them before context is compressed and again when a session ends, so insights survive even when the raw messages are summarized away. -## Walkthrough +## Walkthrough {#walkthrough} "_Teach it your project preferences_" @@ -139,7 +139,7 @@ Let's make this concrete with the pain we opened on: re-explaining your setup to We'll save a convention once and then prove a brand-new session can recall it. This example will touch all four tools along the way. -### Remember +### Remember {#remember} Ask Hermes to commit a convention to long-term memory. Saying "remember in long-term memory" makes sure it lands in the LanceDB store, which shows up as the `⚡ lancedb_r` (`lancedb_remember`) @@ -155,7 +155,7 @@ line below: Remembered. I've stored that project convention: use uv only, never pip, and always add type hints to Python functions. ``` -### Recall +### Recall {#recall} First, take Hermes' built-in notes out of the picture so recall can *only* come from LanceDB — the two layers run side by side otherwise, and either could answer: @@ -189,7 +189,7 @@ Turn the built-in layer back on for everyday use with `hermes config set memory. LanceDB. -### Read +### Read {#read} You can also ask where a fact came from. Hermes attributes the answer to its stored memory rather than guessing from a file in the repo (under the hood, `lancedb_read` can also return @@ -206,7 +206,7 @@ the exact source messages a fact was distilled from): - "For this project, the user only uses uv for Python package management, never pip, and always adds type hints to Python functions." ``` -### Forget +### Forget {#forget} When a preference changes, ask Hermes to drop the old fact. The tool calls tell the whole story: the two `⚡ lancedb_f` (`lancedb_forget`) lines are it previewing matches and then @@ -229,7 +229,7 @@ deleting, and the trailing `⚡ lancedb_r` is it saving the replacement in the s Remember, recall, read, forget: four small operations that between them cover the entire lifecycle of a durable memory. -## Retrieval modes +## Retrieval modes {#retrieval-modes} Recall ships in `vector` mode by default — pure semantic search, which is what survives the paraphrasing you saw above. If you also need exact name or jargon matching, switch to `hybrid` @@ -253,7 +253,7 @@ plugins: The cross-encoder is the one path that pulls in a local ML stack, so it stays opt-in. It defaults to the compact 17M-parameter [ettin reranker](https://huggingface.co/cross-encoder/ettin-reranker-17m-v1). -## Inspect the store +## Inspect the store {#inspect-the-store} Everything lives in one table named `memories` at `~/.hermes/lancedb/memories.lance`. Because it's a plain LanceDB table, you can open it directly and see exactly what the agent has stored @@ -268,7 +268,7 @@ tbl = db.open_table("memories") print(tbl.to_pandas()[["kind", "category", "content"]].head()) ``` -## Configuration +## Configuration {#configuration} The plugin runs on sensible defaults once activated — you don't have to configure anything. `~/.hermes/config.yaml` is purely for overrides. Two common ones: @@ -302,7 +302,7 @@ the table — the plugin fails loudly on a dimension mismatch rather than silent nothing. Every option is documented in the plugin's [`default_config.yaml`](https://github.com/lancedb/hermes-agent-memory/blob/main/src/default_config.yaml). -## Benchmark +## Benchmark {#benchmark} On [LongMemEval-S](https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned), a long-conversation QA benchmark, LanceDB's semantic recall clearly beat Hermes' built-in lexical @@ -312,7 +312,7 @@ per-question-type breakdown, and a reproducible harness, see the [blog post](https://www.lancedb.com/blog/semantic-memory-for-hermes-agent-with-lancedb) and the [benchmark harness](https://github.com/lancedb/hermes-agent-memory/tree/main/benchmarks). -## Why this works well +## Why this works well {#why-this-works-well} - **It's local-first and embedded.** The LanceDB memory table lives on your disk with no server to run; the plugin installs as a dependency of Hermes' own environment. diff --git a/docs/integrations/ai/huggingface.mdx b/docs/integrations/ai/huggingface.mdx index 58be5d4..62ac7e8 100644 --- a/docs/integrations/ai/huggingface.mdx +++ b/docs/integrations/ai/huggingface.mdx @@ -28,13 +28,13 @@ The LAION table includes multimodal columns such as: - `img_emb` (image embedding vector) - metadata fields such as `url` and `similarity` -## Install dependencies +## Install dependencies {#install-dependencies} ```bash pip install lancedb pillow ``` -## Open the dataset with LanceDB +## Open the dataset with LanceDB {#open-the-dataset-with-lancedb} LanceDB can open the dataset directly from the Hub, without needing to download it first. Note that in LanceDB, you need to specify the table name when opening a Lance dataset, @@ -52,7 +52,7 @@ print(f"Opened table: {table.name}") print(f"Rows: {len(table)}") ``` -## Inspect schema and available indexes +## Inspect schema and available indexes {#inspect-schema-and-available-indexes} ```python print(table.schema) @@ -103,7 +103,7 @@ to Hugging Face. You can download the dataset locally, and build the indexes you for instructions on building different types of indexes with LanceDB. -## Projection scan +## Projection scan {#projection-scan} Run a simple scan by projecting relevant columns to get a feel for the dataset. For example, we can run a search without any filters or input parameters to get a small subset of the data: @@ -135,7 +135,7 @@ We get the first three rows and their metadata printed out, which look like this similarity=0.3362061381340027 ``` -## Scan and filter data +## Scan and filter data {#scan-and-filter-data} Filtered search is a common pattern to narrow down interesting subsets of the data during early exploration. Here's an example: @@ -160,7 +160,7 @@ Baby Blue Fitted Short Sleeve T Shirt 3 https://cdn-img.prettylittlething.com/a/ pattern cutting made easy pdf https://i.pinimg.com/736x/7c/6c/a7/7c6ca7361815a8929b3dd6ad34a03ab9.jpg 384 1045 ``` -## Export image bytes to local files +## Export image bytes to local files {#export-image-bytes-to-local-files} To work with a subset of the data locally, you can export the image bytes from the table and save them as JPEG files. ```python @@ -185,7 +185,7 @@ for i, row in enumerate(sample): You can now preview the images you just exported on your local machine to get a better sense of the data. -## Vector search +## Vector search {#vector-search} You can use LanceDB to run vector search directly on the data on the Hub, **without needing to download the dataset or build your own vector index**. This makes it incredibly easy to explore the dataset and iterate on your search queries @@ -222,7 +222,7 @@ Note that the LAION dataset is known to contain a lot of duplicate images, so yo showing up multiple times in the search results. -## Full-text search +## Full-text search {#full-text-search} Run an FTS search query that uses BM25 ranking on the `caption` column (on which we already have an FTS index): @@ -241,7 +241,7 @@ fts_results = ( | Dog Running in Water | https://static.wixstatic.com/m… | 14.756516 | | Dogs on the run by heidiannemo… | http://ih2.redbubble.net/image… | 14.756516 | -## Download the full dataset +## Download the full dataset {#download-the-full-dataset} You may hit Hugging Face rate limits when streaming large samples from `hf://`, despite using a Hugging Face token. @@ -255,7 +255,7 @@ Here's how to download the entire dataset via the [Hugging Face CLI](https://hug huggingface-cli download lance-format/laion-1m --repo-type dataset --local-dir ./laion-1m ``` -## Upload your own datasets to Hugging Face in Lance format +## Upload your own datasets to Hugging Face in Lance format {#upload-your-own-datasets-to-hugging-face-in-lance-format} This section shows how you can upload your own Lance datasets to the Hugging Face Hub to share with the community. @@ -270,7 +270,7 @@ hf auth login --token "$HF_TOKEN" A typical sequence of steps is given below. -### 1. Upload your local directory to the Hub +### 1\. Upload your local directory to the Hub {#1-upload-your-local-directory-to-the-hub} Upload the full local directory to a specified repository on the Hugging Face Hub. The command below uploads the contents of your local LanceDB directory at `/path/to/your_local_dir` to a new repository named `your_hf_org/repo_name` under your Hugging Face account. @@ -284,7 +284,7 @@ hf upload-large-folder /path/to/your_local_dir your_hf_org/repo_name \ The `upload-large-folder` command is designed for [uploading large datasets](https://huggingface.co/docs/huggingface_hub/en/guides/upload) (potentially terabytes in size) and will handle multipart uploads, retries, and resuming interrupted uploads. -### 2. Inspect dataset versions +### 2\. Inspect dataset versions {#2-inspect-dataset-versions} Because you can query your remote dataset directly from Hugging Face with `hf://` URIs in LanceDB, you can easily inspect the dataset versions and updates on the Hub without needing to download the data locally. This is very useful to keep track of changes to the dataset and iterate on your data collection and curation process. @@ -299,7 +299,7 @@ print(versions) ``` This will print out the list of versions available for the dataset on the Hub, along with their metadata such as creation date and description. -### 3. Add a dataset card +### 3\. Add a dataset card {#3-add-a-dataset-card} The Hub dataset card allows you to communicate the schema and usage of the dataset to other developers. It sits at the repo's root in a file named `README.md` on the Hub. This project keeps the source card text in `HF_DATASET_CARD.md`, so you can publish updates @@ -312,7 +312,7 @@ hf upload lancedb/magical_kingdom HF_DATASET_CARD.md README.md \ --commit-message "Update dataset card" ``` -### 4. Update the dataset +### 4\. Update the dataset {#4-update-the-dataset} Over time, you may want to add new rows (append) or columns (backfill) to your dataset as your needs evolve. You can make the necessary updates to your local dataset using LanceDB, and then upload the updated version back to the Hub with the same `hf upload-large-folder` command. @@ -325,7 +325,7 @@ The CLI will only upload the new data that has changed since the last upload, av That's it! Your dataset is now updated on the Hub with the new data and schema changes, and other users can query the latest version of the dataset directly from Hugging Face with `hf://` URIs in LanceDB. -## Explore more Lance datasets on Hugging Face +## Explore more Lance datasets on Hugging Face {#explore-more-lance-datasets-on-hugging-face} The LanceDB team is actively uploading useful and interesting datasets in Lance format to the Hugging Face Hub under the [lance-format](https://huggingface.co/lance-format) organization. We actively encourage the Hugging Face diff --git a/docs/integrations/ai/kiln.mdx b/docs/integrations/ai/kiln.mdx index a944e6b..c93f31e 100644 --- a/docs/integrations/ai/kiln.mdx +++ b/docs/integrations/ai/kiln.mdx @@ -6,7 +6,7 @@ sidebarTitle: "Kiln" [**Kiln**](https://kiln.tech) is a free tool for building production-ready AI systems, combining an intuitive desktop application and an open-source Python library. It supports RAG pipelines, evaluations, agents, MCP tool-calling, synthetic data generation, and fine-tuning. Kiln provides deep integration with LanceDB for vector search, full-text search (BM25), and hybrid search. -## Quick Start: Build a RAG Pipeline in 5 Minutes with Kiln & LanceDB +## Quick Start: Build a RAG Pipeline in 5 Minutes with Kiln and LanceDB {#quick-start-build-a-rag-pipeline-in-5-minutes-with-kiln-and-lancedb} Watch the [quick start overview on Vimeo](https://vimeo.com/1119945690). @@ -18,7 +18,7 @@ Kiln's [app](https://kiln.tech/download) makes it easy to: - Load your data from Kiln into [LanceDB Enterprise](/enterprise) for production use - Iterate with confidence by evaluating new content, prompts, models, and embeddings in minutes instead of weeks -## Find the Best RAG Pipeline for Your Use Case +## Find the Best RAG Pipeline for Your Use Case {#find-the-best-rag-pipeline-for-your-use-case} There is no universal best RAG solution—only the best solution for your specific use case. Kiln makes it easy to compare state-of-the-art configurations and find which works best for you. @@ -32,13 +32,13 @@ Start with pre-configured templates for state-of-the-art RAG at various performa |Embeddings|Embedding models from Gemini, OpenAI, Nomic, Qwen, and more|Find the embedding model best suited to your use case.| |Chunking|LlamaIndex|Find the ideal chunk size and method.| -## Get Started +## Get Started {#get-started} To get started, download the [Kiln App](https://kiln.tech/download), create a project, and navigate to "Docs & Search". See the [Kiln documentation for creating a RAG system](https://docs.kiln.tech/docs/documents-and-search-rag) for details on each step of the process. -## More Information +## More Information {#more-information} - [Kiln Homepage](https://kiln.tech) - [Download the Kiln App](https://kiln.tech/download) diff --git a/docs/integrations/ai/langchain.mdx b/docs/integrations/ai/langchain.mdx index 21548fa..3633310 100644 --- a/docs/integrations/ai/langchain.mdx +++ b/docs/integrations/ai/langchain.mdx @@ -25,14 +25,14 @@ LangChain streamlines these stages (in figure above) by providing pre-built comp Integration of **Langchain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs. -## Quick Start +## Quick Start {#quick-start} You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. {PyFrameworksLangchainQuickStart} -## Documentation +## Documentation {#documentation} In the above example `LanceDB` vector store class object is created using `from_documents()` method which is a `classmethod` and returns the initialized class object. You can also use `LanceDB.from_texts(texts: List[str],embedding: Embeddings)` class method. @@ -61,7 +61,7 @@ The exhaustive list of parameters for `LanceDB` vector store are : {PyFrameworksLangchainVectorStoreConfig} -### Methods +### Methods {#methods} ##### `add_texts()` diff --git a/docs/integrations/ai/llamaIndex.mdx b/docs/integrations/ai/llamaIndex.mdx index 3fba447..a38e2c6 100644 --- a/docs/integrations/ai/llamaIndex.mdx +++ b/docs/integrations/ai/llamaIndex.mdx @@ -11,7 +11,7 @@ import { PyFrameworksLlamaindexQuickStart, } from '/snippets/integrations.mdx'; -## Quickstart +## Quickstart {#quickstart} LlamaIndex is a well-known framework for building LLM-powered agents over your data with LLMs and workflows. You can build your LlamaIndex pipeline and persist your metadata and embeddings in LanceDB via the `LanceDBVectorStore` class. @@ -30,13 +30,13 @@ Run the below script as an example. The vector store connector will open an existing LanceDB directory or create the directory if it does not exist. -### Filtering +### Filtering {#filtering} For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex: {PyFrameworksLlamaindexFiltering} -### Hybrid Search +### Hybrid Search {#hybrid-search} For complete documentation, refer [here](https://docs.lancedb.com/search/hybrid-search). This example uses the `colbert` reranker. Make sure to install necessary dependencies for the reranker you choose. {PyFrameworksLlamaindexHybridSearch} @@ -45,7 +45,7 @@ For complete documentation, refer [here](https://docs.lancedb.com/search/hybrid- In the snippet above, you can change/specify `query_type` when creating the engine/retriever to use different search strategies, such as vector search or FTS. -## API reference +## API reference {#api-reference} `, so the table path is `lance_ns.main.lance_duck`. -## Write Lance table +## Write Lance table {#write-lance-table} Create the `lance_duck` table using SQL and populate it with sample data: @@ -52,7 +52,7 @@ The examples below show SQL entered in the DuckDB CLI. You can run the same SQL Python as well, using LanceDB and DuckDB's Python clients in your application code. -## Query the table with SQL +## Query the table with SQL {#query-the-table-with-sql} ```sql SQL icon="database" SELECT * @@ -60,7 +60,7 @@ SELECT * LIMIT 5; ``` -## Vector search +## Vector search {#vector-search} ```sql SQL icon="database" SELECT animal, noise, vector, _distance @@ -74,7 +74,7 @@ SELECT animal, noise, vector, _distance ORDER BY _distance ASC; ``` -## Full-text search +## Full-text search {#full-text-search} ```sql SQL icon="database" SELECT animal, noise, vector, _score @@ -88,7 +88,7 @@ SELECT animal, noise, vector, _score ORDER BY _score DESC; ``` -## Hybrid search +## Hybrid search {#hybrid-search} ```sql SQL icon="database" SELECT animal, noise, vector, _hybrid_score, _distance, _score @@ -106,14 +106,14 @@ SELECT animal, noise, vector, _hybrid_score, _distance, _score ORDER BY _hybrid_score DESC; ``` -## Directory namespace model +## Directory namespace model {#directory-namespace-model} A directory namespace maps a LanceDB catalog root to namespace-qualified table identifiers in DuckDB. This keeps table discovery and table naming stable as your project grows. To learn more about the catalog and namespace model, see [Namespaces and the Catalog Model](/namespaces). -## Advanced usage +## Advanced usage {#advanced-usage} See the [docs](https://github.com/lance-format/lance-duckdb) directory in the Lance-DuckDB extension repo for more advanced usage on SQL and REST API clients. diff --git a/docs/integrations/data/pandas_and_pyarrow.mdx b/docs/integrations/data/pandas_and_pyarrow.mdx index 02d4f8d..170ccf7 100644 --- a/docs/integrations/data/pandas_and_pyarrow.mdx +++ b/docs/integrations/data/pandas_and_pyarrow.mdx @@ -15,7 +15,7 @@ Because Lance is built on top of [Apache Arrow](https://arrow.apache.org/), LanceDB fits naturally into Pandas-first workflows. You can ingest a `DataFrame`, query it with LanceDB's vector operators, and keep working in Pandas without any glue code. -## Create a dataset +## Create a dataset {#create-a-dataset} Start by importing LanceDB alongside your usual Pandas utilities and connect to a temporary database. @@ -29,7 +29,7 @@ Use the familiar `pd.DataFrame` API to prepare your rows, then pass the entire f {PyPlatformsPandasCreateTable} -## Vector search +## Vector search {#vector-search} Queries can return Pandas frames as well, so you can immediately inspect the results or pipe them into downstream analytics. @@ -37,7 +37,7 @@ Queries can return Pandas frames as well, so you can immediately inspect the res {PyPlatformsPandasVectorSearch} -## Async API +## Async API {#async-api} For web services or background jobs that already rely on `asyncio`, use the asynchronous helpers to keep everything non-blocking. diff --git a/docs/integrations/data/polars_arrow.mdx b/docs/integrations/data/polars_arrow.mdx index 947d7f2..4cb8573 100644 --- a/docs/integrations/data/polars_arrow.mdx +++ b/docs/integrations/data/polars_arrow.mdx @@ -14,7 +14,7 @@ import { LanceDB supports [Polars](https://github.com/pola-rs/polars), a blazingly fast DataFrame library for Python written in Rust. Under the hood, both Lance and Polars speak Arrow, so passing data back and forth stays zero-copy and ergonomic. -## Create & Query a Table +## Create and Query a Table {#create-and-query-a-table} Import the required libraries, including the optional Pydantic helpers if you plan to define schemas. @@ -34,7 +34,7 @@ Run vector search and keep the results as a Polars `DataFrame` for further proce {PyPlatformsPolarsVectorSearch} -## Work with LazyFrames +## Work with LazyFrames {#work-with-lazyframes} When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. @@ -42,7 +42,7 @@ When you want to operate on the entire table (potentially larger than RAM), conv {PyPlatformsPolarsLazyframe} -## Define Schemas with Pydantic +## Define Schemas with Pydantic {#define-schemas-with-pydantic} You can also describe your table via `LanceModel` and continue ingesting data from Polars. This is useful when multiple teams share a schema or when you want validation. diff --git a/docs/integrations/data/pydantic.mdx b/docs/integrations/data/pydantic.mdx index 6f99d55..b6f80ac 100644 --- a/docs/integrations/data/pydantic.mdx +++ b/docs/integrations/data/pydantic.mdx @@ -43,7 +43,7 @@ Now you can create a table, add data, and perform vector search operations: -## Vector Field +## Vector Field {#vector-field} LanceDB provides a `lancedb.pydantic.Vector` method to define a vector Field in a Pydantic Model. @@ -58,7 +58,7 @@ This example demonstrates how LanceDB automatically converts Pydantic field type - `Vector(768)` becomes `pa.list_(pa.float32(), 768)` (fixed-size list of 768 float32 values) - The `False` parameter indicates that the fields are not nullable -## Type Conversion +## Type Conversion {#type-conversion} LanceDB automatically convert Pydantic fields to [Apache Arrow DataType](https://arrow.apache.org/docs/python/generated/pyarrow.DataType.html#pyarrow.DataType). diff --git a/docs/integrations/data/voxel51.mdx b/docs/integrations/data/voxel51.mdx index cf287c1..1ee7823 100644 --- a/docs/integrations/data/voxel51.mdx +++ b/docs/integrations/data/voxel51.mdx @@ -25,7 +25,7 @@ Any developers, data scientists, and researchers who work with computer vision a Let's get started and see how to use **LanceDB** to create a **similarity index** on your FiftyOne datasets. -## Overview +## Overview {#overview} [Embeddings](/embedding/) are foundational to all of the **vector search** features. In FiftyOne, embeddings are managed by the [**FiftyOne Brain**](https://docs.voxel51.com/user_guide/brain.html) that provides powerful machine learning techniques designed to transform how you curate your data from an art into a measurable science. @@ -50,7 +50,7 @@ We'll be doing the following : - A list of IDs (samples or patches) - A text prompt (search semantically) -## Prerequisites: install necessary dependencies +## Prerequisites: install necessary dependencies {#prerequisites-install-necessary-dependencies} 1. **Create and activate a virtual environment** @@ -77,7 +77,7 @@ pip install fiftyone -## Understand basic workflow +## Understand basic workflow {#understand-basic-workflow} The basic workflow shown below uses LanceDB to create a similarity index on your FiftyOne datasets: @@ -91,7 +91,7 @@ The basic workflow shown below uses LanceDB to create a similarity index on your 5. If desired, delete the table. -## Quick Example +## Quick Example {#quick-example} Let's jump on a quick example that demonstrates this workflow. @@ -135,7 +135,7 @@ This means that you can index an entire Dataset once and then perform searches o -## Using LanceDB backend +## Using LanceDB backend {#using-lancedb-backend} By default, calling `compute_similarity()` or `sort_by_similarity()` will use an sklearn backend. To use the LanceDB backend, simply set the optional `backend` parameter of `compute_similarity()` to `"lancedb"`: @@ -177,7 +177,7 @@ This will override the default `brain_config` and will set it according to your {PyPlatformsVoxel51BrainConfig} -## LanceDB config parameters +## LanceDB config parameters {#lancedb-config-parameters} The LanceDB backend supports query parameters that can be used to customize your similarity queries. These parameters include: diff --git a/docs/integrations/embedding/ibm.mdx b/docs/integrations/embedding/ibm.mdx index 78e2f59..cec6c32 100644 --- a/docs/integrations/embedding/ibm.mdx +++ b/docs/integrations/embedding/ibm.mdx @@ -7,7 +7,7 @@ import { PyEmbeddingIbmUsage } from '/snippets/integrations.mdx'; Generate text embeddings using IBM's watsonx.ai platform. -## Supported Models +## Supported Models {#supported-models} You can find a list of supported models at [IBM watsonx.ai Documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). The currently supported model names are: @@ -23,7 +23,7 @@ You can find a list of supported models at [IBM watsonx.ai Documentation](https: For new tables, `ibm/granite-embedding-278m-multilingual` is the recommended default. Older model IDs (such as `ibm/slate-125m-english-rtrvr` and `sentence-transformers/all-minilm-l12-v2`) remain resolvable for tables whose stored metadata references them, but they are no longer advertised for new use. -## Parameters +## Parameters {#parameters} The following parameters can be passed to the `create` method: @@ -40,7 +40,7 @@ The following parameters can be passed to the `create` method: You must supply exactly one of `project_id` or `space_id` (either as an argument or via its environment variable). Setting both, or neither, raises a `ValueError`. -## Usage Example +## Usage Example {#usage-example} First, the watsonx.ai library is an optional dependency, so must be installed separately: diff --git a/docs/integrations/embedding/imagebind.mdx b/docs/integrations/embedding/imagebind.mdx index 51b3065..4010de6 100644 --- a/docs/integrations/embedding/imagebind.mdx +++ b/docs/integrations/embedding/imagebind.mdx @@ -30,20 +30,20 @@ Below is an example demonstrating how the API works: Now, we can search using any modality: -#### image search +#### image search {#image-search} {PyEmbeddingImagebindImageSearch} -#### audio search +#### audio search {#audio-search} {PyEmbeddingImagebindAudioSearch} -#### Text search +#### Text search {#text-search} You can add any input query and fetch the result as follows: diff --git a/docs/integrations/embedding/jina.mdx b/docs/integrations/embedding/jina.mdx index c22f0ba..7045718 100644 --- a/docs/integrations/embedding/jina.mdx +++ b/docs/integrations/embedding/jina.mdx @@ -8,7 +8,7 @@ import { PyEmbeddingJinaMultimodal, } from '/snippets/integrations.mdx'; -## Text Embedding Models +## Text Embedding Models {#text-embedding-models} Jina embeddings are used to generate embeddings for text and image data. You also need to set the `JINA_API_KEY` environment variable to use the Jina API. @@ -29,7 +29,7 @@ Usage Example: -## Multimodal Embedding Models +## Multimodal Embedding Models {#multimodal-embedding-models} Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list under [https://jina.ai/embeddings/](https://jina.ai/embeddings/) diff --git a/docs/integrations/embedding/superlinked.mdx b/docs/integrations/embedding/superlinked.mdx index 0a483f6..f357962 100644 --- a/docs/integrations/embedding/superlinked.mdx +++ b/docs/integrations/embedding/superlinked.mdx @@ -5,7 +5,7 @@ sidebarTitle: Superlinked [Superlinked](https://superlinked.com) is a self-hosted inference engine (SIE) for embedding, reranking, and extraction. The `sie-lancedb` package registers SIE as a first-class embedding function in LanceDB's embeddings registry, so embeddings are computed automatically on insert and search. You need a running SIE instance - see the [Superlinked quickstart](https://superlinked.com/docs) for deployment options. -## Installation +## Installation {#installation} ```bash Python icon=Python @@ -17,7 +17,7 @@ npm install @superlinked/sie-lancedb @lancedb/lancedb ``` -## Registered functions +## Registered functions {#registered-functions} Importing `sie_lancedb` registers two embedding functions in LanceDB's registry: @@ -33,7 +33,7 @@ Supported parameters on `.create()`: | `model` | `str` | Any of 85+ SIE-supported models (e.g. `BAAI/bge-m3`, `NovaSearch/stella_en_400M_v5`, `jinaai/jina-colbert-v2`) | | `base_url` | `str` | URL of the SIE endpoint (e.g. `http://localhost:8080`) | -## Usage +## Usage {#usage} ```py Python icon=Python import lancedb @@ -64,7 +64,7 @@ results = table.search("What is deep learning?").limit(3).to_list() LanceDB handles embedding generation for both inserts and queries automatically, based on the `SourceField` / `VectorField` declarations on the schema. -## Hybrid search with reranker +## Hybrid search with reranker {#hybrid-search-with-reranker} `SIEReranker` plugs into LanceDB's hybrid search pipeline. It uses SIE's cross-encoder `score()` to rerank combined vector + full-text search results. You need a full-text search index on the column first: @@ -87,8 +87,7 @@ for r in results: The reranker also works with pure vector or pure FTS search via `.rerank()`. -## ColBERT / multivector - +## ColBERT and multivector {#colbert-and-multivector} `SIEMultiVectorEmbeddingFunction` (registered as `"sie-multivector"`) works with LanceDB's native `MultiVector` type and MaxSim scoring for ColBERT and ColPali models: ```py Python icon=Python @@ -110,7 +109,7 @@ table.add([{"text": "Machine learning is a subset of AI."}]) results = table.search("What is ML?").limit(5).to_list() ``` -## Entity extraction +## Entity extraction {#entity-extraction} `SIEExtractor` adds entity extraction to LanceDB's data-enrichment workflows. Extract entities from a text column and merge the results back as a structured Arrow column - enabling filtered search on extracted entities: @@ -133,7 +132,7 @@ extractor.enrich_table( The `entities` column stores structured Arrow data (`list>`), so you can filter on extracted entities in queries. -## Links +## Links {#links} - [`sie-lancedb` on PyPI](https://pypi.org/project/sie-lancedb/) - [`@superlinked/sie-lancedb` on npm](https://www.npmjs.com/package/@superlinked/sie-lancedb) diff --git a/docs/integrations/embedding/voyageai.mdx b/docs/integrations/embedding/voyageai.mdx index 471196c..54d155d 100644 --- a/docs/integrations/embedding/voyageai.mdx +++ b/docs/integrations/embedding/voyageai.mdx @@ -49,7 +49,7 @@ Usage Example: -### Multimodal Example +### Multimodal Example {#multimodal-example} The `voyage-multimodal-3.5` model can embed text alongside images. You can use image URLs, file paths, or PIL Image objects: diff --git a/docs/integrations/lerobotdataset.mdx b/docs/integrations/lerobotdataset.mdx index 5919c02..33c28cd 100644 --- a/docs/integrations/lerobotdataset.mdx +++ b/docs/integrations/lerobotdataset.mdx @@ -17,13 +17,13 @@ import { Lance pairs well with LeRobot when you need high-performance random access, lazy multimodal blob reads, and a single table interface for curation, search, and training data preparation. The `lerobot-lancedb` package ships Lance-backed `LeRobotDataset` subclasses, and LanceDB can open Lance-formatted LeRobot datasets on the Hub directly through `hf://` URIs. -## Install +## Install {#install} ```bash pip install lancedb lance lerobot-lancedb ``` -## Use Lance-backed LeRobotDataset loaders +## Use Lance-backed LeRobotDataset loaders {#use-lance-backed-lerobotdataset-loaders} `LeRobotLanceDataset` is useful when your Lance-backed dataset stores decoded image observations. It's a drop-in replacement for `LeRobotDataset`, so existing policy training code keeps working with the usual PyTorch dataset and dataloader patterns. @@ -41,7 +41,7 @@ For datasets that store camera observations as MP4 video segments, use `LeRobotL Use the image loader for Lance-backed repos that store image frames. Use the video loader for MP4-backed LeRobot datasets such as `lance-format/lerobot-pusht-lance`. -## Open LeRobot Lance tables with LanceDB +## Open LeRobot Lance tables with LanceDB {#open-lerobot-lance-tables-with-lancedb} Lance-formatted LeRobot datasets published by `lance-format` expose each `.lance` file under `data/` as a LanceDB table. The PushT dataset, for example, has `frames`, `episodes`, and `videos` tables. @@ -51,7 +51,7 @@ Lance-formatted LeRobot datasets published by `lance-format` expose each `.lance Opening the tables directly is handy for inspecting schemas, counting rows, sampling metadata, or building curation workflows before any data reaches the training loop. -## Filter a frame window +## Filter a frame window {#filter-a-frame-window} Most robotics workflows want a deterministic slice by `episode_index`, `frame_index`, or task metadata long before training begins. LanceDB filters those rows without touching the video blobs. @@ -61,7 +61,7 @@ Most robotics workflows want a deterministic slice by `episode_index`, `frame_in With the filtered set in hand, you can materialize a smaller local LanceDB database, add derived columns, attach embeddings, or build vector and scalar indexes for faster repeated access. -## Example Lance-formatted LeRobot datasets +## Example Lance-formatted LeRobot datasets {#example-lance-formatted-lerobot-datasets} @@ -72,7 +72,7 @@ With the filtered set in hand, you can materialize a smaller local LanceDB datab -## More resources +## More resources {#more-resources} @@ -83,7 +83,7 @@ With the filtered set in hand, you can materialize a smaller local LanceDB datab -## When to use each interface +## When to use each interface {#when-to-use-each-interface} | Interface | Best for | |:---|:---| diff --git a/docs/integrations/reranking/answerdotai.mdx b/docs/integrations/reranking/answerdotai.mdx index 7e6e982..e3f500d 100644 --- a/docs/integrations/reranking/answerdotai.mdx +++ b/docs/integrations/reranking/answerdotai.mdx @@ -20,8 +20,7 @@ This integration uses [AnswersDotAI's rerankers](https://github.com/AnswerDotAI/ -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_type` | `str` | `"colbert"` | The type of model to use. Supported model types can be found here: https://github.com/AnswerDotAI/rerankers. | @@ -31,22 +30,22 @@ Accepted Arguments -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | -### Vector Search +### Vector Search {#vector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | -### FTS Search +### FTS Search {#fts-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | diff --git a/docs/integrations/reranking/cohere.mdx b/docs/integrations/reranking/cohere.mdx index d0945f7..e484f3e 100644 --- a/docs/integrations/reranking/cohere.mdx +++ b/docs/integrations/reranking/cohere.mdx @@ -24,8 +24,7 @@ pip install cohere -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"rerank-english-v2.0"` | The name of the reranker model to use. Available cohere models are: rerank-english-v2.0, rerank-multilingual-v2.0 | @@ -36,22 +35,22 @@ Accepted Arguments -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column | | `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) | -### Vector Search +### Vector Search {#vector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) | -### FTS Search +### FTS Search {#fts-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column | diff --git a/docs/integrations/reranking/colbert.mdx b/docs/integrations/reranking/colbert.mdx index c8533b8..463b3d7 100644 --- a/docs/integrations/reranking/colbert.mdx +++ b/docs/integrations/reranking/colbert.mdx @@ -19,8 +19,7 @@ This reranker uses ColBERT model to rerank the search results. You can use this -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"colbert-ir/colbertv2.0"` | The name of the reranker model to use.| @@ -29,22 +28,22 @@ Accepted Arguments | `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. | -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | -### Vector Search +### Vector Search {#vector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | -### FTS Search +### FTS Search {#fts-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | diff --git a/docs/integrations/reranking/jina.mdx b/docs/integrations/reranking/jina.mdx index f2eb83d..4624149 100644 --- a/docs/integrations/reranking/jina.mdx +++ b/docs/integrations/reranking/jina.mdx @@ -21,8 +21,7 @@ This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the searc -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"jina-reranker-v2-base-multilingual"` | The name of the reranker model to use. You can find the list of available models in https://jina.ai/reranker. | @@ -33,22 +32,22 @@ Accepted Arguments -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | -### Vector Search +### Vector Search {#vector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | -### FTS Search +### FTS Search {#fts-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | diff --git a/docs/integrations/reranking/openai.mdx b/docs/integrations/reranking/openai.mdx index a68f1b9..413f95e 100644 --- a/docs/integrations/reranking/openai.mdx +++ b/docs/integrations/reranking/openai.mdx @@ -20,8 +20,7 @@ This reranker uses OpenAI chat model to rerank the search results. You can use t -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"gpt-4-turbo-preview"` | The name of the reranker model to use.| @@ -30,22 +29,22 @@ Accepted Arguments | `api_key` | `str` | `None` | The API key to use. If None, will use the OPENAI_API_KEY environment variable. -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | -### Vector Search +### Vector Search {#vector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | -### FTS Search +### FTS Search {#fts-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | diff --git a/docs/integrations/reranking/voyageai.mdx b/docs/integrations/reranking/voyageai.mdx index 704d2c5..62e9a16 100644 --- a/docs/integrations/reranking/voyageai.mdx +++ b/docs/integrations/reranking/voyageai.mdx @@ -23,8 +23,7 @@ This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `None` | The name of the reranker model to use. Available models are: rerank-2, rerank-2-lite | @@ -35,22 +34,22 @@ Accepted Arguments | `truncation` | `bool` | `None` | Whether to truncate the input to satisfy the "context length limit" on the query and the documents. | -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Returns only have the `_relevance_score` column | | `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) | -### Vector Search +### Vector Search {#vector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Returns only have the `_relevance_score` column | | `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) | -### FTS Search +### FTS Search {#fts-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Returns only have the `_relevance_score` column | diff --git a/docs/integrations/reranking/watsonx.mdx b/docs/integrations/reranking/watsonx.mdx index 5949f7f..775af4f 100644 --- a/docs/integrations/reranking/watsonx.mdx +++ b/docs/integrations/reranking/watsonx.mdx @@ -23,8 +23,7 @@ pip install ibm-watsonx-ai -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"cross-encoder/ms-marco-minilm-l-12-v2"` | The rerank model ID. See [supported rerank models](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx#rerank). | @@ -41,23 +40,23 @@ Accepted Arguments You must supply exactly one of `project_id` or `space_id` (either as an argument or via its environment variable). Setting both, or neither, raises a `ValueError`. -## Supported scores for each query type +## Supported scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | -### Vector Search +### Vector Search {#vector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | -### FTS Search +### FTS Search {#fts-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | diff --git a/docs/integrations/stable-worldmodel.mdx b/docs/integrations/stable-worldmodel.mdx index 1e884b4..ee35e21 100644 --- a/docs/integrations/stable-worldmodel.mdx +++ b/docs/integrations/stable-worldmodel.mdx @@ -17,7 +17,7 @@ The LanceDB integration is built into Stable World Model's data format registry. Random access speed is the bottleneck for world model training, since the loop repeatedly samples temporal windows from high-dimensional observations, actions, and rewards. The faster those windows arrive, the more GPU time goes into training rather than waiting on the data loader. -## Install +## Install {#install} ```bash pip install stable-worldmodel @@ -25,7 +25,7 @@ pip install stable-worldmodel Datasets and checkpoints are stored under `$STABLEWM_HOME`, which defaults to `~/.stable_worldmodel/`. -## Collect data into Lance +## Collect data into Lance {#collect-data-into-lance} Stable World Model uses Lance by default when you collect to a `.lance` path. Replace `your_expert_policy` with the expert or scripted policy you use to collect demonstrations. @@ -36,7 +36,7 @@ Replace `your_expert_policy` with the expert or scripted policy you use to colle Every writer accepts a `mode` argument such as `append`, `overwrite`, or `error`. The default is append, so re-running collection extends the existing dataset. -## Load a Lance dataset for training +## Load a Lance dataset for training {#load-a-lance-dataset-for-training} The dataset loader autodetects the Lance format from the path. @@ -46,7 +46,7 @@ The dataset loader autodetects the Lance format from the path. Your model code stays focused on the world model objective while LanceDB handles the storage layout and read path. -## Evaluate with model-predictive control +## Evaluate with model-predictive control {#evaluate-with-model-predictive-control} After training a world model on the Lance-backed dataset, Stable World Model can evaluate it with planning solvers such as CEM. Replace `world_model` with the trained model object from your training loop. @@ -55,7 +55,7 @@ Replace `world_model` with the trained model object from your training loop. {PyFrameworksStableWorldmodelEvaluate} -## Convert between formats +## Convert between formats {#convert-between-formats} Stable World Model can convert between registered dataset formats. A common workflow is to collect in Lance for fast training reads, then export to the video layout for compact inspection artifacts. @@ -63,7 +63,7 @@ Stable World Model can convert between registered dataset formats. A common work {PyFrameworksStableWorldmodelConvert} -## Throughput +## Throughput {#throughput} The Stable World Model README reports the following PushT benchmark results from `scripts/benchmark/compare_h5_lance.py`: @@ -83,7 +83,7 @@ In that benchmark, local LanceDB reached about **3.4x** the no-cache throughput These numbers come from the Stable World Model project's own benchmark setup, so they're best read as a reproducible directional baseline that may shift across environments, models, and storage configurations. -## Storage +## Storage {#storage} The same README reports these local storage sizes for the benchmark dataset: @@ -95,7 +95,7 @@ The same README reports these local storage sizes for the benchmark dataset: LanceDB used about **69% less local storage than HDF5** in the reported benchmark, while preserving a table interface built for fast training reads and append-heavy collection. -## More resources +## More resources {#more-resources} diff --git a/docs/lance.mdx b/docs/lance.mdx index e7a650e..d6db5f1 100644 --- a/docs/lance.mdx +++ b/docs/lance.mdx @@ -23,7 +23,7 @@ instead of moving data across separate storage, search, feature, and training sy Visit the Lance format documentation to learn more about its design, features, and how it enables the multimodal lakehouse. -## Capabilities of the Lance format +## Capabilities of the Lance format {#capabilities-of-the-lance-format} Capability | What it enables --- | --- @@ -35,7 +35,7 @@ Versioned tables | Reproduce experiments, restore previous states, and tie downs Hybrid search and indexing | Combine vector search, full-text search, and scalar filters on the same dataset with Lance indexes. Open lakehouse interoperability | Build on object storage and connect Lance tables to open engines such as PyTorch, Ray, Spark, Trino, DuckDB and Polars. -## Key concepts +## Key concepts {#key-concepts} The following concepts are core to the Lance format: @@ -52,7 +52,7 @@ The following concepts are core to the Lance format: -### Data versioning +### Data versioning {#data-versioning} Data in Lance tables are versioned -- this helps keep LanceDB scalable and consistent. We do not immediately blow away old versions when creating new ones because other clients might be @@ -63,7 +63,7 @@ Each version contains metadata and just the new/updated data in your transaction versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries. -### Data compaction +### Data compaction {#data-compaction} As you insert more data, your dataset will grow and you'll need to perform compaction to maintain query throughput (i.e., keep latencies down to a minimum). Compaction is the process of merging fragments @@ -80,7 +80,7 @@ Compaction focuses on read performance, not immediate disk reclamation. During c new compacted files while older files are still referenced by previous table versions. This means disk usage can increase temporarily until old versions are cleaned up. -### Data deletion and recovery +### Data deletion and recovery {#data-deletion-and-recovery} Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. It simply marks the row as deleted in the `DataFile` that represents a fragment. diff --git a/docs/namespaces/index.mdx b/docs/namespaces/index.mdx index 097172f..e7f4adc 100644 --- a/docs/namespaces/index.mdx +++ b/docs/namespaces/index.mdx @@ -19,7 +19,7 @@ This is why many SDK methods in LanceDB, like `create_table`, `open_table`, `dro language: Python uses `namespace_path`, Rust uses builder methods like `.namespace(...)`, and TypeScript uses `namespacePath` arguments. -## Namespace hierarchy +## Namespace hierarchy {#namespace-hierarchy} Namespaces are generalizations of catalog specs that give platform developers a clean way to present Lance tables in the structures users expect. The diagram below shows how the hierarchy can go beyond a single level. A namespace can contain a collection of tables, and it can also contain namespaces recursively. @@ -29,7 +29,7 @@ A namespace can contain a collection of tables, and it can also contain namespac Before diving into examples, it helps to keep two terms in mind: the **namespace client** is the abstraction that presents a consistent namespace API, while the **namespace implementation** is the concrete backend that resolves namespaces and table locations (for example, a local directory or an external catalog). If you want to go deeper, see the Lance format [namespace documentation](https://lance.org/format/namespace/). -## Namespace paths and names +## Namespace paths and names {#namespace-paths-and-names} A namespace path is a list of components. For example, `["prod", "search"]` means the `search` namespace inside the `prod` namespace. The empty path, `[]`, means the root namespace. @@ -38,7 +38,7 @@ Each component is a name, not a filesystem path segment. Namespace names can't b component can contain only letters, numbers, underscores, hyphens, and periods. That keeps the same identifier usable across local directory namespaces and REST namespace identifiers. -## Directory namespaces +## Directory namespaces {#directory-namespaces} The simplest namespace model in LanceDB is a single root namespace, often represented by one directory: @@ -142,7 +142,7 @@ println!("Created table: {}", table.name()); - To integrate LanceDB with external catalogs and to use it as a true **multimodal lakehouse**, it's useful to understand the different namespace implementations and how to use them in your organization's setup. -## Remote or external catalog namespaces +## Remote or external catalog namespaces {#remote-or-external-catalog-namespaces} The example above showed local directory-based namespaces. LanceDB also supports namespaces backed by remote object stores and external catalogs, via the REST namespace implementation. @@ -217,7 +217,7 @@ let db = lancedb::connect_namespace("rest", properties) contract can be used to interact with it. For authentication examples in LanceDB Enterprise, visit the [Namespaces in SDKs](/namespaces/usage#namespaces-in-lancedb-enterprise) page. -## Best practices +## Best practices {#best-practices} Below, we list some best practices for working with namespaces: - For simple use cases and single, stand-alone applications, the directory-based root namespace is sufficient and requires no special configuration. diff --git a/docs/namespaces/usage.mdx b/docs/namespaces/usage.mdx index 009ce9f..5c9096c 100644 --- a/docs/namespaces/usage.mdx +++ b/docs/namespaces/usage.mdx @@ -19,7 +19,7 @@ As your table organization needs grow over time and your projects become more co As described in the [Namespaces and Catalog Model](/namespaces) section, namespaces are LanceDB's way of generalizing catalog specs, providing developers a clean way to manage hierarchical organization of tables in the catalog. The SDKs treat a namespace as a path and can use it for table resolution when you use LanceDB outside the root namespace. -## Table operations with namespace paths +## Table operations with namespace paths {#table-operations-with-namespace-paths} Let's imagine a scenario where your table management needs have evolved, and you now have the following multi-level structure to organize your tables outside the root namespace. ``` @@ -63,7 +63,7 @@ An empty namespace (`[]`), which is the default, means "root namespace", and the the `data/` directory under the specified root path. -## Namespace management APIs +## Namespace management APIs {#namespace-management-apis} You can open/create/drop tables inside a namespace path (like `["prod", "search"]`). All three SDKs expose namespace lifecycle operations directly. @@ -102,7 +102,7 @@ Listing APIs return the immediate children of the requested namespace path. Use returned `page_token` to page through large catalogs; pass an empty namespace path (`[]`) when you want to list from the root namespace. -## Namespaces in LanceDB Enterprise +## Namespaces in LanceDB Enterprise {#namespaces-in-lancedb-enterprise} In LanceDB Enterprise deployments, configure namespace-backed federated databases in a TOML file under your deployment's `config` directory. LanceDB Enterprise supports both directory-based (`ns_impl = "dir"`) and REST-based (`ns_impl = "rest"`) namespace implementations. @@ -142,7 +142,7 @@ For the LanceDB REST API itself, requests use `x-api-key` for API-key authentica serves more than one database, LanceDB can also use headers such as `x-lancedb-database` or `x-lancedb-database-prefix` to route the request to the right database context. -## Related references +## Related references {#related-references} - [Client SDK API references](/api-reference) - [REST API Reference](/api-reference/rest) diff --git a/docs/performance.mdx b/docs/performance.mdx index c7008ed..0d54f7e 100644 --- a/docs/performance.mdx +++ b/docs/performance.mdx @@ -20,7 +20,7 @@ LanceDB is performant by default. This page covers performance best practices th When using Python with multiprocessing, use `spawn` rather than `fork`. LanceDB is multi-threaded internally, and `fork` plus a multi-threaded process is unsafe. -## Ingestion +## Ingestion {#ingestion} If ingestion is taking longer than expected on a large dataset, the cause is almost always how `add()` is called: each call commits a new version and a new fragment, so a per-row loop pays that per-call overhead at every row. The best practice is to pick the ingestion mode that matches your data shape — bulk ingestion when the data is already materialized, or iterator ingestion when it's streamed or computed on the fly. @@ -32,7 +32,7 @@ A merge has to scan existing data to find matches on the join key (or look them Use `add()` for pure appends, and reach for `merge_insert()` only when you need upsert or conditional-insert logic. When you do use it, build a scalar index on the join column first — otherwise the match step falls back to a full column scan, which is the dominant cost at scale. -### Bulk ingestion: for data you already have +### Bulk ingestion: for data you already have {#bulk-ingestion-for-data-you-already-have} For materialized inputs (Arrow Tables, DataFrames) and file-backed sources (`pyarrow.dataset(...)`), LanceDB auto-parallelizes the write across workers, estimating the partition count from the data size — more partitions means more concurrent writes and higher throughput, up to the CPU core count. @@ -56,7 +56,7 @@ For larger-than-memory data, prefer scanning a file-backed dataset (`ds.dataset( For very large initial loads, create the table empty first; passing data directly to `create_table(name, data)` skips the auto-parallel path. -### Iterator ingestion: for data you transform on the fly +### Iterator ingestion: for data you transform on the fly {#iterator-ingestion-for-data-you-transform-on-the-fly} If each row needs work before it can be written — applying a custom transformation as you ingest, for example — the data doesn't exist as a file you can point `ds.dataset(...)` at. The best practice is to pass an iterator of `pyarrow.RecordBatch` instead; LanceDB consumes one batch at a time as you produce them. @@ -83,7 +83,7 @@ table.add(stream(), write_parallelism=4) Each partition becomes its own fragment, so don't over-allocate on a small input — budget one unit of parallelism per ~100K rows or ~1 GB of data as a rule of thumb. -### Bulk ingests into a remote table +### Bulk ingests into a remote table {#bulk-ingests-into-a-remote-table} Enterprise @@ -111,9 +111,9 @@ export LANCE_CLIENT_MAX_REQUEST_DURATION=120 # 2 minutes Splitting into more parts does not change the final table. The server stages every part under the shared upload id and merges them atomically when the write completes. -## Indexing +## Indexing {#indexing} -### Vector indexes +### Vector indexes {#vector-indexes} If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), the cause is the default brute-force scan over every vector. That works fine below ~100K vectors, but past that you should build a dedicated vector index. Pick the type by your data shape: @@ -126,7 +126,7 @@ If vector search latency climbs with table size (i.e., queries that ran in milli The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). For parameter tuning, see [Vector Indexing](/indexing/vector-index). -### Scalar indexes +### Scalar indexes {#scalar-indexes} If filtered queries slow down as the table grows — even when the filter is selective — the cause is a full column scan: without a scalar index, LanceDB evaluates the `where(...)` predicate on every row, and the same applies to `merge_insert` join keys. The best practice is to build a scalar index on every column you filter or join on, picking the type by the column's shape: @@ -138,11 +138,11 @@ If filtered queries slow down as the table grows — even when the filter is sel See [Scalar Indexing](/indexing/scalar-index). -### Full-text search +### Full-text search {#full-text-search} If your full-text index is much larger than expected, or takes longer than expected to build, the cause is usually phrase-query flags being enabled when they aren't needed: `with_position=True` and `remove_stop_words=False` both significantly inflate index size and build time. The best practice is to keep the defaults for most workloads, and only enable those flags when you actually need to search for phrases. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options. -## Compaction and cleanup +## Compaction and cleanup {#compaction-and-cleanup} Two things accumulate on a long-lived table as more and more data gets added to it: @@ -161,7 +161,7 @@ The best practice is to run `optimize()` after large writes or on a schedule. It LanceDB Enterprise handles both compaction and cleanup automatically. -## Querying +## Querying {#querying} Three knobs materially affect query latency, memory use, and recall. Be deliberate about each one on every query: @@ -187,7 +187,7 @@ Three knobs materially affect query latency, memory use, and recall. Be delibera For hybrid search, the default `RRFReranker()` combines vector and FTS results into a single ranking via reciprocal rank fusion. See [Hybrid Search](/search/hybrid-search). -### Avoid materializing the whole table +### Avoid materializing the whole table {#avoid-materializing-the-whole-table} If you need every row in the table (for training, export, or migration), calling `to_pandas()` or `to_arrow()` will run you out of memory on any non-trivial dataset — both materialize the full table at once. The best practice is to iterate via `table.search(...)` or `table.query(...)`, which work the same way in both LanceDB OSS and Enterprise. @@ -205,7 +205,7 @@ for batch in ds.to_batches(columns=["id", "text"], batch_size=10000): LanceDB OSS exposes the `to_pandas()`, `to_arrow()`, and `table.to_lance()` methods for direct Lance dataset access. Enterprise's `RemoteTable` exposes none of these. Only `table.search(...)` and `table.query(...)`. In general, it's always best to go through `search()` and `query()` to keep your code portable across both OSS and Enterprise. -### Diagnostics +### Diagnostics {#diagnostics} To analyze a slow query, inspect what the query engine actually did and the state of the indexes it touched. These two tools surface that information — use them in this order: @@ -223,7 +223,7 @@ print(table.index_stats("vector_idx")) # num_unindexed_rows should be ~0 [Optimize Query Performance](/search/optimize-queries) walks through a fully worked before/after example, including how `KNNVectorDistance` and `output_batches` change once indexes are in place. -## Where to go next +## Where to go next {#where-to-go-next} diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 188e0fc..a60a347 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -52,7 +52,7 @@ features, you'll work with the same underlying table and search primitives. Let's get started in just a few steps! -## 1. Install LanceDB +## 1\. Install LanceDB {#1-install-lancedb} Install LanceDB in your client SDK. @@ -77,7 +77,7 @@ cargo add lancedb ``` -### Python pre-release builds +### Python pre-release builds {#python-pre-release-builds} To pick up the latest features and bug fixes before the next stable release, install a pre-release from LanceDB's Fury index. @@ -118,7 +118,7 @@ The two packages share the same `lancedb/` namespace and conflict at install tim uninstall the other first (`pip uninstall lancedb && pip install lancedb-compat`). -## 2. Connect to a LanceDB database +## 2\. Connect to a LanceDB database {#2-connect-to-a-lancedb-database} LanceDB supports several URI patterns to connect to a database. @@ -126,7 +126,7 @@ LanceDB supports several URI patterns to connect to a database. - A `db://...` URI (when using LanceDB Enterprise) - An object storage URI: `s3://...`, `gs://...`, or `az://...` (when connecting directly from the client SDK) -### Connect via local directory path +### Connect via local directory path {#connect-via-local-directory-path} The simplest way to begin is to use LanceDB as an embedded library. Import LanceDB in your client SDK of choice and point to a local directory path. @@ -152,7 +152,7 @@ client SDK of choice and point to a local directory path. -### Connect via object storage URIs +### Connect via object storage URIs {#connect-via-object-storage-uris} You can also connect directly to object storage from the client SDK: @@ -177,7 +177,7 @@ You can also connect directly to object storage from the client SDK: For credentials, endpoints, and provider-specific options, see [Configuring storage](/storage/configuration). -### Connect to LanceDB Enterprise +### Connect to LanceDB Enterprise {#connect-to-lancedb-enterprise} If you're using LanceDB Enterprise, you can connect to the remote database using the `db://` URI along with the API key, region, and cluster endpoint you received from the @@ -211,7 +211,7 @@ endpoint, [contact the LanceDB team](mailto:contact@lancedb.com). To learn more about `RemoteTable` semantics and how Enterprise differs operationally from embedded LanceDB, see the [Enterprise overview](/enterprise). -## 3. Create a new table +## 3\. Create a new table {#3-create-a-new-table} Let's create a small table of characters from the kingdom of Camelot. Each row stores source text, metadata, structured fields, and a vector embedding in the same LanceDB table. @@ -274,7 +274,7 @@ with the appropriate schema and ingests the data. -## 4. Semantic search +## 4\. Semantic search {#4-semantic-search} Search is a useful capability for all kinds of AI data pipelines. Below, we do a vector similarity search for samples similar to a "_wise magical advisor_" (transforming the natural language query to @@ -313,7 +313,7 @@ to be used downstream in your application. -## 5. Curation +## 5\. Curation {#5-curation} Searching for relevant results can be more useful when combined with metadata filters. In this tiny example, we filter to examples with high `magic` stats. @@ -335,7 +335,7 @@ In this tiny example, we filter to examples with high `magic` stats. When working with large datasets, it's common to use the same pattern to filter on quality labels, train/eval splits, numeric fields, categorical values, timestamp windows, or generated tags and labels. -## 6. Add a derived feature +## 6\. Add a derived feature {#6-add-a-derived-feature} Feature engineering is the process of cleaning up your data and creating new signals that help your model learn, make better predictions, or your agent retrieve more useful information. @@ -382,7 +382,7 @@ Next, you can query a compact view of the new feature: The same workflow is used for data preparation tasks when adding derived features, cached model signals, review scores, or dataset quality indicators. -## 7. Store multimodal data +## 7\. Store multimodal data {#7-store-multimodal-data} Multimodal data is a first-class citizen in LanceDB. Binary data (image, audio, video, etc.) is stored as blobs or inline Arrow binary types in a LanceDB column, and they benefit from the same @@ -424,15 +424,14 @@ These snippets load the local image file and store the bytes in an `image` colum For more examples, see the [multimodal data](/tables/multimodal) section. -## Code +## Code {#code} See the full code for these examples (including helper functions) in the `quickstart` file for the appropriate client language in the [files provided in the repo](https://github.com/lancedb/docs/tree/main/tests). -## What's next? - +## Next steps {#next-steps} You've learned how to install LanceDB, connect, create one table for AI data, retrieve related examples, curate with metadata, add a derived feature, and represent multimodal records. These same primitives apply across the AI data lifecycle, from data preparation and feature engineering to diff --git a/docs/reranking/cross_encoder.mdx b/docs/reranking/cross_encoder.mdx index de90435..1bb9660 100644 --- a/docs/reranking/cross_encoder.mdx +++ b/docs/reranking/cross_encoder.mdx @@ -19,8 +19,7 @@ This reranker uses Cross Encoder models from sentence-transformers to rerank the -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.| @@ -32,22 +31,22 @@ Accepted Arguments The reranker loads the model locally through `sentence-transformers`, so install the local model runtime dependencies you need, such as PyTorch and any device-specific acceleration packages. -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | -### Vector Search +### Vector Search {#vector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | -### FTS Search +### FTS Search {#fts-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | diff --git a/docs/reranking/custom-reranker.mdx b/docs/reranking/custom-reranker.mdx index 853d74e..f19108b 100644 --- a/docs/reranking/custom-reranker.mdx +++ b/docs/reranking/custom-reranker.mdx @@ -20,7 +20,7 @@ The Python base class exposes hybrid, vector-only, and FTS-only rerank hooks. Ty currently expose the custom reranker interface for hybrid reranking. In Rust, a custom reranker must also satisfy the trait bounds `Debug + Send + Sync`. -## Interface +## Interface {#interface} The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates @@ -79,7 +79,7 @@ class MyReranker(Reranker): ``` -## Example +## Example {#example} As an example, let's build custom reranker that enhances the Cohere Reranker by accepting a filter query, and accepts any other `CohereReranker` params as `kwargs`. diff --git a/docs/reranking/eval.mdx b/docs/reranking/eval.mdx index facf0d7..ab4c1b7 100644 --- a/docs/reranking/eval.mdx +++ b/docs/reranking/eval.mdx @@ -15,7 +15,7 @@ Before evaluating hybrid search, build an FTS index on the text column and use a metadata or explicit query vectors for the vector side. Otherwise the evaluation is measuring setup fallbacks or errors rather than the reranker. -## Reranking strategies +## Reranking strategies {#reranking-strategies} There are two common approaches for reranking search results from multiple sources. @@ -56,14 +56,14 @@ ones that work well for all cases, because reranking quality is dataset and appl Evaluating whether a reranking strategy is a good fit is also a challenge. In the next section, we discuss an example evaluation of different reranking strategies on a sample dataset. -## Example evaluation +## Example evaluation {#example-evaluation} The table below shows our evaluation results from an experiment comparing multiple rerankers on ~800 hybrid search queries. This is a modified version of an evaluation script by [LlamaIndex](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate \@ top-k. -### Using OpenAI `text-embedding-ada-002` +### Using OpenAI `text-embedding-ada-002` {#using-openai-text-embedding-ada-002} Vector Search baseline: **0.64** @@ -77,7 +77,7 @@ Vector Search baseline: **0.64** -### Using OpenAI `text-embedding-3-small` +### Using OpenAI `text-embedding-3-small` {#using-openai-text-embedding-3-small} Vector Search baseline: **0.59** @@ -90,7 +90,7 @@ Vector Search baseline: **0.59** -## Conclusion +## Conclusion {#conclusion} The results show that the reranking methods can significantly improve the search relevance. However, the improvement we saw was not consistent across all rerankers. In reality, the choice of reranker diff --git a/docs/reranking/index.mdx b/docs/reranking/index.mdx index 722fe2b..eb12449 100644 --- a/docs/reranking/index.mdx +++ b/docs/reranking/index.mdx @@ -12,7 +12,7 @@ Reranking is the process of re-ordering search results to improve relevance, oft different model than the one used for the initial search. LanceDB has built-in support for reranking with models from Cohere, Sentence-Transformers, and more. -### Quickstart +### Quickstart {#quickstart} To use a reranker, you run a search and pass the results to the `rerank()` method. The examples below move from the simplest, model-free rerankers to a model-based one. Each is a complete, runnable script. @@ -51,7 +51,7 @@ Reach for the model-free rerankers (`LinearCombinationReranker`, `RRFReranker`) matter most; reach for a model-based one like `CohereReranker` or `CrossEncoderReranker` when you need higher relevance and can afford to score every query and document pair with a model. -### Supported Rerankers +### Supported Rerankers {#supported-rerankers} LanceDB supports the following rerankers out of the box. The first three are score-based and run no model; the rest are model-based. The built-in rerankers are documented in this section; the hosted @@ -99,7 +99,7 @@ yourself. -### Multi-vector reranking +### Multi-vector reranking {#multi-vector-reranking} Most rerankers support reranking based on multiple vectors. To rerank based on multiple vectors, you can pass a list of vectors to the `rerank` method. Here's an example of how to rerank based on multiple vector columns using the `CrossEncoderReranker`: @@ -126,7 +126,7 @@ input result sets is missing the `_rowid` column. Therefore, it's recommended to - `RRFReranker.rerank_multivector(...)` always requires `_rowid` on its inputs, regardless of the `deduplicate` flag. -## Creating Custom Rerankers +## Creating Custom Rerankers {#creating-custom-rerankers} LanceDB also allows you to create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of diff --git a/docs/reranking/linear_combination.mdx b/docs/reranking/linear_combination.mdx index 7797502..8917815 100644 --- a/docs/reranking/linear_combination.mdx +++ b/docs/reranking/linear_combination.mdx @@ -22,8 +22,7 @@ The Linear Combination Reranker combines the results of semantic and full-text s -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `weight` | `float` | `0.7` | The weight to use for the semantic search score. The weight for the full-text search score is `1 - weight`. | @@ -34,10 +33,10 @@ Accepted Arguments returns the non-empty side with `_relevance_score` attached rather than failing. -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column | diff --git a/docs/reranking/mrr.mdx b/docs/reranking/mrr.mdx index cb6cbbe..b517f8b 100644 --- a/docs/reranking/mrr.mdx +++ b/docs/reranking/mrr.mdx @@ -19,8 +19,7 @@ This reranker uses the Mean Reciprocal Rank (MRR) algorithm to combine and reran -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `weight_vector` | `float` | `0.5` | Weight for vector search results (0.0 to 1.0). | @@ -33,16 +32,16 @@ For multivector reranking, input result sets need `_rowid` so LanceDB can identi across the ranked lists. Add `.with_row_id(True)` to each vector search before passing the results to the reranker. -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | | `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | -### Multivector Search +### Multivector Search {#multivector-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | diff --git a/docs/reranking/rrf.mdx b/docs/reranking/rrf.mdx index 7df3477..3a58cec 100644 --- a/docs/reranking/rrf.mdx +++ b/docs/reranking/rrf.mdx @@ -29,8 +29,7 @@ why it's the default reranker for LanceDB hybrid search. The implementation foll -Accepted Arguments ----------------- +## Accepted Arguments {#accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical. | @@ -39,7 +38,7 @@ Accepted Arguments `K` must be greater than `0`. In TypeScript, construct the built-in reranker with `await RRFReranker.create(k)` before passing it to `.rerank(...)`. -## Multi-vector reranking +## Multi-vector reranking {#multi-vector-reranking} `RRFReranker` can also fuse the results of several vector searches with `rerank_multivector`, applying the same rank-fusion algorithm across more than two lists. Every input result set must include the @@ -47,10 +46,10 @@ the same rank-fusion algorithm across more than two lists. Every input result se otherwise the call raises a `ValueError`. See [multi-vector reranking](/reranking#multi-vector-reranking) for a full example. -## Supported Scores for each query type +## Supported Scores for each query type {#supported-scores-for-each-query-type} You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: -### Hybrid Search +### Hybrid Search {#hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | | `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column. | diff --git a/docs/search/filtering.mdx b/docs/search/filtering.mdx index 350b302..b8ebf37 100644 --- a/docs/search/filtering.mdx +++ b/docs/search/filtering.mdx @@ -14,7 +14,7 @@ with filtering capabilities even on datasets containing billions of records. On hybrid queries, the same `where(...)` filter is applied to both the vector and full-text halves of the query. The prefilter or postfilter choice controls whether that happens before each subquery scores candidates or after the subquery top-k is produced. -## Chaining `where` clauses +## Chaining `where` clauses {#chaining-where-clauses} In more recent LanceDB SDK versions (see the callout box below for the exact version numbers), you can call `where(...)` (Python and TypeScript) or `only_if(...)` (Rust) more than once on the same query builder. Each additional filter is combined with the previous one using logical `AND`, so `where("a > 0").where("b < 10")` is equivalent to `where("(a > 0) AND (b < 10)")`. @@ -47,7 +47,7 @@ const result = await table In Python SDK versions before `0.34.0` and Rust/TypeScript SDK versions before `0.31.0`, a second `where(...)` or `only_if(...)` call replaced the first filter, so only the last predicate was applied. If your code needs to run on those older versions, write a single predicate with `AND` instead of chaining calls. When upgrading to the latest SDKs, review any existing chained filters and drop earlier calls you no longer want to apply. -## Example: Metadata Filtering +## Example: Metadata Filtering {#example-metadata-filtering} To illustrate filtering capabilities, let's try four data points with combinations of vectors and metadata: @@ -77,7 +77,7 @@ const table = await db.createTable(tableName, data, { ``` -### Filtering Without Vector Search +### Filtering Without Vector Search {#filtering-without-vector-search} You can always filter your data without search. This is useful when you need to query based on metadata: @@ -104,7 +104,7 @@ const filteredResult = await table If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set. -### Pre-Filtering with Vector Search +### Pre-Filtering with Vector Search {#pre-filtering-with-vector-search} ```python Python icon="python" @@ -124,7 +124,7 @@ const results = await table ``` -### Post-Filtering with Vector Search +### Post-Filtering with Vector Search {#post-filtering-with-vector-search} ```python Python icon="python" @@ -150,7 +150,7 @@ When querying large tables, omitting a `limit` clause may overwhelm resources an to be mindful of the potential impact on performance and costs when working with really large tables. -## Filtering with SQL +## Filtering with SQL {#filtering-with-sql} Because it's built on top of DataFusion, LanceDB embraces the utilization of standard SQL expressions as predicates for filtering operations. SQL can be used during vector search, update, and deletion operations. @@ -168,7 +168,7 @@ LanceDB supports a growing list of SQL expressions: | `regexp_match(column, pattern)` | Regular expression matching | | [DataFusion Functions](https://datafusion.apache.org/user-guide/sql/scalar_functions.html) | Additional SQL functions | -### Simple SQL Filters +### Simple SQL Filters {#simple-sql-filters} For example, the following filter string is acceptable: @@ -187,7 +187,7 @@ await table ``` -### Advanced SQL Filters +### Advanced SQL Filters {#advanced-sql-filters} If your column name contains special characters, upper-case characters, or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html), you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the @@ -202,7 +202,7 @@ AND `nested with space`.`inner with space` < 2 Field names containing periods (.) are NOT supported. -### Dates, Timestamps, Decimals +### Dates, Timestamps, Decimals {#dates-timestamps-decimals} Literals for dates, timestamps, and decimals can be written by writing the string value after the type name. For example: @@ -225,7 +225,7 @@ parameter. Microsecond precision (6) is the default. | `timestamp(6)` | Microseconds | | `timestamp(9)` | Nanoseconds | -## Apache Arrow Mapping +## Apache Arrow Mapping {#apache-arrow-mapping} LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format. The mapping from SQL types to Arrow types is: @@ -246,7 +246,7 @@ The mapping from SQL types to Arrow types is: | `binary` | `Binary` | -## Best Practices +## Best Practices {#best-practices} **Scalar Indexes**: We strongly recommend creating scalar indices on columns used for filtering, whether combined with a search operation or applied independently (e.g., for updates or deletions). @@ -259,7 +259,7 @@ For best performance with large tables or high query volumes: For a column of type LIST(T), you can use `LABEL_LIST` to create a scalar index. Then you should leverage DataFusion's [array functions](https://datafusion.apache.org/user-guide/sql/scalar_functions.html#array-functions) like `array_has_any` or `array_has_all` for optimized filtering. -## Limitations +## Limitations {#limitations} Both **pre-filtering** and **post-filtering** can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to call `bypass_vector_index()` if you know that the filter is highly selective. diff --git a/docs/search/fts-examples.mdx b/docs/search/fts-examples.mdx index 17f13a2..dff4426 100644 --- a/docs/search/fts-examples.mdx +++ b/docs/search/fts-examples.mdx @@ -7,9 +7,9 @@ icon: "book-open" These worked examples build on the concepts from the [Full-Text Search guide](/search/full-text-search). They walk through creating sample tables, building FTS indices, and running fuzzy, phrase, boosted, boolean, and substring queries. -## Fuzzy Search and Boosting Example +## Fuzzy Search and Boosting Example {#fuzzy-search-and-boosting-example} -### Generate Data +### Generate Data {#generate-data} First, let's create a table with sample text data for testing fuzzy search: @@ -71,7 +71,7 @@ const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) ``` -### Create Table +### Create Table {#create-table} ```python Python icon="python" @@ -105,7 +105,7 @@ const table = await db.createTable(tableName, data, { mode: "overwrite" }); ``` -### Construct FTS Index +### Construct FTS Index {#construct-fts-index} Create a full-text search index on the first text column: @@ -139,11 +139,11 @@ await waitForIndex(table, "text2_idx"); ``` -### Basic and Fuzzy Search +### Basic and Fuzzy Search {#basic-and-fuzzy-search} Now we can perform basic, fuzzy, and prefix match searches: -#### Basic Exact Search +#### Basic Exact Search {#basic-exact-search} ```python Python icon="python" @@ -170,7 +170,7 @@ const basicMatchResults = await table.query() ``` -#### Fuzzy Search with Typos +#### Fuzzy Search with Typos {#fuzzy-search-with-typos} ```python Python icon="python" @@ -195,7 +195,7 @@ const fuzzyResults = await table.query() ``` -#### Prefix based Match +#### Prefix based Match {#prefix-based-match} Prefix-based match allows you to search for documents containing words that start with a specific prefix. @@ -222,7 +222,7 @@ const fuzzyResults = await table.query() ``` -### Phrase Match +### Phrase Match {#phrase-match} Phrase matching enables you to search for exact sequences of words. Unlike regular text search which matches individual terms independently, phrase matching requires words to appear in the @@ -263,7 +263,7 @@ const phraseResults = await table.query() ``` -#### Flexible Phrase Match +#### Flexible Phrase Match {#flexible-phrase-match} To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, even if they are not directly adjacent or in the exact order, as long as they are within the specified `slop` value. For example, the phrase query "puppy merrily" would not return any results by default. However, if you set `slop=1`, it will match phrases like "puppy jumps merrily", "puppy runs merrily", and similar variations where one word appears between "puppy" and "merrily". @@ -293,7 +293,7 @@ const phraseResults = await table.query() ``` -### Search with Boosting +### Search with Boosting {#search-with-boosting} Boosting allows you to control the relative importance of different search terms or fields in your queries. This feature is particularly useful when you need to: @@ -388,10 +388,10 @@ const multiMatchBoostingResults = await table.query() - For complex queries, use SQL to combine FTS with other filter conditions -### Boolean Queries +### Boolean Queries {#boolean-queries} LanceDB supports boolean logic in full-text search, allowing you to combine multiple queries using `and` and `or` operators. This is useful when you want to match documents that satisfy multiple conditions (intersection) or at least one of several conditions (union). -#### Combining Two Match Queries +#### Combining Two Match Queries {#combining-two-match-queries} In Python, you can combine two MatchQuery objects using either the `and` function or the `&` operator (e.g., `MatchQuery("puppy", "text") and MatchQuery("merrily", "text")`); both methods are supported and yield the same result. Similarly, you can use either the `or` function or the `|` operator to perform an or query. @@ -473,11 +473,11 @@ const shouldResults = await table - Use `or`/`|`(Python), `Occur.Should`(Typescript) for union (documents must match at least one query). -## Substring Search Example +## Substring Search Example {#substring-search-example} LanceDB supports searching for substrings in text columns using n-gram tokenization. This is useful for finding partial matches within text content. -### Setting Up the Table +### Setting Up the Table {#setting-up-the-table} First, create a table with sample text data and configure n-gram tokenization: @@ -494,7 +494,7 @@ table.create_fts_index("text", base_tokenizer="ngram") ``` -### Basic Substring Search +### Basic Substring Search {#basic-substring-search} With the default n-gram settings (minimum length of 3), you can search for substrings of length 3 or more: @@ -512,7 +512,7 @@ assert set(r["text"] for r in results) == {"lance database", "lance is cool"} ``` -### Handling Short Substrings +### Handling Short Substrings {#handling-short-substrings} By default, the minimum n-gram length is 3, so shorter substrings like "la" won't match: @@ -523,7 +523,7 @@ assert len(results) == 0 ``` -### Customizing N-gram Parameters +### Customizing N-gram Parameters {#customizing-n-gram-parameters} You can customize the n-gram behavior by adjusting the minimum length and using prefix-only matching: @@ -539,7 +539,7 @@ table.create_fts_index( ``` -### Testing Custom N-gram Settings +### Testing Custom N-gram Settings {#testing-custom-n-gram-settings} With the new settings, you can now search for shorter substrings and use prefix-only matching: diff --git a/docs/search/full-text-search.mdx b/docs/search/full-text-search.mdx index 9764d46..5063374 100644 --- a/docs/search/full-text-search.mdx +++ b/docs/search/full-text-search.mdx @@ -13,11 +13,11 @@ import { LanceDB provides support for Full-Text Search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. -## Basic Usage +## Basic Usage {#basic-usage} Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords. -### Table Setup +### Table Setup {#table-setup} First, open or create the table you want to search: @@ -61,7 +61,7 @@ let tbl = db ``` -### Construct FTS Index +### Construct FTS Index {#construct-fts-index} Create a full-text search index on your text column: @@ -90,7 +90,7 @@ tbl ``` -### Full-text Search +### Full-text Search {#full-text-search} Perform full-text search and retrieve results: @@ -132,7 +132,7 @@ LanceDB automatically searches on the existing FTS index if the input to the sea If a table has more than one FTS index, specify the indexed text column in the query. In Python you can use `fts_columns` or the query builder's `nearest_to_text(..., columns=...)`; in TypeScript, use `query().nearestToText(..., columns)`. The newer Lance-native FTS does not accept legacy Tantivy-only index parameters. -### Keeping the index up to date +### Keeping the index up to date {#keeping-the-index-up-to-date} Rows you add after building an FTS index aren't part of the index until you optimize the table. Until then, queries fall back to a flat scan over the unindexed fragments to keep results complete, which slows them down as the unindexed tail grows. Call `table.optimize()` to fold new rows into the existing index — it's the same operation used for vector indexes: @@ -154,9 +154,9 @@ tbl.optimize(OptimizeAction::All).await?; A useful rule of thumb is to call `optimize()` after roughly 100,000 row changes or 20 data-modification operations, whichever comes first. For tables with continuous ingest, schedule it on a cadence that keeps `num_unindexed_rows` (from `table.index_stats(...)`) close to zero. If you want to skip the flat scan over unindexed rows entirely — for example, on a hot read path where stale results are acceptable — call `.fast_search()` on the query so the search returns only indexed results. -## Advanced Usage +## Advanced Usage {#advanced-usage} -### Tokenize Table Data +### Tokenize Table Data {#tokenize-table-data} By default, the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are longer than 40 characters. All words are converted to lowercase. @@ -228,7 +228,7 @@ table.create_fts_index( ``` -### Filtering Options +### Filtering Options {#filtering-options} LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported. @@ -294,7 +294,7 @@ table ``` -### Phrase vs. Terms Queries +### Phrase vs. Terms Queries {#phrase-vs-terms-queries} Lance-based FTS doesn't support queries using boolean operators `OR`, `AND` in the search string. @@ -313,7 +313,7 @@ table.create_fts_index("text", with_position=True, replace=True) This will allow you to search for phrases, but it will also significantly increase the index size and indexing time. -### Fuzzy Search +### Fuzzy Search {#fuzzy-search} Fuzzy search allows you to find matches even when the search terms contain typos or slight variations. LanceDB uses the classic [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) @@ -326,7 +326,7 @@ to find similar terms within a specified edit distance. For a complete walkthrough that creates a sample table and demonstrates fuzzy search and relevance boosting, see the [fuzzy search example](/search/fts-examples#fuzzy-search-and-boosting-example). -### Search for Substring +### Search for Substring {#search-for-substring} LanceDB supports searching for substrings in the text column, you can set the `base_tokenizer` parameter to `"ngram"` to enable this feature, and use the parameters `ngram_min_length` and `ngram_max_length` to control the length of the substrings: @@ -337,15 +337,15 @@ LanceDB supports searching for substrings in the text column, you can set the `b | prefix_only | bool | false | Whether to only search for prefixes of the n-grams | -## More Examples +## More Examples {#more-examples} For complete worked examples of fuzzy search, prefix matching, phrase matching, boosting, boolean queries, and substring search — including sample data generation and index setup — see [Full-Text Search Examples](/search/fts-examples). -## Full-Text Search on Array Fields +## Full-Text Search on Array Fields {#full-text-search-on-array-fields} LanceDB supports full-text search on string array columns, enabling efficient keyword-based search across multiple values within a single field (e.g., tags, keywords). -### Setting Up the Connection +### Setting Up the Connection {#setting-up-the-connection} Connect to your LanceDB instance: @@ -372,7 +372,7 @@ const db = await lancedb.connect({ ``` -### Defining the Schema +### Defining the Schema {#defining-the-schema} Create a schema that includes an array field for tags: @@ -398,7 +398,7 @@ const schema = new Schema([ ``` -### Creating Sample Data +### Creating Sample Data {#creating-sample-data} Generate sample data with array fields containing tags: @@ -469,7 +469,7 @@ const data = makeArrowTable( ``` -### Creating the Table and Adding Data +### Creating the Table and Adding Data {#creating-the-table-and-adding-data} Create the table and populate it with the sample data: @@ -488,7 +488,7 @@ console.log(`Created table: ${tableName}`); ``` -### Building the Full-Text Search Index +### Building the Full-Text Search Index {#building-the-full-text-search-index} Create an FTS index on the tags column to enable efficient text search: @@ -512,7 +512,7 @@ await waitForIndex(table, ftsIndexName); ``` -### Performing Fuzzy Search +### Performing Fuzzy Search {#performing-fuzzy-search} Search for terms with typos using fuzzy matching: @@ -540,7 +540,7 @@ console.log(fuzzyResults); ``` -### Performing Phrase Search +### Performing Phrase Search {#performing-phrase-search} Search for exact phrases within the array fields: diff --git a/docs/search/hybrid-search.mdx b/docs/search/hybrid-search.mdx index b0eadda..d4109c6 100644 --- a/docs/search/hybrid-search.mdx +++ b/docs/search/hybrid-search.mdx @@ -11,9 +11,9 @@ multiple search techniques. For detailed examples, look at this [Python Notebook](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/saas_examples/python_notebook/Hybrid_search.ipynb) or the [**TypeScript Example**](https://github.com/lancedb/vectordb-recipes/tree/main/examples/saas_examples/ts_example/hybrid-search) -## Example: Hybrid Search +## Example: Hybrid Search {#example-hybrid-search} -### 1. Setup +### 1\. Setup {#1-setup} Import the necessary libraries and dependencies for working with LanceDB, OpenAI embeddings, and reranking. @@ -32,7 +32,7 @@ import { Utf8 } from "apache-arrow"; ``` -### 2. Connect to LanceDB +### 2\. Connect to LanceDB {#2-connect-to-lancedb} Establish a connection to your LanceDB instance, with different options for Enterprise setups or open source. OSS @@ -87,7 +87,7 @@ const db = await lancedb.connect(uri, { -### 3. Configure Embedding Model +### 3\. Configure Embedding Model {#3-configure-embedding-model} Set up the any embedding model that will convert text into vector representations for semantic search. @@ -102,7 +102,7 @@ const embedFunc = lancedb.embedding.getRegistry().get("openai")?.create({ ``` -### 4. Create Table & Schema +### 4\. Create Table and Schema {#4-create-table-and-schema} Define the data structure for your documents, including both the text content and its vector representation. @@ -128,7 +128,7 @@ const table = await db.createEmptyTable(tableName, documentSchema, { ``` -### 5. Add Data +### 5\. Add Data {#5-add-data} Insert sample documents into your table, which will be used for both semantic and keyword search. @@ -154,7 +154,7 @@ console.log(`Created table: ${tableName} with ${data.length} rows`); ``` -### 6. Build Full Text Index +### 6\. Build Full Text Index {#6-build-full-text-index} Create a full-text search index on the text column to enable keyword-based search capabilities. @@ -172,7 +172,7 @@ await waitForIndex(table as any, "text_idx"); ``` -### 7. Set Reranker [Optional] +### 7\. Set Reranker [Optional] {#7-set-reranker-optional} Initialize the reranker that will combine and rank results from both semantic and keyword search. By default, lancedb uses RRF reranker, but you can choose other rerankers like `Cohere`, `CrossEncoder`, or others lister in integrations section. @@ -185,7 +185,7 @@ const reranker = await lancedb.rerankers.RRFReranker.create(); ``` -### 8. Hybrid Search +### 8\. Hybrid Search {#8-hybrid-search} Perform a hybrid search query that combines semantic similarity with keyword matching, using the specified reranker to merge and rank the results. @@ -223,7 +223,7 @@ console.log(hybridResults); ``` -### 9. Hybrid Search - Explicit Vector and Text Query pattern +### 9\. Hybrid Search - Explicit Vector and Text Query pattern {#9-hybrid-search-explicit-vector-and-text-query-pattern} You can also pass the vector and text query explicitly. This is useful if you're not using the embedding API or if you're using a separate embedder service. @@ -240,7 +240,7 @@ text_query = "flower moon" ``` -## Query controls +## Query controls {#query-controls} Hybrid queries inherit the same builder API as vector and FTS queries, so the same knobs for filtering, distance bounds, and row identity apply. These compose with `.rerank(...)` and the explicit `.vector()` / `.text()` form shown above. @@ -249,7 +249,7 @@ Always set `.limit(...)` on production hybrid queries. LanceDB's default search explicit cap gives you a clear top-k contract to tune before reranking. -### Returning row IDs +### Returning row IDs {#returning-row-ids} Pass `with_row_id(True)` (Python) or `withRowId()` (TypeScript) to include the internal `_rowid` column in the results. This is useful for joining hybrid results back to a primary table, or for deduping across multiple queries: @@ -275,7 +275,7 @@ const results = await table ``` -### Bounding vector distance +### Bounding vector distance {#bounding-vector-distance} `distance_range(lower, upper)` (Python) and `distanceRange(lower, upper)` (TypeScript) constrain the vector half of the hybrid query to the half-open interval `[lower, upper)`. This is helpful when you want to cap how far semantic candidates can drift from the query vector before reranking: @@ -302,7 +302,7 @@ const results = await table Either bound can be omitted to leave that side unbounded. -### Prefilter vs. postfilter +### Prefilter vs. postfilter {#prefilter-vs-postfilter} When the query carries a metadata filter via `where(...)`, you can choose whether the filter runs before or after the vector and FTS sub-queries. **Prefiltering** (the default) applies `where` to the candidate set before scoring, which is usually what you want — it shrinks the working set and benefits from any scalar indexes on the filter columns. **Postfiltering** runs the filter on the already-ranked top-k from each sub-query; this can be faster when the filter is non-selective or unindexed, but it may return fewer than `limit` rows because some of the top-k may be filtered out. @@ -343,7 +343,7 @@ await table.query() The choice gets baked into both sub-queries, so the vector and FTS halves see the filter applied the same way. Use [`explain_plan`](/search/optimize-queries#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. -## More on Reranking +## More on Reranking {#more-on-reranking} You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB comes with [**built-in rerankers**](https://docs.lancedb.com/reranking) and you can implement your own **custom reranker** as well. diff --git a/docs/search/index.mdx b/docs/search/index.mdx index cec4613..b6f168d 100644 --- a/docs/search/index.mdx +++ b/docs/search/index.mdx @@ -14,7 +14,7 @@ icon: "list" | [Filtering](/search/filtering/) | Filter results based on metadata fields | | [SQL Queries](/search/sql/index) | SQL query capabilities for data exploration and analytics | -## Before you search +## Before you search {#before-you-search} - Vector search can run without an ANN index as an exhaustive scan. That's useful while prototyping, but build a vector index before relying on low-latency searches over larger tables. - Full-text and hybrid text search require an FTS index on the text column you query. If a table has multiple FTS indexes, specify the target column. FTS also supports phrase, boolean, boosted, multi-match, and fuzzy query forms when you need more than plain terms. diff --git a/docs/search/multivector-search.mdx b/docs/search/multivector-search.mdx index 429ce25..a11ca48 100644 --- a/docs/search/multivector-search.mdx +++ b/docs/search/multivector-search.mdx @@ -11,7 +11,7 @@ This capability is particularly valuable when working with late-interaction mode In this tutorial, you'll create a table with multiple vector embeddings per document and learn how to perform multivector search. For more end-to-end examples, see the [VectorDB recipes repository](https://github.com/lancedb/vectordb-recipes/tree/main/examples). -## Multivector Support +## Multivector Support {#multivector-support} Each item in your dataset can have a column containing multiple vectors, which LanceDB can efficiently index and search. When performing a search, you can query with either a single vector embedding or multiple vector embeddings. @@ -21,7 +21,7 @@ Currently, only the `cosine` metric is supported for multivector search. The vec Each query vector must match the inner vector dimension in the multivector column. This applies to both single-vector queries and multi-vector query matrices. -## Computing Similarity +## Computing Similarity {#computing-similarity} MaxSim (Maximum Similarity) is a key concept in late-interaction models that: @@ -43,9 +43,9 @@ $$ $Q$ represents the query embeddings, and $D = \{d_1, d_2, ..., d_{|D|}\}$ represents the document embeddings. -## Using Multivector Search +## Using Multivector Search {#using-multivector-search} -### 1. Setup +### 1\. Setup {#1-setup} Connect to LanceDB and import the required libraries. @@ -63,7 +63,7 @@ db = lancedb.connect( ``` -### 2. Define Schema +### 2\. Define Schema {#2-define-schema} Define a schema that specifies a multivector field. A multivector field is a nested list structure in which each document contains multiple vectors. In this case, we'll create a schema with: @@ -86,7 +86,7 @@ schema = pa.schema( ``` -### 3. Generate Multivectors +### 3\. Generate Multivectors {#3-generate-multivectors} Generate sample data where each document contains multiple vector embeddings, which can represent different aspects or views of the same document. @@ -104,7 +104,7 @@ data = [ ``` -### 4. Create a Table +### 4\. Create a Table {#4-create-a-table} Create a table with the defined schema and sample data, which will store multiple vectors per document for similarity search. @@ -114,7 +114,7 @@ tbl = db.create_table("multivector_example", data=data, schema=schema) ``` -### 5. Build an Index +### 5\. Build an Index {#5-build-an-index} Only cosine similarity is supported for multivector search operations. For faster search, build the standard `IVF_PQ` index over your vectors: @@ -134,7 +134,7 @@ In LanceDB OSS, the query will just run, so a large unindexed multivector table On LanceDB Enterprise, the brute-force KNN safety check applies a stricter row threshold to multivector columns — roughly 10× lower than for single-vector columns. So an unindexed multivector table will start being rejected with a "vector search would use brute-force KNN" error well before a comparable single-vector table would. Build the index before you start hitting it from production traffic, even if the dataset is small enough that you'd skip indexing for a single-vector workload. -### 6. Query a Single Vector +### 6\. Query a Single Vector {#6-query-a-single-vector} When searching with a single query vector, it will be compared against all vectors in each document, and the similarity scores will be aggregated to find the most relevant documents. @@ -145,7 +145,7 @@ results_single = tbl.search(query).limit(5).to_pandas() ``` -### 7. Query Multiple Vectors +### 7\. Query Multiple Vectors {#7-query-multiple-vectors} With multiple query vectors, LanceDB calculates similarity using late interaction, a late-interaction technique that computes relevance by finding the best-matching pairs between query and document vectors. This approach provides more nuanced matching while maintaining fast retrieval speeds. @@ -159,7 +159,7 @@ results_multi = tbl.search(query_multi).limit(5).to_pandas() Visit the [Hugging Face embedding integration](/integrations/embedding/huggingface/) page for information on embedding models. -## Simple Example: ColBERT Embeddings +## Simple Example: ColBERT Embeddings {#simple-example-colbert-embeddings} [ColBERT](https://arxiv.org/abs/2004.12832) is the most well-known late-interaction retrieval model that represents each document and query as multiple token embeddings and scores matches by taking the best @@ -228,7 +228,7 @@ print(out[["doc_id", "text"]]) Late-interaction model implementations evolve rapidly, so it's a good idea to check the latest popular models when trying multivector search. -## Advanced Example: XTR Embeddings +## Advanced Example: XTR Embeddings {#advanced-example-xtr-embeddings} [ConteXtualized Token Retriever (XTR)](https://arxiv.org/abs/2304.01982) is a late-interaction retrieval model that represents text as token-level vectors instead of a single embedding. This lets search score token-to-token matches (MaxSim), which can improve fine-grained relevance. diff --git a/docs/search/optimize-queries.mdx b/docs/search/optimize-queries.mdx index b9608c3..244e876 100644 --- a/docs/search/optimize-queries.mdx +++ b/docs/search/optimize-queries.mdx @@ -12,9 +12,9 @@ LanceDB provides two powerful tools for query analysis and optimization: `explai | `explain_plan` | Query Analysis | Print the resolved query plan to understand how the query will be executed. Helpful for identifying slow queries or unexpected query results. | | `analyze_plan` | Performance Tuning | Execute the query and return a physical execution plan annotated with runtime metrics including execution time, number of rows processed, and I/O stats. Essential for performance tuning and debugging. | -## Query Analysis Tools +## Query Analysis Tools {#query-analysis-tools} -### explain_plan +### Inspect a query plan {#inspect-a-query-plan} Reveals the logical query plan before execution, helping you identify potential issues with query structure and index usage. This tool is useful for: @@ -23,7 +23,7 @@ Reveals the logical query plan before execution, helping you identify potential - Understanding query execution order - Detecting missing indices -### analyze_plan +### Analyze a query plan {#analyze-a-query-plan} Executes the query and provides detailed runtime metrics, including: - Operation duration (`_elapsed_compute_`) @@ -38,7 +38,7 @@ Metadata filters are prefiltered by default, which usually shows the filter push search instead; that can be useful for some expensive filters, but it changes both latency and the number of rows available after filtering. -## Reading the Execution Plan +## Reading the Execution Plan {#reading-the-execution-plan} To demonstrate query performance analysis, we'll use a table containing 1.2M rows sampled from the [Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). Initially, the table has no indices, allowing us to observe the impact of optimization. @@ -71,7 +71,7 @@ const explainPlan = await table ``` -### Execution Plan Components +### Execution Plan Components {#execution-plan-components} The execution plan reveals the sequence of operations performed to execute your query. Let's examine each component: @@ -87,7 +87,7 @@ ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier LanceScan: uri=***, projection=[vector, identifier], row_id=true, row_addr=false, ordered=false ``` -#### 1. Base Layer (LanceScan) +#### 1\. Base Layer (LanceScan) {#1-base-layer-lancescan} - Initial data scan loading only specified columns to minimize I/O - Unordered scan enabling parallel processing @@ -98,7 +98,7 @@ LanceScan: - row_id=true, row_addr=false, ordered=false ``` -#### 2. First Filter +#### 2\. First Filter {#2-first-filter} - Apply requested filter on `identifier` column - Reduces the number of vectors that need KNN computation @@ -107,7 +107,7 @@ LanceScan: FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 ``` -#### 3. Vector Search +#### 3\. Vector Search {#3-vector-search} - Computes L2 (Euclidean) distances between query vector and all vectors that passed the filter @@ -115,7 +115,7 @@ FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 KNNVectorDistance: metric=l2 ``` -#### 4. Results Processing +#### 4\. Results Processing {#4-results-processing} - Filters out null distance results - Sorts by distance and takes top 100 results @@ -130,7 +130,7 @@ GlobalLimitExec: skip=0, fetch=100 CoalesceBatchesExec: target_batch_size=1024 ``` -#### 5. Data Retrieval +#### 5\. Data Retrieval {#5-data-retrieval} - `RemoteTake` is a key component of Lance's I/O cache - Handles efficient data retrieval from remote storage locations @@ -141,7 +141,7 @@ CoalesceBatchesExec: target_batch_size=1024 RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" ``` -#### 6. Final Output +#### 6\. Final Output {#6-final-output} - Returns only requested columns and maintains column ordering @@ -151,7 +151,7 @@ ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier This plan demonstrates a basic search without index optimizations: it performs a full scan and filter before vector search. -## Performance Analysis +## Performance Analysis {#performance-analysis} Let's use `analyze_plan` to run the query and analyze the query performance, which will help us identify potential bottlenecks: @@ -178,7 +178,7 @@ const analyzePlan = await table ``` -### Performance Metrics Analysis +### Performance Metrics Analysis {#performance-metrics-analysis} ``` ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance], metrics=[output_rows=100, elapsed_compute=1.424µs] @@ -192,21 +192,21 @@ ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier LanceScan: uri=***, projection=[vector, identifier], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1200000, elapsed_compute=21.348178ms, bytes_read=1852931072, iops=78, requests=78] ``` -#### 1. Data Loading (LanceScan) +#### 1\. Data Loading (LanceScan) {#1-data-loading-lancescan} - Scanned 1,200,000 rows from the LanceDB table - Read 1.86GB of data in 78 I/O operations - Only loaded necessary columns (`vector` and `identifier`) - Unordered scan for parallel processing -#### 2. Filtering & Search +#### 2\. Filtering and Search {#2-filtering-and-search} - Applied prefilter condition (`identifier > 0 AND identifier < 1000000`) - Reduced dataset from 1.2M to 1,099,508 rows - KNN search used L2 (Euclidean) distance metric - Vector comparisons processed in 1076 batches -#### 3. Results Processing +#### 3\. Results Processing {#3-results-processing} - KNN results sorted by distance (TopK with fetch=100) - Null distances filtered out @@ -215,7 +215,7 @@ ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier - Remote take operation for 100 results - Final projection of required columns -### Distributed metrics on remote tables +### Distributed metrics on remote tables {#distributed-metrics-on-remote-tables} Enterprise @@ -255,14 +255,14 @@ const plan = await table Use `per_worker` or `full` sparingly — the plan output grows with the size of your worker pool. Stick with the default `aggregate` mode for routine tuning. -### Key Observations +### Key Observations {#key-observations} - Vector search is the primary bottleneck (1,099,508 vector comparisons) - Significant I/O overhead (1.86GB data read) - Full table scan due to lack of indices - Substantial optimization potential through proper index implementation -## Optimized Query Execution +## Optimized Query Execution {#optimized-query-execution} After creating vector and scalar indices, the execution plan shows: @@ -277,9 +277,9 @@ ProjectionExec: expr=[chunk_index@3 as chunk_index, title@4 as title, identifier ScalarIndexQuery: query=AND(identifier > 0,identifier < 1000000) ``` -### Optimized Plan Analysis +### Optimized Plan Analysis {#optimized-plan-analysis} -#### 1. Scalar Index Query +#### 1\. Scalar Index Query {#1-scalar-index-query} ``` ScalarIndexQuery: query=AND(identifier > 0,identifier < 1000000) @@ -297,7 +297,7 @@ metrics=[ - Only 2 index files and 562 scalar index parts loaded - 2.3M index comparisons for matches -#### 2. Vector Search +#### 2\. Vector Search {#2-vector-search} ``` ANNSubIndex: name=vector_idx, k=100, deltas=1 @@ -316,7 +316,7 @@ metrics=[ - 25,893 vector comparisons - 2,000 matching vectors -#### 3. Results Processing +#### 3\. Results Processing {#3-results-processing-2} ``` SortExec: TopK(fetch=100), expr=[_distance@0 ASC NULLS LAST], preserve_partitioning=[false] @@ -328,7 +328,7 @@ CoalesceBatchesExec: target_batch_size=1024 - Limits to top 100 results - Batches into groups of 1024 -#### 4. Data Fetching +#### 4\. Data Fetching {#4-data-fetching} ``` RemoteTake: columns="_distance, _rowid, identifier, chunk_index, title" @@ -338,7 +338,7 @@ metrics=[output_rows=100, elapsed_compute=113.491859ms, output_batches=1, remote - Single output batch - One remote take per row -#### 5. Final Projection +#### 5\. Final Projection {#5-final-projection} ``` ProjectionExec: expr=[chunk_index@3 as chunk_index, title@4 as title, identifier@2 as identifier, _distance@0 as _distance] @@ -346,9 +346,9 @@ ProjectionExec: expr=[chunk_index@3 as chunk_index, title@4 as title, identifier - Returns specified columns: chunk_index, title, identifier, and distance -### Performance Improvements +### Performance Improvements {#performance-improvements} -#### 1. Initial Data Access +#### 1\. Initial Data Access {#1-initial-data-access} ``` ScalarIndexQuery metrics: @@ -361,7 +361,7 @@ ScalarIndexQuery metrics: - After: Only 2 indices and 562 scalar index parts loaded - Benefit: Eliminated table scans for prefilter -#### 2. Vector Search Efficiency +#### 2\. Vector Search Efficiency {#2-vector-search-efficiency} ``` ANNSubIndex: @@ -376,7 +376,7 @@ ANNSubIndex: - 99.8% reduction in vector comparisons - Decreased output batches from 1,076 to 20 -#### 3. Data Retrieval Optimization +#### 3\. Data Retrieval Optimization {#3-data-retrieval-optimization} ``` RemoteTake: @@ -386,17 +386,17 @@ RemoteTake: - RemoteTake operation remains consistent -## Performance Optimization Guide +## Performance Optimization Guide {#performance-optimization-guide} -### 1. Index Implementation +### 1\. Index Implementation {#1-index-implementation} -#### When to Create Indices +#### When to Create Indices {#when-to-create-indices} - Columns used in WHERE clauses - Vector columns for similarity searches - Join columns used in `merge_insert` -#### Index Type Selection +#### Index Type Selection {#index-type-selection} | Data Type | Recommended Index | Use Case | | ----------- | ------------------ | ---------------------------------------- | @@ -410,9 +410,9 @@ Use `table.index_stats()` to monitor index coverage. A well-optimized table should have `num_unindexed_rows ~ 0`. -### 2. Query Plan Optimization +### 2\. Query Plan Optimization {#2-query-plan-optimization} -#### Common Patterns and Fixes +#### Common Patterns and Fixes {#common-patterns-and-fixes} | Plan Pattern | Optimization | | ------------------------------------------ | -------------------------------------------- | @@ -425,18 +425,18 @@ A well-optimized table should have `num_unindexed_rows ~ 0`. Regularly analyze your query plans to identify and address performance bottlenecks. The `analyze_plan` output provides detailed metrics to guide optimization efforts. -### 3. Getting Started with Optimization +### 3\. Getting Started with Optimization {#3-getting-started-with-optimization} For vector search performance: - Create ANN index on your vector column(s) as described in the [index guide](/indexing/vector-index/) - If you often filter by metadata, create [scalar indices](/indexing/scalar-index/) on those columns -## Analyzing non-vector queries +## Analyzing non-vector queries {#analyzing-non-vector-queries} `explain_plan` and `analyze_plan` aren't vector-specific — they're available on every query builder, including FTS and hybrid. The most common reason to look at the plan for a non-vector query is to confirm whether your `where` clause pushed into the scan (good) or ran as a separate `FilterExec` step on top of the search results (often slower, and a hint that the filter column needs a scalar index). -### FTS queries +### FTS queries {#fts-queries} ```python Python icon="python" @@ -461,7 +461,7 @@ const plan = await table In an indexed FTS plan you should see a `MatchQuery` (or other FTS execution node) reading from the inverted index, with the metadata filter pushed down. If the plan shows a `LanceScan` followed by `FilterExec` over the entire text column, the FTS index either isn't covering the column or the filter isn't using a scalar index — both worth investigating. -### Hybrid queries +### Hybrid queries {#hybrid-queries} For hybrid queries, `explain_plan` returns the reranker label followed by the vector and FTS sub-plans, indented for readability: diff --git a/docs/search/sql/fts-sql.mdx b/docs/search/sql/fts-sql.mdx index 91bc70e..7e4d73e 100644 --- a/docs/search/sql/fts-sql.mdx +++ b/docs/search/sql/fts-sql.mdx @@ -17,13 +17,13 @@ LanceDB provides support for full-text search via SQL queries using the `fts()` The SQL `fts()` table function expects exactly two string literals: the table name and the JSON FTS query. Build the JSON query in your application, pass it as a SQL string literal, and keep filtering, grouping, or joining in the surrounding SQL. -## Table Setup +## Table Setup {#table-setup} First, set up your FlightSQL client connection. See [SQL Queries documentation](/search/sql) for detailed client setup instructions. For the examples below, we assume you have a `run_query()` helper function that executes SQL and returns results. -### Creating the Table +### Creating the Table {#creating-the-table} Create a table with text data: @@ -40,7 +40,7 @@ run_query(""" ``` -### Inserting Data +### Inserting Data {#inserting-data} Insert sample documents: @@ -62,7 +62,7 @@ run_query(""" ``` -### Creating FTS Index +### Creating FTS Index {#creating-fts-index} Create a full-text search index on the text column: @@ -84,7 +84,7 @@ CREATE INDEX ON my_docs USING fts (text) WITH (with_position = true) Without position information, phrase queries will not work. See the [Phrase Queries](#phrase-queries) section below for details. -## Basic Full-Text Search +## Basic Full-Text Search {#basic-full-text-search} Use the `fts()` UDTF in SQL queries with JSON-formatted search queries: @@ -135,9 +135,9 @@ LIMIT 5 - `_score` uses the BM25 ranking algorithm to measure relevance -## Advanced Query Types +## Advanced Query Types {#advanced-query-types} -### Fuzzy Search +### Fuzzy Search {#fuzzy-search} Fuzzy search allows you to find matches even when the search terms contain typos: @@ -165,7 +165,7 @@ print(result.to_pandas()) ``` -### Phrase Queries +### Phrase Queries {#phrase-queries} Search for exact phrases in documents: @@ -193,7 +193,7 @@ CREATE INDEX ON my_docs USING fts (text) WITH (with_position = true) ``` -#### Phrase Queries with Slop +#### Phrase Queries with Slop {#phrase-queries-with-slop} Allow some flexibility in phrase matching with the `slop` parameter: @@ -213,11 +213,11 @@ result = run_query(f""" ``` -### Boolean Queries +### Boolean Queries {#boolean-queries} Combine multiple queries using boolean logic: -#### AND Queries +#### AND Queries {#and-queries} ```python Python icon="python" @@ -235,7 +235,7 @@ result = run_query(f""" ``` -#### OR Queries +#### OR Queries {#or-queries} ```python Python icon="python" @@ -262,7 +262,7 @@ print(result.to_pandas()) ``` -### Boost Queries +### Boost Queries {#boost-queries} Control relevance by boosting or demoting certain terms: @@ -286,7 +286,7 @@ result = run_query(f""" ``` -### Multi-Match Queries +### Multi-Match Queries {#multi-match-queries} Search across multiple columns simultaneously: @@ -306,7 +306,7 @@ result = run_query(f""" ``` -#### Multi-Match with Field Boosting +#### Multi-Match with Field Boosting {#multi-match-with-field-boosting} ```python Python icon="python" @@ -324,7 +324,7 @@ result = run_query(f""" ``` -## Combining FTS with SQL +## Combining FTS with SQL {#combining-fts-with-sql} FTS queries can be combined with standard SQL features like WHERE clauses, GROUP BY, and JOINs: @@ -345,11 +345,11 @@ result = run_query(f""" ``` -## Query Parameters Reference +## Query Parameters Reference {#query-parameters-reference} For detailed information about query parameters and options for `MatchQuery`, `PhraseQuery`, `BoostQuery`, and `MultiMatchQuery`, see the [Full-Text Search documentation](/search/full-text-search/). -## Related Documentation +## Related Documentation {#related-documentation} - [Full-text search](/search/full-text-search/) - Learn about FTS capabilities and query types - [SQL queries](/search/sql) - General SQL query documentation diff --git a/docs/search/sql/index.mdx b/docs/search/sql/index.mdx index 6551982..6956b5d 100644 --- a/docs/search/sql/index.mdx +++ b/docs/search/sql/index.mdx @@ -10,7 +10,7 @@ icon: "clipboard-question" [LanceDB Enterprise](/enterprise) comes with an SQL endpoint that can be used for analytical queries and data exploration. The SQL endpoint is designed to be compatible with the [Arrow FlightSQL protocol](https://arrow.apache.org/docs/format/FlightSql.html), which allows you to use any Arrow FlightSQL-compatible client to query your data. -## Installing the client +## Installing the client {#installing-the-client} There are Flight SQL clients available for most languages and tools. If you find that your preferred language or tool is not listed here, please [reach out](mailto:contact@lancedb.com) to us and we can help you find a solution. The following examples demonstrate how to install the Python and TypeScript @@ -34,7 +34,7 @@ npm install --save @lancedb/flightsql-client ``` -## Usage +## Usage {#usage} LanceDB uses the powerful DataFusion query engine to execute SQL queries. This means that you can use a wide variety of SQL syntax and functions to query your data. For more detailed @@ -43,7 +43,7 @@ information on the SQL syntax and functions supported by DataFusion, please refe The FlightSQL endpoint executes one SQL statement per request and is intended for queries. Use the LanceDB SDKs for DDL and table-management operations such as creating tables, adding columns, or building indexes. -### Setting Up the Client +### Setting Up the Client {#setting-up-the-client} Establish a connection to your LanceDB Enterprise SQL endpoint using your preferred FlightSQL client: @@ -72,7 +72,7 @@ const client = await Client.connect({ ``` -### Executing a Query +### Executing a Query {#executing-a-query} Run SQL queries against your LanceDB tables. Different clients may handle the FlightSQL protocol differently: @@ -95,7 +95,7 @@ const result = await client.query("SELECT * FROM flights WHERE origin = 'SFO'"); ``` -### Processing Results +### Processing Results {#processing-results} Handle the query results returned by your FlightSQL client: @@ -118,7 +118,7 @@ console.log(flights); ``` -### Inspecting query plans +### Inspecting query plans {#inspecting-query-plans} The SQL endpoint runs queries through DataFusion, which means DataFusion's `EXPLAIN` family of statements is available unchanged. They're the SQL counterpart of the Python/TypeScript [`explain_plan` and `analyze_plan` methods](/search/optimize-queries) and are useful for the same things: confirming index usage, checking filter pushdown, and finding the slow operator in a query that's underperforming. diff --git a/docs/search/vector-search.mdx b/docs/search/vector-search.mdx index 06381df..185761f 100644 --- a/docs/search/vector-search.mdx +++ b/docs/search/vector-search.mdx @@ -48,7 +48,7 @@ Vector search is a technique used to search for similar items based on their vec Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a multimodal lakehouse like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups. -## Supported distance metrics +## Supported distance metrics {#supported-distance-metrics} Distance metrics determine how LanceDB compares vectors to find similar matches. Euclidean or `l2` is the default, and used for general-purpose similarity, `cosine` for unnormalized embeddings, `dot` for normalized embeddings (best performance), or `hamming` for binary vectors. @@ -77,7 +77,7 @@ For indexed search, supported distance metrics vary by index type: | `IVF_HNSW_PQ` | `["l2", "cosine", "dot"]` | | `IVF_HNSW_SQ` | `["l2", "cosine", "dot"]` | -### Configure Distance Metric +### Configure Distance Metric {#configure-distance-metric} By default, `l2` will be used as metric type. You can specify the metric type as `cosine` or `dot` if required (`hamming` is supported for `IVF_FLAT` index only). @@ -102,7 +102,7 @@ Here you can see the same search but using `cosine` similarity instead of `l2` d Set `.limit(...)` on vector searches you run in applications. You can page through results with `.offset(...)` and include LanceDB's internal row id with `.with_row_id()` / `.withRowId()` when you need a stable handle for follow-up operations. -## Selecting the vector column +## Selecting the vector column {#selecting-the-vector-column} If your table has exactly one vector column, you can omit the column name and LanceDB will pick it for you. This works for both top-level columns (such as `vector`) and vector fields nested inside a struct (such as `image.embedding`). @@ -137,7 +137,7 @@ The same field-path syntax works when creating an index on a nested vector colum When several columns share a name across structs (for example, `image.embedding` and `text.embedding`), LanceDB still picks the one whose dimension matches your query vector. If two candidates have the same dimension, you must pass the column name explicitly. -## Vector Search With ANN Index +## Vector Search With ANN Index {#vector-search-with-ann-index} Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. @@ -149,7 +149,7 @@ Use ANN search for large-scale applications where speed matters more than perfec When a vector index is used, `_distance` is not always the true distance between full vectors. On quantized ANN indexes, LanceDB may compute `_distance` from the compressed representation for speed. Use `refine_factor` when you want reranking on full vectors. -### Exact vs Approximate Distances +### Exact vs Approximate Distances {#exact-vs-approximate-distances} When doing vector search, the meaning of "distance" depends on whether you are using an index and whether `refine_factor` is specified as part of your query. `nprobes` controls how many partitions are searched to find candidates, `approx_mode` controls the query-time speed/recall trade-off for RQ-quantized indexes, and `refine_factor` controls how many candidates are rescored on full vectors for better distance fidelity and reranking quality. @@ -180,7 +180,7 @@ The table below summarizes the behavior of `_distance` in search results based o For deeper tuning guidance on indexing and performance estimation, see the [vector indexes](/indexing/vector-index/#search-configuration) page, For tuning `nprobes`, see below. -### Tuning `approx_mode` +### Tuning approximate mode {#tuning-approximate-mode} Use `approx_mode` when you want to adjust the speed/recall trade-off for approximate vector search at query time. This setting currently applies only to RQ-quantized indexes, such as `IVF_RQ`; other index types ignore it. @@ -194,7 +194,7 @@ The supported values are: You can change `approx_mode` per query without rebuilding the index. For RQ indexes built with `num_bits=1`, `normal` uses the same one-bit scoring path as `fast`. If you also set `refine_factor`, LanceDB first uses `approx_mode` while finding candidates, then reranks the selected candidates on full vectors. -### Tuning `nprobes` +### Tuning `nprobes` {#tuning-nprobes} - `nprobes` controls how many partitions are searched at query time. - `nprobes` improves candidate recall, but does not by itself make `_distance` exact. @@ -208,7 +208,7 @@ For filtered ANN searches, you can also set `minimum_nprobes` and `maximum_nprob with the minimum and can scan more partitions up to the maximum if the filter leaves too few candidates. Calling `nprobes(n)` fixes both values to `n`, which disables that adaptive behavior. -### Vector Search with Prefiltering +### Vector Search with Prefiltering {#vector-search-with-prefiltering} This is the default vector search setting. You can use prefiltering to boost query performance by reducing the search space before vector calculations begin. The system first applies your filter criteria to the dataset, then conducts vector search operations only on the remaining relevant subset. @@ -232,7 +232,7 @@ The `.where("label > 2")` applies a filter before vector search, `.select(["text As a result, you'll see a result with just the data you want from the most similar vectors. -### Vector Search with Postfiltering +### Vector Search with Postfiltering {#vector-search-with-postfiltering} Use postfiltering to prioritize vector similarity by searching the full dataset first, then applying metadata filters to the top results. This approach ensures you get the most similar vectors before filtering, which can be crucial when similarity is more important than metadata constraints. @@ -261,7 +261,7 @@ In the end, you receive a query result with the best matches that also meet your the filter condition after obtaining the nearest neighbors based on vector similarity. -## Multivector Search +## Multivector Search {#multivector-search} Use multivector search when your documents contain multiple embeddings and you need sophisticated matching between query and document vector pairs. The late interaction approach finds the most relevant combinations across all available embeddings and provides nuanced similarity scoring. @@ -278,9 +278,9 @@ Here you can see how to take 2 query vectors and find the best matching pairs be **Read more:** [Multivector search](/search/multivector-search/) -## Advanced Search Scenarios +## Advanced Search Scenarios {#advanced-search-scenarios} -### Search With Distance Range +### Search With Distance Range {#search-with-distance-range} Use `distance_range` search when you need vectors within particular similarity bounds rather than just the closest neighbors. The system filters results to only include vectors that fall within your specified distance thresholds from the query. @@ -304,7 +304,7 @@ The `distance_range()` method filters results by similarity thresholds - the fir Each approach returns Arrow tables with vectors that fall within your specified distance thresholds. -### Search With Binary Vectors +### Search With Binary Vectors {#search-with-binary-vectors} Use binary vector search for scenarios involving binary embeddings, such as those produced by hashing algorithms. The system stores these efficiently as packed uint8 arrays and uses Hamming distance calculations to determine vector similarity. @@ -332,9 +332,9 @@ The schema defines a 32-byte vector field (256 bits ÷ 8), `np.random.randint(0, The search produces an Arrow table with binary vectors ranked by how many bits differ from the query. -## Scaling Vector Search +## Scaling Vector Search {#scaling-vector-search} -### Batch Search +### Batch Search {#batch-search} Use batch search to handle multiple query vectors simultaneously. This gives you significant efficiency gains over individual queries. LanceDB processes all vectors in parallel and organizes results with a `query_index` field that maps each result set back to its originating query. @@ -364,7 +364,7 @@ to explicitly associate each result set with its corresponding query in the input batch. -### Search With Asynchronous Indexing +### Search With Asynchronous Indexing {#search-with-asynchronous-indexing} To optimize for speed over completeness, enable the `fast_search` flag in your query to skip searching unindexed data. @@ -393,9 +393,9 @@ The `fast_search=True` parameter tells LanceDB to only search indexed vectors, s You'll obtain a query result with the top `5` matches from indexed vectors, but might miss data that was just added. -## Brute Force Search +## Brute Force Search {#brute-force-search} -### Search With No Index +### Search With No Index {#search-with-no-index} The simplest way to perform vector search is to perform a brute force search, without an index, where the distance between the query vector and all the vectors in the database are computed, with the top-k closest vectors returned. @@ -423,7 +423,7 @@ This carries out a brute force search through every vector in the table to find As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in. -### Bypass the Vector Index +### Bypass the Vector Index {#bypass-the-vector-index} Use `bypass_vector_index` to get exact, ground-truth results by performing exhaustive searches across all vectors. Instead of relying on approximate methods, the system directly compares your query against every vector in the table, ensuring 100% recall at the cost of increased query time. diff --git a/docs/storage/configuration.mdx b/docs/storage/configuration.mdx index 4113662..45d8bec 100644 --- a/docs/storage/configuration.mdx +++ b/docs/storage/configuration.mdx @@ -42,7 +42,7 @@ When using LanceDB OSS, you can choose where to store your data. The tradeoffs b In LanceDB Enterprise, you connect with `db://...` and the cluster owns the storage credentials, so `storage_options` are not passed at runtime. Cloud auth is set at deployment time. For federated databases, the namespace service vends per-request credentials automatically. See the [quickstart](/quickstart), [Enterprise overview](/enterprise/), and [Azure deployment guide](/enterprise/deployment/azure) for the Enterprise flow. -## Object stores +## Object stores {#object-stores} LanceDB supports AWS S3 (and compatible stores), Azure Blob Storage, and Google Cloud Storage. The URI scheme in your `connect` call selects the backend. @@ -73,7 +73,7 @@ LanceDB supports AWS S3 (and compatible stores), Azure Blob Storage, and Google -### Configuration options +### Configuration options {#configuration-options} When running inside the target cloud with correct IAM bindings, LanceDB often needs no extra configuration. When running elsewhere, provide credentials via environment variables or `storage_options`. @@ -109,7 +109,7 @@ Table-level `storage_options` inherit every key from the connection and override On `AsyncTable`, `await table.initial_storage_options()` returns the options the table was opened with, and `await table.latest_storage_options()` returns the current options after any provider-driven refresh. The deprecated `table.storage_options()` method will be removed in a future release. -#### General object store options +#### General object store options {#general-object-store-options} | Key | Description | | :-- | :-- | @@ -131,7 +131,7 @@ On `AsyncTable`, `await table.initial_storage_options()` returns the options the These are commonly used options. Cloud-specific keys (for example `region`, `endpoint`, `service_account`, and Azure credential keys) are backend-dependent and can be provided in `storage_options` as needed. -#### New table configuration +#### New table configuration {#new-table-configuration} These options control the Lance file format and features used when creating new tables. Pass them via `storage_options` at connection or table level. They are evaluated only at table creation; setting them on an existing connection does not rewrite or alter tables that already exist. @@ -185,7 +185,7 @@ const db = await lancedb.connect("s3://bucket/path", { The `data_storage_version` parameter on `create_table()` is deprecated. Use `new_table_data_storage_version` in `storage_options` instead. -## AWS S3 +## AWS S3 {#aws-s3} ![](/static/assets/images/storage/aws.jpg) @@ -193,7 +193,7 @@ Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optionally `AWS_SESSION_TO Minimum permissions usually include `s3:PutObject`, `s3:GetObject`, `s3:DeleteObject`, `s3:ListBucket`, and `s3:GetBucketLocation` scoped to the relevant bucket/prefix. -### S3-compatible stores +### S3-compatible stores {#s3-compatible-stores} @@ -206,7 +206,7 @@ Minimum permissions usually include `s3:PutObject`, `s3:GetObject`, `s3:DeleteOb If the endpoint is `http://` (common in local development), also set `ALLOW_HTTP=true` or pass `allow_http=True` in `storage_options`. -### S3 Express +### S3 Express {#s3-express} @@ -225,7 +225,7 @@ Consult AWS networking requirements for S3 Express before enabling. LanceDB aborts multipart uploads on graceful shutdown, but crashes can leave incomplete uploads. Add an S3 lifecycle rule to delete in-progress uploads after a few days. -### Server-side encryption with KMS +### Server-side encryption with KMS {#server-side-encryption-with-kms} To encrypt at rest with an AWS KMS key, set `aws_server_side_encryption` to `aws:kms` and `aws_sse_kms_key_id` to the key ID or ARN. The same options apply at connection or table level and combine with bucket-level default encryption. @@ -240,7 +240,7 @@ To encrypt at rest with an AWS KMS key, set `aws_server_side_encryption` to `aws The IAM principal needs `kms:Encrypt`, `kms:Decrypt`, and `kms:GenerateDataKey` on the configured KMS key. -## Google Cloud Storage +## Google Cloud Storage {#google-cloud-storage} ![](/static/assets/images/storage/gcp.jpg) @@ -255,7 +255,7 @@ Provide credentials via `GOOGLE_SERVICE_ACCOUNT` (path to JSON) or include the p -## Azure Blob Storage +## Azure Blob Storage {#azure-blob-storage} ![](/static/assets/images/storage/azure.jpg) @@ -283,7 +283,7 @@ For SAS-token auth, set `azure_storage_account_name` and `azure_storage_sas_toke Other supported keys include service principal credentials (`azure_client_id`, `azure_client_secret`, `azure_tenant_id`), managed identities, and custom endpoints. -## Tigris Object Storage +## Tigris Object Storage {#tigris-object-storage} ![](/static/assets/images/storage/tigris.jpg) @@ -300,7 +300,7 @@ Tigris exposes an S3-compatible API. Configure the endpoint and region: Environment variables `AWS_ENDPOINT=https://t3.storage.dev` and `AWS_DEFAULT_REGION=auto` achieve the same configuration. -## Tencent COS +## Tencent COS {#tencent-cos} [Tencent Cloud Object Storage (COS)](https://www.tencentcloud.com/products/cos) is the primary object store for workloads running in the China region. Use the `cos://` URI scheme to connect directly to a COS bucket. @@ -318,7 +318,7 @@ Supported keys include `secret_id`, `secret_key`, `region`, and `endpoint`. You COS is bundled in the Python wheel by default. To use it from Rust or the Node binding, build with the `cos` Cargo feature enabled. -## GooseFS +## GooseFS {#goosefs} [GooseFS](https://www.tencentcloud.com/document/product/1424) is Tencent Cloud's distributed cache acceleration layer for COS and S3. It is a common choice when the same hot dataset is read repeatedly, such as vector search and AI training workloads. Connect using the `goosefs://` URI scheme. diff --git a/docs/storage/index.mdx b/docs/storage/index.mdx index e0df20f..1d57dce 100644 --- a/docs/storage/index.mdx +++ b/docs/storage/index.mdx @@ -9,7 +9,7 @@ LanceDB's storage layer is built on modular, disk-first components. That design Choosing a backend is a balance between latency, scalability, cost, and operational complexity. Use this guide to pick the right fit for your workload. -## Storage backend selection guide +## Storage backend selection guide {#storage-backend-selection-guide} ![](/static/assets/images/storage/lancedb_storage_tradeoffs.png) @@ -20,12 +20,11 @@ When architecting your system, ask yourself: - **Cost**: What is the all-in cost of storage plus serving? - **Reliability/Availability**: How will replication and disaster recovery work? -## Storage backend comparison +## Storage backend comparison {#storage-backend-comparison} Below is a high-level comparison ordered from lowest cost to lowest latency. -### 1. Object storage (S3 / GCS / Azure Blob) - +### 1\. Object storage (S3, GCS, Azure Blob) {#1-object-storage-s3-gcs-azure-blob} - **Latency**: Highest; expect hundreds of milliseconds and higher p95. - **Scalability**: Effectively unlimited storage; QPS bound by concurrency limits. - **Cost**: Lowest overall. @@ -39,8 +38,7 @@ LanceDB separates storage and compute and writes immutable fragments, making it S3 and S3 Express now support atomic writes natively, so LanceDB handles concurrent writers against the same table out-of-the-box — no external commit coordinator is required. Bucket-level [server-side encryption with KMS](/storage/configuration#server-side-encryption-with-kms) and [S3 Express One Zone](/storage/configuration#s3-express) are also supported on this tier. -### 2. File storage (EFS / GCS Filestore / Azure File) - +### 2\. File storage (EFS, GCS Filestore, Azure File) {#2-file-storage-efs-gcs-filestore-azure-file} - **Latency**: Better than object storage; p95 under ~<100ms is typical. - **Scalability**: High, but limited by provisioned IOPS per volume. - **Cost**: More than object storage but cheaper than in-memory options; cold data can tier down automatically. @@ -48,22 +46,20 @@ S3 and S3 Express now support atomic writes natively, so LanceDB handles concurr Keep a copy of data in object storage for disaster recovery. If zero downtime is required, provision a second network file system with replicated data. -### 3. Third-party storage (e.g., MinIO, WekaFS) +### 3\. Third-party storage (e.g., MinIO, WekaFS) {#3-third-party-storage-e-g--minio-wekafs} - **Latency**: Similar to EFS; typically under <100ms. - **Scalability**: Determined by the chosen vendor’s cluster sizing. - **Cost**: Higher than S3; may edge above EFS at larger scales. - **Reliability/Availability**: Shareable across many nodes; replication depends on vendor capabilities. -### 4. Block storage (EBS / GCP Persistent Disk / Azure Managed Disk) - +### 4\. Block storage (EBS, GCP Persistent Disk, Azure Managed Disk) {#4-block-storage-ebs-gcp-persistent-disk-azure-managed-disk} - **Latency**: Near-local performance; often <30ms. - **Scalability**: Not shareable across instances; shard or copy data when scaling. - **Cost**: Higher than networked file systems, plus potential I/O charges. - **Reliability/Availability**: Persists through instance restarts; backups and sharding must be managed. -### 5. Local storage (SSD / NVMe) - +### 5\. Local storage (SSD, NVMe) {#5-local-storage-ssd-nvme} - **Latency**: Fastest; p95 often under <10ms. - **Scalability**: Hard to scale in cloud environments; requires sharding or additional copies for higher QPS. - **Cost**: Highest; tightly coupling compute and storage makes horizontal scaling difficult. @@ -71,7 +67,7 @@ Keep a copy of data in object storage for disaster recovery. If zero downtime is Use local disk only when you need extremely low latency and are comfortable owning the operational overhead. -## File-format choices that interact with the backend +## File-format choices that interact with the backend {#file-format-choices-that-interact-with-the-backend} A few `storage_options` keys shape new tables in ways that depend on the backend you picked above. They are documented in full on the [configuration page](/storage/configuration#new-table-configuration); the architecture-level summary is: diff --git a/docs/storage/monitoring.mdx b/docs/storage/monitoring.mdx index 017d10e..e62828c 100644 --- a/docs/storage/monitoring.mdx +++ b/docs/storage/monitoring.mdx @@ -14,7 +14,7 @@ The bridge is available in the Python and TypeScript SDKs. It is a thin wrapper This page covers LanceDB OSS. LanceDB Enterprise clusters emit their own Prometheus/OpenTelemetry metrics from the server side — see the [Enterprise overview](/enterprise/) for that flow. -## What you get +## What you get {#what-you-get} Once instrumented, LanceDB registers one observable instrument per metric on your `MeterProvider`. The current catalog covers the object store layer: @@ -32,7 +32,7 @@ The recorder is process-global and pull-based: your configured `MetricReader` co **Histograms are exported Prometheus-style.** OpenTelemetry has no asynchronous histogram instrument, so each histogram surfaces as three observable counters: `_bucket` (with an `le` attribute per bucket boundary, including `+Inf`), `_count`, and `_sum`. Only `_sum` carries the histogram's unit; `_bucket` and `_count` are cumulative sample counts. -## Python +## Python {#python} Install LanceDB with the `otel` extra to pull in the OpenTelemetry API, plus an OpenTelemetry SDK of your choice. The SDK is intentionally not bundled, so you configure it and its readers and exporters however your platform expects. @@ -68,7 +68,7 @@ If you omit `meter_provider`, LanceDB uses the global provider returned by `open `instrument_lancedb_metrics()` returns `False` and emits a warning if another `metrics`-crate recorder is already installed in the process. Only one global recorder is permitted, so instrument LanceDB before any other library that installs its own recorder. -## TypeScript +## TypeScript {#typescript} The Node SDK depends on `@opentelemetry/api` directly, so no extra install step is needed to expose the entry point. You still need an OpenTelemetry SDK to actually export. @@ -93,7 +93,7 @@ const db = await connect("s3://my-bucket/lancedb"); `instrumentLanceDbMetrics()` also accepts no arguments, in which case it uses the global provider from `@opentelemetry/api`. Calling it more than once is safe: instruments are created only on the first successful call. -## What to watch +## What to watch {#what-to-watch} A few starting points for dashboards and alerts: @@ -102,7 +102,7 @@ A few starting points for dashboards and alerts: - **Retryable responses:** a rising `lance_object_store_retryable_responses_total` typically means you are being throttled and should back off or shard writes. - **In-flight requests:** a growing `lance_object_store_in_flight_requests` gauge without a matching rise in throughput indicates queueing. -## Where to go next +## Where to go next {#where-to-go-next} diff --git a/docs/tables-and-namespaces.mdx b/docs/tables-and-namespaces.mdx index becdeea..6899d42 100644 --- a/docs/tables-and-namespaces.mdx +++ b/docs/tables-and-namespaces.mdx @@ -10,7 +10,7 @@ As you dive deeper into LanceDB, it helps to separate two ideas: - A **table** is where your data lives and is queried. - A **namespace** is how groups of tables are organized and resolved at the catalog level. -## Understanding tables +## Understanding tables {#understanding-tables} A table is the core data abstraction in LanceDB: a structured dataset with schema, indexes, and versioned updates. What changes between deployments is how that table is addressed and accessed. @@ -25,7 +25,7 @@ The mental model below clarifies table types by connection mode: From an application perspective, both expose a familiar table API: create/open tables, mutate rows, and query data. The main difference is where resolution and execution happen (directly against storage vs through a remote catalog service). -## Semantic difference between tables and namespaces +## Semantic difference between tables and namespaces {#semantic-difference-between-tables-and-namespaces} The easiest way to think about this is: - A **table** answers: "What data do I store and query?" diff --git a/docs/tables/branching.mdx b/docs/tables/branching.mdx index 12dec53..2f4f0c3 100644 --- a/docs/tables/branching.mdx +++ b/docs/tables/branching.mdx @@ -44,7 +44,7 @@ reads exactly as before. Branches are a natural fit when you want to: `main`. - Hand a collaborator a frozen point-in-time fork while you keep writing to `main`. -## How branches relate to versions and tags +## How branches relate to versions and tags {#how-branches-relate-to-versions-and-tags} Every LanceDB table already tracks a linear history of [versions](/tables/versioning), and you can [tag](/tables/versioning#tag-based-versioning) a version or `checkout` @@ -60,14 +60,14 @@ Branches are supported on local and namespace-backed tables in LanceDB OSS, as well as on LanceDB Enterprise (remote) tables. -## Connect to a table +## Connect to a table {#connect-to-a-table} The branch API is identical no matter how you connect — only the connection itself differs between OSS and Enterprise. Establish a connection (`db`) and open a `table` (see [Create a table](/tables/create)), then use the same branch calls in every example that follows. -### LanceDB OSS +### LanceDB OSS {#lancedb-oss} Point LanceDB at a local directory (or an object-storage URI) to use it as an embedded library. @@ -87,7 +87,7 @@ embedded library. -### LanceDB Enterprise +### LanceDB Enterprise {#lancedb-enterprise} Enterprise @@ -110,13 +110,13 @@ every example that follows. -## Work with branches +## Work with branches {#work-with-branches} The lifecycle of a branch is short and predictable: fork it, write to it, reopen it whenever you need it, and delete it once you're done. The examples below use a small `quotes` table with three rows on `main`. -### Create a branch +### Create a branch {#create-a-branch} Forking from `main` returns a table handle scoped to the new branch. `main` is the reserved default source, so `create` needs only a name; to fork from @@ -136,7 +136,7 @@ somewhere else, pass a branch name, a specific version, or both. -### Write to a branch +### Write to a branch {#write-to-a-branch} Writes go through the branch handle and stay there — the `main` handle keeps reporting its original row count. Listing branches returns a mapping of each @@ -156,7 +156,7 @@ branch name to its metadata, including the version it was forked from. -### Reopen a branch +### Reopen a branch {#reopen-a-branch} A branch outlives the handle that created it. Reopen it later by name — either from an existing table handle or straight from the connection when you open the @@ -176,7 +176,7 @@ table. Both routes give you a writable handle tracking the branch's latest state -### Delete a branch +### Delete a branch {#delete-a-branch} Deleting a branch removes it and its branch-local history; `main` is untouched. Before deleting a branch, make sure you've retained any results you need — see @@ -197,7 +197,7 @@ below. -## Apply branch-tested changes to `main` +## Apply branch-tested changes to `main` {#apply-branch-tested-changes-to-main} A branch has its own writable history. Outside of the [diff and merge @@ -220,7 +220,7 @@ How you apply a validated change depends on the type of work: - **Selected row results:** upsert those rows into `main` using a stable unique key. -### Upsert selected branch rows into `main` +### Upsert selected branch rows into `main` {#upsert-selected-branch-rows-into-main} If the result you want to retain is a set of inserted or updated rows, use [`merge_insert`](/tables/update#merge-incoming-rows-by-key) to write them to @@ -254,7 +254,7 @@ newer values with the same key. Read and upsert the whole branch only when that overwrite is intentional; otherwise, filter the branch read to the rows you intend to apply. -## Compare and merge a branch into `main` +## Compare and merge a branch into `main` {#compare-and-merge-a-branch-into-main} Enterprise @@ -269,7 +269,7 @@ columns; use the [upsert](#upsert-selected-branch-rows-into-main) or rerun patterns above for row and index changes. -### Diff a branch +### Diff a branch {#diff-a-branch} `diff` reads the branch and `main`, and returns a summary of what has changed: which columns were added, removed, or altered; which indexes were added or @@ -300,7 +300,7 @@ matches `main`'s latest), `RowsChanged`, `ColumnRemoved`, `ColumnChanged`, `ParentNotMain`. Newer server codes surface as `Unknown` so older clients keep working. -### Merge a branch +### Merge a branch {#merge-a-branch} `merge` promotes a branch's added columns onto `main`. It is a review-and-land operation: the server re-evaluates the diff at request time, and either lands @@ -352,7 +352,7 @@ see `diff.mergeBlockers`), `notImplemented`, and `unknown` for forward compatibility. Merge requests are not retried on rejection — the response carries everything you need to decide next steps. -## Build indexes on a branch +## Build indexes on a branch {#build-indexes-on-a-branch} One of the most useful things a branch buys you is a safe place to build and validate an index without affecting what's in production on the `main` branch. @@ -385,7 +385,7 @@ Schema changes such as adding, altering, or dropping columns are branch-scoped i the same way, so you can stage and review a larger reshaping of a table before applying the same schema operations to `main`. -## Branches vs. tags vs. versions +## Branches vs. tags vs. versions {#branches-vs-tags-vs-versions} Now that you're familiar with branches, you can see how they complement the other ways LanceDB give you to work with table history. Choose these approaches based on whether you need to diff --git a/docs/tables/consistency.mdx b/docs/tables/consistency.mdx index 715d265..a4152f8 100644 --- a/docs/tables/consistency.mdx +++ b/docs/tables/consistency.mdx @@ -62,7 +62,7 @@ controlled by the cluster-level `weak_read_consistency_interval_seconds` paramet tightens that bound on a per-connection basis. -## Configure Consistency Parameters +## Configure Consistency Parameters {#configure-consistency-parameters} To set strong consistency, set the interval to 0: @@ -127,7 +127,7 @@ a tag, restore a table to a prior version, then return to the live table with `checkout_latest` / `checkoutLatest`. See [Versioning](/tables/versioning/) for the full version and tag workflow. -## Handle bad vectors +## Handle bad vectors {#handle-bad-vectors} This section is currently specific to the Python SDK. diff --git a/docs/tables/create.mdx b/docs/tables/create.mdx index 0e422df..167a0d4 100644 --- a/docs/tables/create.mdx +++ b/docs/tables/create.mdx @@ -47,7 +47,7 @@ In LanceDB, tables store records with a defined schema that specifies column nam - PyArrow schemas for explicit schema control - `LanceModel` for Pydantic-based validation -## Create a table with data +## Create a table with data {#create-a-table-with-data} Initialize a LanceDB connection and create a table @@ -67,7 +67,7 @@ Initialize a LanceDB connection and create a table Depending on the SDK, LanceDB can ingest arrays of records, Arrow tables or record batches, and Arrow batch iterators or readers. Let's take a look at some of the common patterns. -### From list of objects +### From list of objects {#from-list-of-objects} You can provide a list of objects to create a table. The Python and TypeScript SDKs support lists/arrays of dictionaries, while the Rust SDK supports lists of structs. @@ -88,7 +88,7 @@ In Python, pass a list or other batch-like object; a single bare `dict` or singl -### Handle existing tables +### Handle existing tables {#handle-existing-tables} By default, `create_table` raises an error if a table with the same name already exists. You can change this behavior with two parameters that resolve the conflict in different ways: @@ -121,7 +121,7 @@ You can change this behavior with two parameters that resolve the conflict in di contains specific rows, prefer the [empty-table-then-add pattern](#create-empty-table). -### From a custom schema +### From a custom schema {#from-a-custom-schema} You can define a custom Arrow schema for the table. This is useful when you want to have more control over the column types and metadata. @@ -148,7 +148,7 @@ For Python ingest, malformed vector values fail by default. If you expect occasi null, or NaN vectors, choose an `on_bad_vectors` policy: `"drop"` removes those rows, `"fill"` writes `fill_value`, and `"null"` writes nulls. -### From an Arrow Table +### From an Arrow Table {#from-an-arrow-table} You can also create LanceDB tables directly from Arrow tables. Rust uses an Arrow `RecordBatchReader` for the same Arrow-native ingest flow. @@ -167,7 +167,7 @@ Rust uses an Arrow `RecordBatchReader` for the same Arrow-native ingest flow. -### From a Pandas DataFrame +### From a Pandas DataFrame {#from-a-pandas-dataframe} Python Only @@ -184,7 +184,7 @@ Data is converted to Arrow before being written to disk. For maximum control ove The **`vector`** column needs to be a [Vector](/integrations/data/pydantic#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type. -### From a Polars DataFrame +### From a Polars DataFrame {#from-a-polars-dataframe} Python Only LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library @@ -198,7 +198,7 @@ is on the way. -### From Pydantic Models +### From Pydantic Models {#from-pydantic-models} Python Only When you create an empty table without data, you must specify the table schema. @@ -219,7 +219,7 @@ LanceDB only understands subclasses of `lancedb.pydantic.LanceModel` -#### Nested schemas +#### Nested schemas {#nested-schemas} Sometimes your data model may contain nested objects. For example, you may want to store the document string and the document source name as a nested Document object: @@ -251,7 +251,7 @@ document: struct not null child 1, source: string not null ``` -#### Validators +#### Validators {#validators} Because `LanceModel` inherits from Pydantic's `BaseModel`, you can combine them with Pydantic's [field validators](https://docs.pydantic.dev/latest/concepts/validators). The example @@ -266,7 +266,7 @@ for a `created_at` field. When you run this code it, should raise the `ValidationError`. -### Loading Large Datasets +### Loading Large Datasets {#loading-large-datasets} When ingesting large datasets, use `table.add()` on an existing table rather than passing all data to `create_table()`. The `add()` method auto-parallelizes large @@ -277,7 +277,7 @@ For best performance with large datasets, create an empty table first and then c `table.add()`. This enables automatic write parallelism for materialized data sources. -#### From files (Parquet, CSV, etc.) +#### From files (Parquet, CSV, etc.) {#from-files-parquet-csv-etc} Python Only For file-based data, pass a `pyarrow.dataset.Dataset` to `table.add()`. This streams @@ -295,7 +295,7 @@ file-based dataset ingestion is tracked in [lancedb#3173](https://github.com/lancedb/lancedb/issues/3173). -#### From iterators (custom batch generation) +#### From iterators (custom batch generation) {#from-iterators-custom-batch-generation} When you need custom batch logic — generating embeddings on the fly, transforming rows from an external source, etc. — use an iterator of `RecordBatch` objects. @@ -322,7 +322,7 @@ Use this pattern when: Python can also consume iterators of other supported types like Pandas DataFrames or Python lists. -#### Write parallelism +#### Write parallelism {#write-parallelism} For materialized data (`pa.Table`, `pd.DataFrame`, `pa.dataset()`), LanceDB @@ -335,7 +335,7 @@ but not yet exposed in Python or TypeScript ([tracking issue](https://github.com/lancedb/lancedb/issues/3173)). -#### Tracking ingestion progress +#### Tracking ingestion progress {#tracking-ingestion-progress} TypeScript Only For long-running writes, pass a `progress` callback to `table.add()` to surface @@ -366,7 +366,7 @@ A few things to know before you wire this up: - Errors swallowed: anything your callback throws is logged with `console.warn` and won't abort the write, so keep the callback side-effect-only and don't rely on it for control flow. - Row totals: `totalRows` is only populated when the input source can report it up front (for example, a materialized `arrow.Table`). For streaming sources it stays `undefined` until the final callback, where it falls back to the actual rows written. -## Create empty table +## Create empty table {#create-empty-table} You can create an empty table for scenarios where you want to add data to the table later. An example would be when you want to collect data from a stream/external file and then add it to a table in batches. @@ -400,7 +400,7 @@ that has been extended to support LanceDB specific types like `Vector`. Once the empty table has been created, you can append to it or modify its contents, as explained in the [updating and modifying tables](/tables/update) section. -## Open an existing table +## Open an existing table {#open-an-existing-table} You can open an existing table by specifying the name of the table to the `open_table` / `openTable` method. If you forget the name of your table, you can always get a listing of all table names. @@ -419,7 +419,7 @@ If you forget the name of your table, you can always get a listing of all table -## Drop a table +## Drop a table {#drop-a-table} Use the `drop_table()` method on the database to remove a table. diff --git a/docs/tables/index.mdx b/docs/tables/index.mdx index ec370a0..de34d4e 100644 --- a/docs/tables/index.mdx +++ b/docs/tables/index.mdx @@ -73,7 +73,7 @@ Use the example below as a template, and see [Quickstart](/quickstart#python-syn for example snippets on both sync and async Python usage. -## Dataset +## Dataset {#dataset} We'll work with this small dataset based on characters from the legends of Camelot. Note that the `vector` column holds 4-dimensional embeddings, and the `stats` column is a nested struct @@ -153,9 +153,9 @@ The `vector` arrays here are synthetic and for demonstration purposes only. In y applications, you'd generate these vectors from the raw text fields using a suitable embedding model. -## Connect to a database +## Connect to a database {#connect-to-a-database} -### Option 1: Direct table access +### Option 1: Direct table access {#option-1-direct-table-access} We start by connecting to a LanceDB database path. The example below uses a local path in LanceDB OSS. @@ -176,7 +176,7 @@ We start by connecting to a LanceDB database path. The example below uses a loca You can also connect LanceDB OSS directly to object storage. For credentials, endpoints, and provider-specific options, see [Configuring storage](/storage/configuration). -### Option 2: Remote tables +### Option 2: Remote tables {#option-2-remote-tables} If you're using LanceDB [Enterprise](/enterprise), you can connect using a `db://` URI, along with any necessary credentials. Simply replace the local path with a remote `uri` @@ -207,9 +207,9 @@ methods for bulk data export are not available. To retrieve data, use search queries instead: `table.search(query).limit(n).to_arrow()`. -## Create a table and ingest data +## Create a table and ingest data {#create-a-table-and-ingest-data} -### From JSON +### From JSON {#from-json} LanceDB stores records in Lance tables. Each row is a record and each column holds a field or related metadata. The simplest way to start is to obtain the source data as a list of JSON records that includes a vector column and any metadata @@ -274,7 +274,7 @@ For more ingestion patterns, including PyArrow tables, Python `pyarrow.dataset.D empty tables, and Python `LanceModel` schemas with nested fields, see [Ingesting data](/tables/create/). -### From Pandas DataFrames +### From Pandas DataFrames {#from-pandas-dataframes} Python Only You can create LanceDB tables directly from [Pandas](https://pandas.pydata.org/) DataFrames. Simply @@ -287,7 +287,7 @@ and directly ingest to it. -### From Polars DataFrames +### From Polars DataFrames {#from-polars-dataframes} Python Only You can also create LanceDB tables directly from [Polars](https://www.pola.rs/) DataFrames. Simply @@ -300,7 +300,7 @@ and directly ingest to it. -### From an Arrow schema +### From an Arrow schema {#from-an-arrow-schema} If you want to create an _empty_ table without any data -- say you want to define the schema first and then incrementally add data later -- you can @@ -344,7 +344,7 @@ stats: struct ``` -## Append data to a table +## Append data to a table {#append-data-to-a-table} LanceDB tables are mutable, and you can append new records to existing tables. If you're starting with a fresh session, connect to the database and open the @@ -384,7 +384,7 @@ via the `add` method. For the Rust snippet, you can find the helper functions in We now have two new records in the table. Let's begin to query our data! -## Vector search +## Vector search {#vector-search} It's straightforward to run vector similarity search in LanceDB. Let's answer some questions about the data using vector search with projections (returning only @@ -446,7 +446,7 @@ into which you can pass SQL-like expressions. Only three characters have magical abilities greater than 3. Merlin is clearly the most magical of them all! -## Filtered search +## Filtered search {#filtered-search} You can also run traditional analytics-style search queries that do not involve vectors. For example, let's find the strongest characters in @@ -486,7 +486,7 @@ Lance extension to query Lance tables directly with SQL. See the [DuckDB integration guide](/integrations/data/duckdb). -## Add column +## Add column {#add-column} We can also add new columns to an existing LanceDB table using the `add_columns` method. For this example, let's add a new float column named `power` that shows the average @@ -553,7 +553,7 @@ all their abilities! Sir Lancelot and the Lady of the Lake follow closely behind For column renames, type changes, nullability changes, and grouping multiple schema changes into one operation, see [Schema and data evolution](/tables/schema/). -## Delete data +## Delete data {#delete-data} You can delete rows from a LanceDB table using the `delete` method with a filtering expression. @@ -578,7 +578,7 @@ This will delete the row(s) where the `role` value matches "Traitor Knight". You can verify that the row has been deleted by running a search query again, and confirming that Mordred no longer appears in the results. -## Drop column +## Drop column {#drop-column} If you want to remove or delete a column from an existing LanceDB table, you can use the `drop_columns` method. @@ -599,7 +599,7 @@ the `drop_columns` method. This will remove the `power` column we added earlier from the table schema. -## Drop table +## Drop table {#drop-table} If you want to delete an entire table from the database, you can use the `drop_table` method. @@ -626,7 +626,7 @@ See the full code for these examples (including helper functions) in the [docs repo](https://github.com/lancedb/docs/tree/main/tests). -## What about vector indexes? +## What about vector indexes? {#what-about-vector-indexes} LanceDB supports vector indexes to speed up similarity search on large datasets. For datasets up to a few hundred thousand vectors, LanceDB's highly efficient kNN @@ -635,8 +635,7 @@ grows larger, you can create vector indexes on your vector columns to accelerate search. See the [indexing](/indexing/) documentation for details on how to create and use vector indexes in LanceDB. -## What's next? - +## Next steps {#next-steps} Now that you've learned the basics of creating tables, adding data, running vector search, and modifying table schemas, you're ready to explore more advanced features of LanceDB. Below are some suggested next pages. diff --git a/docs/tables/multimodal.mdx b/docs/tables/multimodal.mdx index 19d7214..6da5000 100644 --- a/docs/tables/multimodal.mdx +++ b/docs/tables/multimodal.mdx @@ -39,11 +39,11 @@ LanceDB handles multimodal data—images, audio, video, and PDF files—natively This guide demonstrates how to ingest, store, and retrieve image data using standard binary columns, and also introduces the **Lance Blob API** for optimized handling of larger multimodal files. -## Store binary data +## Store binary data {#store-binary-data} To store binary data, define a binary Arrow field in your schema (`pa.binary()` in Python, `Binary` in TypeScript, and `DataType::Binary` in Rust). -### 1. Setup and imports +### 1\. Setup and imports {#1-setup-and-imports} First, import the necessary libraries for LanceDB and Arrow in your SDK. @@ -61,7 +61,7 @@ First, import the necessary libraries for LanceDB and Arrow in your SDK. -### 2. Prepare data +### 2\. Prepare data {#2-prepare-data} For this example, we'll create some dummy in-memory images. In a real application, you would read these from files or an API. The key is to convert your data (image, audio, etc.) into a raw `bytes` object. @@ -79,7 +79,7 @@ For this example, we'll create some dummy in-memory images. In a real applicatio -### 3. Define the schema +### 3\. Define the schema {#3-define-the-schema} When creating the table, it is **highly recommended** to define the schema explicitly. This ensures that your binary data is correctly interpreted as a `binary` type by Arrow/LanceDB and not as a generic string or list. @@ -97,7 +97,7 @@ When creating the table, it is **highly recommended** to define the schema expli -### 4. Ingest data +### 4\. Ingest data {#4-ingest-data} Now, create the table using the data and the defined schema. @@ -115,7 +115,7 @@ Now, create the table using the data and the defined schema. -## Retrieve and use blobs +## Retrieve and use blobs {#retrieve-and-use-blobs} When you search your LanceDB table, you can retrieve the binary column just like any other metadata. @@ -133,7 +133,7 @@ When you search your LanceDB table, you can retrieve the binary column just like -### Convert bytes back to objects +### Convert bytes back to objects {#convert-bytes-back-to-objects} Once you have the bytes back from the search result, you can decode them into the original format (for example, an image object or audio buffer). @@ -151,11 +151,11 @@ Once you have the bytes back from the search result, you can decode them into th -## Large Blobs (Blob API) +## Large Blobs (Blob API) {#large-blobs-blob-api} For larger files like high-resolution images or videos, Lance provides a specialized **Blob API**. By using a large-binary Arrow type (`pa.large_binary()` in Python, `LargeBinary` in TypeScript, and `DataType::LargeBinary` in Rust) and specific metadata, you enable **lazy loading** and optimized encoding. This allows you to work with massive datasets without loading all binary data into memory upfront. -### 1. Define a blob schema +### 1\. Define a blob schema {#1-define-a-blob-schema} To use the Blob API, you must mark the column with `{"lance-encoding:blob": "true"}` metadata. @@ -173,7 +173,7 @@ To use the Blob API, you must mark the column with `{"lance-encoding:blob": "tru -### 2. Ingest large blobs +### 2\. Ingest large blobs {#2-ingest-large-blobs} You can then ingest data normally, and Lance will handle the optimized storage. @@ -196,7 +196,7 @@ For more advanced usage, including random access and file-like reading of blobs, Lance format's [blob API documentation](https://lance.org/guide/blob/). -### 3. Convert blob tables to pandas +### 3\. Convert blob tables to pandas {#3-convert-blob-tables-to-pandas} When you call `to_pandas()` on a local LanceDB table that contains Blob API columns, the `blob_mode` argument controls how those columns materialize. This is available in the Python SDK on local tables; remote tables raise `NotImplementedError`. @@ -228,7 +228,7 @@ Query builders also accept `blob_mode` on their `to_pandas()` method: -## Other modalities +## Other modalities {#other-modalities} The `pa.binary()` and `pa.large_binary()` types are universal. You can use this same pattern for other types of multimodal data: diff --git a/docs/tables/schema.mdx b/docs/tables/schema.mdx index 7d10120..ef8db19 100644 --- a/docs/tables/schema.mdx +++ b/docs/tables/schema.mdx @@ -65,7 +65,7 @@ LanceDB supports ACID-compliant schema evolution through granular operations (ad * Scale Seamlessly: Handle ML model iterations, regulatory changes, or feature additions * Optimize Continuously: Remove unused fields or enforce new constraints without downtime -## Schema evolution operations +## Schema evolution operations {#schema-evolution-operations} LanceDB supports four primary schema evolution operations: @@ -83,14 +83,14 @@ Each schema evolution operation commits a new table version and returns status m the committed `version`. Run these operations from a mutable table handle; if you checked out an older version for reads, call `checkout_latest` / `checkoutLatest` before modifying the schema. -## Add new columns +## Add new columns {#add-new-columns} You can add new columns to a table with the [`add_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.add_columns) method in Python, [`addColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#addcolumns) in TypeScript/JavaScript, or `add_columns` in Rust. New columns are populated based on SQL expressions you provide. -### Set up the example table +### Set up the example table {#set-up-the-example-table} First, let's create a sample table with product data to demonstrate schema evolution: @@ -108,7 +108,7 @@ First, let's create a sample table with product data to demonstrate schema evolu -### Add derived columns +### Add derived columns {#add-derived-columns} You can add new columns that are derived from existing data using SQL expressions. For feature engineering on large existing tables, group related derived features into @@ -156,7 +156,7 @@ If your transformation cannot be expressed in SQL, compute the values outside `add_columns` before writing them back through another workflow. -### Add columns with default values +### Add columns with default values {#add-columns-with-default-values} Add boolean columns with default values for status tracking: @@ -174,7 +174,7 @@ Add boolean columns with default values for status tracking: -### Add nullable columns +### Add nullable columns {#add-nullable-columns} Add timestamp columns that can contain NULL values: @@ -196,7 +196,7 @@ Add timestamp columns that can contain NULL values: When adding columns that should contain NULL values, be sure to cast the NULL to the appropriate type, e.g., `cast(NULL as timestamp)`. -## Alter existing columns +## Alter existing columns {#alter-existing-columns} You can alter columns using the [`alter_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.alter_columns) method in Python, [`alterColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#altercolumns) in TypeScript/JavaScript, or `alter_columns` in Rust. This allows you to: @@ -206,7 +206,7 @@ method in Python, [`alterColumns`](https://lancedb.github.io/lancedb/js/classes/ - Modify nullability (whether a column can contain NULL values) -### Set up the example table +### Set up the example table {#set-up-the-example-table-2} Create a table with a custom schema to demonstrate column alterations: @@ -224,7 +224,7 @@ Create a table with a custom schema to demonstrate column alterations: -### Rename columns +### Rename columns {#rename-columns} Change column names to better reflect their purpose: @@ -242,7 +242,7 @@ Change column names to better reflect their purpose: -### Change data types +### Change data types {#change-data-types} Convert column data types for better performance or compatibility: @@ -260,7 +260,7 @@ Convert column data types for better performance or compatibility: -### Make columns nullable +### Make columns nullable {#make-columns-nullable} You can alter columns to contain NULL values: @@ -281,7 +281,7 @@ You can alter columns to contain NULL values: Changing a column to nullable affects future writes and merges too: missing values are accepted only when the target column is nullable. -### Multiple changes at once +### Multiple changes at once {#multiple-changes-at-once} Apply several alterations in a single operation: @@ -299,7 +299,7 @@ Apply several alterations in a single operation: -### Expression-based type changes +### Expression-based type changes {#expression-based-type-changes} For transformations that are not simple casts (for example, converting `"$100"` to an integer), use a SQL-expression column add, then drop and rename: @@ -317,7 +317,7 @@ For transformations that are not simple casts (for example, converting `"$100"` -### Alter embedding types and dimensions +### Alter embedding types and dimensions {#alter-embedding-types-and-dimensions} It's quite common to need to change an embedding column's schema, in case a new model becomes available with a different embedding dimension. - In Python, the example shows an in-place type update when the cast is compatible. @@ -350,7 +350,7 @@ For such cases, use `addColumns` / `add_columns` (with `arrow_cast`), then `drop Changing data types requires rewriting the column data and may be resource-intensive for large tables. Renaming columns or changing nullability is more efficient as it only updates metadata. -## Update field metadata +## Update field metadata {#update-field-metadata} Each column in a LanceDB table can carry a small key/value map of Arrow field metadata — useful for annotating columns with units, provenance, PII flags, embedding model versions, or any other @@ -403,7 +403,7 @@ You can pass multiple updates in a single call to change metadata on several fie each call commits a single new table version. -### Canonical metadata keys +### Canonical metadata keys {#canonical-metadata-keys} You can use any string as a metadata key. By convention, LanceDB treats keys prefixed with `lancedb:` as canonical. LanceDB Enterprise displays these keys in the UI, and agents can rely @@ -421,13 +421,13 @@ These keys are conventions, not enforced constraints. LanceDB does not validate but following the conventions keeps your tables consistent with Enterprise tooling. -## Drop columns +## Drop columns {#drop-columns} You can remove columns using the [`drop_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.drop_columns) method in Python, [`dropColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#dropcolumns) in TypeScript/JavaScript, or `drop_columns` in Rust. -### Set Up the example table +### Set Up the example table {#set-up-the-example-table-3} Create a table with temporary columns that we'll remove: @@ -445,7 +445,7 @@ Create a table with temporary columns that we'll remove: -### Drop single columns +### Drop single columns {#drop-single-columns} Remove individual columns that are no longer needed: @@ -463,7 +463,7 @@ Remove individual columns that are no longer needed: -### Drop multiple columns +### Drop multiple columns {#drop-multiple-columns} Remove several columns at once for efficiency: diff --git a/docs/tables/update.mdx b/docs/tables/update.mdx index ad4aaa8..64e169e 100644 --- a/docs/tables/update.mdx +++ b/docs/tables/update.mdx @@ -54,7 +54,7 @@ The `update` method is simpler to use when you already know which rows you want Let's look at an example that demonstrates these operations in practice. -## Connect to LanceDB +## Connect to LanceDB {#connect-to-lancedb} Connect to your local LanceDB instance: @@ -96,7 +96,7 @@ In the Rust snippets, a `make_users_reader` helper is used to build Arrow input -## Create the example table +## Create the example table {#create-the-example-table} We'll start by creating a simple table with `id`, `name`, and `login_count` columns. All examples below use the same table. @@ -123,7 +123,7 @@ Expected table contents: The example above shows a PyArrow schema. You can just as well create the table using other table creation patterns (Pandas, Polars, Pydantic, iterators, etc.) -- see the [ingestion](/tables/create/) guide for more details. -## Choose a write method +## Choose a write method {#choose-a-write-method} | Family | Method | Use this when... | | --------------- | --------------- | ---------------- | @@ -141,7 +141,7 @@ before modifying data. The committed `version` advances even for writes that affect zero rows, such as a delete predicate that matches nothing. -## Update rows +## Update rows {#update-rows} Use `update` when you already know which target rows to modify and you do not need to compare against an incoming dataset. @@ -170,7 +170,7 @@ Expected table contents: Updating nested columns is not yet supported. -## Update rows with SQL expressions +## Update rows with SQL expressions {#update-rows-with-sql-expressions} Use `values_sql` when you want to use SQL-like expressions to update rows. This is useful for operations like incrementing a counter, or setting a column value based on another column. @@ -201,7 +201,7 @@ See the [SQL queries](/search/sql/) page for more information on the supported S When rows are updated, they are moved out of any existing index. The row will still show up in search queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider triggering an index rebuild afterwards. -## Merge incoming rows by key +## Merge incoming rows by key {#merge-incoming-rows-by-key} Merging is different from updating because it involves comparing incoming rows to existing rows by key, and then choosing what to do based on whether the key exists in the target table or not. The `merge_insert(""..."")` method lets you do this. @@ -248,7 +248,7 @@ Primary keys in LanceDB are metadata used by operations such as `merge_insert`; enforced as uniqueness constraints on ordinary writes. Keep using `merge_insert` or your own deduplication logic when you need key-based upsert semantics. -### Update matched rows only +### Update matched rows only {#update-matched-rows-only} This updates keys that already exist in the target table. Source rows with new keys are ignored. @@ -273,7 +273,7 @@ Expected table contents: | 1 | Alice | 10 | | 2 | Bobby | 21 | -### Insert unmatched rows only +### Insert unmatched rows only {#insert-unmatched-rows-only} This inserts only brand-new keys from the source. Existing keys are left unchanged. @@ -299,7 +299,7 @@ Expected table contents: | 2 | Bob | 20 | | 3 | Charlie | 5 | -### Update matched rows and insert unmatched rows +### Update matched rows and insert unmatched rows {#update-matched-rows-and-insert-unmatched-rows} Use both `when_matched_update_all()` and `when_not_matched_insert_all()` when you want to update existing keys and insert missing keys in one operation. @@ -329,7 +329,7 @@ Expected table contents: | 2 | Bobby | 21 | | 3 | Charlie | 5 | -### Delete target rows that are missing from source +### Delete target rows that are missing from source {#delete-target-rows-that-are-missing-from-source} Use `when_not_matched_by_source_delete()` when you want to remove any target row that does not appear in the incoming source data. @@ -356,7 +356,7 @@ Expected table contents: In the example above, LanceDB matches rows by `id`. Rows with `id=2` and `id=3` exist in both the table and incoming data, so they are updated. Row `id=1` exists only in the target, so it is deleted. -### Use partial columns in merge updates +### Use partial columns in merge updates {#use-partial-columns-in-merge-updates} Merge updates do not require you to provide values for all columns. You can provide only a subset of columns in source rows. For matched rows, only the provided columns are updated. @@ -384,7 +384,7 @@ Expected table contents: Note that in the example above, when `merge_insert` creates a new row, any missing columns are written as `null`. If a missing column is non-nullable in your schema, the insert will fail. -## Delete rows +## Delete rows {#delete-rows} Delete operations **soft delete** rows that match a given condition. The underlying data is not immediately removed, but is marked diff --git a/docs/tables/versioning.mdx b/docs/tables/versioning.mdx index e2b961e..ca2f507 100644 --- a/docs/tables/versioning.mdx +++ b/docs/tables/versioning.mdx @@ -41,11 +41,11 @@ import { This page shows the core table-versioning APIs used in the code snippets for Python, TypeScript, and Rust. Each operation below maps directly to methods shown in the examples. -## Basic Versioning Example +## Basic Versioning Example {#basic-versioning-example} Let's create a table with sample data to demonstrate LanceDB's versioning capabilities: -### Set Up the Table +### Set Up the Table {#set-up-the-table} First, let's create a table with some sample data: @@ -71,7 +71,7 @@ First, let's create a table with some sample data: -### Check Initial Version +### Check Initial Version {#check-initial-version} After creating the table, let's check the initial version information: @@ -89,11 +89,11 @@ After creating the table, let's check the initial version information: -## Modify Data +## Modify Data {#modify-data} When you modify data through operations like update or delete, LanceDB automatically creates new versions. -### Update Existing Data +### Update Existing Data {#update-existing-data} Let's update some existing records to see versioning in action: @@ -111,7 +111,7 @@ Let's update some existing records to see versioning in action: -### Add New Data +### Add New Data {#add-new-data} Now let's add more records to the table: @@ -129,7 +129,7 @@ Now let's add more records to the table: -### Check Version Changes +### Check Version Changes {#check-version-changes} Let's see how the versions have changed after our modifications: @@ -147,11 +147,11 @@ Let's see how the versions have changed after our modifications: -## Rollback to Previous Versions +## Rollback to Previous Versions {#rollback-to-previous-versions} LanceDB supports fast rollbacks to any previous version without data duplication. -### View All Versions +### View All Versions {#view-all-versions} First, let's see all the versions we've created: @@ -169,7 +169,7 @@ First, let's see all the versions we've created: -### Restore a Version Snapshot +### Restore a Version Snapshot {#restore-a-version-snapshot} Now let's restore a captured version snapshot: @@ -187,7 +187,7 @@ Now let's restore a captured version snapshot: -## Tag-Based Versioning +## Tag-Based Versioning {#tag-based-versioning} Numeric table versions like `v3` or `v17` are precise but hard to remember. Tags let you attach human-readable labels (e.g., `"prod"`, `"baseline"`, @@ -218,7 +218,7 @@ Deleting a tag only removes the label, not the version it points to. After deletion, the underlying table version becomes eligible for cleanup again. -## Branches +## Branches {#branches} Beyond linear history, LanceDB also supports **branches** — isolated, writable lines of history forked from `main` (or a specific version). Whereas tags and @@ -228,11 +228,11 @@ that you want to keep separate from production reads on `main`. Branches are covered in their own guide: see [Branches](/tables/branching). -## Delete Data From the Table +## Delete Data From the Table {#delete-data-from-the-table} Let's demonstrate how deletions also create new versions: -### Go Back to Latest Version +### Go Back to Latest Version {#go-back-to-latest-version} First, let's return to the latest version: @@ -250,7 +250,7 @@ First, let's return to the latest version: -### Delete Data +### Delete Data {#delete-data} Now let's delete some data to see how it affects versioning: @@ -268,7 +268,7 @@ Now let's delete some data to see how it affects versioning: -### Version History and Operations +### Version History and Operations {#version-history-and-operations} On a fresh table, the snippets in this guide produce this version sequence: diff --git a/docs/training/index.mdx b/docs/training/index.mdx index 18f1bb8..a1510a2 100644 --- a/docs/training/index.mdx +++ b/docs/training/index.mdx @@ -11,7 +11,7 @@ what order. For a more complete solution, LanceDB also provides a streaming data This PyTorch `IterableDataset` adapts the lower-level `Permutation` API and adds prefetching, elastic determinism, resumability, and multithreaded transformations. -## Basic data loading +## Basic data loading {#basic-data-loading} Most model training frameworks iterate through data in batches and feed this data into the model. This process is often referred to as **data loading**. The simplest way to load data into a model is to iterate a LanceDB table in @@ -67,7 +67,7 @@ across cluster sizes, or when you want filtering and prefetching to happen befor Use `Permutation` directly when you need map-style random access instead. Only one iterator can be active on a `StreamingDataset` instance at a time; create a separate instance for each concurrent consumer. -## Advanced data loading +## Advanced data loading {#advanced-data-loading} The `StreamingDataset` wraps a LanceDB `Table` and, by default, adds prefetching and conversion from Arrow to Python. It can also handle more advanced scenarios. To explain these, consider a model trained with stochastic gradient @@ -86,7 +86,7 @@ will use the following PyTorch terms: Other concepts, such as read batch size and `num_workers`, are introduced in the relevant sections below. -### Prefetching +### Prefetching {#prefetching} PyTorch datasets were originally built around in-memory structures like a Pandas DataFrame. When they are iterated, they yield a single sample at a time. This makes sense for a simple in-memory structure, but accessing data on object @@ -110,7 +110,7 @@ ds = StreamingDataset( ``` -### Transformation +### Transformation {#transformation} Many model training workloads require a transformation step between loading the data and training the model. For example, we may need to decode images, tokenize text, or normalize data. A transformation function can be provided @@ -140,7 +140,7 @@ ds = StreamingDataset(table, shuffle_seed=42, transform=normalize) ``` -#### DataLoader workers +#### DataLoader workers {#dataloader-workers} The thread-based transformation model that `StreamingDataset` uses by default is only effective when the transform function releases the GIL. This is true for many Python scientific libraries, including NumPy, PyArrow, and @@ -166,7 +166,7 @@ By default, `StreamingDataset` serializes enough table state to reopen the table setup, pass a picklable `connection_factory` callable that accepts a table name and returns an open table. This avoids serializing connection credentials into worker state. -### Observability & performance +### Observability and performance {#observability-and-performance} Optimizing data loader performance is tricky because it can be difficult to locate the bottleneck. What is often blamed on I/O can be a CPU bottleneck in the transform stage, or vice versa. To help distinguish them, @@ -215,7 +215,7 @@ are cumulative across iterations of the same dataset instance; because work runs can exceed wall-clock time. The queue depths report rows currently waiting in the pipeline, and the progress counters reflect the latest iterator snapshot. -### Filtering data +### Filtering data {#filtering-data} By default, the streaming data loader includes all rows and columns. LanceDB is a columnar database that also supports efficient random access. Reducing the number of columns you load has a direct impact on I/O performance. Reducing the @@ -237,7 +237,7 @@ ds = StreamingDataset( ``` -### Shuffling rows +### Shuffling rows {#shuffling-rows} By default, `StreamingDataset` sets `shuffle=True` and randomly assigns rows to splits. This helps prevent the model from learning artifacts from storage order. Set `shuffle=False` to divide rows into splits sequentially, which is @@ -286,7 +286,7 @@ ds = StreamingDataset( ``` -### Data splits and elasticity +### Data splits and elasticity {#data-splits-and-elasticity} `StreamingDataset` partitions the permutation into a fixed number of equal-sized groups called splits. Each rank gets a contiguous group of splits, and each DataLoader worker gets a contiguous subgroup of its rank's splits. Samples are @@ -338,7 +338,7 @@ for sample in ds: ``` -### Checkpointing and resumability +### Checkpointing and resumability {#checkpointing-and-resumability} Model training is expensive, and failures can occur partway through a run. A model checkpoint is not enough for an exact resume: the streaming data loader must also continue from the same position. `StreamingDataset.state_dict()` @@ -409,7 +409,7 @@ exact resume, also use the same table snapshot, `epoch`, `shuffle`, `filter`, an `world_size` and number of DataLoader workers may change as long as the split and batch-size divisibility constraints still hold. -#### Checkpointing with multiple DataLoader workers +#### Checkpointing with multiple DataLoader workers {#checkpointing-with-multiple-dataloader-workers} The plain `torch.utils.data.DataLoader` only produces a safe checkpoint when `num_workers=0`. With `num_workers > 0`, PyTorch runs `StreamingDataset.__iter__` in separate worker processes and prefetches batches ahead of the trainer, so @@ -460,7 +460,7 @@ With more than one worker, call `state_dict()` at a complete logical step bounda rank has the same consumed-sample count. Calling it mid-step raises `RuntimeError` asking you to consume more batches first. -#### Resuming across different topologies +#### Resuming across different topologies {#resuming-across-different-topologies} When training across ranks, each rank owns its own subset of splits and only its own splits have exact progress. To resume on a different `world_size`, collect the `state_dict()` from every rank of the previous run and merge them with @@ -490,14 +490,14 @@ dataset.load_state_dict(merged) ``` -## Permutations +## Permutations {#permutations} In more complicated scenarios, you may want the flexibility to shuffle, split, and select data without using the full iterable streaming data loader. In these cases, use `Permutation`, the lower-level class on which `StreamingDataset` is built. A `Permutation` defines a custom ordering of the data and supports map-style access through `__getitem__()` and batched access through `__getitems__()`. -### Base table version pinning +### Base table version pinning {#base-table-version-pinning} A `Permutation` is pinned to the version of the base table at the time it was built, and every read (including reads from a `StreamingDataset` that wraps it, and every DataLoader worker after a `fork`) resolves against that pinned diff --git a/docs/training/object-detection.mdx b/docs/training/object-detection.mdx index ff16cb3..2b516d6 100644 --- a/docs/training/object-detection.mdx +++ b/docs/training/object-detection.mdx @@ -9,7 +9,7 @@ This example walks through fine-tuning an autonomous vehicle (AV) perception mod The full pipeline lives in the [lancedb/training](https://github.com/lancedb/training/tree/main/object-detection) repository. This page focuses on the parts most relevant to training: defining curated splits as materialized views, loading them through the [`Permutation`](/training/) API, and pinning checkpoints to an exact data version. -## What you get +## What you get {#what-you-get} Fine-tuning Faster R-CNN ResNet50 FPN v2 for 10 epochs on each curated slice (batch size 64, AMP, A100), starting from the same COCO-pretrained checkpoint and evaluating on the matching validation view: @@ -32,7 +32,7 @@ No external data added — only training-distribution correction via SQL filters The rest of the page walks through the pipeline that produced these checkpoints. -## The failure modes +## The failure modes {#the-failure-modes} A perception model fine-tuned on a generic dataset typically misses the long-tail scenarios that matter most in deployment. Three common failure modes drive this example: @@ -44,7 +44,7 @@ A perception model fine-tuned on a generic dataset typically misses the long-tai Each curated slice becomes a [materialized view](/geneva/jobs/materialized-views) — a named, refreshable SQL filter over the source table — and the training script loads it by name. New footage flows in through `add()` → `backfill()` → `refresh()`; no manifests, no exports, no reshuffling on disk. -## 1. Schema +## 1\. Schema {#1-schema} The source table holds raw image bytes alongside structured annotations. Bounding boxes are stored as a parallel list (one element per box) rather than a nested struct so they remain directly queryable with SQL. @@ -72,7 +72,7 @@ BDD_SCHEMA = pa.schema([ Ingestion streams `pa.RecordBatch`es of raw frames + annotations directly into a Lance table — no intermediate preprocessing job. The table can live on local disk, S3, GCS, or Azure; everything downstream (backfills, views, the training loader) opens it in place via `lancedb.connect("s3://...")` with no local copy step. -## 2. Backfill curation features with Geneva +## 2\. Backfill curation features with Geneva {#2-backfill-curation-features-with-geneva} Curation signals are added as columns on the same table using [Geneva UDFs](/geneva/). Backfills are incremental and checkpointed: re-running the command after new footage arrives only computes the new rows. @@ -120,7 +120,7 @@ with gconn.local_ray_context(): Because the curation features are flat scalar columns on the same table, all four retrieval modes — SQL, full-text search, vector search, and SQL-filtered vector search — work directly without joins or exports. See the [Geneva end-to-end example](/geneva/end-to-end) for more on the backfill pattern. -## 3. Define training splits as materialized views +## 3\. Define training splits as materialized views {#3-define-training-splits-as-materialized-views} A training split is a named SQL filter, not a CSV manifest. Each view stays in sync with the source table and bumps its `version` on every refresh — the link between a checkpoint and the exact data that produced it. @@ -153,7 +153,7 @@ with gconn.local_ray_context(): print(f"[{name}] {mv.count_rows()} rows (version {mv.version})") ``` -## 4. PyTorch DataLoader via the Permutation API +## 4\. PyTorch DataLoader via the Permutation API {#4-pytorch-dataloader-via-the-permutation-api} The training script doesn't know about the filter — it opens a view by name and reads through the [`Permutation`](/training/) API. Each DataLoader worker reopens its own connection lazily, reads Arrow batches directly from Lance (zero-copy, no intermediate file format), and the collate function decodes the whole batch in one pass. `Permutation` provides random-access indexing over the table, so shuffling is a cheap pointer rewrite rather than a full-dataset shuffle on disk. @@ -249,7 +249,7 @@ def make_loader(uri, table_name, batch_size=64, num_workers=8, shuffle=False): `with_format("arrow")` keeps batches as zero-copy `pa.RecordBatch`es — no per-row Python boxing, no pickling between worker and main. Each DataLoader worker reopens its own `Permutation` after fork (the Rust async handle is cleared in `__getstate__`), so reads scale with `num_workers` and stream straight from the underlying object store. JPEG decode overlaps with GPU compute via `pin_memory` + `prefetch_factor`, which is what keeps the loader from becoming the bottleneck on a fast GPU. -## 5. Fine-tune Faster R-CNN +## 5\. Fine-tune Faster R-CNN {#5-fine-tune-faster-r-cnn} The training loop is plain PyTorch — the Lance integration ends at the loader. Mixed precision is enabled on CUDA for ~2× speedup on Ampere GPUs. @@ -310,7 +310,7 @@ for epoch in range(1, 11): print(f"epoch {epoch} ({time.time() - t0:.1f}s)") ``` -## 6. Pin the checkpoint to a data version +## 6\. Pin the checkpoint to a data version {#6-pin-the-checkpoint-to-a-data-version} Every Lance table — including a materialized view — exposes a monotonically increasing `version`. Logging it next to the weights gives a permanent, deterministic link between a checkpoint and the exact data snapshot that produced it. @@ -338,7 +338,7 @@ tbl = lancedb.connect("data/bdd100k/lancedb").open_table("bdd100k_rider_train") tbl.checkout(version=7) # exact snapshot the checkpoint was trained on ``` -## 7. Continuous updates +## 7\. Continuous updates {#7-continuous-updates} When new footage arrives, the same three calls update every downstream view — no view definitions change, no training-script edits required: @@ -363,6 +363,6 @@ for view_name in gconn.table_names(): The next training run picks up the new data automatically — and pins itself to the new `version`. -## Full source +## Full source {#full-source} The complete code, including a synthetic-data mode for pipeline verification (`--synthetic 500`), GPU UDFs for CLIP embeddings and dHash deduplication, and the EDA notebook, is in this [GitHub repository](https://github.com/lancedb/training/tree/main/object-detection). diff --git a/docs/training/torch.mdx b/docs/training/torch.mdx index c7bd63d..39c778a 100644 --- a/docs/training/torch.mdx +++ b/docs/training/torch.mdx @@ -7,7 +7,7 @@ icon: fire LanceDB provides a seamless integration with PyTorch for training and inference. This allows you to use LanceDB as a backend for your PyTorch models, and to use PyTorch for training and inference. You can use LanceDB to store your data, and PyTorch to train your models. -## Quickstart +## Quickstart {#quickstart} The `Table` class in LanceDB implements a contract for a PyTorch [Dataset](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.Dataset). @@ -41,7 +41,7 @@ permutation = Permutation.identity(table) dataloader = torch.utils.data.DataLoader(permutation) ``` -## Output Formats +## Output Formats {#output-formats} By default, a `Table` data loader will emit Arrow data. `collate_fn` is PyTorch's batching hook: PyTorch calls it to turn the fetched items into one batch. PyTorch's default collate function only knows how to combine tensors, NumPy @@ -61,7 +61,7 @@ the Arrow data in different ways. The `arrow` and `polars` formats will always `pandas`, and `torch_col` formats will also avoid data copies in most cases. The `python`, `python_col`, and `torch` formats will all require at least one full copy of the data and are the slowest options. -### Using the torch_col format with a torch data loader +### Using the torch column format with a torch data loader {#using-the-torch-column-format-with-a-torch-data-loader} The `torch_col` format is the most efficient way to convert from Arrow to a `torch.Tensor`. It will convert the entire Arrow batch to a _column-major_ `torch.Tensor`. In other words, given C columns and R rows, the resulting @@ -86,7 +86,7 @@ dataloader = torch.utils.data.DataLoader(permutation, collate_fn=lambda x: x) This will now output a single two-dimensional tensor for each batch. -## Selecting columns +## Selecting columns {#selecting-columns} By default, the `Table` class will return all columns in the table when used as input to PyTorch. If you only need a subset of columns, you can significantly reduce your I/O requirements by selecting only the columns you need. The @@ -104,7 +104,7 @@ for batch in dataloader: print(batch.schema) ``` -## Using multiple DataLoader workers +## Using multiple DataLoader workers {#using-multiple-dataloader-workers} Set `num_workers > 0` to read from LanceDB in multiple PyTorch worker processes. LanceDB tables and `Permutation` objects are picklable, so each worker reopens the table after it starts. @@ -129,7 +129,7 @@ dataloader = torch.utils.data.DataLoader( ) ``` -### Remote tables in DataLoader workers +### Remote tables in DataLoader workers {#remote-tables-in-dataloader-workers} Remote LanceDB Enterprise tables (`db://...`) work the same way: workers reopen the table from the pickled connection state. @@ -158,7 +158,7 @@ dataloader = torch.utils.data.DataLoader( This sends the connection state, including the API key, to each worker. Use a connection factory if credentials should be loaded inside the worker or your `client_config` contains a non-serializable `header_provider`. -### Providing a custom connection factory +### Providing a custom connection factory {#providing-a-custom-connection-factory} `Permutation.with_connection_factory` lets each worker reopen the base table with custom logic. The factory takes the table name, returns a LanceDB table, and must be picklable. diff --git a/docs/training/vlm-finetuning.mdx b/docs/training/vlm-finetuning.mdx index 5547849..39abf24 100644 --- a/docs/training/vlm-finetuning.mdx +++ b/docs/training/vlm-finetuning.mdx @@ -30,7 +30,7 @@ Because the vision tower's weights do not change during fine-tuning, its output The Colab notebook uses a pre-baked subset of the TextVQA dataset: it downloads a curated Lance subset whose expensive feature columns have already been computed. This page explains the complete end-to-end pipeline that produced that subset, then shows how the notebook applies it to produce a fine-tuned model that improves performance on the TextVQA task. -## What you get +## What you get {#what-you-get} On the curated `text_dense` TextVQA slice, the demo fine-tunes `Qwen2.5-VL-3B-Instruct` with QLoRA and evaluates on held-out images: @@ -46,7 +46,7 @@ The larger point is not the absolute score, because you could just as well fine- 2. **Read fixed-size model features efficiently** for shuffled PyTorch batches. 3. **Iterate quickly** from feature idea to scalable CPU/GPU backfill, using Geneva UDFs. -## Why LanceDB fits this workflow +## Why LanceDB fits this workflow {#why-lancedb-fits-this-workflow} VLM fine-tuning pipelines spend a lot of time between "I have an experiment idea" and "I trained the model." LanceDB shortens that loop in three places. @@ -64,7 +64,7 @@ VLM fine-tuning pipelines spend a lot of time between "I have an experiment idea In this pipeline, those three properties combine into the core optimization: compute the VLM vision features once, store them cheaply, then train by reading only the cached columns the model needs. -## Pipeline overview +## Pipeline overview {#pipeline-overview} The runnable demo uses the exact Colab subset hosted at [`lance-format/textvqa-lance-colab`](https://huggingface.co/datasets/lance-format/textvqa-lance-colab). It is derived from the Lance-formatted TextVQA corpus and stores inline JPEG bytes, questions, answers, OCR tokens, object classes, CLIP image/question embeddings, and the cached training features used by this example. The full demo pipeline adds three tiers of derived features on top. @@ -82,7 +82,7 @@ The runnable demo uses the exact Colab subset hosted at [`lance-format/textvqa-l The Colab notebook's workflow starts after all three tiers have been computed. It downloads a small curated subset and runs the training/evaluation path without needing to run Geneva or the vision-tower backfill on the notebook GPU. -## 1. Start with a multimodal LanceDB table +## 1\. Start with a multimodal LanceDB table {#1-start-with-a-multimodal-lancedb-table} The base schema comes from the TextVQA Lance dataset. One row contains the image bytes, natural-language question, reference answers, OCR tokens, scene tags, and retrieval embeddings. @@ -107,7 +107,7 @@ BASE_SCHEMA = pa.schema([ Because the raw image, text, OCR, and embedding features live together, the same table supports curation, retrieval, feature engineering, and training. For example, the notebook can run a text-to-image retrieval demo by searching `image_emb` with a question embedding that already exists in the row. -## 2. Add feature columns with Geneva +## 2\. Add feature columns with Geneva {#2-add-feature-columns-with-geneva} Geneva turns feature engineering into UDF definitions plus backfills. The UDFs can be simple text functions, image-processing functions, or stateful GPU model calls. @@ -189,7 +189,7 @@ The same Tier 3 work can be done manually by creating PyArrow batches and callin See the full UDF registry in [`vlm/geneva_udfs.py`](https://github.com/lancedb/tmls-2026-demo/blob/main/vlm/geneva_udfs.py) and the backfill driver in [`vlm/backfill_geneva.py`](https://github.com/lancedb/tmls-2026-demo/blob/main/vlm/backfill_geneva.py). -## 3. Curate a training slice +## 3\. Curate a training slice {#3-curate-a-training-slice} The demo uses a `text_dense` slice: TextVQA examples whose images contain many OCR tokens. The slice was chosen empirically because it gave the clearest LoRA lift over the already-strong base model. @@ -214,7 +214,7 @@ python -m vlm.colab_prepare \ The train table contains cached Tier 3 columns because training reads them directly. The validation table keeps raw images because evaluation should run the full VLM on unseen images. -## 4. Explore the prepared table +## 4\. Explore the prepared table {#4-explore-the-prepared-table} Before training, it helps to look at the actual task. Each row pairs an image with a question whose answer is often visible as text in the image: a product label, phone screen, sign, book spine, or package. @@ -301,7 +301,7 @@ hits = ( This is the same table that later feeds training. There is no separate feature store, image directory, Parquet export, or manifest to keep synchronized. -## 5. Benchmark Lance vs Parquet-style reads +## 5\. Benchmark Lance vs Parquet-style reads {#5-benchmark-lance-vs-parquet-style-reads} Many training pipelines start with Parquet. Parquet is excellent for columnar analytics, but training commonly needs shuffled batches and fixed-size tensor columns. The notebook compares Lance and Parquet on two access patterns: @@ -355,7 +355,7 @@ The takeaways are workload-specific: The numbers shown above are central to the example. The Tier 3 feature is only useful if the storage format can read it efficiently in the way a trainer actually needs: projected columns, repeated scans, and shuffled batches. **Lance specializes in exactly that access pattern**, including fixed-size list columns stored on disk. -## 6. Load cached columns with the Permutation API +## 6\. Load cached columns with the Permutation API {#6-load-cached-columns-with-the-permutation-api} The training DataLoader projects only the columns needed by the cached training loop: @@ -409,7 +409,7 @@ The training batch contains: | `attention_mask` | `int64[B, 512]` | | `labels` | `int64[B, 512]` | -## 7. Fine-tune without loading the vision tower +## 7\. Fine-tune without loading the vision tower {#7-fine-tune-without-loading-the-vision-tower} The training process loads the language-model side of Qwen2.5-VL in 4-bit, deletes the vision tower, and wraps the LLM projections with LoRA adapters. @@ -470,7 +470,7 @@ saved adapter to runs/colab_lora/lora | peak VRAM 5.3 GB The training loop pays zero per-step cost for image decode, vision-tower forward, or prompt tokenization. Those costs were moved into feature engineering, where LanceDB and Geneva make them durable, incremental, and reusable. -## 8. Evaluate on held-out images +## 8\. Evaluate on held-out images {#8-evaluate-on-held-out-images} Evaluation uses the held-out validation table and loads the full VLM, including the vision tower. That is intentional: inference should see raw unseen images, not the cached train features. @@ -502,7 +502,7 @@ The tuned adapter is not meant to be a state-of-the-art TextVQA checkpoint. It i The notebook renders side-by-side examples: image, question, base answer, tuned answer, and ground truth. This closes the loop from feature idea to trained model while keeping the source data, derived features, training batches, and evaluation split in Lance. -## Full source +## Full source {#full-source} The complete demo implementation with helper scripts and usage instructions is in [this repo](https://github.com/lancedb/tmls-2026-demo). diff --git a/docs/training/why-lancedb.mdx b/docs/training/why-lancedb.mdx index 0adfc81..a74b0ee 100644 --- a/docs/training/why-lancedb.mdx +++ b/docs/training/why-lancedb.mdx @@ -17,7 +17,7 @@ pin versions, and read batches without rewriting the original data. LanceDB gives these stages one platform, so curation, feature engineering, retrieval, and training stay connected. -## A connected data lifecycle +## A connected data lifecycle {#a-connected-data-lifecycle} Training pipelines usually need more than a pile of files. They need curation, derived features, reproducible splits, fast random access, and a clean path into frameworks such as PyTorch. LanceDB keeps these pieces connected through @@ -42,7 +42,7 @@ the same table model, whether you organize a workflow as one table or several re -## Lance as the foundation +## Lance as the foundation {#lance-as-the-foundation} LanceDB is built on [Lance](https://lance.org/), an open-source lakehouse format designed for multimodal AI data. The table below highlights the Lance features that enable the multimodal lakehouse on top. @@ -56,7 +56,7 @@ The table below highlights the Lance features that enable the multimodal lakehou | **Versioning** | Reproduce experiments against the same table snapshot, even as the dataset evolves. | | **Search and filtering** | Find and materialize useful training slices directly from the table. | -## Search inside training workflows +## Search inside training workflows {#search-inside-training-workflows} Search is not limited to QA systems, agents, or production retrieval apps. It is also a practical way to inspect, curate, and improve training data: @@ -69,7 +69,7 @@ curate, and improve training data: In LanceDB, retrieval and training workflows can operate over the same multimodal tables instead of forcing teams to manage separate data systems for each stage. -## Projects using LanceDB for training workflows +## Projects using LanceDB for training workflows {#projects-using-lancedb-for-training-workflows} diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index 96fcbea..bd46b83 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -5,11 +5,11 @@ description: "Tips for troubleshooting basic LanceDB issues." icon: "tools" --- -## Frequently-asked questions +## Frequently-asked questions {#frequently-asked-questions} For commonly asked questions about LanceDB, please refer to our [FAQ section](/faq). -## Getting technical support +## Getting technical support {#getting-technical-support} If you're using LanceDB OSS, the best place to get help is in our [Discord community](https://discord.gg/AUEWnJ7Txb), @@ -19,9 +19,9 @@ and our engineering team. If you are a LanceDB Enterprise user, please contact our support team at [support@lancedb.com](mailto:support@lancedb.com) for dedicated assistance. -## General issues +## General issues {#general-issues} -### Slow or unexpected query results +### Slow or unexpected query results {#slow-or-unexpected-query-results} If you have slow queries or unexpected query results, it can be helpful to print the resolved query plan. @@ -30,6 +30,5 @@ LanceDB provides two powerful tools for query analysis and optimization: `explai Read the full guide on [Query Optimization](/search/optimize-queries/). -### Python's multiprocessing module - +### The Python multiprocessing module {#the-python-multiprocessing-module} Multiprocessing with `fork` is not supported. You should use `spawn` instead. diff --git a/docs/tutorials/agents/multimodal-agent/index.mdx b/docs/tutorials/agents/multimodal-agent/index.mdx index 0957d6e..039f01c 100644 --- a/docs/tutorials/agents/multimodal-agent/index.mdx +++ b/docs/tutorials/agents/multimodal-agent/index.mdx @@ -10,7 +10,7 @@ description: "Build an AI agent that understands both text and images to help us Ever wanted to combine the power of text and images in a single AI agent? In this tutorial, you'll build an agent that can understand both text and images to help users discover recipes that are relevant to them. The approach shown combines LanceDB's multimodal capabilities with [Pydantic AI](https://ai.pydantic.dev/) for the agentic workflow. -## Key Technologies +## Key Technologies {#key-technologies} - **LanceDB**: Embedded retrieval library and multimodal lakehouse for efficient storage and retrieval - **PydanticAI**: Modern AI agent framework with type safety @@ -18,9 +18,9 @@ you'll build an agent that can understand both text and images to help users dis - **CLIP**: Vision-language model for image understanding - **Streamlit**: Interactive web application framework -## Tutorial Overview +## Tutorial Overview {#tutorial-overview} -### Option 1: Notebook +### Option 1: Notebook {#option-1-notebook} The notebook shows how to work through the steps and prepare a small sample recipe dataset, generate both text and image embeddings, store everything efficiently in LanceDB, and then build a PydanticAI agent with custom tools to query it. You'll finish by testing the agent against a few example questions to see the full multimodal flow @@ -32,7 +32,7 @@ No local setup required - just click and start learning about multimodal agents. [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1pxavAGoXa-KSh_4HxNpvP2AjHPcIRpbq?usp=sharing) -### Option 2: Demo Application (Local Setup) +### Option 2: Demo Application (Local Setup) {#option-2-demo-application-local-setup} The demo application is the full codebase: you'll download and process a real recipe dataset with thousands of items, run a Streamlit chat interface that supports image upload, and follow a structure that includes production-minded touches like error handling, logging, and monitoring. Everything you need to deploy is @@ -42,12 +42,12 @@ included. Download the files for the full demo application here. -### Dataset Information +### Dataset Information {#dataset-information} - **Source**: [Kaggle Recipe Dataset](https://www.kaggle.com/datasets/pes12017000148/food-ingredients-and-recipe-dataset-with-images) - **Size**: Thousands of recipes with images - **Format**: CSV file with recipe data and image references -### Setup +### Setup {#setup} ```bash bash icon="code" diff --git a/docs/tutorials/agents/nvidia-rag-blueprint/index.mdx b/docs/tutorials/agents/nvidia-rag-blueprint/index.mdx index 3807d40..667c876 100644 --- a/docs/tutorials/agents/nvidia-rag-blueprint/index.mdx +++ b/docs/tutorials/agents/nvidia-rag-blueprint/index.mdx @@ -4,7 +4,7 @@ sidebarTitle: "NVIDIA RAG Blueprint" description: "Use LanceDB as the retrieval layer for NVIDIA RAG Blueprint with a Docker-first, retrieval-only reference integration." --- -## What this tutorial shows +## What this tutorial shows {#what-this-tutorial-shows} If you are using [NVIDIA RAG Blueprints](https://build.nvidia.com/blueprints) and want to evaluate LanceDB in that stack, this tutorial gives you a concrete starting point. It shows how to use LanceDB as the retrieval layer for a Docker-based NVIDIA RAG deployment with a small, script-driven reference integration where LanceDB OSS is embedded directly in the NVIDIA containers, the collection is prepared ahead of time, and the RAG server retrieves from it for search and generation. The example is intentionally retrieval-only, but it also includes hybrid search and reranker selection so you can see how LanceDB fits into a realistic NVIDIA retrieval workflow. @@ -14,7 +14,7 @@ The runnable example for this tutorial lives in the -## How NVIDIA organizes vector databases +## How NVIDIA organizes vector databases {#how-nvidia-organizes-vector-databases} NVIDIA's [RAG Blueprint documentation](https://docs.nvidia.com/rag/latest/readme.html) effectively describes three different patterns for vector database support. 1. There are built-in backends such as Milvus, where NVIDIA already owns both ingestion and @@ -29,7 +29,7 @@ The LanceDB example shown below fits into the third category. More specifically, RAG Blueprint is then pointed at that existing collection for search and generation. It does not yet teach NVIDIA's ingestor how to write new documents into LanceDB automatically. -## Deployment model +## Deployment model {#deployment-model} This reference integration uses **LanceDB OSS as an embedded retrieval library**, not as a separate database service. In practice, `APP_VECTORSTORE_NAME` is set to `lancedb`, `APP_VECTORSTORE_URL` @@ -37,7 +37,7 @@ points to a local filesystem path inside the NVIDIA containers, the LanceDB coll ahead of time, and the NVIDIA RAG server loads the LanceDB adapter to retrieve directly from that local dataset. -## What the recipe contains +## What the recipe contains {#what-the-recipe-contains} The recipe at [`examples/nvidia-rag-blueprint-lancedb`](https://github.com/lancedb/vectordb-recipes/tree/main/examples/nvidia-rag-blueprint-lancedb) @@ -48,9 +48,9 @@ retrieval-only integration point for NVIDIA RAG Blueprint, while the Docker over change guide show the minimal configuration and source changes needed to run the example against NVIDIA's containers. -## End-to-end flow +## End-to-end flow {#end-to-end-flow} -### 1. Prepare the LanceDB collection +### 1\. Prepare the LanceDB collection {#1-prepare-the-lancedb-collection} From the [recipe directory](https://github.com/lancedb/vectordb-recipes/tree/main/examples/nvidia-rag-blueprint-lancedb): @@ -69,7 +69,7 @@ That script creates: The default embedder is an offline demo embedder so the example stays easy to run. If you want a more realistic setup, the same script can switch to a sentence-transformers embedder. -### 2. Patch the NVIDIA blueprint +### 2\. Patch the NVIDIA blueprint {#2-patch-the-nvidia-blueprint} Follow the instructions in the recipe's [`nvidia_blueprint_changes.md`](https://github.com/lancedb/vectordb-recipes/tree/main/examples/nvidia-rag-blueprint-lancedb/nvidia_blueprint_changes.md). @@ -81,7 +81,7 @@ The essential changes are: NVIDIA's [RAG blueprint documentation](https://docs.nvidia.com/rag/latest/readme.html) and custom-VDB guide provide useful background if you want more context before applying the LanceDB-specific changes. -### 3. Start the Docker deployment +### 3\. Start the Docker deployment {#3-start-the-docker-deployment} Set the absolute path to the recipe directory: @@ -111,9 +111,9 @@ The key environment values are: - `APP_VECTORSTORE_SEARCHTYPE=hybrid` - `LANCEDB_RERANKER=mrr` -## Verifying the integration +## Verifying the integration {#verifying-the-integration} -### Search +### Search {#search} ```bash curl -X POST http://localhost:8081/v1/search \ @@ -127,7 +127,7 @@ curl -X POST http://localhost:8081/v1/search \ }' ``` -### Generate +### Generate {#generate} ```bash curl -N -X POST http://localhost:8081/v1/generate \ @@ -141,7 +141,7 @@ curl -N -X POST http://localhost:8081/v1/generate \ }' ``` -## Hybrid retrieval and rerankers +## Hybrid retrieval and rerankers {#hybrid-retrieval-and-rerankers} This example is meant to prove more than a trivial vector lookup. @@ -153,7 +153,7 @@ This example is meant to prove more than a trivial vector lookup. That matters for NVIDIA partner workloads because product names, storage platforms, and technical jargon often need exact lexical matching as well as semantic retrieval. -## How this can be extended +## How this can be extended {#how-this-can-be-extended} The current example follows NVIDIA's **custom retrieval-only backend** path. In practice, that means the LanceDB collection is created ahead of time and NVIDIA RAG Blueprint is then pointed at diff --git a/docs/tutorials/agents/time-travel-rag/index.mdx b/docs/tutorials/agents/time-travel-rag/index.mdx index fb29888..3f33533 100644 --- a/docs/tutorials/agents/time-travel-rag/index.mdx +++ b/docs/tutorials/agents/time-travel-rag/index.mdx @@ -9,7 +9,7 @@ All the scripts and code for this tutorial are available in the [vectorDB recipes](https://github.com/lancedb/vectordb-recipes/tree/main/examples/time-travel-rag) repository. -## Use case: Financial services regulatory knowledge base +## Use case: Financial services regulatory knowledge base {#use-case-financial-services-regulatory-knowledge-base} Imagine you're a major investment bank. Your team is tasked with building a critical Retrieval-Augmented Generation (RAG) system. This system must provide instant, accurate answers to compliance officers about ever-changing financial regulations. A wrong or out-of-date answer isn't just an inconvenience—it could lead to multi-million dollar fines, reputational damage, and regulatory audits. @@ -22,7 +22,7 @@ Your knowledge base is a living entity, constantly evolving with: This dynamic environment creates a series of high-stakes challenges that traditional vector databases are ill-equipped to handle. -## Pain points solved by LanceDB +## Pain points solved by LanceDB {#pain-points-solved-by-lancedb} 1. "Our RAG gave different answers yesterday versus today. Which version was used in the official compliance report?" Without versioning, you can't prove what the AI knew at a specific point in time, making audits impossible. @@ -34,7 +34,7 @@ vector databases are ill-equipped to handle. LanceDB's [zero-cost data evolution](/tables/schema) and [time-travel capabilities](https://docs.lancedb.com/tables/versioning) directly address these critical enterprise pain points, providing the foundation for a reliable, auditable, and production-ready RAG system. -## Dataset: The U.S. Federal Register +## Dataset: The U.S. Federal Register {#dataset-the-us-federal-register} To make this use case realistic, we'll use a perfect real-world dataset: The U.S. Federal Register, the official daily journal of the United States Government. diff --git a/docs/tutorials/feature-engineering/index.mdx b/docs/tutorials/feature-engineering/index.mdx index 4a5c915..ad18e99 100644 --- a/docs/tutorials/feature-engineering/index.mdx +++ b/docs/tutorials/feature-engineering/index.mdx @@ -11,7 +11,7 @@ layout: wide | **Materialized Views**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/feature-engineering/materialized-views.ipynb) | This example shows how to create materialized views: query results persisted as physical tables. -## Read the docs +## Read the docs {#read-the-docs} The relevant section of the documentation are listed below. diff --git a/docs/tutorials/index.mdx b/docs/tutorials/index.mdx index 4824c89..ec25dc0 100644 --- a/docs/tutorials/index.mdx +++ b/docs/tutorials/index.mdx @@ -12,7 +12,7 @@ Explore tutorials organized by use case: | [Agents](/tutorials/agents/) | Build Retrieval-Augmented Generation (RAG) applications and agents with LanceDB. | | [Working with tables in LanceDB](/tables/) | Learn the basics of working with tables in LanceDB: creation, ingestion and schema evolution. | -## Recipes +## Recipes {#recipes} If you're looking for ideas and hands-on code examples, we've worked on a collection of practical projects in the repository linked below. diff --git a/docs/tutorials/search/index.mdx b/docs/tutorials/search/index.mdx index 90e7d41..035ab38 100644 --- a/docs/tutorials/search/index.mdx +++ b/docs/tutorials/search/index.mdx @@ -18,7 +18,7 @@ The table below shows examples of applications built with LanceDB for search use | **Needle-in-a-haystack multi-vector search**

[Read the tutorial](/tutorials/search/multivector-needle-in-a-haystack/)
| This tutorial complements the XTR multi-vector example by comparing several retrieval strategies on a token-level "needle in a haystack" benchmark. It shows when full multi-vector search, pooling, and reranking help or hurt when the goal is to find an exact page rather than just a relevant document. | -## Read the docs +## Read the docs {#read-the-docs} The relevant section of the documentation are listed below. diff --git a/docs/tutorials/search/multivector-needle-in-a-haystack.mdx b/docs/tutorials/search/multivector-needle-in-a-haystack.mdx index 3e97e60..7a030ae 100644 --- a/docs/tutorials/search/multivector-needle-in-a-haystack.mdx +++ b/docs/tutorials/search/multivector-needle-in-a-haystack.mdx @@ -14,11 +14,11 @@ This guide provides a technical analysis of multivector search for high-precisio To reproduce the work below, see the code [here](https://github.com/lancedb/research/tree/main/multivector-needle-haystack-bench). -## The Dataset +## The Dataset {#the-dataset} This task is different from benchmarks like BEIR, which focus on text-based doc retrieval, finding the most relevant documents from a large collection. Here, we want *intra-document localization*, where the goal is to find a precise piece of information within a single, dense document, in a multimodal setting. -### The Task: The Document Haystack Dataset +### The Task: The Document Haystack Dataset {#the-task-the-document-haystack-dataset} Our benchmark is built on the **[AmazonScience/document-haystack](https://huggingface.co/datasets/AmazonScience/document-haystack)** dataset, which contains 25 visually complex source documents (e.g., financial reports, academic papers). To create a rigorous test, our evaluation follows a per-document methodology: @@ -41,18 +41,17 @@ What is the secret object #3 in the document? The intention of this task is to find the page which has the text needle that answers this questions -## Models and Architectures +## Models and Architectures {#models-and-architectures} Our testbed includes a baseline single-vector model and a family of advanced multivector models. -### Single-Vector (Bi-Encoder) Baseline: `openai/clip-vit-base-patch32` - +### Single-Vector (Bi-Encoder) Baseline: CLIP ViT-Base-Patch32 {#single-vector-bi-encoder-baseline-clip-vit-base-patch32} A bi-encoder maps an entire piece of content (a query, a document page) to a *single* vector. The search process is simple: pre-compute one vector for every page, and at query time, find the page vector closest to the query vector. * **Strength:** Speed and simplicity. * **Weakness:** This creates an **information bottleneck**. All the nuanced details, keywords, and semantic relationships on a page must be compressed into a single, fixed-size vector. For finding a needle, this is like trying to describe a specific person's face using only one word. -### Multi-vector (Late-Interaction) Models +### Multi-vector (Late-Interaction) Models {#multi-vector-late-interaction-models} Multi-vector models, pioneered by ColBERT, take a different approach. Instead of one vector per page, they generate a *set of vectors* for each page—one for every token (or image patch). @@ -61,12 +60,12 @@ Multi-vector models, pioneered by ColBERT, take a different approach. Instead of ![](/static/assets/images/search/multivector/multivector-4.png) -## Different Retrieval Strategies Used +## Different Retrieval Strategies Used {#different-retrieval-strategies-used} A full multivector search is powerful but computationally intensive. Here are five strategies for managing it, complete with LanceDB implementation details. -### 1. `base`: The Gold Standard (Full Multi-vector Search) +### 1\. `base`: The Gold Standard (Full Multi-vector Search) {#1-base-the-gold-standard-full-multi-vector-search} This is the pure, baseline late-interaction search. It offers the highest potential for accuracy by considering every token. @@ -96,7 +95,7 @@ tbl.add([{"page_num": 1, "vector": multi_token_embeddings.tolist()}]) results = tbl.search(query_multi_vector).limit(5).to_list() ``` -### 2. `flatten`: Mean Pooling +### 2\. `flatten`: Mean Pooling {#2-flatten-mean-pooling} This strategy "flattens" the set of token vectors into a single vector by averaging them. This transforms the search into a standard, fast approximate nearest neighbor (ANN) search. @@ -119,12 +118,11 @@ query_mean_vector = query_multi_vector.mean(axis=0) results = tbl_flat.search(query_mean_vector).limit(5).to_list() ``` -### 3. `max_pooling` +### 3\. Max pooling {#3-max-pooling} This is a variation of `flatten`. `max_pooling` takes the element-wise max across all token vectors instead of the mean. The implementation is identical to `flatten`, just with a different aggregation method (`.max(axis=0)`). -### 4. `flatten and multivector rerank`: The Hybrid "Optimization" - +### 4\. `flatten and multivector rerank`: The Hybrid Optimization {#4-flatten-and-multivector-rerank-the-hybrid-optimization} This two-stage strategy aims for the best of both worlds. First, use a fast, pooled-vector search to find a set of promising candidates. Then, run the full, accurate multivector search on *only* those candidates. **LanceDB Implementation:** @@ -164,7 +162,7 @@ final_results = tbl_rerank.search(query_multi_vector, vector_column_name="vector .to_list() ``` -### 5. `hierarchical token pooling`: Compressing the Haystack +### 5\. `hierarchical token pooling`: Compressing the Haystack {#5-hierarchical-token-pooling-compressing-the-haystack} This is an indexing-time strategy that aims to reduce the storage footprint and computational cost of multivector search by reducing the number of vectors per document. Instead of using every token vector, it clusters semantically similar tokens together and replaces them with a single, averaged vector. @@ -200,11 +198,11 @@ tbl_hierarchical.add([{"page_num": 1, "vector": pooled_embeddings.tolist()}]) results = tbl_hierarchical.search(query_multi_vector).limit(5).to_list() ``` -## The Results +## The Results {#the-results} For a "needle in a haystack" task, retrieval accuracy is the primary metric of success. The benchmark results reveal a significant performance gap between the full multivector search strategy and common optimization techniques. -### Baseline Performance: Single-Vector Bi-Encoder +### Baseline Performance: Single-Vector Bi-Encoder {#baseline-performance-single-vector-bi-encoder} First, we establish a baseline using a standard single-vector bi-encoder model, `openai/clip-vit-base-patch32`. This represents a common approach to semantic search but, as the data shows, is ill-suited for this task's precision requirements. @@ -214,7 +212,7 @@ First, we establish a baseline using a standard single-vector bi-encoder model, With a Hit@20 rate of just under 12%, the baseline model struggles to reliably locate the correct page. This performance level is insufficient for applications requiring high precision. -### Multi-vector Model Performance +### Multi-vector Model Performance {#multi-vector-model-performance} We now examine the performance of multivector models using different strategies. The following table compares the `base` (full multivector), `flatten` (mean pooling), and `rerank` (hybrid) strategies across several late-interaction models. @@ -234,7 +232,7 @@ We now examine the performance of multivector models using different strategies. The data shows a consistent pattern: the `base` strategy outperforms all other techniques. The flattned pooling and reranking strategies perform no better than the single-vector baseline. However, hierarchical token pooling seems like a decent alternative to base considering speed vs accuracy tradeoff. Let's look at the numbers in detail. -### In-Depth Analysis of Pooling Strategies +### In-Depth Analysis of Pooling Strategies {#in-depth-analysis-of-pooling-strategies} To further understand the failure of optimization techniques, we compared different methods for pooling token vectors into a single vector: `mean` (`flatten`), `max`. @@ -255,7 +253,7 @@ All flattened pooling methods perform poorly, confirming that the aggregation of 3. **`hierarchical token pooling`:** By clustering and pooling tokens at indexing time, it reduces the number of vectors per page (in our case, by a factor of 4). This intelligently compresses the data, while preserving enough token-level detail in multivector setting. It achieves a **Hit@20 of 91.6%**, only slightly behind the `base` strategy's 95.5%, but is significantly faster. 4. **`base` multivector Search:** The vanilla, un-optimized `base` multivector search remains the most accurate strategy. Preserving every token vector provides the highest guarantee of finding the needle, but this comes at the highest computational cost. -### Latency: +### Latency: {#latency} ![](/static/assets/images/search/multivector/multivector-6.png) @@ -270,11 +268,11 @@ The "optimizations" are not all created equal. While simple pooling is fast, its | `base` (Most Accurate) | 0.668 s | **95.5%** | _Latency reported is as seen on NVIDIA H100 GPUs_ -## Practical Considerations +## Practical Considerations {#practical-considerations} The accuracy of `base` multivector search is impressive, but its computational intensity has historically limited its use. `hierarchical token pooling` as a viable strategy creates a new, practical sweet spot on the accuracy-latency curve, making high-precision search accessible for a wider range of applications. -### Search Latency and Computational Complexity +### Search Latency and Computational Complexity {#search-latency-and-computational-complexity} As the benchmark data shows, the search latency for `base` multivector search is orders of magnitude higher than for single-vector (or pooled-vector) search. It's important to note that the reported ~670ms latency is an average from per-document evaluations. In this benchmark, each of the 25 documents is processed independently. All pages from a single document's variants (ranging from 5 to 200 pages) are ingested into a temporary table, resulting in a table size of approximately **1,230 rows (pages)** per evaluation. The search is performed on this table, and then the table is discarded. This highlights a significant performance cost even on a relatively small, per-document scale. This stems from a fundamental difference in computational complexity: @@ -282,7 +280,7 @@ As the benchmark data shows, the search latency for `base` multivector search is * **Modern ANN Search (for single vectors):** Algorithms like HNSW (Hierarchical Navigable Small World) provide sub-linear search times, often close to `O(log N)`, where `N` is the number of items in the index. This allows them to scale to billions of vectors with millisecond-level latency. * **Late-Interaction Search (Multi-vector):** The search process is far more intensive. For each query, it must compute similarity scores between query tokens and the tokens of many candidate documents. The complexity is closer to `O(M * Q * D)`, where `M` is the number of candidate documents to score, `Q` is the number of query tokens, and `D` is the average number of tokens per document. `Hierarchical token pooling` directly attacks this problem by reducing `D`, leading to a significant reduction in search latency. -### When to Use Multi-Vector Search +### When to Use Multi-Vector Search {#when-to-use-multi-vector-search} Given these constraints, the choice of strategy depends on the specific requirements of the application. @@ -290,7 +288,7 @@ Given these constraints, the choice of strategy depends on the specific requirem * **For a Balance of Precision and Performance (`hierarchical token pooling`):** This is the ideal choice for many applications. It makes high-precision search practical for larger datasets and more interactive use cases where the sub-second latency of the `base` search may be too high. It significantly lowers the barrier to entry for adopting multivector search. It should still not be seen as a drop-in replacement for ANN, as it still requires more computational resources than single-vector search. * **For General-Purpose Document Retrieval (`flatten` / single-vector):** For large-scale retrieval where understanding the "gist" is sufficient or where in cases where large-context text-based models suffice, single-vector search remains the most practical and scalable solution. -## Appendix: Full Benchmark Results +## Appendix: Full Benchmark Results {#appendix-full-benchmark-results} The full benchmark results are shown below. diff --git a/scripts/add_anchors.py b/scripts/add_anchors.py new file mode 100644 index 0000000..7db8bba --- /dev/null +++ b/scripts/add_anchors.py @@ -0,0 +1,297 @@ +""" +Give every section heading a stable anchor. + +An anchor is the identity a section keeps when it is reworded or moved, and the +key Enterprise overlays attach to from A5: an overlay says "put this after +`{#branch-create}`" and must still land correctly after someone rewrites the +heading above it. Heading-derived slugs cannot do that — they change with the +text — so the anchor is written down once and then never regenerated. + +That "never regenerated" is the whole point, and it shapes this script: + + * headings that already carry an anchor are left exactly as they are, so + re-running is safe and an anchor edited by hand survives; + * names come from the ids the site *already renders*, not from a slug rule of + our own. That is what keeps every existing deep link working: a reader's + bookmark to `#what's-next` must still resolve afterwards. Deriving the name + independently looked equivalent and was not — Mintlify keeps a curly + apostrophe in the id where a naive slug turns it into a hyphen, so + `tables/index` would have silently changed its anchor. + + Names are still readable, because Mintlify derives them from the heading + text too. They are simply frozen at today's value rather than recomputed: + a later reworded heading keeps the original anchor, and that divergence is + the point, not drift. + +Fenced code blocks are skipped: a `## comment` inside a shell example is not a +heading. + +The rendered ids come from a `mint export` bundle, so run one first: + + cd docs && mint export --output /tmp/site.zip + +Usage: + + python scripts/add_anchors.py --export /tmp/site.zip docs/tables + python scripts/add_anchors.py --export /tmp/site.zip --check docs/tables +""" + +from __future__ import annotations + +import argparse +import html as html_module +import re +import sys +import tempfile +import zipfile +from pathlib import Path + +# `## Heading`, capturing any existing `{#anchor}` so it can be preserved. +HEADING_RE = re.compile( + r"^(?P#{2,6})\s+(?P.+?)(?:\s+\{#(?P[^}]+)\})?\s*$" +) +FENCE_RE = re.compile(r"^\s*```") +# `### 1. Setup` renders with its number until an explicit `{#anchor}` is added, +# at which point Mintlify re-parses the text and treats the number as an ordered +# list marker, silently dropping it from the heading and the table of contents. +# 117 headings across 17 pages start this way. Escaping the period keeps the +# rendered text and the id identical; the backslash does not reach the output. +LEADING_NUMBER_RE = re.compile(r"^(\d+)\. (?=\S)") +# Eleven h2s in the reranking pages are Setext -- text over a rule of dashes -- +# rather than ATX. Mintlify renders them with ids like any other heading, so a +# parser that only saw ATX would consume the rendered ids out of order and give +# every later heading on the page the wrong anchor: plausible names silently +# attached to the wrong sections. They are rewritten to ATX, which renders +# identically and makes the anchor syntax uniform. +SETEXT_UNDERLINE_RE = re.compile(r"^-{3,}\s*$") +# Mintlify emits heading ids down to h4 and no further. The eight h5 headings in +# the corpus -- all API method names on integrations/ai/langchain.mdx -- render +# with no id at all, so there is no existing anchor to preserve and nothing for +# an overlay to have been written against. They are left alone rather than given +# an invented anchor, which would also change the rendered HTML. +MAX_ANCHORED_LEVEL = 4 +# Mintlify keeps `&` in an auto-generated id but strips it from an explicit +# `{#anchor}`, so `observability-&-performance` cannot be written down: any +# anchor we set changes the id and breaks links to it. Five headings across the +# corpus join two words with an ampersand. They keep their generated id and go +# without an explicit anchor; if A5 ever needs to attach to one, reword the +# heading then rather than silently move it now. +UNWRITABLE_IN_ANCHOR = "&" +FRONTMATTER_DELIM = "---" + +# Inline markup to strip before deriving a name, so `## Use \`add_columns()\`` +# becomes `use-add-columns` rather than carrying backticks into the anchor. +INLINE_CODE_RE = re.compile(r"`([^`]*)`") +LINK_RE = re.compile(r"\[([^\]]*)\]\([^)]*\)") +JSX_RE = re.compile(r"<[^>]+>") +NON_SLUG_RE = re.compile(r"[^a-z0-9]+") + + +def slugify(text: str) -> str: + text = LINK_RE.sub(r"\1", text) + text = INLINE_CODE_RE.sub(r"\1", text) + text = JSX_RE.sub(" ", text) + text = NON_SLUG_RE.sub("-", text.lower()).strip("-") + return text or "section" + + +HEADING_ID_RE = re.compile(r']*\bid="([^"]+)"') + + +def rendered_ids(export: Path) -> dict[str, list[str]]: + """Map page path -> heading ids, in document order, from a mint export. + + Mintlify emits an id on every heading. Reusing those ids as anchors is what + makes the pass invisible to readers and safe for existing links. + """ + with tempfile.TemporaryDirectory() as td: + root = Path(td) + if export.is_dir(): + root = export + else: + with zipfile.ZipFile(export) as zf: + zf.extractall(root) + ids: dict[str, list[str]] = {} + for page in root.rglob("index.html"): + rel = page.parent.relative_to(root).as_posix() + # The site root renders to index.html at the top level; everything + # else keeps its full path, including a page literally named index. + if rel == ".": + rel = "index" + found = [ + # Unescape: an id containing `&` appears in the HTML as `&`, + # and writing that back verbatim produced `...-amp-...`, changing + # the id on five pages whose headings join two words with an + # ampersand. + html_module.unescape(anchor) + for _level, anchor in HEADING_ID_RE.findall( + page.read_text(encoding="utf-8", errors="replace") + ) + # React-generated ids are per-build noise, not heading slugs. + if not anchor.startswith("_R_") + ] + ids[rel] = found + return ids + + +def take_anchor(path: Path, available: list[str], used: set[str], index: int) -> str: + """Consume the next rendered id, in document order.""" + if index >= len(available): + raise SystemExit( + f"{path}: more headings in source than rendered ids " + f"({index + 1} > {len(available)}); re-run mint export" + ) + anchor = available[index] + if anchor in used: + raise SystemExit(f"{path}: rendered id {anchor!r} appears twice") + used.add(anchor) + return anchor + + +def anchor_file( + path: Path, docs_root: Path, ids: dict[str, list[str]], apply: bool +) -> tuple[int, int, list[str]]: + """Return (added, existing, sample) for one page.""" + page = str(path.relative_to(docs_root)).removesuffix(".mdx") + available = list(ids.get(page, [])) + + lines = path.read_text(encoding="utf-8").split("\n") + out: list[str] = [] + used: set[str] = set() + added = existing = skipped = unwritable = 0 + unwritable_ids: list[str] = [] + sample: list[str] = [] + + in_fence = False + in_frontmatter = False + for index, line in enumerate(lines): + # Frontmatter opens on the very first line and is not content. + if index == 0 and line.strip() == FRONTMATTER_DELIM: + in_frontmatter = True + out.append(line) + continue + if in_frontmatter: + if line.strip() == FRONTMATTER_DELIM: + in_frontmatter = False + out.append(line) + continue + if FENCE_RE.match(line): + in_fence = not in_fence + out.append(line) + continue + if in_fence: + out.append(line) + continue + + # A Setext h2 is the *previous* line plus this rule. Detect it when the + # underline arrives, rewrite the pair as ATX, and drop the rule. + if ( + SETEXT_UNDERLINE_RE.match(line) + and out + and out[-1].strip() + and not out[-1].lstrip().startswith(("#", "-", "|", "<", ":")) + and "|" not in out[-1] + ): + heading_text = out.pop().strip() + anchor = take_anchor(path, available, used, len(used) + unwritable) + added += 1 + if len(sample) < 3: + sample.append(f"{heading_text[:44]} -> {{#{anchor}}} (setext)") + out.append(f"## {heading_text} {{#{anchor}}}") + continue + + match = HEADING_RE.match(line) + if not match: + out.append(line) + continue + + if match.group("anchor"): + used.add(match.group("anchor")) + existing += 1 + out.append(line) + continue + + if len(match.group("hashes")) > MAX_ANCHORED_LEVEL: + skipped += 1 + out.append(line) + continue + + # Take the id the site already renders for this heading. Falling back to + # a derived slug would reintroduce exactly the divergence this avoids, so + # a missing id is an error rather than a guess. + anchor = available[len(used) + unwritable] if len(used) + unwritable < len(available) else None + if anchor is not None and UNWRITABLE_IN_ANCHOR in anchor: + unwritable += 1 + unwritable_ids.append(anchor) + out.append(line) + continue + anchor = take_anchor(path, available, used, len(used) + unwritable) + added += 1 + if len(sample) < 3: + sample.append(f"{match.group('text')[:44]} -> {{#{anchor}}}") + text = LEADING_NUMBER_RE.sub(r"\1\\. ", match.group("text")) + out.append(f"{match.group('hashes')} {text} {{#{anchor}}}") + + # Anchors are assigned by position, so a count mismatch means every anchor + # after the divergence is attached to the wrong section. Fail rather than + # write plausible-looking nonsense. + if unwritable_ids: + print(f"{path}: left unanchored, id not writable: {', '.join(unwritable_ids)}") + if added + existing + unwritable != len(available): + raise SystemExit( + f"{path}: {added + existing + unwritable} headings accounted for but " + f"{len(available)} rendered ids — anchors would be misaligned" + + (f" ({skipped} deeper than h{MAX_ANCHORED_LEVEL} skipped)" if skipped else "") + ) + + if apply and added: + path.write_text("\n".join(out), encoding="utf-8") + return added, existing, sample + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + parser.add_argument("paths", nargs="+", type=Path, help="files or directories") + parser.add_argument( + "--export", + type=Path, + required=True, + help="mint export zip or directory supplying the rendered heading ids", + ) + parser.add_argument( + "--docs-root", type=Path, default=Path("docs"), help="docs root for page paths" + ) + parser.add_argument( + "--check", + action="store_true", + help="report headings without an anchor and exit non-zero if any remain", + ) + args = parser.parse_args() + + targets: list[Path] = [] + for path in args.paths: + targets.extend(sorted(path.rglob("*.mdx")) if path.is_dir() else [path]) + + ids = rendered_ids(args.export) + total_added = total_existing = 0 + for target in targets: + added, existing, sample = anchor_file( + target, args.docs_root, ids, apply=not args.check + ) + total_added += added + total_existing += existing + if added: + verb = "missing" if args.check else "anchored" + print(f"{target}: {verb} {added}, already anchored {existing}") + for line in sample: + print(f" {line}") + + if args.check: + print(f"\n{total_added} headings without an anchor, {total_existing} with one") + return 1 if total_added else 0 + print(f"\nanchored {total_added} headings, left {total_existing} unchanged") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/compare_exports.py b/scripts/compare_exports.py index 10c8c13..d64ccfa 100644 --- a/scripts/compare_exports.py +++ b/scripts/compare_exports.py @@ -27,23 +27,27 @@ rb"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" ) -# Pages that fail to generate request examples are not reproducible: the failure -# renders "A valid request URL is required to generate request examples" in place -# of the code samples, and *which* pages fail varies per run. That was an upstream -# spec defect -- every `servers` entry was templated -- fixed in -# lance-format/lance-namespace@3b167e4c. +# Mintlify renders the OpenAPI reference non-deterministically. Two exports of a +# byte-identical tree intermittently disagree on these pages: response code blocks +# come out syntax-highlighted on one run and plain on the next, a ~2 KB difference +# across ~78 fragments, on top of per-build React keys. It is intermittent -- one +# comparison passes, the next fails -- so comparing this subtree by content makes +# the harness flaky, and a gate that fails at random is one people learn to ignore. # -# Quarantine is therefore keyed on the placeholder itself, not on the path. It -# retires automatically: once the tree carries the fixed spec no page contains the -# placeholder, nothing is quarantined, and the whole reference is compared like any -# other page. Verified against exports built from the fixed spec -- all 54 REST -# pages that differ between runs differ *only* by per-build UUIDs, with zero -# content differences. -UNSTABLE_PREFIX = "api-reference/rest/" +# The contents are therefore compared for *presence* but not for *bytes*, and only +# here. This gives up nothing about the assembler: it passes `openapi.yml` through +# byte-identically, both sides feed Mintlify the same spec, and the assembler has +# no way to influence one render differently from the other. What it could break -- +# a page appearing or disappearing -- is still caught, because the file set is +# compared exactly. +# +# The separate, stronger guarantee is that the assembled tree is byte-identical to +# its source, which `make assemble` proves directly and which covers the spec file. +GENERATED_PREFIX = "api-reference/rest/" PLACEHOLDER = b"A valid request URL is required" -# Observed jitter on identical input: 43-48 placeholders across runs. A hard -# gate on this number would fail at random, so it is reported always and warned -# on only outside the jitter band -- never allowed to decide the verdict. +# Reported every run: with the spec fix in place the count should be zero, and a +# non-zero count means the reference has regressed to unusable examples. Not a +# hard gate, because the count itself jitters on identical input. PLACEHOLDER_JITTER = 6 @@ -100,25 +104,24 @@ def main() -> int: # UUID normalization is scoped to the generated reference, whose React # keys are regenerated per build. Applying it to authored pages would # silently accept a genuine UUID-only edit. - if not rel.startswith(UNSTABLE_PREFIX): + if not rel.startswith(GENERATED_PREFIX): real_diffs.append(f"{rel} (content)") continue na, ca = normalize(ra) nb, cb = normalize(rb) if na == nb and ca == cb: uuid_only.append(rel) - elif PLACEHOLDER in ra or PLACEHOLDER in rb: - # Example generation failed on at least one side; not reproducible. - quarantined.append(rel) else: - why = "content" if na != nb else f"uuid count {ca} vs {cb}" - real_diffs.append(f"{rel} ({why})") + # Generated reference page whose render is not reproducible. + # Present on both sides, contents not compared. See + # GENERATED_PREFIX for why that gives up nothing. + quarantined.append(rel) def placeholders(m: dict[str, Path]) -> int: return sum( 1 for rel, p in m.items() - if rel.startswith(UNSTABLE_PREFIX) and PLACEHOLDER in p.read_bytes() + if rel.startswith(GENERATED_PREFIX) and PLACEHOLDER in p.read_bytes() ) ph_a, ph_b = placeholders(a), placeholders(b) @@ -128,8 +131,8 @@ def placeholders(m: dict[str, Path]) -> int: identical = len(shared) - len(uuid_only) - len(real_diffs) - len(quarantined) print(f"identical bytes : {identical}") print(f"equivalent (uuid-only): {len(uuid_only)}") - label = "quarantined (broken examples)" - print(f"{label:<22}: {len(quarantined)}" + (" — retires once the fixed spec lands" if quarantined else " — none: reference is fully compared")) + label = "generated (presence only)" + print(f"{label:<22}: {len(quarantined)} — OpenAPI reference, see GENERATED_PREFIX") print(f"REAL DIFFERENCES : {len(real_diffs)}") print(f"only in baseline : {len(only_base)}") print(f"only in new : {len(only_new)}") @@ -161,13 +164,12 @@ def placeholders(m: dict[str, Path]) -> int: # request examples. A quarantined page now means the reference has # regressed rather than that it was never working -- fail instead of # noting it. - equivalent = not (real_diffs or only_base or only_new or quarantined) + equivalent = not (real_diffs or only_base or only_new) print("\nVERDICT:", "EQUIVALENT" if equivalent else "DIFFERENT") - if quarantined and equivalent: + if ph_a or ph_b: print( - f"note: {len(quarantined)} reference pages could not generate request " - "examples and were not compared. Expected until the tree carries " - "lance-format/lance-namespace@3b167e4c; zero after." + "warning: reference pages could not generate request examples " + f"({ph_a} baseline, {ph_b} new) — check the OpenAPI spec pin" ) return 0 if equivalent else 1