diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index bbc8b53..0279ea0 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -35,6 +35,16 @@ jobs: steps: - uses: actions/checkout@v4 + # The open-source pages live in lancedb/lancedb and are assembled from + # there. Pinned to main: this repository publishes what that repository + # has merged, not what is in flight. + - name: Check out the open-source documentation + uses: actions/checkout@v4 + with: + repository: lancedb/lancedb + ref: main + path: lancedb + - uses: actions/setup-node@v4 with: node-version: 22 @@ -49,9 +59,13 @@ jobs: # have drifted the site would document something that was never released. - name: Check the OpenAPI spec matches its release run: make check-spec + env: + LANCEDB_DOCS_ROOT: lancedb/docs/web - name: Assemble run: make assemble + env: + LANCEDB_DOCS_ROOT: lancedb/docs/web - name: Check links in the assembled tree working-directory: build/site @@ -65,8 +79,17 @@ jobs: working-directory: build/site run: mint export --output "$RUNNER_TEMP/assembled.zip" - - name: Export the source tree - working-directory: docs + # Nothing to compare against directly any more: the site no longer exists + # as one tree anywhere. Assembling twice proves the pipeline is + # deterministic, which is what the comparison can still establish. + - name: Assemble again + run: make assemble + env: + LANCEDB_DOCS_ROOT: lancedb/docs/web + ASSEMBLE_OUTPUT: build/site-again + + - name: Export the second assembly + working-directory: build/site-again run: mint export --output "$RUNNER_TEMP/direct.zip" - name: Compare diff --git a/.gitignore b/.gitignore index d6d7e03..5f7e44b 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ scratch # Assembled output — generated by scripts/assemble.py, published to the # `assembled` branch by CI rather than committed here. /build/ + +# Second assembly, used by CI to prove the pipeline is deterministic. +/build/ diff --git a/assemble.yaml b/assemble.yaml index d319613..2e5edf9 100644 --- a/assemble.yaml +++ b/assemble.yaml @@ -1,35 +1,29 @@ # Assembler configuration. # -# The site is built from one or more content roots rather than published -# directly from this repository. Today there is a single root — this repo's own -# `docs/` — and the assembled output is byte-identical to it. Later phases add -# roots without changing the program: +# The published site is built from several content roots. The first `reference` +# root owns the navigation and ships a complete `docs.json`, so it can also be +# served on its own with `mint dev` — that is how a contributor previews the +# open-source documentation without this repository at all. Every other root +# contributes a `docs.nav.json` fragment, whose tabs are merged by name. # -# A3 the open-source pages move to lancedb/lancedb, added here as a second -# `reference` root -# A5 the Enterprise overlays land in sophon, added as an `overlay` root whose -# fragments merge onto reference anchors -# -# Adding a root is an edit to this file, not to scripts/assemble.py. That is the -# point of keeping it declarative. +# A5 adds sophon as an `overlay` root, whose Enterprise fragments merge onto +# reference anchors. That is a config edit, not a code change. -output: build/site +output: ${ASSEMBLE_OUTPUT:-build/site} roots: - - name: oss + # Open-source pages, and the navigation for the whole site. Override the path + # with LANCEDB_DOCS_ROOT when the checkout is somewhere else; CI sets it. + - name: lancedb + path: ${LANCEDB_DOCS_ROOT:-../lancedb/docs/web} + role: reference + + # What this repository still owns: the Geneva pages until Stage G replaces + # them, the generated dataset cards, and the OpenAPI spec. + - name: build path: docs role: reference -# Generated inputs the assembler owns rather than trusting a bot to commit. -# -# The REST reference is generated from the Lance Namespace OpenAPI spec. It is -# tracked at a *release tag*, not a commit: the site should document something -# that was released. `make sync-spec` writes the released spec into the tree and -# CI fails if the two have drifted, so the pin cannot rot silently. -# -# v0.12.0 is the first release carrying the fix for the broken request examples -# (lance-format/lance-namespace#362). Anything older renders "A valid request -# URL is required to generate request examples" on most endpoint pages. openapi: repo: lance-format/lance-namespace release: v0.12.0 diff --git a/docs/agent-branch-experiments.mdx b/docs/agent-branch-experiments.mdx deleted file mode 100644 index 59bcf68..0000000 --- a/docs/agent-branch-experiments.mdx +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: "Run experiments on branches" -sidebarTitle: "Run experiments on branches" -description: "Use LanceDB branches to isolate agent-driven experiments from main, evaluate them on a fixed test set, and promote only the winner." -icon: "flask" -keywords: ["AI agents", "branches", "experiments", "embeddings", "evaluation"] ---- - -LanceDB supports [branching](/tables/branching) to isolate experiments from the main table when working with agents. -Branches are useful when you instruct an agent to try several approaches without -affecting the production data on the `main` branch. -Each experiment gets its own writable table history. - -For example, you can generate and compare embeddings from two text embedding -models on the given table: - -| Branch | Model | Added column | -| --- | --- | --- | -| `embed-minilm` | `sentence-transformers/all-MiniLM-L6-v2` | `vector_minilm` | -| `embed-nomic` | `nomic-embed-text` from Ollama | `vector_nomic` | - -The branches share a logical table, but each has its own schema and history. -The original table handle still points to `main`. LanceDB does not have a -process-wide current branch. - -Install the model packages, then download the `nomic-embed-text` model from Ollama -(or any other model you prefer): - -```bash -uv add sentence-transformers ollama -ollama pull nomic-embed-text -``` - -It's always recommended to measure the results of an experiment on a fixed evaluation set. -Here's an example of how you could ask the agent to run both experiments and compare the results: - -```text Agent prompt -Use the lancedb plugin and the LanceDB branching documentation to compare two -embedding experiments on the `camelot_multimodal` table. - -1. Fork `embed-minilm` from `main`. Build normalized embeddings from - `"{role}. {description}"` with - `sentence-transformers/all-MiniLM-L6-v2` and add them as `vector_minilm`. -2. Fork `embed-nomic` from `main`. Embed the identical text with Ollama - `nomic-embed-text` and add the results as `vector_nomic`. -3. Process rows in batches and write only through each branch-scoped table - handle. Verify that neither new column appears on `main`. -4. Evaluate both branches with the queries "wise magical advisor", - "treacherous rebel", and "virtuous Grail knight". Report the top three names, - latency, and whether the expected character ranks first. -5. Do not modify `main` and do not delete either branch. Recommend a winner - based on the results. -``` - - -The vector dimensions may differ because the columns live on separate branches. -Use a different column name for each model. Within an experiment, use the same -model for the stored text and the search queries. - - -## Apply the winning experiment to `main` {#apply-the-winning-experiment-to-main} - -The branch experiments leave you with the results side by side, and they -deliberately never touch `main`. Once you've picked a winner, apply it to -`main` yourself by rerunning that experiment's validated transformation -directly against the `main` table. Rerunning the reviewed operation is the -reliable path on both OSS and Enterprise: it replays exactly the transformation -you validated on the branch, and it works the same regardless of how the branch -evolved. - -Suppose `nomic-embed-text` wins. Rerun the same Nomic embedding transformation -you validated on `embed-nomic` against `main` in batches, then call -`table.optimize()` on OSS. Verify a bounded vector search on `main`, and keep -the losing branch around until you're confident in the result. - -See [Branches](/tables/branching) for more on how branches, versions, and tags -relate. - -## More experiments you can run {#more-experiments-you-can-run} - -Swapping embedding models on a branch while working with agents is only one example -of what you can do with the LanceDB plugin. The table below shows other experiments -you can run with the plugin. - -| Hypothesis | Change on the branch | What to evaluate | -| --- | --- | --- | -| A different embedding model improves retrieval | Add a new vector column | Recall, ranking quality, latency, and cost | -| A new parser or OCR model improves source data | Reprocess files and backfill metadata | Validation failures and a reviewed sample | -| A new search setup works better | Build an index or change hybrid-search and reranking settings | Relevance and latency on a fixed query set | -| A curation rule improves the corpus | Deduplicate, classify, or filter records | False positives, false negatives, and row counts | -| A migration is safe to deploy | Add columns or run a backfill | Schema compatibility and application checks | - -For each experiment, ask the agent to state the hypothesis, use a fixed -evaluation set, report the comparison, and, if you want, work on an alternate -branch that's isolated from `main` until you decide you want the new derived -column in `main`. - -Once you've chosen a winner, rerun that validated transformation to ingest it into `main` -yourself, on either LanceDB OSS or Enterprise. See our documentation on [branches](/tables/branching) -for more information. diff --git a/docs/api-reference/index.mdx b/docs/api-reference/index.mdx deleted file mode 100644 index 5075602..0000000 --- a/docs/api-reference/index.mdx +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: SDKs and REST API Reference -sidebarTitle: "SDKs" -description: "SDK and REST API reference for LanceDB Enterprise and OSS." - ---- - -For detailed information of the available functions and methods in your preferred language's SDKs, -refer to the API documentation linked below. - -If you're looking for a REST API reference, visit the [REST API](/api-reference/rest) page. - -If you're looking for conceptual and practical namespace guidance before diving into method signatures, see -[Namespaces and Catalog Model](/namespaces) and [Using Namespaces in SDKs](/namespaces/usage). - -## Supported SDKs {#supported-sdks} - -Python, Typescript and Rust SDKs are officially supported by LanceDB. You can use these SDKs to interact with both LanceDB OSS and Enterprise deployments. - -| Reference | Description | -|:--------------|-------------------| -| [Python](https://lancedb.github.io/lancedb/python/python/) | Full-featured Python client with pandas & numpy integration | -| [Typescript](https://lancedb.github.io/lancedb/js/) | A TypeScript wrapper around the Rust library, built with `napi-rs` -| [Rust](https://docs.rs/lancedb/latest/lancedb/index.html) | Native Rust library with persistent-storage and high performance | - -## REST API SDKs {#rest-api-sdks} - -Enterprise - -REST API-based SDKs provide a convenient way to interact with LanceDB Enterprise deployments using the Lance REST Namespace API. - -| Reference | Description | -|:--------------|-------------------| -| [Java](https://lancedb.github.io/lancedb/java/java/)| REST API Enterprise SDK in Java | - -## Community-driven SDKs {#community-driven-sdks} - -In addition to the officially supported SDKs, the LanceDB community may contribute SDKs in other languages. -These SDKs may not have the same level of support or feature parity as the official ones supported by LanceDB, but they can be an option -for users working in languages other than those listed above. - -| Reference | Description | -|:--------------|-------------------| -| [Go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) | Community-contributed Go SDK for LanceDB | -| [Ruby](https://github.com/scientist-labs/lancelot) | Community-contributed Ruby bindings for LanceDB | -| [Swift](https://github.com/RyanLisse/LanceDbSwiftKit) | Community-contributed Swift SDK for LanceDB | -| [R](https://github.com/CathalByrneGit/lancedb) | Community-contributed R package for LanceDB | -| [Flutter](https://github.com/Alexcn/flutter_lancedb) | Community-contributed Flutter bindings for LanceDB | diff --git a/docs/api-reference/rest/index.mdx b/docs/api-reference/rest/index.mdx deleted file mode 100644 index 1442225..0000000 --- a/docs/api-reference/rest/index.mdx +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: REST API Reference -sidebarTitle: "Overview" -description: "API reference for LanceDB" - ---- - -[Lance REST Namespace](https://lance.org/format/namespace/) spec -is an OpenAPI protocol that enables reading, writing and managing Lance tables by connecting -those metadata services or building a custom metadata server in a standardized way. - -LanceDB OSS allows you to interface with Lance tables via the REST Namespace. -LanceDB Enterprise provides an extended REST API with -additional endpoints for managing tables and data. -If you have specific needs or questions about the Enterprise REST API Namespace, -please [contact us](mailto:support@lancedb.com). - -## Authentication {#authentication} - -Enterprise - -All HTTP requests to LanceDB APIs must contain an x-api-key header that specifies a valid API key and -must be encoded as JSON or Arrow RPC. - -To authenticate to the Enterprise REST API, you need the endpoint for your deployment and a valid API key for that deployment. - -### Get your Enterprise credentials {#get-your-enterprise-credentials} - -1. Obtain the following values from your LanceDB administrator or the LanceDB team that provisioned your Enterprise deployment: - - an API key - - your Enterprise REST endpoint - - your database name, if your deployment uses a private endpoint or `host_override` - -2. Export those values in your terminal: - -```bash -export LANCEDB_API_KEY="" -export LANCEDB_URI="https://your-enterprise-endpoint.com" -export LANCEDB_DATABASE="your-database-name" -``` - -3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint. - -### Verify authentication {#verify-authentication} - -4. Check that you can reach the deployment and list tables: - -```bash -curl -X GET "$LANCEDB_URI/v1/tables" \ - -H "Content-Type: application/json" \ - -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" -``` - -If your deployment endpoint already includes the database host name, you can omit the `x-lancedb-database` header. - -5. Create a table to confirm write access. Let's call it `words`. - -```bash -curl -X POST "$LANCEDB_URI/v1/tables/words" \ - -H "Content-Type: application/vnd.apache.arrow.stream" \ - -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" -``` - -6. Check that the table has been created: - -```bash -curl -X GET "$LANCEDB_URI/v1/tables" \ - -H "Content-Type: application/json" \ - -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" -``` - -That's it -- you're connected! Now, you can start adding data and querying it. -You can visit the tutorial section to build your own applications with LanceDB. - - - Check out our tutorials on building various applications with LanceDB. - diff --git a/docs/build-with-ai-agents.mdx b/docs/build-with-ai-agents.mdx deleted file mode 100644 index 8296400..0000000 --- a/docs/build-with-ai-agents.mdx +++ /dev/null @@ -1,270 +0,0 @@ ---- -title: "Tutorial: Use the LanceDB agent plugin" -sidebarTitle: "LanceDB agent plugin" -description: "Install the LanceDB plugin and use an AI coding agent to quickly build a multimodal ingestion pipeline." -icon: "robot" -keywords: ["AI agents", "agent plugins", "multimodal", "Pydantic", "branches", "embeddings"] ---- - -import { - PyCamelotSchema, - PyCamelotBatches, - PyCamelotOssIngestion, -} from '/snippets/build_with_ai_agents.mdx'; - -The LanceDB agent plugin gives coding agents a maintained reference for the -Python and TypeScript APIs. It also covers portable OSS and Enterprise code, -ingestion performance, and branch operations. It ships as the `lancedb` plugin in -the [lancedb-agent-plugins](https://github.com/lancedb/lancedb-agent-plugins) -repository, which is also a plugin marketplace. Install it with your agent's -plugin manager: - - -```text Claude Code -/plugin marketplace add lancedb/lancedb-agent-plugins -/plugin install lancedb@lancedb -``` - -```bash Codex icon="terminal" -codex plugin marketplace add lancedb/lancedb-agent-plugins -codex plugin add lancedb@lancedb -``` - -```bash Other agents icon="terminal" -npx plugins add lancedb/lancedb-agent-plugins -``` - - -The [`plugins`](https://www.npmjs.com/package/plugins) installer shown under -**Other agents** is a cross-tool option. It detects which agent CLIs are on your -`PATH` — Cursor, GitHub Copilot CLI, VS Code, Grok Build, and Kimi Code, as well -as Claude Code and Codex — and installs through each one's native plugin system. - -The plugin supplements the agent's training data with current LanceDB -instructions. To pick up later revisions, refresh the marketplace: - - -```text Claude Code -/plugin marketplace update lancedb -``` - -```bash Codex icon="terminal" -codex plugin marketplace upgrade -``` - -```bash Other agents icon="terminal" -npx plugins add lancedb/lancedb-agent-plugins -``` - - - -Each plugin manager clones the repository, reads its marketplace manifest to find -the `lancedb` plugin, and hands the plugin to the agent's own plugin store rather -than copying files into your project. Claude Code and Codex keep it under -`~/.claude/plugins` and `~/.codex/plugins` respectively, so the plugin is available -in every project on the machine. - -The `npx plugins` installer defaults to the same user-wide scope. Pass -`--scope project` to record the plugin in the current repository instead, so that -everyone working in it gets the same plugin, or `-t ` to install for a -single agent rather than every one it detects: - -```bash -npx plugins add lancedb/lancedb-agent-plugins --scope project -t cursor -``` - -Not every agent supports project scope; `npx plugins targets` lists what it -found and what each target supports. - - -## Get started with the LanceDB agent plugin {#get-started-with-the-lancedb-agent-plugin} - -This tutorial uses the Camelot dataset from the [quickstart](/quickstart), with -a portrait added for each character. Each LanceDB row contains validated -metadata and raw JPEG bytes. Text, images, and any embeddings you add later -remain in the same table. - -### 1\. Download the multimodal dataset {#1-download-the-multimodal-dataset} - -From a new project directory, download the JSON file and portraits: - -```bash -mkdir -p data/img - -BASE_URL="https://docs.lancedb.com/static/assets/tutorials/build-with-ai-agents/camelot/data" -curl -fsSL "$BASE_URL/camelot.json" -o data/camelot.json - -for image in \ - arthur.jpg \ - guinevere.jpg \ - merlin.jpg \ - mordred.jpg \ - sir_galahad.jpg \ - sir_gawain.jpg \ - sir_lancelot.jpg \ - sir_percival.jpg -do - curl -fsSL "$BASE_URL/img/$image" -o "data/img/$image" -done -``` - -Each JSON record has this shape: - -```json -{ - "id": 2, - "name": "Merlin", - "role": "Wizard and Advisor", - "description": "A powerful wizard and prophet who mentors Arthur.", - "stats": { - "strength": 2, - "courage": 4, - "magic": 5, - "wisdom": 5 - }, - "img": "data/img/merlin.jpg" -} -``` - -JSON input may have missing fields, unexpected fields, or values of the wrong -type. The LanceDB plugin tells the agent to validate each record with strict -Pydantic models before writing it. - -After the agent writes the pipeline, inspect the schema, batching, and write -path rather than assuming it followed the plugin's guidance correctly. - -### 2\. Prompt your agent to build the pipeline {#2-prompt-your-agent-to-build-the-pipeline} - -Install the Python packages used by the example: - -```bash icon="terminal" -uv init -uv add lancedb pyarrow pydantic -``` - -If you're using LanceDB Enterprise, ask the agent to ingest into an Enterprise table, -provide the relevant environment variables for connecting to your Enterprise deployment -in a local `.env` file, and point the agent to it. - -```text .env -LANCEDB_URI=db://your_project_name -LANCEDB_API_KEY=your_api_key_here -LANCEDB_REGION=us-east-1 -LANCEDB_HOST_OVERRIDE=https://your-enterprise-endpoint.com -``` - -If you're using LanceDB OSS, no connection settings are required, as it runs as an -embedded retrieval library. A simple prompt like this should work: - -```text Agent prompt -# If using OSS -Use the lancedb plugin to ingest the dataset in `data/` into a LanceDB -OSS table. - -# If using enterprise -Use the lancedb plugin to ingest the dataset in `data/` into a LanceDB -Enterprise table using the connection information in `.env`. -``` - -Because the plugin is registered with the agent's own plugin system, the agent -should pick it up on its own once you restart the session. If it does not, -simply ask it to use the `lancedb` plugin in the prompt, as shown above. - -That should be enough! The agent will create `ingest_multimodal.py`, or similar. -The following sections inspect the script to verify that it follows the plugin's guidance. - -#### Data validation {#data-validation} - -The plugin encourages the agent to validate each record with Pydantic before writing it. -The agent should ideally define a schema for the table and a nested schema for the -`stats` field. - - - {PyCamelotSchema} - - -In this case, our agent correctly defined `Character` and `Stats` Pydantic models -and validated the JSON before adding it to the table. - -#### Batched ingestion {#batched-ingestion} - -Naively calling `table.add()` once per row is slow, and is considered an anti-pattern -in LanceDB. The plugin encourages the agent to collect incoming rows into batches and -write them with a single `table.add()` call. When you use the plugin, the agent should -produce something like this: - - - {PyCamelotBatches} - - -The script calls `Character.model_validate(...)` before adding a record to the -batch. If validation fails, that batch is never written. The function yields up -to `batch_size` rows at a time, providing an iterable of batches for the ingestion step, -shown next. - -#### Table maintenance {#table-maintenance} -For LanceDB OSS, the plugin instructs the agent to call -`table.optimize()` after the ingestion loop. This compacts small fragments, cleans up -old versions according to the retention policy, and incorporates new data into indexes. - - - {PyCamelotOssIngestion} - - -If you're using LanceDB Enterprise, the plugin mentions that this step is not needed -because LanceDB Enterprise handles maintenance automatically. - - - -This example dataset has only eight rows, so the default batch size writes it in -one call. Larger inputs should still avoid single-row write commits. - - -### 3\. Run the OSS pipeline {#3-run-the-oss-pipeline} - -```bash -uv run python ingest_multimodal.py -``` - -The table now contains the validated character records and their JPEG bytes. -Here are the first three rows: - -| Image | Character | Role | -| --- | --- | --- | -| King Arthur | King Arthur | King of Camelot | -| Merlin | Merlin | Wizard and Advisor | -| Queen Guinevere | Queen Guinevere | Queen of Camelot | - -Once your pipeline works, you can [run experiments on branches](/agent-branch-experiments) -to try new embedding models, parsers, or search settings without touching `main`. - -## Takeaways {#takeaways} - -The example in this tutorial was small, but similar ideas apply to other workflows, too. -Give the agent the data source, the constraints it must respect, and the -artifacts it should return. - -The plugin supplies LanceDB-specific guidance, but it's the user's responsibility to -ensure the output makes sense for the application. - -### Try the plugin with your own dataset {#try-the-plugin-with-your-own-dataset} - -The plugin shown in this tutorial should generalize reasonably well to other use cases. -If you find any issues, open [an issue](https://github.com/lancedb/lancedb-agent-plugins/issues) -on GitHub, clearly describing the intended behavior. - -You can choose OSS or Enterprise based on how the work will run: - -Start with [LanceDB OSS](/quickstart) during the early stages of a project -when an agent is helping you prototype, -explore a dataset, or run small workflows on a subset of the data. The application owns -the storage and lifecycle work, so ask the agent to validate inputs, write in -batches, and it will include table maintenance operations such as `optimize()` -where appropriate. - -Choose [LanceDB Enterprise](/enterprise) when the resulting table becomes -shared production infrastructure, and the workload needs distributed capacity, -private deployment, or platform-managed operations. The underlying data format and table -API stay the same, so the pipeline does not need to be redesigned. The agent -instead connects to a remote `db://` table and lets the cluster handle -maintenance and background work. diff --git a/docs/demos/index.mdx b/docs/demos/index.mdx deleted file mode 100644 index 95f1ab3..0000000 --- a/docs/demos/index.mdx +++ /dev/null @@ -1,78 +0,0 @@ ---- -title: "Demo Application Gallery" -sidebarTitle: "App Gallery" -description: "Demo apps showcasing end-to-end applications built with LanceDB for production use cases." ---- - -Explore the demo applications built with LanceDB below. - -App | Description ---- | --- -[Semantic.Art](#semantic-art) | A multimodal art discovery platform using feelings, phrases, and images. -[Wikipedia 41M Hybrid Search](#wikipedia-41m-hybrid-search) | An interactive hybrid search demo combining full-text search and vector search. -[Video Search](#video-search) | A video search application that allows searching through a library of videos using natural language queries. - - -## Semantic.art {#semantic-art} - -multimodal -hybrid-search -vector-search -semantic-routing - - -Semantic.art turns real, human-made art discovery into a multimodal search experience using -feelings, phrases, and images. It's built with LanceDB hybrid search and semantic routing. - - -Read in detail about how Semantic.art is built in this blog post. - - - - -## Wikipedia 41M Hybrid Search {#wikipedia-41m-hybrid-search} - -multimodal -hybrid-search -vector-search -fts - - -Interactive hybrid search, full-text search (FTS) and vector search demo with 41M+ Wikipedia entries. -Explore the power of combining FTS with vector search for more relevant results. - - -Read in detail about how the Wikipedia 41M hybrid search demo is built in this blog post. - - - - -## Video Search {#video-search} - -multimodal -video-search -vector-search - - -Search through a library of videos using natural language queries. This demo showcases how to use LanceDB -to perform semantic search on video content. - - \ No newline at end of file diff --git a/docs/docs.json b/docs/docs.json deleted file mode 100644 index f8931c1..0000000 --- a/docs/docs.json +++ /dev/null @@ -1,577 +0,0 @@ -{ - "$schema": "https://mintlify.com/docs.json", - "appearance": { - "strict": false - }, - "theme": "mint", - "name": "LanceDB", - "banner": { - "content": "[Why Multimodal Data Needs a Better Lakehouse? \u2014 Download the Research Study](https://lancedb.com/download/)", - "dismissible": true - }, - "colors": { - "primary": "#FF6B35", - "light": "#FF8A5C", - "dark": "#E55A2B" - }, - "fonts": { - "family": "Inter" - }, - "styling": { - "codeblocks": { - "theme": { - "light": "vitesse-light", - "dark": "catppuccin-mocha" - } - } - }, - "favicon": "/static/favicon.ico", - "navigation": { - "tabs": [ - { - "tab": "Documentation", - "groups": [ - { - "group": "Get started", - "pages": [ - "quickstart", - { - "group": "What is LanceDB?", - "pages": [ - "index", - "lance", - "tables-and-namespaces" - ] - }, - { - "group": "LanceDB Enterprise", - "pages": [ - "enterprise/index", - "enterprise/architecture", - "enterprise/security", - "enterprise/authentication", - "enterprise/benchmarks", - { - "group": "Deployment", - "pages": [ - "enterprise/deployment/index", - "enterprise/deployment/azure" - ] - } - ] - }, - "performance" - ] - }, - { - "group": "Model training", - "pages": [ - "training/why-lancedb", - "training/index", - "training/torch", - "training/object-detection", - "training/vlm-finetuning" - ] - }, - { - "group": "Guides", - "pages": [ - { - "group": "Table operations", - "pages": [ - "tables/index", - "tables/create", - "tables/multimodal", - "tables/schema", - "tables/update", - "tables/versioning", - "tables/branching", - "tables/consistency" - ] - }, - { - "group": "Namespaces", - "pages": [ - "namespaces/index", - "namespaces/usage" - ] - }, - { - "group": "Embeddings", - "pages": [ - "embedding/index", - "embedding/quickstart" - ] - }, - { - "group": "Indexing", - "pages": [ - "indexing/index", - "indexing/vector-index", - "indexing/fts-index", - "indexing/scalar-index", - "indexing/gpu-indexing", - "indexing/quantization", - "indexing/reindexing" - ] - }, - { - "group": "Search", - "pages": [ - "search/index", - "search/vector-search", - "search/multivector-search", - "search/full-text-search", - "search/fts-examples", - "search/hybrid-search", - "search/filtering", - "search/optimize-queries", - { - "group": "Enterprise SQL", - "pages": [ - "search/sql/index", - "search/sql/fts-sql" - ] - } - ] - }, - { - "group": "Reranking", - "pages": [ - "reranking/index", - "reranking/rrf", - "reranking/linear_combination", - "reranking/mrr", - "reranking/cross_encoder", - "reranking/custom-reranker", - "reranking/eval" - ] - }, - { - "group": "Storage", - "pages": [ - "storage/index", - "storage/configuration", - "storage/monitoring" - ] - }, - { - "group": "Build with AI agents", - "expanded": true, - "pages": [ - "build-with-ai-agents", - "agent-branch-experiments" - ] - } - ] - }, - { - "group": "Feature Engineering (Geneva)", - "pages": [ - "geneva/index", - "geneva/overview/index", - "geneva/getting-started", - { - "group": "Transforms", - "pages": [ - "geneva/udfs/index", - "geneva/udfs/udfs", - "geneva/udfs/scalar-udtfs", - "geneva/udfs/batch-udtfs", - "geneva/udfs/error_handling", - "geneva/udfs/profiling-memory", - "geneva/udfs/blobs" - ] - }, - { - "group": "Built-in Transforms", - "pages": [ - "geneva/udfs/providers/index", - "geneva/udfs/providers/openai", - "geneva/udfs/providers/gemini", - "geneva/udfs/providers/sentence-transformers" - ] - }, - { - "group": "Running Jobs", - "pages": [ - "geneva/jobs/index", - "geneva/jobs/backfilling", - "geneva/jobs/bulk-load-columns", - "geneva/jobs/materialized-views", - "geneva/jobs/advanced-job-configuration", - "geneva/jobs/lifecycle", - "geneva/jobs/conflicts", - "geneva/jobs/performance", - "geneva/jobs/job_metrics", - "geneva/jobs/console", - "geneva/jobs/troubleshooting", - "geneva/jobs/contexts" - ] - }, - { - "group": "Deployment", - "pages": [ - "geneva/deployment/helm", - "geneva/jobs/startup", - "geneva/deployment/dependency-verification", - "geneva/udfs/advanced-configuration", - "geneva/deployment/index", - "geneva/deployment/troubleshooting" - ] - }, - "geneva/end-to-end", - "geneva/reference" - ] - }, - { - "group": "Support", - "pages": [ - "troubleshooting", - { - "group": "FAQ", - "pages": [ - "faq/index", - "faq/faq-oss", - "faq/faq-enterprise" - ] - } - ] - } - ] - }, - { - "tab": "Integrations", - "groups": [ - { - "group": "Integrations", - "pages": [ - "integrations/index", - { - "group": "Embedding Providers", - "pages": [ - "integrations/embedding/huggingface", - "integrations/embedding/aws", - "integrations/embedding/cohere", - "integrations/embedding/colpali", - "integrations/embedding/gemini", - "integrations/embedding/ibm", - "integrations/embedding/imagebind", - "integrations/embedding/instructor", - "integrations/embedding/jina", - "integrations/embedding/ollama", - "integrations/embedding/openai", - "integrations/embedding/openclip", - "integrations/embedding/sentence-transformers", - "integrations/embedding/voyageai", - "integrations/embedding/superlinked" - ] - }, - { - "group": "Rerankers", - "pages": [ - "integrations/reranking/answerdotai", - "integrations/reranking/cohere", - "integrations/reranking/colbert", - "integrations/reranking/jina", - "integrations/reranking/openai", - "integrations/reranking/voyageai", - "integrations/reranking/watsonx" - ] - }, - { - "group": "Data Platforms & Frameworks", - "expanded": true, - "pages": [ - "integrations/data/pydantic", - "integrations/data/duckdb", - "integrations/data/pandas_and_pyarrow", - "integrations/data/polars_arrow", - "integrations/data/dlt", - "integrations/data/voxel51" - ] - }, - { - "group": "AI Platforms & Frameworks", - "expanded": true, - "pages": [ - "integrations/ai/agno", - "integrations/ai/hermes-agent", - "integrations/ai/huggingface", - "integrations/ai/langchain", - "integrations/ai/llamaIndex", - "integrations/ai/genkit", - "integrations/ai/kiln", - "integrations/ai/prompttools", - "integrations/ai/synthetic-data-kit" - ] - } - ] - } - ] - }, - { - "tab": "Tutorials", - "groups": [ - { - "group": "Tutorials", - "pages": [ - "tutorials/index", - { - "group": "Search", - "expanded": true, - "pages": [ - "tutorials/search/index", - "tutorials/search/multivector-needle-in-a-haystack" - ] - }, - { - "group": "RAG & Agents", - "expanded": true, - "pages": [ - "tutorials/agents/index", - "tutorials/agents/nvidia-rag-blueprint/index", - "tutorials/agents/time-travel-rag/index", - "tutorials/agents/multimodal-agent/index" - ] - }, - "tutorials/feature-engineering/index" - ] - } - ] - }, - { - "tab": "Demos", - "groups": [ - { - "group": "Demos", - "pages": [ - "demos/index" - ] - } - ] - }, - { - "tab": "Datasets", - "groups": [ - { - "group": "Overview", - "pages": [ - "datasets/index" - ] - }, - { - "group": "Image Classification", - "pages": [ - "datasets/mnist", - "datasets/cifar10", - "datasets/fashion-mnist", - "datasets/food101", - "datasets/oxford-pets", - "datasets/stanford-cars", - "datasets/imagenet-1k-val", - "datasets/eurosat" - ] - }, - { - "group": "OCR & Handwriting", - "pages": [ - "datasets/handwriting-ocr" - ] - }, - { - "group": "Object Detection & Segmentation", - "pages": [ - "datasets/coco-detection-2017", - "datasets/pascal-voc-2012-segmentation", - "datasets/ade20k", - "datasets/kitti-2d-detection" - ] - }, - { - "group": "Image Retrieval", - "pages": [ - "datasets/coco-captions-2017", - "datasets/flickr30k", - "datasets/laion-1m" - ] - }, - { - "group": "Visual Question Answering", - "pages": [ - "datasets/chartqa", - "datasets/docvqa", - "datasets/textvqa", - "datasets/vqav2", - "datasets/gqa-testdev-balanced" - ] - }, - { - "group": "Text QA", - "pages": [ - "datasets/squad-v2", - "datasets/trivia-qa", - "datasets/hotpotqa-distractor", - "datasets/natural-questions-val", - "datasets/ms-marco-v2" - ] - }, - { - "group": "Text Corpora", - "pages": [ - "datasets/fineweb-edu" - ] - }, - { - "group": "Speech", - "pages": [ - "datasets/librispeech-clean" - ] - }, - { - "group": "Video", - "pages": [ - "datasets/openvid" - ] - }, - { - "group": "Robotics", - "pages": [ - "datasets/lerobot-pusht", - "datasets/lerobot-xvla-soft-fold" - ] - } - ] - }, - { - "tab": "Use Cases", - "groups": [ - { - "group": "Robotics", - "pages": [ - "integrations/lerobotdataset" - ] - }, - { - "group": "World Models", - "pages": [ - "integrations/stable-worldmodel" - ] - } - ] - }, - { - "tab": "API Reference", - "groups": [ - { - "group": "API Reference", - "pages": [ - "api-reference/index", - { - "group": "REST API", - "expanded": true, - "pages": [ - "api-reference/rest/index" - ], - "openapi": { - "source": "api-reference/rest/openapi.yml", - "directory": "api-reference/rest" - } - } - ] - } - ] - } - ] - }, - "logo": { - "light": "/static/assets/logo/dark-lancedb-logo.svg", - "dark": "/static/assets/logo/light-lancedb-logo.svg", - "href": "https://lancedb.com" - }, - "background": { - "color": { - "light": "#FAF5F0", - "dark": "#000000" - } - }, - "navbar": { - "links": [ - { - "label": "Support", - "href": "mailto:contact@lancedb.com" - } - ] - }, - "footer": { - "socials": { - "github": "https://github.com/lancedb", - "linkedin": "https://www.linkedin.com/company/lancedb", - "x": "https://x.com/lancedb", - "discord": "https://discord.gg/AUEWnJ7Txb" - } - }, - "integrations": { - "ga4": { - "measurementId": "G-1TMC2PR69E" - } - }, - "redirects": [ - { - "source": "/integrations/reranking/rrf", - "destination": "reranking/rrf" - }, - { - "source": "/integrations/reranking/linear_combination", - "destination": "reranking/linear_combination" - }, - { - "source": "/integrations/reranking/mrr", - "destination": "reranking/mrr" - }, - { - "source": "/integrations/reranking/cross_encoder", - "destination": "reranking/cross_encoder" - }, - { - "source": "/integrations/platforms/:slug*", - "destination": "integrations/data/:slug*" - }, - { - "source": "/integrations/frameworks/:slug*", - "destination": "integrations/ai/:slug*" - }, - { - "source": "/integrations/data/phidata", - "destination": "integrations/ai/agno" - }, - { - "source": "/tutorials/rag/:slug*", - "destination": "tutorials/agents/:slug*" - }, - { - "source": "/tutorials/vector-search/:slug*", - "destination": "tutorials/search/:slug*" - }, - { - "source": "/geneva/udfs/built-in", - "destination": "/geneva/udfs/providers" - }, - { - "source": "/enterprise/performance", - "destination": "/enterprise/benchmarks" - }, - { - "source": "/enterprise/quickstart", - "destination": "/quickstart" - }, - { - "source": "/huggingface/overview", - "destination": "/integrations/ai/huggingface" - }, - { - "source": "/huggingface/datasets", - "destination": "/datasets" - } - ] -} diff --git a/docs/docs.nav.json b/docs/docs.nav.json new file mode 100644 index 0000000..629da2d --- /dev/null +++ b/docs/docs.nav.json @@ -0,0 +1,298 @@ +{ + "insert": [ + { + "into": [ + "Documentation", + "Get started" + ], + "after": "What is LanceDB?", + "entry": { + "group": "LanceDB Enterprise", + "pages": [ + "enterprise/index", + "enterprise/architecture", + "enterprise/security", + "enterprise/authentication", + "enterprise/benchmarks", + { + "group": "Deployment", + "pages": [ + "enterprise/deployment/index", + "enterprise/deployment/azure" + ] + } + ] + } + }, + { + "into": [ + "Documentation", + "Guides", + "Indexing" + ], + "after": "indexing/scalar-index", + "entry": "indexing/gpu-indexing" + }, + { + "into": [ + "Documentation", + "Guides", + "Search" + ], + "after": "search/optimize-queries", + "entry": { + "group": "Enterprise SQL", + "pages": [ + "search/sql/index", + "search/sql/fts-sql" + ] + } + }, + { + "into": [ + "Documentation" + ], + "after": "Guides", + "entry": { + "group": "Feature Engineering (Geneva)", + "pages": [ + "geneva/index", + "geneva/overview/index", + "geneva/getting-started", + { + "group": "Transforms", + "pages": [ + "geneva/udfs/index", + "geneva/udfs/udfs", + "geneva/udfs/scalar-udtfs", + "geneva/udfs/batch-udtfs", + "geneva/udfs/error_handling", + "geneva/udfs/profiling-memory", + "geneva/udfs/blobs" + ] + }, + { + "group": "Built-in Transforms", + "pages": [ + "geneva/udfs/providers/index", + "geneva/udfs/providers/openai", + "geneva/udfs/providers/gemini", + "geneva/udfs/providers/sentence-transformers" + ] + }, + { + "group": "Running Jobs", + "pages": [ + "geneva/jobs/index", + "geneva/jobs/backfilling", + "geneva/jobs/bulk-load-columns", + "geneva/jobs/materialized-views", + "geneva/jobs/advanced-job-configuration", + "geneva/jobs/lifecycle", + "geneva/jobs/conflicts", + "geneva/jobs/performance", + "geneva/jobs/job_metrics", + "geneva/jobs/console", + "geneva/jobs/troubleshooting", + "geneva/jobs/contexts" + ] + }, + { + "group": "Deployment", + "pages": [ + "geneva/deployment/helm", + "geneva/jobs/startup", + "geneva/deployment/dependency-verification", + "geneva/udfs/advanced-configuration", + "geneva/deployment/index", + "geneva/deployment/troubleshooting" + ] + }, + "geneva/end-to-end", + "geneva/reference" + ] + } + }, + { + "into": [ + "Documentation", + "Support", + "FAQ" + ], + "after": "faq/faq-oss", + "entry": "faq/faq-enterprise" + }, + { + "into": [], + "after": "Demos", + "entry": { + "tab": "Datasets", + "groups": [ + { + "group": "Overview", + "pages": [ + "datasets/index" + ] + }, + { + "group": "Image Classification", + "pages": [ + "datasets/mnist", + "datasets/cifar10", + "datasets/fashion-mnist", + "datasets/food101", + "datasets/oxford-pets", + "datasets/stanford-cars", + "datasets/imagenet-1k-val", + "datasets/eurosat" + ] + }, + { + "group": "OCR & Handwriting", + "pages": [ + "datasets/handwriting-ocr" + ] + }, + { + "group": "Object Detection & Segmentation", + "pages": [ + "datasets/coco-detection-2017", + "datasets/pascal-voc-2012-segmentation", + "datasets/ade20k", + "datasets/kitti-2d-detection" + ] + }, + { + "group": "Image Retrieval", + "pages": [ + "datasets/coco-captions-2017", + "datasets/flickr30k", + "datasets/laion-1m" + ] + }, + { + "group": "Visual Question Answering", + "pages": [ + "datasets/chartqa", + "datasets/docvqa", + "datasets/textvqa", + "datasets/vqav2", + "datasets/gqa-testdev-balanced" + ] + }, + { + "group": "Text QA", + "pages": [ + "datasets/squad-v2", + "datasets/trivia-qa", + "datasets/hotpotqa-distractor", + "datasets/natural-questions-val", + "datasets/ms-marco-v2" + ] + }, + { + "group": "Text Corpora", + "pages": [ + "datasets/fineweb-edu" + ] + }, + { + "group": "Speech", + "pages": [ + "datasets/librispeech-clean" + ] + }, + { + "group": "Video", + "pages": [ + "datasets/openvid" + ] + }, + { + "group": "Robotics", + "pages": [ + "datasets/lerobot-pusht", + "datasets/lerobot-xvla-soft-fold" + ] + } + ] + } + } + ], + "set": [ + { + "into": [ + "API Reference", + "API Reference", + "REST API" + ], + "key": "openapi", + "value": { + "source": "api-reference/rest/openapi.yml", + "directory": "api-reference/rest" + } + } + ], + "redirects": [ + { + "source": "/hybrid-search", + "destination": "/search/hybrid-search" + }, + { + "source": "/integrations/reranking/rrf", + "destination": "reranking/rrf" + }, + { + "source": "/integrations/reranking/linear_combination", + "destination": "reranking/linear_combination" + }, + { + "source": "/integrations/reranking/mrr", + "destination": "reranking/mrr" + }, + { + "source": "/integrations/reranking/cross_encoder", + "destination": "reranking/cross_encoder" + }, + { + "source": "/integrations/platforms/:slug*", + "destination": "integrations/data/:slug*" + }, + { + "source": "/integrations/frameworks/:slug*", + "destination": "integrations/ai/:slug*" + }, + { + "source": "/integrations/data/phidata", + "destination": "integrations/ai/agno" + }, + { + "source": "/tutorials/rag/:slug*", + "destination": "tutorials/agents/:slug*" + }, + { + "source": "/tutorials/vector-search/:slug*", + "destination": "tutorials/search/:slug*" + }, + { + "source": "/geneva/udfs/built-in", + "destination": "/geneva/udfs/providers" + }, + { + "source": "/enterprise/performance", + "destination": "/enterprise/benchmarks" + }, + { + "source": "/enterprise/quickstart", + "destination": "/quickstart" + }, + { + "source": "/huggingface/overview", + "destination": "/integrations/ai/huggingface" + }, + { + "source": "/huggingface/datasets", + "destination": "/datasets" + } + ] +} diff --git a/docs/embedding/index.mdx b/docs/embedding/index.mdx deleted file mode 100644 index 30ad260..0000000 --- a/docs/embedding/index.mdx +++ /dev/null @@ -1,239 +0,0 @@ ---- -title: "Managing Embeddings" -sidebarTitle: Overview -description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support." -icon: "bars" ---- - -import { - PyOpenaiEmbeddings, - PyCreateEmbeddingFunction, - PyManualQuerySearch, - PyEmbeddingFunction, - PyRegisterDevice, - PyRegisterSecret, - TsOpenaiEmbeddings, - TsCreateEmbeddingFunction, - TsManualQuerySearch, - TsEmbeddingFunction, - TsRegisterModelFallback, - TsRegisterSecret, - RsOpenaiEmbeddings, - RsCreateEmbeddingFunction, - RsManualQuerySearch, - RsEmbeddingFunction, -} from '/snippets/embedding.mdx'; - -Modern machine learning models can be trained to convert raw data into embeddings, which are vectors -of floating point numbers. The position of an embedding in vector space captures the semantics of -the data, so vectors that are close to each other are considered similar. - -LanceDB provides an embedding function registry in OSS as well as its Enterprise versions -([see below](#embeddings-in-lancedb-enterprise)) -that automatically generates vector embeddings during data ingestion. Automatic query-time embedding -generation is available in LanceDB OSS, with SDK-specific query ergonomics. The API abstracts -embedding generation, allowing you to focus on your application logic. - -## Embedding Registry {#embedding-registry} - -You can get a supported embedding function from the registry, and then use it in your table schema. -Once configured, the embedding function will automatically generate embeddings when you insert data -into the table. Query-time behavior depends on SDK: Python/TypeScript can query with text directly, -while Rust examples typically compute query embeddings explicitly before vector search. - - - - {PyOpenaiEmbeddings} - - - - {TsOpenaiEmbeddings} - - - - {RsOpenaiEmbeddings} - - - -### Using an embedding function {#using-an-embedding-function} - -Create an embedding function before you attach it to table or schema metadata. Python and TypeScript fetch -provider implementations from the embedding registry, while Rust constructs the provider embedding function -directly and registers it on the connection before using it in an `EmbeddingDefinition`. - - - - {PyCreateEmbeddingFunction} - - - - {TsCreateEmbeddingFunction} - - - - {RsCreateEmbeddingFunction} - - - -Provider configuration is SDK-specific, so copy the option names from the provider page for the SDK you use. -For example, the OpenAI model is selected with `name` in Python, `model` in TypeScript, and the model argument -to `OpenAIEmbeddingFunction::new_with_model` in Rust. - -| Concept | Python | TypeScript | Rust | -| --- | --- | --- | --- | -| Model | `name="text-embedding-3-small"` | `{ model: "text-embedding-3-small" }` | `new_with_model(api_key, "text-embedding-3-small")` | -| Retry count | `max_retries=7` | Provider/client-specific | Provider/client-specific | -| API key | `api_key="..."`, environment variables, or `$var:` | `apiKey: "..."`, environment variables, or `$var:` | Constructor argument or environment variable | -| Device | Provider-specific, for example `device="cuda"` | Provider-specific | Provider-specific | - -When ingesting data with an embedding definition, LanceDB only computes the vector column if that -column is missing from the incoming batch or present but entirely null. If you provide any non-null -values in the vector column, LanceDB treats the column as user-supplied and does not backfill the -remaining rows in that batch. - -For reusable runtime configuration, the registry also supports `$var:` placeholders in embedding-function config. -This is useful for provider secrets and environment-specific settings in Python and TypeScript. - -- Python uses `registry.set_var(...)`. -- TypeScript uses `registry.setVar(...)`. -- You can provide a fallback with `$var:name:default`. -- Sensitive values such as API keys should be passed through registry variables instead of hardcoding them in config. - - - - {PyRegisterSecret} - - - - {TsRegisterSecret} - - - -For non-sensitive settings such as inference device selection, you can also use a default fallback: - - - - {PyRegisterDevice} - - - - {TsRegisterModelFallback} - - - -Find the full list of arguments for each provider in the [integrations](/integrations/embedding) section. - -## Multiple embedding columns {#multiple-embedding-columns} - -A single table can include more than one embedding definition when you want to store multiple semantic views -of the same data, or generate embeddings from different source columns. In practice, each embedding definition -maps one source column to one vector column, and the table schema can contain multiple such pairs. - -The exact setup differs by SDK, but the underlying pattern is the same: define a distinct source/vector pair -for each embedding function you want applied during ingest. - -In TypeScript, automatic query embedding currently uses the first embedding function stored in the -table metadata. If a table has multiple embedding definitions and you need to query a specific vector -column, compute the query embedding explicitly and pass the vector to the search builder. - -## Embedding model providers {#embedding-model-providers} - -LanceDB supports most popular embedding providers. - -### Text embeddings {#text-embeddings} - -| Provider | Model ID | Default Model | -|----------|----------|---------------| -| OpenAI | `openai` | `text-embedding-ada-002` | -| Sentence Transformers | `sentence-transformers` | `all-MiniLM-L6-v2` | -| Hugging Face | `huggingface` | `colbert-ir/colbertv2.0` | -| Cohere | `cohere` | `embed-english-v3.0` | -| ... | ... | ... | - -### Multimodal embedding {#multimodal-embedding} - -| Provider | Model ID | Supported Inputs | -|----------|----------|------------------| -| OpenCLIP | `open-clip` | Text, Images | -| ImageBind | `imagebind` | Text, Images, Audio, Video | -| ... | ... | ... | - -You can find all supported embedding models in the [integrations](/integrations/embedding) section. - -## Embeddings in LanceDB Enterprise {#embeddings-in-lancedb-enterprise} -Enterprise - -In LanceDB Enterprise, embedding generation during data ingestion is client-side and the resulting vectors are -stored on the remote table. - - -The Enterprise server does not currently generate embeddings from query text on its own. Any automatic -query-time embedding happens on the client side. - - -### How string queries are interpreted {#how-string-queries-are-interpreted} - -For the Python remote client, `table.search("hello")` can take two different paths: - -- If the selected vector column has embedding metadata - (i.e., the table schema stores the source-column, vector-column, and - embedding-function mapping created from fields like `SourceField()` and - `VectorField()` during table creation), then the embeddings are computed in the Python client process. - The client uses the same local LanceDB embedding registry used by OSS tables to - reconstruct the embedding function from schema metadata, compute the query vector in - the client process, and send that vector to Enterprise for search. - ```python - result = table.search("hello").limit(1).to_list() - # The Python client computes the query embedding locally, then sends a vector search. - ``` -- If the table does not have embedding metadata for that search, `table.search("hello")` in `auto` mode is - treated as an FTS query instead. - ```python - result = table.search("hello").limit(5).to_list() - # In auto mode this is treated as an FTS query, not a vector query. - ``` - -If you want explicit vector or hybrid behavior and the client cannot resolve an embedding function from the -table metadata, generate the query embedding yourself and pass the vector directly. - -TypeScript has a similar string-query distinction in `auto` mode: if no embedding providers have been -registered in the process, `search("text")` falls back to FTS. Once an embedding provider has been -imported and registered, the client expects table embedding metadata for automatic vector search and -raises an error if the table has none. - -The manual query-embedding flow below works across Enterprise SDKs and is an explicit path you can use when you -want full control over query-time behavior. - - - - {PyManualQuerySearch} - - - - {TsManualQuerySearch} - - - - {RsManualQuerySearch} - - - -## Custom Embedding Functions {#custom-embedding-functions} - -You can always implement your own embedding function: -- Python/TypeScript: subclass `TextEmbeddingFunction` (text) or `EmbeddingFunction` (multimodal). -- Rust: implement the `EmbeddingFunction` trait. - - - - {PyEmbeddingFunction} - - - - {TsEmbeddingFunction} - - - - {RsEmbeddingFunction} - - diff --git a/docs/embedding/quickstart.mdx b/docs/embedding/quickstart.mdx deleted file mode 100644 index bcaf90f..0000000 --- a/docs/embedding/quickstart.mdx +++ /dev/null @@ -1,155 +0,0 @@ ---- -title: "Embeddings: Quickstart" -sidebarTitle: "Quickstart" -description: "Quickstart guide for generating and working with embeddings." -icon: "rocket" ---- - -import { - TsQuickstartConnect, - TsQuickstartCreateTable, - TsQuickstartImports, - TsQuickstartInitModel, - TsQuickstartQuery, - TsQuickstartSchema, -} from '/snippets/sentence-transformers.mdx'; - -LanceDB will automatically vectorize the data both at ingestion and query time. All you need to do is specify which model to use. -Popular embedding models like OpenAI, Hugging Face, Sentence Transformers, CLIP, and more, are supported. - -## Step 1: Import Required Libraries {#step-1-import-required-libraries} - -First, import the necessary LanceDB components: - - -```python Python icon="python" -import lancedb -from lancedb.pydantic import LanceModel, Vector -from lancedb.embeddings import get_registry -``` - - - {TsQuickstartImports} - - - -- `lancedb`: The main database connection and operations -- `LanceModel`: Pydantic model for defining table schemas -- `Vector`: Field type for storing vector embeddings -- `get_registry()`: Access to the embedding function registry. It has all the supported as well as custom embedding functions registered by the user -- TypeScript uses `lancedb.embedding.getRegistry()` and `lancedb.embedding.LanceSchema()` for the same registry/schema workflow -- In TypeScript, import the provider module before calling `getRegistry().get(...)`; the provider import is what registers names such as `"huggingface"` or `"openai"` - -## Step 2: Connect to LanceDB {#step-2-connect-to-lancedb} - -Establish a connection to your LanceDB OSS directory or Enterprise cluster: - - -```python Python icon="python" -# Enter your LanceDB connection URI for OSS or Enterprise here -db = lancedb.connect(...) -``` - - - {TsQuickstartConnect} - - - -## Step 3: Initialize the Embedding Function {#step-3-initialize-the-embedding-function} - -Choose and configure your embedding model: - - -```python Python icon="python" -model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", ) -``` - - - {TsQuickstartInitModel} - - - -This creates an embedding function from the local embedding registry. The Python snippet uses the -`sentence-transformers` provider with the BGE model; the TypeScript snippet uses the Transformers-backed -`huggingface` provider. You can: -- Change `"sentence-transformers"` to other providers like `"openai"`, `"cohere"`, etc. -- Modify the model name for different embedding models -- Set `device="cuda"` for GPU acceleration if available - -## Step 4: Define Your Schema {#step-4-define-your-schema} - -Create a Pydantic model that defines your table structure: - - -```python Python icon="python" -class Words(LanceModel): - text: str = model.SourceField() - vector: Vector(model.ndims()) = model.VectorField() -``` - - - {TsQuickstartSchema} - - - -- `SourceField()`: This field will be embedded -- `VectorField()`: This stores the embeddings -- `model.ndims()`: Sets vector dimensions for your model -- In TypeScript, use `model.sourceField(...)` and `model.vectorField()` inside `LanceSchema(...)` - -## Step 5: Create Table and Ingest Data {#step-5-create-table-and-ingest-data} - -Create a table with your schema and add data: - - -```python Python icon="python" -table = db.create_table("words", schema=Words) -table.add([ - {"text": "hello world"}, - {"text": "goodbye world"} -]) -``` - - - {TsQuickstartCreateTable} - - - -The `table.add()` call automatically: -- Takes the text from each document -- Generates embeddings using your chosen model -- Stores both the original text and the vector embeddings - -If your input already includes the vector column, automatic embedding only runs when that column is -absent or entirely null for the batch. Partially supplied vectors are treated as manual data, so -LanceDB preserves them instead of filling only the missing rows. - -## Step 6: Query with Automatic Embedding {#step-6-query-with-automatic-embedding} - -Note: On LanceDB Enterprise, the server does not generate embeddings from query text. In the Python remote -client, `table.search("greetings")` can still work when the table schema includes embedding metadata, because -the client computes the query embedding before sending the vector search. If there is no embedding metadata for -that search, `search("greetings")` in `auto` mode is treated as FTS instead. - -Search your data using natural language queries: - - -```python Python icon="python" -query = "greetings" -actual = table.search(query).limit(1).to_pydantic(Words)[0] -print(actual.text) -``` - - - {TsQuickstartQuery} - - - -The search process: -1. Automatically converts your query text to embeddings -2. Finds the most similar vectors in your table -3. Returns the matching documents - -Automatic text search depends on the table's embedding metadata. If the client cannot reconstruct an -embedding function from that metadata, compute the query embedding yourself and search with the -vector directly. diff --git a/docs/enterprise/architecture.mdx b/docs/enterprise/architecture.mdx index 4f78f3b..8c11286 100644 --- a/docs/enterprise/architecture.mdx +++ b/docs/enterprise/architecture.mdx @@ -11,25 +11,6 @@ At a high level, it helps to think of LanceDB Enterprise as a set of layers that This architecture matters because enterprise workloads are rarely shaped like a single benchmark. Some need thousands of concurrent queries. Others need large-scale ingestion, continuous indexing, or strict operational boundaries between user traffic and background work. LanceDB Enterprise is designed so those workloads do not all compete for the same machine, disk, or process. -```mermaid -flowchart TB - RT[Remote tables] - - subgraph DP[Data plane] - QS[Query serving] - IX[Indexers] - end - - CP[Control plane] - OS[(Object storage)] - - RT --> DP - DP --> OS - IX --> OS - CP -.->|Govern and configure| DP - CP -.->|Govern and configure| IX -``` - ## Compute-storage separation {#compute-storage-separation} In LanceDB Enterprise, storage and compute are deliberately decoupled. Table data and index artifacts live in object storage, while query-serving and background workers read from and write to that shared durable layer. This means compute can be replaced, scaled, or specialized without making any individual node the owner of the dataset. diff --git a/docs/faq/faq-oss.mdx b/docs/faq/faq-oss.mdx deleted file mode 100644 index 7c1b4ce..0000000 --- a/docs/faq/faq-oss.mdx +++ /dev/null @@ -1,90 +0,0 @@ ---- -title: "LanceDB: Frequently Asked Questions" -sidebarTitle: "LanceDB OSS" -description: "Commonly asked questions about LanceDB OSS." -icon: "code" -mode: wide ---- - -This section covers some common questions and issues that you may encounter when using LanceDB. - -### Is LanceDB open source? {#is-lancedb-open-source} - -LanceDB OSS is a permissively licensed embedded retrieval library available under an Apache 2.0 license. We also have a LanceDB Enterprise, a commercial product that can be deployed on a private cloud or a bring-your-own-cloud (BYOC) solution. LanceDB Enterprise transforms your data lake into a high-performance multimodal lakehouse. - -### What is the difference between Lance and LanceDB? {#what-is-the-difference-between-lance-and-lancedb} - -[Lance](https://github.com/lancedb/lance) is a modern lakehouse format for multimodal AI. It's perfect for building search engines, feature stores and being the foundation of large-scale ML training jobs requiring high performance IO and shuffles. It also has native support for storing, querying, and inspecting deeply nested data for robotics or large blobs like images, point clouds, and more. - -LanceDB is the multimodal lakehouse that's built on top of Lance, and utilizes the underlying optimized storage format to build efficient disk-based indexes that power semantic search & retrieval applications, from RAGs to QA bots to recommender systems. - -### Why invent another data format instead of using Parquet? {#why-invent-another-data-format-instead-of-using-parquet} - -As we mention in our talk titled "[Lance, a modern columnar data format](https://www.youtube.com/watch?v=ixpbVyrsuL8)", Parquet and other tabular formats that derive from it are rather dated (Parquet is over 10 years old), especially when it comes to random access on vectors. We needed a format that's able to handle the complex trade-offs involved in shuffling, scanning, OLAP and filtering large datasets involving vectors, and our extensive experiments with Parquet didn't yield sufficient levels of performance for modern ML. [Our benchmarks](https://lancedb.com/blog/benchmarking-random-access-in-lance/) show that Lance is up to 1000x faster than Parquet for random access, which we believe justifies our decision to create a new data format for AI. - -### Why build in Rust? {#why-build-in-rust} - -We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust's safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database. - -### What makes LanceDB different? {#what-makes-lancedb-different} - -LanceDB is among the few embedded vector DBs out there that we believe can unlock a whole new class of LLM-powered applications in the browser or via edge functions. Lance's multimodal nature allows you to store the raw data, metadata and the embeddings all at once, unlike other solutions that typically store just the embeddings and metadata. - -The Lance data format that powers our storage system also provides true zero-copy access and seamless interoperability with numerous other data formats (like Pandas, Polars, Pydantic) via Apache Arrow, as well as automatic data versioning and data management without needing extra infrastructure. - -### How large of a dataset can LanceDB handle? {#how-large-of-a-dataset-can-lancedb-handle} - -LanceDB and its underlying data format, Lance, are built to scale to really large amounts of data. LanceDB OSS can comfortably handle millions of vectors on a single node, making it a great fit for most applications. Its disk-based indexes keep performance strong without requiring expensive infrastructure. - -If you need to scale to hundreds of millions of vectors or work with terabytes of data, we recommend [LanceDB Enterprise](/enterprise). Enterprise customers regularly operate on billions of rows, backed by distributed infrastructure designed for large-scale production workloads. - -### Do I need to build a vector index to run vector search? {#do-i-need-to-build-a-vector-index-to-run-vector-search} - -No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, a vector index is usually not necessary. - -For large-scale (>1M) or higher dimension vectors, it is beneficial to create a vector index. See the [Vector Indexes](/indexing/vector-index/) section for more details. - -### How can I speed up data inserts? {#how-can-i-speed-up-data-inserts} - -LanceDB auto-parallelizes large writes when you call `table.add()` with materialized -data such as `pa.Table`, `pd.DataFrame`, or `pa.dataset()`. No extra configuration -is needed — writes are automatically split into partitions of ~1M rows or 2GB. - -For best results: - -- **Create an empty table first**, then call `table.add()`. The `add()` path enables - automatic write parallelism, while passing data directly to `create_table()` does not. -- **For file-based data**, use `pyarrow.dataset.dataset("path/to/data/", format="parquet")` - so LanceDB can stream from disk without loading everything into memory. -- **Avoid inserting one row at a time.** Each insert creates a new data fragment on - disk. Batch your data into Arrow tables, DataFrames, or use iterators. - -See [Loading Large Datasets](/tables/create#loading-large-datasets) for full examples. - -### Do I need to set a refine factor when using an index? {#do-i-need-to-set-a-refine-factor-when-using-an-index} - -Yes. LanceDB uses PQ, or Product Quantization, to compress vectors and speed up search when using an ANN index. However, because PQ is a lossy compression algorithm, it tends to reduce recall while also reducing the index size. To address this trade-off, we introduce a process called **refinement**. The normal process computes distances by operating on the compressed PQ vectors. The refinement factor (*rf*) is a multiplier that takes the top-k similar PQ vectors to a given query, fetches `rf * k` *full* vectors and computes the raw vector distances between them and the query vector, reordering the top-k results based on these scores instead. - -For example, if you're retrieving the top 10 results and set `refine_factor` to 25, LanceDB will fetch the 250 most similar vectors (according to PQ), compute the distances again based on the full vectors for those 250 and then re-rank based on their scores. This can significantly improve recall, with a small added latency cost (typically a few milliseconds), so it's recommended you set a `refine_factor` of anywhere between 5-50 and measure its impact on latency prior to deploying your solution. - -### How can I improve IVF-PQ recall while keeping latency low? {#how-can-i-improve-ivf-pq-recall-while-keeping-latency-low} - -When using an IVF-PQ index, there's a trade-off between recall and latency at query time. You can improve recall by increasing the number of probes and the `refine_factor`. In our benchmark on the GIST-1M dataset, we show that it's possible to achieve >0.95 recall with a latency of under 10 ms on most systems, using ~50 probes and a `refine_factor` of 50. This is, of course, subject to the dataset at hand and a quick sensitivity study can be performed on your own data. You can find more details on the benchmark in a past [blog post](https://medium.com/etoai/benchmarking-lancedb-92b01032874a). - -![](/static/assets/images/faq/recall-vs-latency.webp) - -### How much data can LanceDB practically manage without affecting performance? {#how-much-data-can-lancedb-practically-manage-without-affecting-performance} - -We target good performance on ~10-50 billion rows and ~10-30 TB of data. For the best performance and -scalability guarantees, check out [LanceDB Enterprise](/enterprise). - -### Does LanceDB support concurrent operations? {#does-lancedb-support-concurrent-operations} - -LanceDB can handle concurrent reads very well, and can scale horizontally. The main constraint is how well the storage layer you've chosen, scales. For writes, we support concurrent writing, though too many concurrent writers can lead to failing writes as there is a limited number of times a writer retries a commit. - - -If you use Python's multiprocessing, you should probably not use `fork` as Lance is multi-threaded -internally and `fork` and multi-threaded Python do not work well together. -[Refer to this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555) -for more information. - diff --git a/docs/faq/index.mdx b/docs/faq/index.mdx deleted file mode 100644 index c4bc36a..0000000 --- a/docs/faq/index.mdx +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: Frequently Asked Questions -sidebarTitle: FAQ -description: "Common questions about LanceDB" -icon: "question-circle" ---- - -Find answers to common questions about LanceDB across different deployment options and use cases. - - -Reach out on [Discord](https://discord.gg/AUEWnJ7Txb) for community support or [contact us](mailto:support@lancedb.com) for Enterprise assistance. - - -| Category | Description | -|:---------|:------------| -| [LanceDB OSS](/faq/faq-oss) | Questions about LanceDB open source deployment, installation, and community support | -| [LanceDB Enterprise](/faq/faq-enterprise) | Questions about LanceDB Enterprise features, security, compliance, and support | \ No newline at end of file diff --git a/docs/index.mdx b/docs/index.mdx deleted file mode 100644 index e59402a..0000000 --- a/docs/index.mdx +++ /dev/null @@ -1,124 +0,0 @@ ---- -title: LanceDB -sidebarTitle: "LanceDB" -description: "Multimodal lakehouse for AI." -icon: "/static/assets/logo/lancedb-icon-gray.svg" -keywords: ["multimodal lakehouse", "training", "feature engineering", "search", "open source", "oss"] ---- - -**LanceDB** is a [multimodal lakehouse](https://lancedb.com/blog/multimodal-lakehouse/) for AI teams that need -one data layer for curation, feature engineering, search and retrieval, and model training. -It is built on top of [Lance](/lance), an open-source lakehouse format designed for multimodal AI data. - -Move from data exploration to model training on one, unified platform without needing to manage a -fragmented stack of storage, feature, retrieval, and training systems. - -## Build better models, faster {#build-better-models-faster} - -Training data and experimentation slow down when raw data, metadata, embeddings, features, and governance -artifacts live in separate systems. LanceDB keeps them together in one versioned multimodal table, so AI teams spend less -time stitching infrastructure together and more time improving datasets, testing features, and keeping GPUs fed. - -![Training data lifecycle: Curation, Feature Engineering, Search and Retrieval, Training](/static/assets/images/overview/training-data-lifecycle.svg) - -Use the same table to curate training data, add derived features, retrieve examples, and feed training jobs that rely on expensive GPUs. -Training workloads can sample, shuffle, and scan projected columns from local storage or object storage, then assemble -GPU-ready batches from a tagged dataset version. - -For a deeper look at how this works in training pipelines, start with [Why LanceDB for training](/training/why-lancedb). - -## LanceDB suite {#lancedb-suite} - -The LanceDB suite includes LanceDB OSS, an open-source embedded retrieval library, and LanceDB Enterprise, -a multimodal lakehouse platform for the full AI data lifecycle. -OSS is easy to set up on a local machine for search and regular-scale workflows. LanceDB Enterprise is built -for teams that need scale without building bespoke infrastructure for curation, -feature engineering, search and retrieval, and efficient training data access. - -![LanceDB suite: OSS search and Enterprise multimodal lakehouse on Lance format](/static/assets/images/overview/lancedb-suite.svg) - -## Why teams use LanceDB {#why-teams-use-lancedb} - - - - Store images, video, audio, text, annotations, embeddings, and model-generated features together in one schema-enforced table. - The same table can support dataset curation, feature backfills, experiment splits, retrieval, and training. - - - Training workloads mix fast random access with high-throughput sequential scans. LanceDB is designed for both, so - teams can shuffle data into GPU-ready batches more efficiently, improve input throughput, and iterate on experiments faster. - - - Whether the end user is a human or an agent, LanceDB powers production retrieval workloads such as semantic search, - hybrid search, RAG, agent memory, and recommendation systems. Retrieval runs against the same LanceDB tables used - for curation, feature engineering, and training workflows. - - - -## Start with your workload {#start-with-your-workload} - - - - Learn why LanceDB works well as the data layer for training workloads. - - - Use LanceDB tables and permutations for projected, shuffled, random-access training reads. - - - Explore Lance-formatted multimodal datasets with raw bytes, metadata, embeddings, and indices. - - - Use vector search, full-text search, hybrid search, reranking, filtering, and SQL. - - - -## From local development to production scale {#from-local-development-to-production-scale} - -LanceDB OSS and LanceDB Enterprise share the same Lance format and table model. Start locally with the embedded OSS -library, then move to Enterprise when your team needs distributed scale, managed infrastructure, private deployment, -or higher-throughput curation, feature engineering, search and retrieval, and training workflows. - -### 1\. LanceDB OSS {#1-lancedb-oss} -The fastest way to get started is the open-source embedded library, with client SDKs in Python, TypeScript -and Rust. Run it locally in just a few steps, which lets you explore datasets, curate data, and run search and retrieval workloads -for agents. Start here: - - - - Get started with LanceDB in minutes. - - - Create tables, evolve schemas, version data, and modify rows in LanceDB. - - - -### 2\. LanceDB Enterprise {#2-lancedb-enterprise} - -[LanceDB Enterprise](/enterprise) is a petabyte-scale (and beyond), distributed **multimodal lakehouse** platform built for -search, curation, feature engineering, and high-throughput training data access workflows on top of the same core table -abstraction. This eliminates the need for teams to build bespoke infrastructure to manage large multimodal datasets. -To set up LanceDB Enterprise in your organization, reach out to us at -[contact@lancedb.com](mailto:contact@lancedb.com). - - -**Built with scale, performance, and security in mind.** - -LanceDB Enterprise is designed for very large-scale, high-performance, distributed workloads in -private deployments, and can operate under strict [security requirements](/enterprise/security). - - - - Get started with LanceDB in minutes, including Enterprise `db://` connections. - diff --git a/docs/indexing/fts-index.mdx b/docs/indexing/fts-index.mdx deleted file mode 100644 index 2d62ad8..0000000 --- a/docs/indexing/fts-index.mdx +++ /dev/null @@ -1,165 +0,0 @@ ---- -title: "Full-Text Search (FTS) Index" -sidebarTitle: "FTS index" -description: "Create and tune BM25-based full-text search indexes in LanceDB." -icon: "book" ---- -import { PyFtsIndexAsync as FtsIndexAsync, PyFtsIndexCreate as FtsIndexCreate, PyFtsIndexNested as FtsIndexNested, PyFtsIndexWait as FtsIndexWait } from '/snippets/indexing.mdx'; - -LanceDB provides performant full-text search based on BM25, allowing you to incorporate keyword-based search in your retrieval solutions. This page shows -examples on how to create and configure FTS indexes in LanceDB OSS and Enterprise, using the synchronous and asynchronous APIs. - - -In LanceDB Enterprise, `create_fts_index` API returns immediately, but index building happens asynchronously. - - -## Creating FTS Indexes {#creating-fts-indexes} - -### Synchronous API {#synchronous-api} - -Use `create_fts_index` with synchronous LanceDB connections: - - - - {FtsIndexCreate} - - - -Check FTS index status using the API: - - - - {FtsIndexWait} - - - -`wait_for_index(...)` waits until the named FTS index exists and `index_stats(...)` reports `num_unindexed_rows == 0`. It can time out if writes keep adding rows faster than the index catches up. If a table has multiple FTS indexes, specify the target text column when querying instead of relying on implicit selection. - -### Asynchronous API {#asynchronous-api} - -When using async connections (`connect_async`), use `create_index` with the `FTS` configuration: - - - - {FtsIndexAsync} - - - - -The `create_fts_index` method is not available on `AsyncTable`. Use `create_index` with `FTS` config instead. - - -The current FTS implementation is Lance-native. Legacy Tantivy-only options, including -`use_tantivy`, are no longer accepted by the index creation APIs. - -## Nested field paths {#nested-field-paths} - -FTS indexes can target text leaves inside struct columns by passing a dotted path (for example, `payload.text`). The same path works for [`MatchQuery`](/search/full-text-search) and [`PhraseQuery`](/search/full-text-search), and for the `columns` argument on async `nearest_to_text` queries. - -You can point an index at any string leaf nested in a struct, regardless of depth. The struct container itself isn't indexable: you have to name a specific text field. - - - - {FtsIndexNested} - - - -LanceDB rejects paths that don't resolve to a text leaf: - -- A struct container (for example, `payload`): raises `ValueError: FTS index cannot be created ...`. -- A non-text leaf such as an integer or float (for example, `payload.count`): raises the same error. -- A path that doesn't exist in the schema (for example, `payload.missing`): raises `ValueError: Field path ... not found`. - -The async API accepts the same dotted paths through `create_index`: - -```python Python icon="python" -from lancedb.index import FTS - -await async_table.create_index("payload.text", config=FTS(with_position=True)) -``` - -## Configuration Options {#configuration-options} - -### FTS Parameters {#fts-parameters} - -| Parameter | Type | Default | Description | -|:----------|:-----|:--------|:------------| -| `with_position` | bool | `False` | Store token positions (required for phrase queries) | -| `base_tokenizer` | str | `"simple"` | Text splitting method (`simple`, `whitespace`, `raw`, `ngram`, `icu`, `jieba/*`, or `lindera/*`) | -| `language` | str | `"English"` | Language for stemming and stop-word filters. Choose CJK and mixed-language segmentation with `base_tokenizer`. | -| `max_token_length` | int | `40` | Maximum token size; longer tokens are omitted | -| `lower_case` | bool | `True` | Lowercase tokens | -| `stem` | bool | `True` | Apply stemming (`running` → `run`) | -| `remove_stop_words` | bool | `True` | Drop common stop words | -| `ascii_folding` | bool | `True` | Normalize accented characters | -| `custom_stop_words` | list[str] | `None` | Extra stop words to drop in addition to the language defaults. Requires `remove_stop_words=True`. | -| `ngram_min_length` | int | `3` | Minimum n-gram length. Applies only when `base_tokenizer="ngram"`. | -| `ngram_max_length` | int | `3` | Maximum n-gram length. Applies only when `base_tokenizer="ngram"`. | -| `prefix_only` | bool | `False` | Index only prefix n-grams rather than all substrings. Applies only when `base_tokenizer="ngram"`. | -| `block_size` | int | `128` | Number of documents per compressed posting block. Supported values are `128` and `256`. Setting this to `256` opts in to the experimental FTS V3 layout. | - - -- `max_token_length` can filter out base64 blobs or long URLs. -- Disabling `with_position` reduces index size but disables phrase queries. -- `ascii_folding` helps with international text (e.g., “café” → “cafe”). - - -### Tokenizer choices {#tokenizer-choices} - -`base_tokenizer` controls segmentation before token filters run: - -- `simple`, `whitespace`, and `raw` cover common tokenization strategies for space-delimited text. -- `ngram` indexes overlapping character spans for substring-style matching. -- `icu` uses bundled ICU4X word segmentation for mixed-language text and scripts where whitespace splitting is not enough. ICU stands for [International Components for Unicode](https://icu.unicode.org/), and this tokenizer does not need external model files. -- `jieba/*` is for Chinese word segmentation with Jieba. -- `lindera/*` loads a compiled Lindera dictionary, such as `lindera/ipadic` for Japanese or `lindera/ko-dic` for Korean. - -Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `/jieba/default/...`. - -`language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`. - -### Posting block size {#posting-block-size} - -`block_size` controls the number of documents packed into each compressed posting block on disk. The default of `128` matches the current FTS layout and is the right choice for most workloads. Setting it to `256` opts in to the experimental FTS V3 format, which changes how postings are encoded and may introduce breaking changes in future releases. Any other value is rejected at index creation time. - -You can set the option through either the synchronous or asynchronous API. In the async Python API, pass it on the `FTS` config, and in the TypeScript API use the camelCase `blockSize` field on `FtsOptions`: - -```python Python icon="python" -from lancedb.index import FTS - -await async_table.create_index("text", config=FTS(block_size=256)) -``` - -```typescript TypeScript icon="square-js" -await table.createIndex("text", { - config: lancedb.Index.fts({ blockSize: 256 }), -}); -``` - -### Phrase Query Configuration {#phrase-query-configuration} - -Enable phrase queries by setting: - -| Parameter | Required Value | Purpose | -|:----------|:---------------|:--------| -| `with_position` | `True` | Track token positions for phrase matching | -| `remove_stop_words` | `False` | Preserve stop words for exact phrase matching | - -## Indexing nested string fields {#indexing-nested-string-fields} - -You can build an FTS index on a string field inside a struct by passing its full dotted path, like `nested.text`. The same path is used when you query the index through `fts_columns`, and the indexed column is reported back as the full path from `list_indices()`. - -```python -# Schema: pa.struct([pa.field("text", pa.string())]) stored under the `nested` column. -table.create_fts_index("nested.text") - -results = ( - table.search("puppy", query_type="fts", fts_columns="nested.text") - .limit(5) - .to_list() -) -``` - - -Use the canonical Lance path: dot-separate each struct field from root to leaf (for example, `metadata.author.name`). The same convention applies to scalar and vector indexes. - diff --git a/docs/indexing/index.mdx b/docs/indexing/index.mdx deleted file mode 100644 index d3b123f..0000000 --- a/docs/indexing/index.mdx +++ /dev/null @@ -1,161 +0,0 @@ ---- -title: "Indexing Data" -sidebarTitle: "Overview" -description: "Optimize search performance in LanceDB using vector indexes, full-text search, and scalar indexes. Understand IVF-PQ indexing for efficient vector similarity search." -icon: "list" ---- - -An **index** is a data structure that facilitates efficient scans and lookups on the embeddings of a given dataset. LanceDB provides a comprehensive suite of indexes to optimize query performance across diverse workloads: - -- **Vector Index**: Optimized for searching high-dimensional data (like images, audio, or text embeddings) by efficiently finding the most similar vectors -- **Full-Text Search Index**: Enables fast keyword-based searches by indexing words and phrases -- **Scalar Index**: Accelerates filtering and sorting of structured numeric or categorical data (e.g., timestamps, prices) - - -Scalar indices serve as a foundational optimization layer, accelerating filtering across diverse search workloads. They can be combined with: - -- Vector search (prefilter or post-filter results using metadata) -- Full-text search (combining keyword matching with structured filters) -- SQL scans (optimizing WHERE clauses on scalar columns) -- Key-value lookups (enabling rapid primary key-based retrievals) - - -## Supported Index Types {#supported-index-types} - -LanceDB provides a comprehensive suite of indexing strategies for different data types and use cases: - -| Index | Use Case | Description | -| :--------- | :------- | :---------- | -| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. Supports binary vectors with hamming distance. | Inverted File Index—a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search.
Distance metrics: `l2` `cosine` `dot` `hamming`
Quantizations: `None/Flat` `PQ` `SQ` `RQ`| -| `IVF_HNSW` (Vector) | Large-scale vector search requiring both high recall and efficient partitioning. Combines the scalability of IVF with the search quality of HNSW. | Hybrid index combining IVF partitioning with HNSW graphs built within each partition. Provides improved search quality over pure IVF while maintaining scalability.
Distance metrics: `l2` `cosine` `dot`
Quantizations: `None/Flat` `SQ` `PQ`| -| `FTS` (Full-text search) | String columns (e.g., title, description, content) requiring keyword-based search with BM25 ranking. | Full-text search index using BM25 ranking algorithm. Tokenizes text with configurable tokenization, stemming, stop word removal, and language-specific processing. | -| `BTree` (Scalar) | Numeric, temporal, and string columns with mostly distinct values. Best for selective equality, inequality, and range predicates. | Sorted index storing sorted copies of scalar columns with block headers in a btree cache. Header entries map to blocks of rows (4096 rows per block) for efficient disk reads. | -| `Bitmap` (Scalar) | Low-cardinality columns with few thousand or fewer distinct values. Accelerates equality and range filters. | Stores a bitmap for each distinct value in the column, with one bit per row indicating presence. Memory-efficient for low-cardinality data. | -| `LabelList` (Scalar) | List columns (e.g., tags, categories, keywords) requiring `array_contains_all` or `array_contains_any` filters. | Scalar index for `List` and `LargeList` columns of primitive values, using an underlying bitmap index structure to enable fast array membership lookups. | -| `FM` (Scalar) | String or binary columns that need raw substring search. | FM-Index over `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary` data for filters such as `contains(path, 'needle')`. Use FTS instead for tokenized word search and BM25 ranking. | - - -TypeScript currently doesn't support `IvfSq` (IVF with Scalar Quantization). - - - -**Operational checks** - -For vector indexes, use the same distance metric when creating the index and searching it. After appends or other writes, use `optimize()` to fold new rows into existing indexes, then check `index_stats(...)` or `wait_for_index(...)` if you need to confirm the index has caught up. `wait_for_index(...)` waits until the named indexes exist and report `num_unindexed_rows == 0`; it can time out if writes keep adding unindexed rows. - -By default, automatic vector indexing creates `IVF_PQ`, and scalar index creation defaults to -`BTree` unless you pass another scalar index config. `BTree` and `Bitmap` indexes target scalar -columns, not list columns; use `LabelList` for list containment filters. - - -### Quantization Types {#quantization-types} - -Vector indexes can use different quantization methods to compress vectors and improve search performance: - -| Quantization | Use Case | Description | -| :----------- | :------- | :---------- | -| `PQ` (Product Quantization) | Default choice for most vector search scenarios. Use when you need to balance index size and recall. | Divides vectors into subvectors and quantizes each subvector independently. Provides a good balance between compression ratio and search accuracy. | -| `SQ` (Scalar Quantization) | Use when you need faster indexing or when vector dimensions have consistent value ranges. | Quantizes each dimension independently. Simpler than PQ but typically provides less compression. | -| `RQ` (RabitQ Quantization) | Use when you need maximum compression or have specific per-dimension requirements. | Per-dimension quantization using a RabitQ codebook. Provides fine-grained control over compression per dimension. For `IVF_RQ`, vector dimensions must be divisible by `8`. | -| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization—stores raw vectors. Provides the highest accuracy but requires more storage and memory. | - -## Understanding the IVF-PQ Index {#understanding-the-ivf-pq-index} - -An ANN (Approximate Nearest Neighbors) index is a data structure that quickly produces an approximate solution to the **k-nearest neighbors (kNN)** problem. -It greatly improves upon the runtime of a brute-force kNN search, while admitting a slight decrease in accuracy. LanceDB uses the disk-based indexing technique IVF-PQ, discussed below. - -LanceDB differs from other vector databases in that it is built on top of [Lance](https://github.com/lancedb/lance), an open-source columnar data format designed for performant ML workloads and fast random access. Due to the design of Lance, LanceDB's indexing philosophy adopts a primarily *disk-based* indexing philosophy. - -## IVF-PQ {#ivf-pq} - -LanceDB uses **IVF-PQ** indexing, which combines the clustering-based **Inverted File Index (IVF)** with **Product Quantization (PQ)** to efficiently compress embeddings. -The implementation provides several parameters to fine-tune the index's size, query throughput, latency, and recall. - -### Product Quantization {#product-quantization} - -Quantization is a compression technique used to reduce the dimensionality of an embedding to speed up search. - -Product quantization (PQ) works by dividing a large, high-dimensional vector of size into equally sized subvectors. Each subvector is assigned a "reproduction value" that maps to the nearest centroid of points for that subvector. The reproduction values are then assigned to a codebook using unique IDs, which can be used to reconstruct the original vector. - -![](/static/assets/images/indexing/ivfpq_pq_desc.png) - -It's important to remember that quantization is a *lossy process*, i.e., the reconstructed vector is not identical to the original vector. This results in a trade-off between the size of the index and the accuracy of the search results. - -As an example, consider starting with 128-dimensional vector consisting of 32-bit floats. Quantizing it to an 8-bit integer vector with 4 dimensions as in the image above, we can significantly reduce memory requirements. - - -Original: `128 × 32 = 4096` bits -Quantized: `4 × 8 = 32` bits - -Quantization results in a **128x** reduction in memory requirements for each vector in the index, which is substantial. - - -### Inverted File Index (IVF) Implementation {#inverted-file-index-ivf-implementation} - -While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space. - -In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index. - -![](/static/assets/images/indexing/ivfpq_ivf_desc.webp) - -During query time, depending on where the query lands in vector space, it may be close to the border of multiple Voronoi cells, which could make the top-k results ambiguous and span across multiple cells. To address this, the IVF-PQ introduces the `nprobe` parameter, which controls the number of Voronoi cells to search during a query. The higher the `nprobe`, the more accurate the results, but the slower the query. -![](/static/assets/images/indexing/ivfpq_query_vector.webp) - -## HNSW Index Implementation {#hnsw-index-implementation} - -Approximate Nearest Neighbor (ANN) search is a method for finding data points near a given point in a dataset, though not always the exact nearest one. HNSW is one of the most accurate and fastest Approximate Nearest Neighbour search algorithms, It's beneficial in high-dimensional spaces where finding the same nearest neighbor would be too slow and costly. - -### Types of ANN Search Algorithms {#types-of-ann-search-algorithms} - -Approximate Nearest Neighbor (ANN) search is a method for finding data points near a given point in a dataset, though not always the exact nearest one. -For example, HNSW is an ANN index that performs well in high-dimensional spaces where other techniques prove too slow and costly. - -There are three main types of ANN search algorithms: - -* **Tree-based search algorithms**: Use a tree structure to organize and store data points. -* **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice. -* **Graph-based search algorithms**: Use a graph structure to store data points, which can be a bit complex. - -HNSW is a graph-based algorithm. All graph-based search algorithms rely on the idea of a k-nearest neighbor (or k-approximate nearest neighbor) graph, which we outline below. -HNSW also combines this with the ideas behind a classic 1-dimensional search data structure: the skip list. - -### Understanding k-Nearest Neighbor Graphs {#understanding-k-nearest-neighbor-graphs} - -The k-nearest neighbor graph actually predates its use for ANN search. Its construction is quite simple: - -* Each vector in the dataset is given an associated vertex. -* Each vertex has outgoing edges to its k nearest neighbors. That is, the k closest other vertices by Euclidean distance between the two corresponding vectors. This can be thought of as a "friend list" for the vertex. -* For some applications (including nearest-neighbor search), the incoming edges are also added. - -Eventually, it was realized that the following greedy search method over such a graph typically results in good approximate nearest neighbors: - -* Given a query vector, start at some fixed "entry point" vertex (e.g. the approximate center node). -* Look at that vertex's neighbors. If any of them are closer to the query vector than the current vertex, then move to that vertex. -* Repeat until a local optimum is found. - -The above algorithm also generalizes to e.g. top 10 approximate nearest neighbors. - -Computing a k-nearest neighbor graph is actually quite slow, taking quadratic time in the dataset size. It was quickly realized that near-identical performance can be achieved using a k-approximate nearest neighbor graph. That is, instead of obtaining the k-nearest neighbors for each vertex, an approximate nearest neighbor search data structure is used to build much faster. -In fact, another data structure is not needed: This can be done "incrementally". -That is, if you start with a k-ANN graph for n-1 vertices, you can extend it to a k-ANN graph for n vertices as well by using the graph to obtain the k-ANN for the new vertex. - -One downside of k-NN and k-ANN graphs alone is that one must typically build them with a large value of k to get decent results, resulting in a large index. - -### Hierarchical Navigable Small Worlds (HNSW) {#hierarchical-navigable-small-worlds-hnsw} - -HNSW builds on k-ANN in two main ways: - -* Instead of getting the k-approximate nearest neighbors for a large value of k, it sparsifies the k-ANN graph using a carefully chosen "edge pruning" heuristic, allowing for the number of edges per vertex to be limited to a relatively small constant. -* The "entry point" vertex is chosen dynamically using a recursively constructed data structure on a subset of the data, similarly to a skip list. - -This recursive structure can be thought of as separating into layers: - -* At the bottom-most layer, a k-ANN graph on the whole dataset is present. -* At the second layer, a k-ANN graph on a fraction of the dataset (e.g. 10%) is present. -* At the Lth layer, a k-ANN graph is present. It is over a (constant) fraction (e.g. 10%) of the vectors/vertices present in the L-1th layer. - -Then the greedy search routine operates as follows: - -* At the top layer (using an arbitrary vertex as an entry point), use the greedy local search routine on the k-ANN graph to get an approximate nearest neighbor at that layer. -* Using the approximate nearest neighbor found in the previous layer as an entry point, find an approximate nearest neighbor in the next layer with the same method. -* Repeat until the bottom-most layer is reached. Then use the entry point to find multiple nearest neighbors (e.g. top 10). diff --git a/docs/indexing/quantization.mdx b/docs/indexing/quantization.mdx deleted file mode 100644 index 8acc1fc..0000000 --- a/docs/indexing/quantization.mdx +++ /dev/null @@ -1,79 +0,0 @@ ---- -title: "Quantization" -sidebarTitle: "Quantization" -description: "Learn about quantization when creating an index in LanceDB." -icon: "compress" -keywords: ["quantization", "quantize", "rabitq"] ---- - -Quantization compresses high-dimensional float vectors into a smaller, approximate representation, where instead of storing every vector as a float32 or float64, it's stored in compressed form, without too much of a compromise in search quality. - -Use quantization when: - -- You have a large dataset with relatively high-dimensional vectors (512, 768, 1024+) -- Index build time and query latency matter - -LanceDB currently exposes multiple quantized vector index types, including: -- `IVF_PQ` -- Inverted File index with Product Quantization (default). See the [vector indexing guide](/indexing/vector-index) for `IVF_PQ` examples. -- `IVF_SQ` -- Inverted File index with Scalar Quantization. This is available in Python and Rust; TypeScript does not currently expose `IvfSq`. -- `IVF_RQ` -- Inverted File index with **RaBitQ** quantization (binary, 1 bit per dimension). Requires vector dimensions divisible by `8`. See [below](#rabitq-quantization) for details. -- `IVF_HNSW_SQ` -- IVF partitions with an **HNSW graph per partition** plus **Scalar Quantization**. Strong recall/latency/size trade-off for most workloads. -- `IVF_HNSW_PQ` -- IVF partitions with an **HNSW graph per partition** plus **Product Quantization**. Prefer when PQ-level compression matters and you still want HNSW-style in-partition search. - -Two axes are being combined here: whether partitions are searched flatly or via an HNSW graph (`IVF_*` vs. `IVF_HNSW_*`), and which quantizer compresses the vectors (`PQ`, `RQ`, or `SQ`). `IVF_PQ` is the default and works well in many cases. For more drastic compression, RaBitQ (`IVF_RQ`) is a reasonable option. For higher recall at low latency, the HNSW-backed variants are usually the right pick. The ["Choose the Right Index"](/indexing/vector-index#choose-the-right-index) table on the vector indexing page is the canonical decision tool. - -Use the same distance metric when training the index and running queries against it. For IVF-based indexes, `num_partitions` controls the number of groups and `sample_rate` controls how many training vectors are sampled per partition, so the training sample is roughly `sample_rate * num_partitions`. - -## RaBitQ quantization {#rabitq-quantization} - -RaBitQ is a binary quantization method that represents each normalized embedding using **1 bit per dimension**, plus a couple of small corrective scalars. In practice, a 1,024-dimensional `float32` vector that would normally take 4 KB can be compressed to roughly a few hundred bytes with RaBitQ, while still maintaining reasonable recall. - -### How RaBitQ works {#how-rabitq-works} - -- Embeddings are grouped around centroids (as in other IVF indexes). -- Each residual vector is normalized and mapped to the nearest vertex of a randomly rotated hypercube on the unit sphere. -- The sign pattern of that vector is stored as bits (1 bit per dimension). -- Two small corrective factors are stored: - 1. The distance from the original vector to its centroid - 2. The dot product between the normalized vector and its quantized version - -Compared to `IVF_PQ`, RaBitQ: -- Avoids training expensive PQ codebooks -- Builds indexes faster and handles updates more easily -- Maintains or improves recall at high dimensionality under the same storage budget - -For a deeper dive into the theory and some benchmark results, see the blog post: [LanceDB's RaBitQ Quantization for Blazing Fast Vector Search](https://lancedb.com/blog/feature-rabitq-quantization/). - -### Using RaBitQ {#using-rabitq} - -You can create an RaBitQ-backed vector index by setting `index_type="IVF_RQ"` when calling `create_index`. - - -When using `IVF_RQ`, vector dimensions must be divisible by `8`. - - -`num_bits` controls how many bits per dimension are used: - -1 bit is the classic RaBitQ setting. You can set it to 2, 4, or 8 bits to improve fidelity for better precision or recall — the main trade-off is additional storage for the extra bits per dimension, with only a modest increase in query-time compute. -It's also possible to tune the number of IVF partitions in `IVF_RQ`, similar to how you would do in `IVF_PQ`. - - -Indexes built with `num_bits >= 2` use an updated on-disk layout. Older LanceDB versions cannot read them and will fail with a clear missing-column error rather than returning incorrect results. Existing indexes keep working and upgrade automatically when they are rewritten (for example, during compaction, optimize, or remap). `num_bits=1` indexes are unaffected in both directions. - - -## API Reference {#api-reference} - -The full list of parameters to the algorithm are listed below. - -- `distance_type`: Literal["l2", "cosine", "dot"], defaults to "l2" - The distance metric to use for similarity comparison. Choose "l2" for Euclidean, "cosine" for cosine similarity, or "dot" for dot product. -- `num_partitions`: Optional[int], defaults to None - Number of IVF partitions (affects index build time and query accuracy). More partitions can improve recall but may increase build time. When unset, LanceDB chooses roughly the square root of the row count. -- `num_bits`: int, defaults to 1 - Bits per dimension for quantization (1 is standard RaBitQ). Higher values improve fidelity, mainly at the cost of additional storage. -- `max_iterations`: int, defaults to 50 - Maximum number of iterations for training the quantizer. Increase for larger datasets or to improve quantization quality. -- `sample_rate`: int, defaults to 256 - Number of samples per partition during training. Higher values may improve accuracy but increase training time. -- `target_partition_size`: Optional[int], defaults to None - Target number of vectors per partition. Adjust to control partition granularity and memory usage. If `num_partitions` is also set, `num_partitions` takes precedence. diff --git a/docs/indexing/reindexing.mdx b/docs/indexing/reindexing.mdx deleted file mode 100644 index 2a1cabc..0000000 --- a/docs/indexing/reindexing.mdx +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: "Keeping Indexes Up-to-Date with Reindexing" -sidebarTitle: "Reindexing" -description: "Learn how to keep your indexes up-to-date in LanceDB using incremental indexing, including best practices for adding new records without full reindexing." -icon: "refresh" -keywords: ["reindexing", "re-index", "incremental indexing", "reindex"] ---- -import { PyReindexingIncremental as ReindexingIncremental } from '/snippets/indexing.mdx'; - -As you add new data to your LanceDB tables, your indexes may become outdated. -Reindexing is the process of updating the index to account for new data -- this applies to either a full-text search (FTS) index or a vector index. Reindexing is an important operation to run periodically as your data grows, as it has performance implications. - -As data is being added and a reindex operation is running, LanceDB will combine results from the existing index with exhaustive/flat search on the new data. This is done to ensure that you're still retrieving results over all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable. - -Rather than dropping an existing index entirely and reindexing from scratch, LanceDB supports **incremental indexing**. - -## Incremental Reindexing {#incremental-reindexing} - -You can manually trigger an incremental indexing operation on updated data -using the `optimize()` method on a table. - -Table optimization performs three maintenance operations: - -1. **Compaction**: merges small fragments into larger ones to improve read performance -2. **Pruning/Cleanup**: removes files from versions older than a retention window (7 days by default) -3. **Index update**: adds newly-ingested data to existing vector, scalar, and FTS indexes - - - - {ReindexingIncremental} - - - -Enterprise - -LanceDB Enterprise support incremental reindexing through an automated background process. When new data is added to a table, the system automatically triggers a new index build. As the dataset grows, indexes are asynchronously updated in the background. - -- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. To avoid this, set `fast_search=True` to search only indexed data. -- Use `index_stats()` to view the number of unindexed rows. This will be zero when indexes are fully up-to-date. If you call `wait_for_index(...)`, it polls the same status and can time out while continuous writes keep adding unindexed rows. - -The benefit of using LanceDB Enterprise is that it automates the reindexing process -and operates continuously in the background, minimizing the impact on latency under high loads. -In OSS, you must manually manage the reindexing cadence based on your data growth and performance needs. - -## Disk utilization {#disk-utilization} - -Compaction by itself does not immediately free disk space, and can temporarily increase it because new -compacted files are written before old-version files are deleted. Disk space is reclaimed when old versions -are pruned during cleanup. Set retention only as low as your rollback and time-travel requirements allow. - -If you need to reclaim space more aggressively in OSS, use a shorter retention window: - - ```python Python icon=Python - from datetime import timedelta - - table.optimize(cleanup_older_than=timedelta(days=1)) - ``` - - diff --git a/docs/indexing/scalar-index.mdx b/docs/indexing/scalar-index.mdx deleted file mode 100644 index 68bffdd..0000000 --- a/docs/indexing/scalar-index.mdx +++ /dev/null @@ -1,243 +0,0 @@ ---- -title: "Scalar Indexes" -sidebarTitle: "Scalar Index" -description: "Learn how to use scalar indexes in LanceDB for efficient metadata filtering and query optimization." -icon: "tree" ---- -import { - PyScalarIndexBuild as ScalarIndexBuild, - PyScalarIndexWait as ScalarIndexWait, - PyScalarIndexOptimize as ScalarIndexOptimize, - PyScalarIndexFilter as ScalarIndexFilter, - PyScalarIndexPrefilter as ScalarIndexPrefilter, - PyScalarIndexUuidType as ScalarIndexUuidType, - PyScalarIndexUuidData as ScalarIndexUuidData, - PyScalarIndexUuidTable as ScalarIndexUuidTable, - PyScalarIndexUuidWait as ScalarIndexUuidWait, - PyScalarIndexUuidUpsert as ScalarIndexUuidUpsert, - PyScalarIndexNestedFields as ScalarIndexNestedFields, -} from '/snippets/indexing.mdx'; - -Scalar indexes organize data by scalar attributes (e.g., numbers, categories) and enable fast filtering of vector data. They accelerate retrieval of scalar data associated with vectors, thus enhancing query performance. - -LanceDB supports four types of scalar indexes: - -- `BTREE`: Stores column data in sorted order for binary search. Best for columns with many unique values. -- `BITMAP`: Uses bitmaps to track value presence. Ideal for columns with few unique values (e.g., categories, tags). -- `LABEL_LIST`: Special index for `List` and `LargeList` columns of primitive values supporting `array_contains_all` and `array_contains_any` queries. -- `FM`: FM-Index over string or binary columns that accelerates substring search via `contains(col, 'needle')`. - -## Choosing the Right Index Type {#choosing-the-right-index-type} - -| Data Type | Filter | Index Type | -|:----------------------------------------------------------------|:------------------------------------------|:-------------| -| Numeric, String, Temporal | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE` | -| Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP` | -| List of low cardinality of numbers or strings | `array_has_any`, `array_has_all` | `LABEL_LIST` | -| String or binary (`Utf8`, `LargeUtf8`, `Binary`, `LargeBinary`) | `contains(col, 'needle')` | `FM` | - -## Scalar Index Operations {#scalar-index-operations} - -### 1\. Build the Index {#1-build-the-index} - -You can create multiple scalar indexes within a table. By default, the index will be `BTREE`, but you can always configure another type like `BITMAP` - - - - {ScalarIndexBuild} - - - - -If you are using LanceDB Enterprise, the `create_scalar_index` API returns immediately, but the building of the scalar index is asynchronous. To wait until all data is fully indexed, you can specify the `wait_timeout` parameter on `create_scalar_index()` or call `wait_for_index()` on the table. - - -### 2\. Check Index Status {#2-check-index-status} - - - - {ScalarIndexWait} - - - -`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. - -### 3\. Update the Index {#3-update-the-index} - -Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index. - - - - {ScalarIndexOptimize} - - - - -New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise automates the optimize process, minimizing the impact on search speed. - - -### 4\. Run Indexed Searches {#4-run-indexed-searches} - -The following scan will be faster if the column `book_id` has a scalar index: - - - - {ScalarIndexFilter} - - - -Scalar indexes can also speed up scans containing a vector search or full text search, and a prefilter: - - - - {ScalarIndexPrefilter} - - - -## Indexing nested fields {#indexing-nested-fields} - -Scalar indexes can target a scalar field inside a struct by passing its full dotted path. The path is preserved end to end: it's the value you pass to `create_scalar_index`, it's what `list_indices()` reports under `columns`, and it's the column reference you use in filter predicates. - -```python -# Schema: pa.struct([pa.field("user_id", pa.int32())]) stored under the `metadata` column. -table.create_scalar_index("metadata.user_id", name="metadata_user_id_idx") - -# The same dotted path works in WHERE clauses. -table.search().where("metadata.user_id = 42").limit(1).to_list() -``` - - -Nested paths follow Lance field-path semantics: dot-separate each struct field from root to leaf (for example, `metadata.author.name`). The same convention applies to FTS and vector indexes. - - -## FM-Index for substring search {#fm-index-for-substring-search} - -The `FM` index is a scalar index built over string or binary columns that -accelerates substring lookups expressed as `contains(col, 'needle')`. Unlike the -tokenized [FTS index](/indexing/fts-index), which matches whole words after -tokenization, the FM-Index matches arbitrary substrings of the raw bytes — so it -works well for URLs, file paths, identifiers, log lines, or any column where you -search for a fragment rather than a word. - -Use the FM-Index when: - -- Filters use `contains(col, 'needle')` (substring), not equality or word search. -- The column is `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary`. -- You want substring matches without paying for tokenization, language analysis, - or BM25 scoring. - -Pick `FTS` instead when you need word-level relevance ranking, phrase queries, -or language-aware tokenization. - -### Create an FM-Index {#create-an-fm-index} - -Build an FM-Index with the async `create_index` API by passing the `Fm` config in -Python or `Index.fm()` in TypeScript. In Rust, use `Index::Fm(FmIndexBuilder::default())`. - - -```python Python icon="python" -from lancedb.index import Fm - -await tbl.create_index("text", config=Fm()) -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import { Index } from "@lancedb/lancedb"; - -await tbl.createIndex("text", { config: Index.fm() }); -``` - -```rust Rust icon="rust" -use lancedb::index::Index; -use lancedb::index::scalar::FmIndexBuilder; - -tbl.create_index(&["text"], Index::Fm(FmIndexBuilder::default())) - .execute() - .await?; -``` - - -After the index is built, substring filters use it automatically: - -```python -table.search().where("contains(text, 'needle')").limit(10).to_pandas() -``` - -`list_indices()` reports the index type as `"Fm"`. - -## Index UUID Columns {#index-uuid-columns} - -LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)`), enabling efficient lookups and filtering on UUID-based primary keys. - - -**To use `FixedSizeBinary`, ensure you have:** - -- Python SDK version `0.22.0` or later -- TypeScript SDK version `0.19.0` or later - - -### 1\. Define UUID Type {#1-define-uuid-type} - - - - {ScalarIndexUuidType} - - - -### 2\. Generate UUID Data {#2-generate-uuid-data} - - - - {ScalarIndexUuidData} - - - -### 3\. Create Table with UUID Column {#3-create-table-with-uuid-column} - - - - {ScalarIndexUuidTable} - - - -### 4\. Create and Wait for the Index {#4-create-and-wait-for-the-index} - - - - {ScalarIndexUuidWait} - - - -### 5\. Perform Operations with the UUID Index {#5-perform-operations-with-the-uuid-index} - - - - {ScalarIndexUuidUpsert} - - - -## Index nested fields {#index-nested-fields} - -You can build a scalar index on a field inside a struct column by passing the -canonical dot-separated path to `create_index`. This is useful when filters -target attributes nested under a `metadata`-style column, for example -`metadata.user_id` or `metadata.event.type`. - -If a literal segment of the path itself contains a dot (for example a column -named `user.id` nested inside `metadata`), wrap that segment in backticks so -LanceDB can tell the dot apart from the path separator: `` metadata.`user.id` ``. - -`list_indices()` echoes the same canonical path back, so the column you pass in -round-trips through index metadata regardless of nesting depth or escaping. - - - - {ScalarIndexNestedFields} - - - - -Composite indexes that cover multiple columns aren't supported yet. Each -`create_index` call must target a single (possibly nested) field path. - diff --git a/docs/indexing/vector-index.mdx b/docs/indexing/vector-index.mdx deleted file mode 100644 index febe764..0000000 --- a/docs/indexing/vector-index.mdx +++ /dev/null @@ -1,379 +0,0 @@ ---- -title: "Vector Indexes" -sidebarTitle: "Vector Index" -description: "Build and optimize LanceDB vector indexes, including IVF, HNSW and binary quantized indexes." -icon: "arrow-up-right-dots" ---- -import { - PyVectorIndexConfigureIvf as VectorIndexConfigureIvf, - PyVectorIndexSetup as VectorIndexSetup, - PyVectorIndexBuildIvf as VectorIndexBuildIvf, - PyVectorIndexNestedField as VectorIndexNestedField, - PyVectorIndexAsyncConfig as VectorIndexAsyncConfig, - PyVectorIndexQueryIvf as VectorIndexQueryIvf, - PyVectorIndexBuildHnsw as VectorIndexBuildHnsw, - PyVectorIndexQueryHnsw as VectorIndexQueryHnsw, - PyVectorIndexBinarySchema as VectorIndexBinarySchema, - PyVectorIndexBinaryAddData as VectorIndexBinaryAddData, - PyVectorIndexBinaryBuildIndex as VectorIndexBinaryBuildIndex, - PyVectorIndexBinarySearch as VectorIndexBinarySearch, - PyVectorIndexCheckStatus as VectorIndexCheckStatus, - PyVectorIndexNprobes as VectorIndexNprobes, - PyVectorIndexDistanceRange as VectorIndexDistanceRange, - PyVectorIndexBypassRecall as VectorIndexBypassRecall, - PyVectorIndexCustomName as VectorIndexCustomName, -} from '/snippets/indexing.mdx'; - -You can create and manage multiple vector indexes on any Lance dataset. LanceDB offers two kinds of vector indexing algorithms: **Inverted File (IVF)** and **Hierarchical Navigable Small World (HNSW)**. - - -**IVF + HNSW** - -In LanceDB, HNSW is not exposed as a top-level vector index. Instead, it's available as a sub-index inside IVF partitions. What this means in practice is that vectors are first partitioned by IVF, then each selected partition is searched using an HNSW graph. LanceDB supports the unquantized variant `IVF_HNSW_FLAT`, along with quantized variants such as `IVF_HNSW_PQ` and `IVF_HNSW_SQ`. This combines IVF's scalability with HNSW's higher-recall ANN search within partitions. - - -### Manual Indexing {#manual-indexing} - -If using LanceDB OSS, you will have to create the vector index manually, by calling `table.create_index()`, and updating the index as new data arrives and tuning its parameters is also a manual process. - -### Automatic Indexing {#automatic-indexing} - - Enterprise-only -Vector indexing is managed **automatically** in LanceDB Enterprise. As soon as data is updated, the system updates the index and optimizates it. *This is done asynchronously as a background process*. - -When you create a table in LanceDB Enterprise, LanceDB automatically: - -- Infers the vector columns from the schema -- Create an optimized `IVF_PQ` index without manual configuration -- Automatically configure indexing parameters - -The default distance is `l2` (Euclidean). - - -You can call `create_index()` with different parameters to create a new index -- this replaces any existing index. -Although the `create_index` API returns immediately, the building of the vector index is asynchronous. To wait until all data is fully indexed, you can specify the `wait_timeout` parameter. - - -Use the same distance metric for index creation and search. Once a vector index exists, queries use the metric stored with that index. If you need to confirm an async build or refresh is finished, `wait_for_index(...)` waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`; it can time out if new writes keep arriving. - -Rows appended after an index build remain outside that index until optimization refreshes it. Normal -search still checks those unindexed rows with a slower fallback path; `fast_search()` skips that -fallback and searches only indexed rows. - -## Choose the Right Index {#choose-the-right-index} - -Use this table as a quick starting point for choosing the right index type and quantization method for your use case: - -| If your top priority is... | Use this index | Why | Typical compressed size vs. raw vectors | -| :--- | :--- | :--- | :--- | -| Highest recall / no quantization | `IVF_HNSW_FLAT` | Uses raw vectors inside the IVF+HNSW structure, avoiding quantization loss. | Around raw vector size plus HNSW graph overhead | -| Best recall/latency trade-off | `IVF_HNSW_SQ` | Combines IVF partitioning with HNSW graph search for strong quality at low latency. | Typically a little larger than `1/4` of raw size | -| Maximum compression | `IVF_RQ` | RaBitQ-style quantization with very strong compression. | Around `1/32` of raw size | -| Higher accuracy at small dimensions (`dimension <= 256`) | `IVF_PQ` | On small-dimensional vectors, `IVF_PQ` often provides higher accuracy with similar performance compared to `IVF_RQ`. | Usually `1/64` to `1/16` of raw size (depends on `num_sub_vectors`) | - - -If your vector search frequently includes metadata filters (`where(...)`), prefer `IVF_RQ` or `IVF_PQ`. In filtered workloads, HNSW-backed IVF indexes such as `IVF_HNSW_FLAT` and `IVF_HNSW_SQ` can show higher latency variance. - - -Compression ratios are practical rules of thumb and can vary with vector distribution, metric, and configuration. -For small dimensions, choose `IVF_PQ` for accuracy, not for guaranteed higher compression than `IVF_RQ`. - -### Index Tuning {#index-tuning} - -Start with these values, then tune for your workload: - -- HNSW-backed IVF indexes (`IVF_HNSW_FLAT`, `IVF_HNSW_SQ`, `IVF_HNSW_PQ`) - - `num_partitions`: start at `num_rows // 1,048,576` (rounded to an integer) - - Lower `num_partitions` can reduce search latency, but index build may become slower because partitions are larger. - - `ef_construction`: start at `150`; increase for better recall, decrease for faster indexing. -- `IVF_RQ` - - `num_partitions`: start at `num_rows // 4096` (rounded to an integer). This is a strong default for most datasets. -- `IVF_PQ` - - `num_partitions`: start at `num_rows // 4096` (rounded to an integer). - - `num_sub_vectors`: start at `dimension // 8`. Increase for better recall, decrease for faster search and smaller indexes. - - For small dimensions (`dimension <= 256`), `IVF_PQ` is often preferred over `IVF_RQ` for better accuracy at similar query performance. - -## Example: Construct an IVF Index {#example-construct-an-ivf-index} - -In this example, we will create an index for a table containing 1536-dimensional vectors. The index will use IVF_PQ with L2 distance, which is well-suited for high-dimensional vector search. - -Make sure you have enough data in your table (at least a few thousand rows) for effective index training. - -### Index Configuration {#index-configuration} - -Sometimes you need to configure the index beyond default parameters: - -- Index Types: - - `IVF_HNSW_FLAT`: highest recall, with no vector quantization - - `IVF_HNSW_SQ`: best recall/latency trade-off - - `IVF_RQ`: best compression for large, high-dimensional datasets - - `IVF_PQ`: often higher accuracy than `IVF_RQ` for small dimensions (`<= 256`) at similar query performance -- `metrics`: default is `l2`, other available are `cosine` or `dot` - - When using `cosine` similarity, distances range from 0 (identical vectors) to 2 (maximally dissimilar) -- `num_partitions`: use index-specific starting points from the section above: - - HNSW-backed IVF indexes (`IVF_HNSW_FLAT`, `IVF_HNSW_SQ`, `IVF_HNSW_PQ`): `num_rows // 1,048,576` - - `IVF_RQ` and `IVF_PQ`: `num_rows // 4096` -- `target_partition_size`: alternative IVF sizing knob that asks LanceDB to derive the partition - count from a target number of rows per partition. If you set both `num_partitions` and - `target_partition_size`, `num_partitions` takes precedence. -- `num_sub_vectors`: applies to `IVF_PQ`; start with `dimension // 8`. Larger values often improve recall but can slow search. - -Let's take a look at a sample request for an IVF index: - - - - - {VectorIndexConfigureIvf} - - - -### 1\. Setup {#1-setup} - -Connect to LanceDB and open the table you want to index. - - - - {VectorIndexSetup} - - - -### 2\. Construct an IVF Index {#2-construct-an-ivf-index} - -Create an `IVF_PQ` index with `cosine` similarity. Specify `vector_column_name` if you use multiple vector columns or non-default names. For a vector field nested inside a struct, use dot notation (e.g. `image.embedding`); see [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full syntax. You can switch `index_type` to `IVF_RQ`, `IVF_HNSW_SQ`, or `IVF_HNSW_FLAT` depending on your recall/latency/compression target. - - - - {VectorIndexBuildIvf} - - - -#### Indexing nested vector fields {#indexing-nested-vector-fields} - -If your vector column lives inside a struct, pass its full dotted path as `vector_column_name`. The same path is used at query time and is what `list_indices()` reports under `columns`: - - - - {VectorIndexNestedField} - - - - -Nested paths follow Lance field-path semantics: dot-separate each struct field from root to leaf (for example, `image.thumbnail.embedding`). The same convention applies to FTS and scalar indexes. - - -### Async API and Config Objects {#async-api-and-config-objects} - -With asynchronous Python connections, create vector indexes with `await table.create_index("vector", config=...)`. The `config` object carries the same index choices you configure in the synchronous API, such as distance metric, partition count, and quantization settings: - - - - {VectorIndexAsyncConfig} - - - -Use these Python config classes for the index types shown on this page: - -| Index type | Python config class | -| :--- | :--- | -| `IVF_FLAT` | `IvfFlat` | -| `IVF_PQ` | `IvfPq` | -| `IVF_RQ` | `IvfRq` | -| `IVF_SQ` | `IvfSq` | -| `IVF_HNSW_FLAT` | `IvfHnswFlat` | -| `IVF_HNSW_PQ` | `IvfHnswPq` | -| `IVF_HNSW_SQ` | `IvfHnswSq` | - -### 3\. Query the IVF Index {#3-query-the-ivf-index} - -Search using a random 1,536-dimensional embedding. - - - - {VectorIndexQueryIvf} - - - -#### Search Configuration {#search-configuration} - -Core knobs available on a vector search call: - -| Parameter | Description | -| :--- | :--- | -| `limit` | Number of results to return (`k`). | -| `nprobes` | Shorthand that sets both `minimum_nprobes` and `maximum_nprobes` to the same value. LanceDB auto-tunes this by default. | -| `minimum_nprobes` | Partitions that are *always* scanned. Higher values raise recall at the cost of latency. | -| `maximum_nprobes` | Upper bound on partitions scanned. The partitions above `minimum_nprobes` are only searched if the initial pass does not return enough results — useful for narrow filters. Set to `0` to remove the cap. | -| `ef` | HNSW search-time exploration factor. Relevant for `IVF_HNSW_FLAT` and `IVF_HNSW_SQ`; start around `1.5 * k` and increase up to `10 * k` for higher recall. | -| `refine_factor` | Reads additional candidates and reranks them in memory to recover recall lost to quantization. | - - -**Filtered queries and adaptive nprobes.** When a `where(...)` filter is active, LanceDB starts by scanning `minimum_nprobes` partitions and only extends toward `maximum_nprobes` if fewer than `limit` rows survive the filter. Setting `minimum_nprobes == maximum_nprobes` (or calling `nprobes(n)`) disables this adaptive behavior and fixes the partition count. - - - - - {VectorIndexNprobes} - - - -Recommended `nprobes` behavior by index type: - -| Index type | Guidance | -| :--- | :--- | -| `IVF_HNSW_FLAT`, `IVF_HNSW_SQ` | Keep the auto-tuned `nprobes`, then tune `ef` first. Expect higher latency variance under filtered search. | -| `IVF_RQ` | Keep auto-tuned `nprobes`; raise only when recall is insufficient. | -| `IVF_PQ` | Keep auto-tuned `nprobes`; raise when recall is insufficient. Often preferred over `IVF_RQ` when `dimension <= 256`. | - -#### Advanced Search Controls {#advanced-search-controls} - -These controls are useful for thresholded retrieval, recall measurement, and working around index-level metric constraints. - -| Method | Description | -| :--- | :--- | -| `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. | -| `bypass_vector_index()` | Skip the ANN index and perform an exhaustive (flat) scan. Primary uses: (1) compute ground-truth results to measure ANN recall@k, and (2) query with a metric the index was not built for (e.g., a non-cosine query on a multivector column). | - -**Thresholding with `distance_range`:** - - - - {VectorIndexDistanceRange} - - - -**Measuring recall with `bypass_vector_index`:** - -Compare ANN results against a flat-scan ground truth to compute recall@k. This is the standard way to pick `nprobes` for your workload. - - - - {VectorIndexBypassRecall} - - - - -Flat search is $O(n)$ — reserve `bypass_vector_index()` for sampled recall measurements or small tables, not production queries. - - - -Multivector indexing currently requires `distance_type="cosine"` — `l2` is rejected at index-creation time. That restriction is why `bypass_vector_index()` is the escape hatch for non-cosine queries on a multivector column: the metric you want at query time cannot be served by the index, so you fall back to a flat scan. See [Multivector Search](/search/multivector-search) for the full rules. - - -## Example: Construct an HNSW Index {#example-construct-an-hnsw-index} - -### Index Configuration {#index-configuration-2} - -There are four key parameters to set when constructing an HNSW index: - -- `index_type`: choose `IVF_HNSW_SQ` for a strong recall/latency/size trade-off, or `IVF_HNSW_FLAT` when you want the IVF+HNSW structure without vector quantization. -- `metric`: The default is `l2` euclidean distance metric. Other available are `dot` and `cosine`. -- `m`: The number of neighbors to select for each vector in the HNSW graph. -- `ef_construction`: The number of candidates to evaluate during the construction of the HNSW graph. - -### 1\. Construct an HNSW Index {#1-construct-an-hnsw-index} - -The snippet below uses `IVF_HNSW_SQ`. If you want the unquantized variant, change `index_type` to `IVF_HNSW_FLAT`. - - - - {VectorIndexBuildHnsw} - - - -### 2\. Query the HNSW Index {#2-query-the-hnsw-index} - - - - {VectorIndexQueryHnsw} - - - -## Example: Construct a Binary Vector Index {#example-construct-a-binary-vector-index} - -Binary vectors are useful for hash-based retrieval, fingerprinting, or any scenario where data can be represented as bits. - -### Index Configuration {#index-configuration-3} - -- Store binary vectors as fixed-size binary data (uint8 arrays, with 8 bits per byte). For storage, pack binary vectors into bytes to save space. -- Index Type: `IVF_FLAT` is used for indexing binary vectors -- `metric`: the `hamming` distance is used for similarity search -- The dimension of binary vectors must be a multiple of 8. For example, a 128-dimensional vector is stored as a uint8 array of size 16. - - -**`IVF_FLAT` + `hamming` is the only supported path for binary vectors.** - -- `hamming` distance is only valid on packed binary (uint8) data; it is rejected on float vector columns. -- Quantized index types (`IVF_PQ`, `IVF_RQ`, `IVF_SQ`, `IVF_HNSW_PQ`, `IVF_HNSW_SQ`) do not accept binary inputs — their `distance_type` is restricted to `l2`, `cosine`, or `dot`. - - -### 1\. Create Table and Schema {#1-create-table-and-schema} - - - - {VectorIndexBinarySchema} - - - -### 2\. Generate and Add Data {#2-generate-and-add-data} - - - - {VectorIndexBinaryAddData} - - - -### 3\. Construct the Binary Index {#3-construct-the-binary-index} - - - - {VectorIndexBinaryBuildIndex} - - - -### 4\. Vector Search {#4-vector-search} - - - - {VectorIndexBinarySearch} - - - -## Check Index Status {#check-index-status} - -Vector index creation runs in the background and may take some time to complete. While it is ongoing, you can check its status either programmatically through the API or from the **LanceDB Enterprise UI**. - -In the LanceDB Enterprise UI, navigate to your table page - the "Index" column reflects each column's index status: it is blank when no index exists, shows an "in progress" label while the index is being built, and shows the index type once the build completes. - -Programmatically, use `list_indices()` and `index_stats()`. **By default**, the index name is formed by appending `_idx` to the column name (e.g., a `keywords_embeddings` column produces `keywords_embeddings_idx`). Note that `list_indices()` only returns information after the index is fully built. -To wait until all data is fully indexed, you can specify the `wait_timeout` parameter on `create_index()` or call `wait_for_index()` on the table. - -Each entry returned by `list_indices()` also carries detailed per-index metadata, so you can inspect an index without a follow-up `index_stats()` call. Node.js exposes the same fields in camelCase (`num_indexed_rows` → `numIndexedRows`): - -| Field | What it tells you | -| :--- | :--- | -| `num_indexed_rows`, `num_unindexed_rows` | Index coverage over the table | -| `size_bytes` | Total size of the index files on disk | -| `num_segments`, `index_version` | On-disk layout and format version | -| `created_at` | Creation time (ms since the Unix epoch in Node.js) | -| `index_uuid`, `type_url` | Internal identifiers for the index segment | -| `index_details` | Type-specific details (e.g. IVF partition counts or quantization settings) | - - -These fields are populated for local and embedded tables. On LanceDB Enterprise remote tables they are returned as `None` / `undefined` until the server response surfaces them. - - - - - {VectorIndexCheckStatus} - - - -## Custom Index Names {#custom-index-names} - -The `{column}_idx` suffix is a default convention, not the only supported naming path. Pass `name=...` to `create_index()` to override it — useful when you want to manage multiple indexes on the same column (for example, side-by-side `IVF_PQ` and `IVF_HNSW_SQ` builds) or when you script index replacement by name. Once set, `list_indices()`, `index_stats(name)`, and `wait_for_index([name])` all reference the custom name. - - - - {VectorIndexCustomName} - - diff --git a/docs/integrations/ai/agno.mdx b/docs/integrations/ai/agno.mdx deleted file mode 100644 index b5edef0..0000000 --- a/docs/integrations/ai/agno.mdx +++ /dev/null @@ -1,151 +0,0 @@ ---- -title: "Agno" -sidebarTitle: "Agno" -description: "Build a search assistant using the Agno agent framework with LanceDB as the knowledge backend." ---- - -import { - PyFrameworksAgnoAgent, - PyFrameworksAgnoCliChat, - PyFrameworksAgnoIngestYoutube, - PyFrameworksAgnoSetup, -} from '/snippets/integrations.mdx'; - -[Agno](https://docs.agno.com/introduction) is a framework for building agentic AI applications. -It supports LanceDB as a knowledge backend, allowing you to easily ingest and retrieve external content for your agents. - -When you pair Agno's `Knowledge` system with LanceDB, you get a clean Agentic RAG setup. -We'll walk through the steps below to build a YouTube transcript-aware Agno assistant that can: -- Ingest a transcript from a YouTube video via the YouTube API -- Store embeddings and metadata in LanceDB -- Retrieve context during responses with hybrid search -- Ask questions about the video content in a CLI chat loop - -## Prerequisites {#prerequisites} - -Install dependencies: - - -```bash pip icon="terminal" -pip install -U agno openai lancedb youtube-transcript-api beautifulsoup4 -``` - -```bash uv icon="terminal" -uv add agno openai lancedb youtube-transcript-api beautifulsoup4 -``` - - -## Step 1: Configure LanceDB-backed knowledge {#step-1-configure-lancedb-backed-knowledge} - -First, you can initialize the core `Knowledge` object that your agent will use for retrieval. -It configures LanceDB as the vector store, enables hybrid search with native LanceDB FTS, and sets the embedding model. - - - {PyFrameworksAgnoSetup} - - -## Step 2: Fetch and ingest the YouTube transcript {#step-2-fetch-and-ingest-the-youtube-transcript} - -Next, extract a YouTube video ID, fetch the full transcript, and flatten it into text for indexing. -The snippet shown below then inserts that transcript text into the Agno knowledge base, which writes vectors and metadata to LanceDB. - - - {PyFrameworksAgnoIngestYoutube} - - - -This path explicitly fetches the transcript first, then inserts transcript text into LanceDB through Agno. - - -## Step 3: Create an Agno agent with knowledge search {#step-3-create-an-agno-agent-with-knowledge-search} - -The next step is to construct an Agno `Agent` and attach the knowledge base you just populated. -With `search_knowledge=True`, the agent performs retrieval before answering, so responses stay grounded in transcript context. - -In Agno, retrieval is exposed as a tool call that the model can invoke at runtime. -When `search_knowledge=True`, Agno makes a knowledge-search tool (shown in output as `search_knowledge_base(...)`) available to the model; the model decides when to call it, Agno executes the tool, and the returned context is fed back into the final answer. - - - {PyFrameworksAgnoAgent} - - -## Step 4: Start a CLI chat loop {#step-4-start-a-cli-chat-loop} - -You can now ask an initial question and then start an interactive loop for follow-up queries. -Each prompt runs through the same retrieval pipeline, so you can iteratively inspect what the transcript contains. - - - {PyFrameworksAgnoCliChat} - - - -Want local-first inference? Replace OpenAI model/embedder classes with Agno's Ollama providers. See Agno's Ollama knowledge examples: [docs.agno.com/examples/models/ollama/chat/knowledge](https://docs.agno.com/examples/models/ollama/chat/knowledge). - - -### Question 1 {#question-1} - -The following question is asked in the CLI chat loop: -``` -┏━ Message ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ ┃ -┃ Q: What kinds of data can LanceDB handle? ┃ -┃ ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -┏━ Tool Calls ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ ┃ -┃ • search_knowledge_base(query=What kinds of data can LanceDB handle?) ┃ -┃ • search_knowledge_base(query=LanceDB images audio video handle kinds of data ┃ -┃ can handle 'LanceDB can handle' 'kinds of data' 'images audio video' transcript) ┃ -┃ ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -┏━ Response (19.1s) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ ┃ -┃ ┃ -┃ • Images, audio, video — i.e., multimodal AI data and “all manners of things ┃ -┃ you don't put into traditional databases” (per the transcript). ┃ -┃ ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -``` - -We get the response based on the transcript's contents as expected. - -### Question 2 {#question-2} - -Let's ask a more specific question about the CEO of LanceDB, which is also in the transcript: - -``` -You: What is the name of the CEO of LanceDB? -INFO Found 10 documents -┏━ Message ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ ┃ -┃ What is the name of the CEO of LanceDB? ┃ -┃ ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -┏━ Tool Calls ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ ┃ -┃ • search_knowledge_base(query=CEO of LanceDB) ┃ -┃ ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -┏━ Response (16.7s) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ ┃ -┃ ┃ -┃ • According to the retrieved YouTube transcript/title, the CEO of LanceDB is ┃ -┃ Chang She. ┃ -┃ ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -``` - -We get the response based on the transcript's contents and title as expected. - -## Why this works well {#why-this-works-well} - -To start, LanceDB OSS can run from a local directory, so transcript data can stay on your machine when you are using the OSS stack. - -- You do not need to maintain a separate transcript parser in your application code. -- You do not need to hand-roll chunking and retrieval orchestration across multiple modules. -- One explicit Agno `Knowledge` object, backed by LanceDB, defines both ingestion and search behavior in one place. -- Fewer moving parts means the tutorial stays readable and the same pattern is easier to carry into production code. - -As your application needs grow, you can migrate to LanceDB [Enterprise](/enterprise) for -convenience features like automatic compaction and reindexing and the ability to scale to -really large datasets. diff --git a/docs/integrations/ai/genkit.mdx b/docs/integrations/ai/genkit.mdx deleted file mode 100644 index 6314842..0000000 --- a/docs/integrations/ai/genkit.mdx +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: "GenKit" -sidebarTitle: "GenKit" - ---- - -import { - TsFrameworksGenkitCustomIndexer, - TsFrameworksGenkitCustomRetriever, - TsFrameworksGenkitUsage, -} from '/snippets/integrations.mdx'; - -### genkitx-lancedb {#genkitx-lancedb} -Genkit is an open-source framework for building end-to-end AI and RAG pipelines with a clean, TypeScript-first -developer experience. The genkitx-lancedb plugin lets you use LanceDB as a high-performance vector store -inside your Genkit flows, so you can index, search, and retrieve data efficiently as part of your AI -applications. - -### Installation {#installation} -```bash -pnpm install genkitx-lancedb -``` - -### Usage {#usage} - -Adding LanceDB plugin to your genkit instance. - - - {TsFrameworksGenkitUsage} - - -You can run this app with the following command: -```bash -genkit start -- tsx --watch src/index.ts -``` - -This'll add LanceDB as a retriever and indexer to the genkit instance. You can see it in the GUI view -Screenshot 2025-05-11 at 7 21 05 PM - -**Testing retrieval on a sample table** -Let's see the raw retrieval results - -Screenshot 2025-05-11 at 7 21 05 PM -On running this query, you'll get 5 results fetched from the lancedb table, where each result looks something like this: -Screenshot 2025-05-11 at 7 21 18 PM - - - -## Creating a custom RAG flow {#creating-a-custom-rag-flow} - -Now that we've seen how you can use LanceDB in a Genkit pipeline, let's refine the flow and create a RAG. A RAG flow will consist of an index and a retriever with its outputs postprocessed and fed into an LLM for final response - -### Creating custom indexer flows {#creating-custom-indexer-flows} -You can also create custom indexer flows, utilizing more options and features provided by LanceDB. - - - {TsFrameworksGenkitCustomIndexer} - - -Screenshot 2025-05-11 at 8 35 56 PM - -In your console, you can see the logs - -Screenshot 2025-05-11 at 7 19 14 PM - -### Creating custom retriever flows {#creating-custom-retriever-flows} -You can also create custom retriever flows, utilizing more options and features provided by LanceDB. - - {TsFrameworksGenkitCustomRetriever} - -Now using our retrieval flow, we can ask a question about the ingested PDF -Screenshot 2025-05-11 at 7 18 45 PM - diff --git a/docs/integrations/ai/hermes-agent.mdx b/docs/integrations/ai/hermes-agent.mdx deleted file mode 100644 index a5521a9..0000000 --- a/docs/integrations/ai/hermes-agent.mdx +++ /dev/null @@ -1,327 +0,0 @@ ---- -title: "Hermes Agent" -sidebarTitle: "Hermes Agent" -description: "Use LanceDB as a persistent, semantic memory backend for Hermes Agent. Get durable recall across sessions with vector and hybrid search." ---- - -[Hermes Agent](https://github.com/NousResearch/hermes-agent) is a self-hosted, open-source -personal agent from [Nous Research](https://nousresearch.com). You can talk to it from a -terminal UI or reach the same agent from Telegram, Discord, and Slack, and it exposes a -dedicated slot for external *memory providers* that run alongside its built-in notes. - -The [LanceDB memory plugin](https://github.com/lancedb/hermes-agent-memory) fills that slot. -It gives Hermes durable, semantic recall across sessions: state a preference or a project -convention once, and the agent can retrieve it weeks later in a brand-new session — even when -you ask for it in completely different words. Everything runs inside Hermes' own Python -process, storing a single LanceDB table on local disk. There's no memory server to operate. - - -**The mental model is clean** - -- Hermes owns the agent loop -- LanceDB manages the durable long-term memory and offers semantic recall. - - -## Why LanceDB fits agent memory {#why-lancedb-fits-agent-memory} - -Out of the box, Hermes remembers with a small curated notes file frozen into the system -prompt, plus lexical (keyword) search over past sessions. Both are useful, but keyword search -misses paraphrases of what you originally typed — the exact thing you need when recalling a -fact you phrased differently months ago. - -LanceDB is an embedded retrieval library, which makes it a natural fit here: - -- **No server to stand up** — it reads and writes a table on local disk, so the plugin ships - as a dependency rather than a service to operate. -- **One table holds everything** — content, metadata, and embeddings live together. A memory - becomes a structured row with a category, tags, timestamps, and provenance, not just a text - blob. -- **Query it any way you need** — vector similarity for meaning, BM25 full-text for exact - names and jargon, a hybrid of the two, or plain metadata filters to keep recall scoped to - the right workspace. -- **It scales up** — the same table abstraction carries over to larger LanceDB deployments - later, so the local setup is never a dead end. - -## Install and activate {#install-and-activate} - - -Want to try this without touching your existing Hermes setup? Run everything in an isolated -profile: `hermes profile create demo`, then add `-p demo` to the commands below. When you're -done, `rm -rf ~/.hermes/profiles/demo` removes all trace. - - - - -Skip this if you already have Hermes installed. - -```bash -curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash -``` - - - -This shallow-clones the plugin into `~/.hermes/plugins/lancedb/`. - -```bash -hermes plugins install lancedb/hermes-agent-memory -``` - - - -Hermes loads plugins inside its own Python interpreter, so the dependencies go *there* — not -into a separate virtualenv. (This interpreter is shared across profiles, so you only install -once.) - -```bash -uv pip install --python ~/.hermes/hermes-agent/venv/bin/python3 lancedb openai pyyaml -``` - - - -The plugin turns conversations into embeddings, so it needs an embeddings key. By default that -is OpenAI, so set `OPENAI_API_KEY` in your environment or in `~/.hermes/.env`. - - -Prefer a local or non-OpenAI model? The plugin uses an OpenAI-compatible client, so you can -point it at any compatible endpoint (OpenRouter, Ollama, vLLM, …) in your config — no code -change needed. See [Configuration](#configuration) below. - - - - -Switch memory on and pick this plugin: - -```bash -hermes memory setup # choose "lancedb" -``` - -Then confirm it's actually active before you start chatting — this is the one step worth not -skipping, because Hermes quietly falls back to its built-in notes if the provider isn't set: - -```bash -hermes memory status -``` - -```text -Memory status -──────────────────────────────────────── - Built-in: always active - Provider: lancedb - - Plugin: installed ✓ - Status: available ✓ -``` - -You want to see `Provider: lancedb` with both `installed ✓` and `available ✓`. - - - -## The memory tools {#the-memory-tools} - -Once activated, the agent has four tools for working with long-term memory: - -| Tool | What it does | -|:--|:--| -| `lancedb_recall` | Semantic (vector, the default) or hybrid search over your workspace memory. Returns matching facts with scores and provenance. | -| `lancedb_remember` | Stores a durable fact when you explicitly ask. Deduplicated by content hash, so remembering the same thing twice doesn't pile up rows. | -| `lancedb_read` | Fetches a single memory by ID, optionally with the original conversation messages it was distilled from. | -| `lancedb_forget` | Deletes safely: previews candidates first, then deletes by exact ID, so nothing disappears by accident. | - -Beyond these tools, the plugin also captures durable facts from your conversations -automatically — an auxiliary model distills them before context is compressed and again when a -session ends, so insights survive even when the raw messages are summarized away. - -## Walkthrough {#walkthrough} - -"_Teach it your project preferences_" - -Let's make this concrete with the pain we opened on: re-explaining your setup to the agent every session. -We'll save a convention once and then prove a brand-new session can recall it. This example will touch all four -tools along the way. - -### Remember {#remember} - -Ask Hermes to commit a convention to long-term memory. Saying "remember in long-term memory" -makes sure it lands in the LanceDB store, which shows up as the `⚡ lancedb_r` (`lancedb_remember`) -line below: - -```text -● Remember in long-term memory: for this project I only use uv, never pip, and I always add type hints to Python functions. - - ┊ 🧠 memory +memory: "For this project, the user only uses uv for Python package management, never pip, and always adds type hints to Python functions." - ┊ ⚡ lancedb_r 0.0s - ─ ⚕ Hermes ──────────────────────────────────────────────────────────────── - - Remembered. I've stored that project convention: use uv only, never pip, and always add type hints to Python functions. -``` - -### Recall {#recall} - -First, take Hermes' built-in notes out of the picture so recall can *only* come from LanceDB — -the two layers run side by side otherwise, and either could answer: - -```bash -hermes config set memory.memory_enabled false -``` - -Now start a fresh session (`/new`) and ask for the convention back, worded completely -differently from how you saved it: - -```text -● /new -● Look through your long-term memory for my Python project preferences. - - ─ ⚕ Hermes ──────────────────────────────────────────────────────────────── - - I found one durable memory about Python project preferences: - - - Project Python tooling preference is uv; pip is not used; type hints are required. -``` - -This is the whole point in one exchange. You asked about your *preferences* using none of the -words you stored the fact with — no *uv*, no *pip*, no *type hints* — yet a fresh session, with -none of the earlier conversation in context, pulls it straight back. Semantic search matches on -meaning rather than exact wording, which is precisely where keyword search falls short. - - -Turn the built-in layer back on for everyday use with `hermes config set memory.memory_enabled true` -— the two memory layers happily coexist. We only disabled it here to prove the recall came from -LanceDB. - - -### Read {#read} - -You can also ask where a fact came from. Hermes attributes the answer to its stored memory -rather than guessing from a file in the repo (under the hood, `lancedb_read` can also return -the exact source messages a fact was distilled from): - -```text -● Where did that uv preference come from? - - ─ ⚕ Hermes ──────────────────────────────────────────────────────────────── - - It came from the project's persistent memory, not from a codebase file I inspected. - - Specifically, the recalled memory says: - - "For this project, the user only uses uv for Python package management, never pip, and always adds type hints to Python functions." -``` - -### Forget {#forget} - -When a preference changes, ask Hermes to drop the old fact. The tool calls tell the whole -story: the two `⚡ lancedb_f` (`lancedb_forget`) lines are it previewing matches and then -deleting, and the trailing `⚡ lancedb_r` is it saving the replacement in the same breath: - -```text -● I've switched from uv to pixi. Forget the uv preference. - - ┊ ⚡ lancedb_f uv preference python package management never pip type hints 0.3s - ┊ ⚡ lancedb_f 0.0s - ┊ ⚡ lancedb_r 0.2s - ─ ⚕ Hermes ──────────────────────────────────────────────────────────────── - - Done — I removed the uv preference from memory and replaced it with: - - - This project uses pixi for Python package management - - Type hints on Python functions are still preferred -``` - -Remember, recall, read, forget: four small operations that between them cover the entire -lifecycle of a durable memory. - -## Retrieval modes {#retrieval-modes} - -Recall ships in `vector` mode by default — pure semantic search, which is what survives the -paraphrasing you saw above. If you also need exact name or jargon matching, switch to `hybrid` -(vector + BM25) and choose how the two legs are fused: RRF, a vector-biased linear blend, or a -cross-encoder reranker. Mode is set per call; fusion is a config setting. - -```yaml -# ~/.hermes/config.yaml -plugins: - lancedb: - retrieval: - mode: hybrid # vector (default) | hybrid - reranker: - type: rrf # how the vector + BM25 legs are fused - # Swap RRF for a reranking pass (pulls in sentence-transformers + torch): - # type: cross-encoder - # model: cross-encoder/ettin-reranker-17m-v1 - # rerank_top_n: 50 -``` - -The cross-encoder is the one path that pulls in a local ML stack, so it stays opt-in. It -defaults to the compact 17M-parameter [ettin reranker](https://huggingface.co/cross-encoder/ettin-reranker-17m-v1). - -## Inspect the store {#inspect-the-store} - -Everything lives in one table named `memories` at `~/.hermes/lancedb/memories.lance`. Because -it's a plain LanceDB table, you can open it directly and see exactly what the agent has stored -— a `kind` column separates extracted `fact` rows from the raw `turn` rows they were drawn -from: - -```python -import lancedb - -db = lancedb.connect("~/.hermes/lancedb") -tbl = db.open_table("memories") -print(tbl.to_pandas()[["kind", "category", "content"]].head()) -``` - -## Configuration {#configuration} - -The plugin runs on sensible defaults once activated — you don't have to configure anything. -`~/.hermes/config.yaml` is purely for overrides. Two common ones: - -Use a cheaper model for the auxiliary fact-extraction calls: - -```yaml -# ~/.hermes/config.yaml -auxiliary: - lancedb_extraction: - provider: openrouter - model: google/gemini-3-flash -``` - -Point embeddings at a fully local endpoint (for example, Ollama) so nothing leaves your -machine: - -```yaml -# ~/.hermes/config.yaml -plugins: - lancedb: - embedding: - model: nomic-embed-text - base_url: http://localhost:11434/v1 - api_key_env: OLLAMA_API_KEY # any value works for local Ollama -``` - - -Changing the embedding model (or its dimension) against an existing store requires recreating -the table — the plugin fails loudly on a dimension mismatch rather than silently returning -nothing. Every option is documented in the plugin's [`default_config.yaml`](https://github.com/lancedb/hermes-agent-memory/blob/main/src/default_config.yaml). - - -## Benchmark {#benchmark} - -On [LongMemEval-S](https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned), a -long-conversation QA benchmark, LanceDB's semantic recall clearly beat Hermes' built-in lexical -search (0.66 vs. 0.53 answer accuracy) by finding the right messages even when the question was -worded differently from the original conversation. For the full methodology, the -per-question-type breakdown, and a reproducible harness, see the -[blog post](https://www.lancedb.com/blog/semantic-memory-for-hermes-agent-with-lancedb) and the -[benchmark harness](https://github.com/lancedb/hermes-agent-memory/tree/main/benchmarks). - -## Why this works well {#why-this-works-well} - -- **It's local-first and embedded.** The LanceDB memory table lives on your disk with no server to run; - the plugin installs as a dependency of Hermes' own environment. -- **Recall survives paraphrasing.** Semantic search matches meaning, not spelling, which is the - failure mode that sinks keyword-only session search. -- **Memories are structured and traceable.** Each fact is a row with metadata and a link back - to the messages it came from, and `forget` always previews before it deletes. -- **Nothing about it is a dead end.** As your needs grow, the same table abstraction carries - over to LanceDB [Enterprise](/enterprise) for automatic compaction, reindexing, and scale. - -To try it, install the plugin, enable it with `hermes memory setup`, and run the kind of -workflow we walked through above. diff --git a/docs/integrations/ai/huggingface.mdx b/docs/integrations/ai/huggingface.mdx deleted file mode 100644 index 62ac7e8..0000000 --- a/docs/integrations/ai/huggingface.mdx +++ /dev/null @@ -1,341 +0,0 @@ ---- -title: "Hugging Face Hub" -sidebarTitle: "Hugging Face" -description: "Use LanceDB directly on Lance datasets hosted on the Hugging Face Hub for multimodal search and retrieval." ---- - -[Hugging Face Hub](https://huggingface.co/datasets?format=format:lance&sort=trending) is a popular platform for sharing machine learning datasets, models, and other resources. - -LanceDB can directly scan Lance datasets hosted on the [Hugging Face Hub](https://huggingface.co/datasets?format=format:lance) with `hf://` URIs. -This is enabled under the hood by the [lance-huggingface](https://lance.org/integrations/huggingface/) -integration that allows users to stream Lance datasets directly from Hugging Face without needing to -download them first. - -For ML and AI engineers working in LanceDB, this capability is incredibly useful for quickly exploring -multimodal datasets and reusing Lance datasets shared by others, without writing custom data loaders -or preprocessing pipelines. - -The snippets below use the [`lance-format/laion-1m`](https://huggingface.co/datasets/lance-format/laion-1m) -dataset published in Lance format. The dataset includes a million image-caption pairs, and the -Lance dataset can package image embeddings alongside the metadata. This makes it useful for -demonstrating LanceDB's multimodal search capabilities in combination with easy sharing via the -Hugging Face Hub. - -The LAION table includes multimodal columns such as: - -- `image` (inline JPEG bytes) -- `caption` (text) -- `img_emb` (image embedding vector) -- metadata fields such as `url` and `similarity` - -## Install dependencies {#install-dependencies} - -```bash -pip install lancedb pillow -``` - -## Open the dataset with LanceDB {#open-the-dataset-with-lancedb} - -LanceDB can open the dataset directly from the Hub, without needing to download it first. -Note that in LanceDB, you need to specify the table name when opening a Lance dataset, -and the Hugging Face convention is to use `train` and `test` splits for datasets. -The LAION dataset is uploaded as a single split named `train`, so we specify the table name -that contains the `*.lance` files when opening the dataset. - -```python -import lancedb - -db = lancedb.connect("hf://datasets/lance-format/laion-1m/data") -table = db.open_table("train") - -print(f"Opened table: {table.name}") -print(f"Rows: {len(table)}") -``` - -## Inspect schema and available indexes {#inspect-schema-and-available-indexes} - -```python -print(table.schema) -``` -This prints the schema of the LAION table. Note that there's an image embedding column that's -a fixed-size list of 768-dimensional floats, and a binary column containing the raw JPEG bytes of the image. -``` -image_path: string -caption: string -NSFW: string -similarity: double -LICENSE: string -url: string -key: string -status: string -error_message: null -width: int64 -height: int64 -original_width: int64 -original_height: int64 -exif: string -md5: string -img_emb: fixed_size_list[768] - child 0, item: float -image: binary -``` - -When inspecting Lance datasets from Hugging Face, it's also a good idea to check whether the dataset author included -any pre-built indexes that you can use for search. You can check the available indexes with: - -```python -print(table.list_indices()) -``` -``` -[ - Index(IvfPq, columns=["img_emb"], name="img_emb_idx"), - Index(FTS, columns=["caption"], name="caption_idx") -] -``` - -In this case, we see that we have an IVF_PQ vector index on the `img_emb` column, and an FTS index on the `caption` -column, which means we can directly do vector search on the image embeddings and keyword search on the captions -without needing to build the indexes ourselves! - - -If you see an empty list, it may be because the dataset author did not include the index files when uploading -to Hugging Face. You can download the dataset locally, and build the indexes yourself. See the [indexing guide](/indexing/) -for instructions on building different types of indexes with LanceDB. - - -## Projection scan {#projection-scan} - -Run a simple scan by projecting relevant columns to get a feel for the dataset. For example, we -can run a search without any filters or input parameters to get a small subset of the data: - -```python -rows = ( - table.search() - .select(["caption", "url", "similarity"]) - .limit(3) - .to_list() -) - -for i, row in enumerate(rows, start=1): - print(f"{i}. {row['caption']}") - print(f" url={row['url']}") - print(f" similarity={row['similarity']}") -``` - -We get the first three rows and their metadata printed out, which look like this: -``` -1. Cordelia and Dudley on their wedding day last year - url=https://i.dailymail.co.uk/i/pix/2012/01/05/article-2082728-0EF8956600000578-53_233x315.jpg - similarity=0.2926466464996338 -2. Statistics on challenges for automation in 2021 - url=https://verloop.io/wp-content/uploads/2021/02/Challenges.jpg - similarity=0.30174341797828674 -3. Teacher Gifts / Great gifts for your child's teacher. Don't know what to get? Take a look at these gifts that the teacher in your life will love! - url=https://i.pinimg.com/custom_covers/216x146/550494823141083777_1487893945.jpg - similarity=0.3362061381340027 -``` - -## Scan and filter data {#scan-and-filter-data} - -Filtered search is a common pattern to narrow down interesting subsets of the data during early -exploration. Here's an example: - -```python -filtered = ( - table.search() - .where("height > 600") - .select(["caption", "url", "width", "height"]) - .limit(3) - .to_list() -) - -for row in filtered: - print(row["caption"], row["url"], row["width"], row["height"]) -``` - -This prints out the metadata for large images with height greater than 600 pixels: -``` -Luca Trousers, mustard stripe https://cdn.shopify.com/s/files/1/0151/5333/products/IMG_0791_1024x1024.jpg?v=1585142190 384 766 -Baby Blue Fitted Short Sleeve T Shirt 3 https://cdn-img.prettylittlething.com/a/d/d/1/add198cab3ec30a61102437275573f4963642528_cmf6022_3.jpg 384 612 -pattern cutting made easy pdf https://i.pinimg.com/736x/7c/6c/a7/7c6ca7361815a8929b3dd6ad34a03ab9.jpg 384 1045 -``` - -## Export image bytes to local files {#export-image-bytes-to-local-files} - -To work with a subset of the data locally, you can export the image bytes from the table and save them as JPEG files. -```python -from pathlib import Path - -sample = ( - table.search() - .select(["image", "caption"]) - .limit(3) - .to_list() -) - -out_dir = Path("samples") -out_dir.mkdir(exist_ok=True) - -for i, row in enumerate(sample): - out_path = out_dir / f"laion_{i}.jpg" - with open(out_path, "wb") as f: - f.write(row["image"]) - print(f"Saved {out_path} | caption={row['caption']}") -``` - -You can now preview the images you just exported on your local machine to get a better sense of the data. - -## Vector search {#vector-search} - -You can use LanceDB to run vector search directly on the data on the Hub, **without needing to download the dataset -or build your own vector index**. This makes it incredibly easy to explore the dataset and iterate on your search queries -before you decide to download a local copy for further experimentation on your end. - -```python -# Pick an arbitrary image embedding from the dataset -query_embedding = ( - table.search() - .select(["img_emb"]) - .limit(1) - .to_list()[0]["img_emb"] -) - -results = ( - table.search(query_embedding, vector_column_name="img_emb") - .select(["caption", "url", "_distance"]) - .limit(3) - .to_list() -) - -for row in results: - print(row["_distance"], row["caption"]) -``` - -| distance | caption | -| --- | --- | -| 0.17765313386917114 | Cordelia and Dudley on their wedding day last year | -| 0.17765313386917114 | Cordelia and Dudley on their wedding day last year | -| 0.17765313386917114 | Cordelia and Dudley on their wedding day last year | - - -Note that the LAION dataset is known to contain a lot of duplicate images, so you may see the same image -showing up multiple times in the search results. - - -## Full-text search {#full-text-search} - -Run an FTS search query that uses BM25 ranking on the `caption` column (on which we already have an FTS index): - -```python -fts_results = ( - table.search("dog running on beach", query_type="fts") - .select(["caption", "url", "_score"]) - .limit(3) - .to_list() -) -``` - -| caption | url | _score | -| --- | --- | --- | -| running with dog | https://www.doggytastic.com/wp… | 15.73168 | -| Dog Running in Water | https://static.wixstatic.com/m… | 14.756516 | -| Dogs on the run by heidiannemo… | http://ih2.redbubble.net/image… | 14.756516 | - -## Download the full dataset {#download-the-full-dataset} - - -You may hit Hugging Face rate limits when streaming large samples from `hf://`, despite using a Hugging Face token. -For repeated queries or queries that operate on the full dataset, it's recommended to -download the dataset locally and query from disk. - - -Here's how to download the entire dataset via the [Hugging Face CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli): - -```bash -huggingface-cli download lance-format/laion-1m --repo-type dataset --local-dir ./laion-1m -``` - -## Upload your own datasets to Hugging Face in Lance format {#upload-your-own-datasets-to-hugging-face-in-lance-format} - -This section shows how you can upload your own Lance datasets to the Hugging Face Hub to share with the community. - -First, install the [Hugging Face CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli) and export both `OPENAI_API_KEY` and `HF_TOKEN`. -Then, create a Lance dataset using LanceDB on a local machine, and then proceed to upload it to the Hub via a CLI command. - -```bash -export OPENAI_API_KEY=... -export HF_TOKEN=hf_... -hf auth login --token "$HF_TOKEN" -``` - -A typical sequence of steps is given below. - -### 1\. Upload your local directory to the Hub {#1-upload-your-local-directory-to-the-hub} - -Upload the full local directory to a specified repository on the Hugging Face Hub. The command below uploads the contents of your local LanceDB directory at `/path/to/your_local_dir` to a new repository named `your_hf_org/repo_name` under your Hugging Face account. - -```bash bash icon="code" -hf upload-large-folder /path/to/your_local_dir your_hf_org/repo_name \ - --repo-type dataset \ - --revision main -``` - - -The `upload-large-folder` command is designed for [uploading large datasets](https://huggingface.co/docs/huggingface_hub/en/guides/upload) (potentially terabytes in size) and will handle multipart uploads, retries, and resuming interrupted uploads. - - -### 2\. Inspect dataset versions {#2-inspect-dataset-versions} - -Because you can query your remote dataset directly from Hugging Face with `hf://` URIs in LanceDB, you can easily inspect the dataset versions and updates on the Hub without needing to download the data locally. This is very useful to keep track of changes to the dataset and iterate on your data collection and curation process. - -```python Python icon="python" -import lancedb - -db = lancedb.connect("hf://datasets/your_hf_org/repo_name") -table = db.open_table("table_name") - -versions = table.list_versions() -print(versions) -``` -This will print out the list of versions available for the dataset on the Hub, along with their metadata such as creation date and description. - -### 3\. Add a dataset card {#3-add-a-dataset-card} - -The Hub dataset card allows you to communicate the schema and usage of the dataset to other developers. It sits at the repo's root in a file named `README.md` on the Hub. -This project keeps the source card text in `HF_DATASET_CARD.md`, so you can publish updates -to the dataset there and upload it as `README.md` using the following command on the HF CLI: -this requires a regular `hf upload` because it is a single-file upload to a specific target path (a custom commit message can be added if you wish). - -```bash -hf upload lancedb/magical_kingdom HF_DATASET_CARD.md README.md \ - --repo-type dataset \ - --commit-message "Update dataset card" -``` - -### 4\. Update the dataset {#4-update-the-dataset} - -Over time, you may want to add new rows (append) or columns (backfill) to your dataset as your needs evolve. You can make the necessary updates to your local dataset using LanceDB, and then upload the updated version back to the Hub with the same `hf upload-large-folder` command. - -```bash bash icon="code" -hf upload-large-folder /path/to/your_local_dir your_hf_org/repo_name \ - --repo-type dataset \ - --revision main -``` -The CLI will only upload the new data that has changed since the last upload, avoiding wasted I/O while making it easy to keep your dataset up-to-date on the Hub. - -That's it! Your dataset is now updated on the Hub with the new data and schema changes, and other users can query the latest version of the dataset directly from Hugging Face with `hf://` URIs in LanceDB. - -## Explore more Lance datasets on Hugging Face {#explore-more-lance-datasets-on-hugging-face} - -The LanceDB team is actively uploading useful and interesting datasets in Lance format to the Hugging Face Hub -under the [lance-format](https://huggingface.co/lance-format) organization. We actively encourage the Hugging Face -and LanceDB communities to upload their own Lance datasets to the Hub to share with others! - -In the meantime, feel free to check out the Hugging Face Hub to discover more Lance datasets uploaded by the community. - - -Click here to explore the latest trending Lance datasets on 🤗 Hugging Face! - diff --git a/docs/integrations/ai/kiln.mdx b/docs/integrations/ai/kiln.mdx deleted file mode 100644 index c93f31e..0000000 --- a/docs/integrations/ai/kiln.mdx +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: "Kiln AI" -sidebarTitle: "Kiln" - ---- - -[**Kiln**](https://kiln.tech) is a free tool for building production-ready AI systems, combining an intuitive desktop application and an open-source Python library. It supports RAG pipelines, evaluations, agents, MCP tool-calling, synthetic data generation, and fine-tuning. Kiln provides deep integration with LanceDB for vector search, full-text search (BM25), and hybrid search. - -## Quick Start: Build a RAG Pipeline in 5 Minutes with Kiln and LanceDB {#quick-start-build-a-rag-pipeline-in-5-minutes-with-kiln-and-lancedb} - -Watch the [quick start overview on Vimeo](https://vimeo.com/1119945690). - -Kiln's [app](https://kiln.tech/download) makes it easy to: - - - Build a RAG pipeline with a simple drag-and-drop interface - - [Compare](#find-the-best-rag-pipeline-for-your-use-case) search index options (powered by LanceDB), document extractors, embedding models, and chunking strategies - - Create end-to-end [evaluations](https://docs.kiln.tech/docs/evaluations) to determine which search configuration works best for your use case - - Load your data from Kiln into [LanceDB Enterprise](/enterprise) for production use - - Iterate with confidence by evaluating new content, prompts, models, and embeddings in minutes instead of weeks - -## Find the Best RAG Pipeline for Your Use Case {#find-the-best-rag-pipeline-for-your-use-case} - -There is no universal best RAG solution—only the best solution for your specific use case. Kiln makes it easy to compare state-of-the-art configurations and find which works best for you. - -Start with pre-configured templates for state-of-the-art RAG at various performance/quality/cost levels, or experiment with any combination of options: - -|Area|Technologies|Description| -|:----|:----|:----| -|Search Index|LanceDB|Compare LanceDB's vector search, full-text search (BM25), and hybrid search to find the best approach for your use case.| -|Content|Kiln Document Library|Collaborate on a document library with your team to find the best content for your RAG. Track every revision and tag document sets.| -|Document Extraction|Gemini, OpenAI GPT, Qwen VL, and more|Find the most accurate document extraction models for converting PDFs, images, audio, video, and other formats into textual data for RAG.| -|Embeddings|Embedding models from Gemini, OpenAI, Nomic, Qwen, and more|Find the embedding model best suited to your use case.| -|Chunking|LlamaIndex|Find the ideal chunk size and method.| - -## Get Started {#get-started} - -To get started, download the [Kiln App](https://kiln.tech/download), create a project, and navigate to "Docs & Search". - -See the [Kiln documentation for creating a RAG system](https://docs.kiln.tech/docs/documents-and-search-rag) for details on each step of the process. - -## More Information {#more-information} - - - [Kiln Homepage](https://kiln.tech) - - [Download the Kiln App](https://kiln.tech/download) - - [Kiln GitHub Repository](https://github.com/Kiln-AI/Kiln) - - [Building RAG Systems - Kiln Documentation](https://docs.kiln.tech/docs/documents-and-search-rag) - - [Python Library](https://pypi.org/project/kiln-ai/) or `pip install kiln_ai` - diff --git a/docs/integrations/ai/langchain.mdx b/docs/integrations/ai/langchain.mdx deleted file mode 100644 index 3633310..0000000 --- a/docs/integrations/ai/langchain.mdx +++ /dev/null @@ -1,226 +0,0 @@ ---- -title: "LangChain" -sidebarTitle: "LangChain" - ---- - -import { - PyFrameworksLangchainAddImages, - PyFrameworksLangchainAddTexts, - PyFrameworksLangchainCreateIndex, - PyFrameworksLangchainMaxMarginalRelevance, - PyFrameworksLangchainQuickStart, - PyFrameworksLangchainSimilaritySearch, - PyFrameworksLangchainSimilaritySearchByVector, - PyFrameworksLangchainSimilaritySearchByVectorWithScores, - PyFrameworksLangchainSimilaritySearchWithScores, - PyFrameworksLangchainVectorStoreConfig, -} from '/snippets/integrations.mdx'; - -**LangChain** is a framework designed for building applications with large language models (LLMs) by chaining together various components. It supports a range of functionalities including memory, agents, and chat models, enabling developers to create context-aware applications. - -![Illustration](https://raw.githubusercontent.com/lancedb/assets/refs/heads/main/docs/assets/integration/langchain_rag.png) - -LangChain streamlines these stages (in figure above) by providing pre-built components and tools for integration, memory management, and deployment, allowing developers to focus on application logic rather than underlying complexities. - -Integration of **Langchain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs. - -## Quick Start {#quick-start} -You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. - - - {PyFrameworksLangchainQuickStart} - - -## Documentation {#documentation} -In the above example `LanceDB` vector store class object is created using `from_documents()` method which is a `classmethod` and returns the initialized class object. - -You can also use `LanceDB.from_texts(texts: List[str],embedding: Embeddings)` class method. - -The exhaustive list of parameters for `LanceDB` vector store are : - -|Name|type|Purpose|default| -|:----|:----|:----|:----| -|`connection`| (Optional) `Any` |`lancedb.db.LanceDBConnection` connection object to use. If not provided, a new connection will be created.|`None`| -|`embedding`| (Optional) `Embeddings` | Langchain embedding model.|Provided by user.| -|`uri`| (Optional) `str` |It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. |`/tmp/lancedb`| -|`vector_key` |(Optional) `str`| Column name to use for vector's in the table.|`'vector'`| -|`id_key` |(Optional) `str`| Column name to use for id's in the table.|`'id'`| -|`text_key` |(Optional) `str` |Column name to use for text in the table.|`'text'`| -|`table_name` |(Optional) `str`| Name of your table in the database.|`'vectorstore'`| -|`api_key` |(Optional `str`) |API key to use for LanceDB Enterprise deployment.|`None`| -|`region` |(Optional) `str`| Region to use for LanceDB Enterprise deployment.| Only for LanceDB Enterprise : `None`.| -|`mode` |(Optional) `str` | Mode to use for adding data to the table. Valid values are "append" and "overwrite".|`'overwrite'`| -|`table`| (Optional) `Any`| You can connect to an existing table of LanceDB, created outside of langchain, and utilize it.|`None`| -|`distance`|(Optional) `str`| The choice of distance metric used to calculate the similarity between vectors.|`'l2'`| -|`reranker` |(Optional) `Any`| The reranker to use for LanceDB.|`None`| -|`relevance_score_fn` |(Optional) `Callable[[float], float]` | Langchain relevance score function to be used.|`None`| -|`limit`|`int`|Set the maximum number of results to return.| `DEFAULT_K` (it is 4)| - - - {PyFrameworksLangchainVectorStoreConfig} - - -### Methods {#methods} - -##### `add_texts()` - -This method turn texts into embedding and add it to the database. - -|Name|Purpose|defaults| -|:---|:---|:---| -|`texts`|`Iterable` of strings to add to the vectorstore.|Provided by user| -|`metadatas`|Optional `list[dict()]` of metadatas associated with the texts.|`None`| -|`ids`|Optional `list` of ids to associate with the texts.|`None`| -|`kwargs`| Other keyworded arguments provided by the user. |-| - -It returns list of ids of the added texts. - - - {PyFrameworksLangchainAddTexts} - - ------- - - -##### create_index() - -This method creates a scalar(for non-vector cols) or a vector index on a table. - -|Name|type|Purpose|defaults| -|:---|:---|:---|:---| -|`vector_col`|`Optional[str]`| Provide if you want to create index on a vector column. |`None`| -|`col_name`|`Optional[str]`| Provide if you want to create index on a non-vector column. |`None`| -|`metric`|`Optional[str]` |Provide the metric to use for vector index. choice of metrics: 'l2', 'dot', 'cosine'. |`l2`| -|`num_partitions`|`Optional[int]`|Number of partitions to use for the index.|`256`| -|`num_sub_vectors`|`Optional[int]` |Number of sub-vectors to use for the index.|`96`| -|`index_cache_size`|`Optional[int]` |Size of the index cache.|`None`| -|`name`|`Optional[str]` |Name of the table to create index on.|`None`| - -For index creation make sure your table has enough data in it. An ANN index is usually not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index. - - - {PyFrameworksLangchainCreateIndex} - - ------- - -##### similarity_search() - -This method performs similarity search based on **text query**. - -| Name | Type | Purpose | Default | -|---------|----------------------|---------|---------| -| `query` | `str` | A `str` representing the text query that you want to search for in the vector store. | N/A | -| `k` | `Optional[int]` | It specifies the number of documents to return. | `None` | -| `filter` | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria. | `None` | -| `fts` | `Optional[bool]` | It indicates whether to perform a full-text search (FTS). | `False` | -| `name` | `Optional[str]` | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance. | `None` | -| `kwargs` | `Any` | Other keyworded arguments provided by the user. | N/A | - -Return documents most similar to the query **without relevance scores**. - - - {PyFrameworksLangchainSimilaritySearch} - - ------- - -##### similarity_search_by_vector() - -The method returns documents that are most similar to the specified **embedding (query) vector**. - -| Name | Type | Purpose | Default | -|-------------|---------------------------|---------|---------| -| `embedding` | `List[float]` | The embedding vector you want to use to search for similar documents in the vector store. | N/A | -| `k` | `Optional[int]` | It specifies the number of documents to return. | `None` | -| `filter` | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria. | `None` | -| `name` | `Optional[str]` | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance. | `None` | -| `kwargs` | `Any` | Other keyworded arguments provided by the user. | N/A | - -**It does not provide relevance scores.** - - - {PyFrameworksLangchainSimilaritySearchByVector} - - ------- - -##### similarity_search_with_score() - -Returns documents most similar to the **query string** along with their relevance scores. - -| Name | Type | Purpose | Default | -|----------|---------------------------|---------|---------| -| `query` | `str` |A `str` representing the text query you want to search for in the vector store. This query will be converted into an embedding using the specified embedding function. | N/A | -| `k` | `Optional[int]` | It specifies the number of documents to return. | `None` | -| `filter` | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria. This allows you to narrow down the search results based on certain metadata attributes associated with the documents. | `None` | -| `kwargs` | `Any` | Other keyworded arguments provided by the user. | N/A | - -It gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`. - - - {PyFrameworksLangchainSimilaritySearchWithScores} - - ------- - -##### similarity_search_by_vector_with_relevance_scores() - -Similarity search using **query vector**. - -| Name | Type | Purpose | Default | -|-------------|---------------------------|---------|---------| -| `embedding` | `List[float]` | The embedding vector you want to use to search for similar documents in the vector store. | N/A | -| `k` | `Optional[int]` | It specifies the number of documents to return. | `None` | -| `filter` | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria. | `None` | -| `name` | `Optional[str]` | It is used for specifying the name of the table to query. | `None` | -| `kwargs` | `Any` | Other keyworded arguments provided by the user. | N/A | - -The method returns documents most similar to the specified embedding (query) vector, along with their relevance scores. - - - {PyFrameworksLangchainSimilaritySearchByVectorWithScores} - - ------- - -##### max_marginal_relevance_search() - -This method returns docs selected using the maximal marginal relevance(MMR). -Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. - -| Name | Type | Purpose | Default | -|---------------|-----------------|-----------|---------| -| `query` | `str` | Text to look up documents similar to. | N/A | -| `k` | `Optional[int]` | Number of Documents to return.| `4` | -| `fetch_k`| `Optional[int]`| Number of Documents to fetch to pass to MMR algorithm.| `None` | -| `lambda_mult` | `float` | Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. | `0.5` | -| `filter`| `Optional[Dict[str, str]]`| Filter by metadata. | `None` | -|`kwargs`| Other keyworded arguments provided by the user. |-| - -Similarly, `max_marginal_relevance_search_by_vector()` function returns docs most similar to the embedding passed to the function using MMR. instead of a string query you need to pass the embedding to be searched for. - - - {PyFrameworksLangchainMaxMarginalRelevance} - - ------- - -##### add_images() - -This method adds images by automatically creating their embeddings and adds them to the vectorstore. - -| Name | Type | Purpose | Default | -|------------|-------------------------------|--------------------------------|---------| -| `uris` | `List[str]` | File path to the image | N/A | -| `metadatas`| `Optional[List[dict]]` | Optional list of metadatas | `None` | -| `ids` | `Optional[List[str]]` | Optional list of IDs | `None` | - -It returns list of IDs of the added images. - - - {PyFrameworksLangchainAddImages} - - - diff --git a/docs/integrations/ai/llamaIndex.mdx b/docs/integrations/ai/llamaIndex.mdx deleted file mode 100644 index a38e2c6..0000000 --- a/docs/integrations/ai/llamaIndex.mdx +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: "LlamaIndex" -sidebarTitle: "LlamaIndex" - ---- - -import { - PyFrameworksLlamaindexAddReranker, - PyFrameworksLlamaindexFiltering, - PyFrameworksLlamaindexHybridSearch, - PyFrameworksLlamaindexQuickStart, -} from '/snippets/integrations.mdx'; - -## Quickstart {#quickstart} - -LlamaIndex is a well-known framework for building LLM-powered agents over your data with LLMs and workflows. -You can build your LlamaIndex pipeline and persist your metadata and embeddings in LanceDB via the `LanceDBVectorStore` class. - -First, install the LlamaIndex-LanceDB integration. - - -pip install llama-index-vector-stores-LanceDB - - -Run the below script as an example. - - - {PyFrameworksLlamaindexQuickStart} - - -The vector store connector will open an existing LanceDB directory or create the directory if it does not exist. - -### Filtering {#filtering} -For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex: - - {PyFrameworksLlamaindexFiltering} - - -### Hybrid Search {#hybrid-search} -For complete documentation, refer [here](https://docs.lancedb.com/search/hybrid-search). This example uses the `colbert` reranker. Make sure to install necessary dependencies for the reranker you choose. - - {PyFrameworksLlamaindexHybridSearch} - - -In the snippet above, you can change/specify `query_type` when creating the engine/retriever -to use different search strategies, such as vector search or FTS. - -## API reference {#api-reference} - - -See the official LlamaIndex Vector Stores API reference for more details. - \ No newline at end of file diff --git a/docs/integrations/ai/prompttools.mdx b/docs/integrations/ai/prompttools.mdx deleted file mode 100644 index b0edd57..0000000 --- a/docs/integrations/ai/prompttools.mdx +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: "PromptTools" -sidebarTitle: "PromptTools" - ---- - -[PromptTools](https://github.com/hegelai/prompttools) offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice. -{/* -[Evaluating Prompts with PromptTools](./examples/prompttools-eval-prompts/) | Open In Colab -*/} - -Open In Colab - -![Alt text](https://prompttools.readthedocs.io/en/latest/_images/demo.gif "a title") - diff --git a/docs/integrations/ai/synthetic-data-kit.mdx b/docs/integrations/ai/synthetic-data-kit.mdx deleted file mode 100644 index 18b9067..0000000 --- a/docs/integrations/ai/synthetic-data-kit.mdx +++ /dev/null @@ -1,81 +0,0 @@ ---- -title: "Meta Llama Synthetic Data Kit" -sidebarTitle: "Synthetic Data Kit" -description: "Use Meta Llama's Synthetic Data Kit with LanceDB to generate high-quality synthetic datasets for LLM fine-tuning and training." - ---- - - - -[Synthetic Data Kit](https://github.com/meta-llama/synthetic-data-kit) is a tool from Meta LLAMA that helps you generate high-quality synthetic datasets for fine-tuning large language models (LLMs). It simplifies the process of preparing data for fine-tuning by providing a command-line interface (CLI) with a modular four-command flow. - -One of the key features of the `synthetic-data-kit` is its use of the Lance format for storing and ingesting datasets. This allows for efficient storage and retrieval of data, which is crucial when working with large datasets. - -### Key Features: {#key-features} - -* **Data Ingestion:** The toolkit can ingest various file formats, including PDF, HTML, YouTube transcripts, DOCX, PPT, and TXT. -* **Fine-tuning Format Creation:** It can create different fine-tuning formats, such as question-answer (QA) pairs, QA pairs with Chain-of-Thought (CoT), and summarization formats. -* **Data Curation:** The tool uses Llama as a judge to curate high-quality examples, ensuring the quality of the generated dataset. -* **Flexible Saving Options:** You can save the generated datasets in various formats compatible with your fine-tuning workflow, including Hugging Face, JSONL, and JSON. - -### How it Works: {#how-it-works} - -The synthetic-data-kit follows a simple four-step process: - -1. **Ingest:** Import your input files into the toolkit. The data is stored in the Lance format for efficient processing. -2. **Create:** Generate diverse fine-tuning datasets, such as reasoning, summarization, and QA pairs, from the ingested documents. -3. **Curate:** Use Llama to filter and select high-quality examples from the generated dataset. -4. **Save-as:** Export the curated dataset in your preferred format. - -### Usage {#usage} - -The `synthetic-data-kit` uses Lance format to store and manage the data that you ingest. The workflow is a series of commands that build on each other, starting with the `ingest` command. - -Here is an example of the end-to-end workflow: - -1. **Ingest Data into a LanceDB dataset** - - This command takes a directory of source files and creates a LanceDB dataset from them. - - ```bash - synthetic-data-kit ingest docs/report.pdf --multimodal - # This will create a Lance dataset at data/parsed/report.lance - # with 'text' and 'image' columns. - - #Generate multimodal-qa pairs from the ingested data - synthetic-data-kit create data/parsed/report.lance --type multimodal-qa - ``` - -2. **Create fine-tuning data** - - This command uses the LanceDB dataset created in the previous step to generate synthetic data in the desired format. - - ```bash - synthetic_data create data/parsed/report.lance - ``` - -3. **Curate the data** - - This step uses a language model to curate the generated data and ensure its quality. - - ```bash - synthetic_data curate report.json - ``` - -4. **Save the final dataset** - - Finally, save the curated data to a file in the desired format. - - ```bash - synthetic_data save-as report.json --save_path ./my_finetuning_data.jsonl - ``` - -This workflow allows you to go from a collection of documents to a high-quality, fine-tuning dataset with just a few commands. The use of LanceDB in the background makes the process efficient and scalable. - -### Getting Started: {#getting-started} - -To get started with the synthetic-data-kit, you can clone the [GitHub Repository](https://github.com/meta-llama/synthetic-data-kit) and install the necessary dependencies. - -> **Note:** You will also need access to a Llama model, either running locally or via a hosted API. - - diff --git a/docs/integrations/data/dlt.mdx b/docs/integrations/data/dlt.mdx deleted file mode 100644 index f15a4b7..0000000 --- a/docs/integrations/data/dlt.mdx +++ /dev/null @@ -1,97 +0,0 @@ ---- -title: "dlt" -sidebarTitle: "DLT" - ---- - -import { - PyPlatformsDltAdapterImport, - PyPlatformsDltAdapterUsage, - PyPlatformsDltPipeline, -} from '/snippets/integrations.mdx'; - -[dlt](https://dlthub.com/docs/intro) is an open-source library that you can add to your Python scripts to load data from various and often messy data sources into well-structured, live datasets. dlt's [integration with LanceDB](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb) lets you ingest data from any source (databases, APIs, CSVs, dataframes, JSONs, and more) into LanceDB with a few lines of simple python code. The integration enables automatic normalization of nested data, schema inference, incremental loading and embedding the data. dlt also has integrations with several other tools like dbt, airflow, dagster etc. that can be inserted into your LanceDB workflow. - -## How to ingest data into LanceDB {#how-to-ingest-data-into-lancedb} - -In this example, we will be fetching movie information from the [Open Movie Database (OMDb) API](https://www.omdbapi.com/) and loading it into a local LanceDB instance. To implement it, you will need an API key for the OMDb API (which can be created freely [here](https://www.omdbapi.com/apikey.aspx)). - -1. **Install `dlt` with LanceDB extras:** - ```sh - pip install dlt[lancedb] - ``` - -2. **Inside an empty directory, initialize a `dlt` project with:** - ```sh - dlt init rest_api lancedb - ``` - This will add all the files necessary to create a `dlt` pipeline that can ingest data from any REST API (ex: OMDb API) and load into LanceDB. - ```text - ├── .dlt - │ ├── config.toml - │ └── secrets.toml - ├── rest_api - ├── rest_api_pipeline.py - └── requirements.txt - ``` - - dlt has a list of pre-built [sources](https://dlthub.com/docs/dlt-ecosystem/verified-sources/) like [SQL databases](https://dlthub.com/docs/dlt-ecosystem/verified-sources/sql_database), [REST APIs](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api), [Google Sheets](https://dlthub.com/docs/dlt-ecosystem/verified-sources/google_sheets), [Notion](https://dlthub.com/docs/dlt-ecosystem/verified-sources/notion) etc., that can be used out-of-the-box by running `dlt init lancedb`. Since dlt is a python library, it is also very easy to modify these pre-built sources or to write your own custom source from scratch. - - -3. **Specify necessary credentials and/or embedding model details:** - - In order to fetch data from the OMDb API, you will need to pass a valid API key into your pipeline. Depending on whether you're using LanceDB OSS or LanceDB Enterprise, you also may need to provide the necessary credentials to connect to the LanceDB instance. These can be pasted inside `.dlt/secrets.toml`. - - dlt's LanceDB integration also allows you to automatically embed the data during ingestion. Depending on the embedding model chosen, you may need to paste the necessary credentials inside `.dlt/secrets.toml`: - ```toml - [sources.rest_api] - api_key = "api_key" # Enter the API key for the OMDb API - - [destination.lancedb] - embedding_model_provider = "sentence-transformers" - embedding_model = "all-MiniLM-L6-v2" - [destination.lancedb.credentials] - uri = ".lancedb" - api_key = "api_key" # API key to connect to LanceDB Enterprise. Leave out if you are using LanceDB OSS. - embedding_model_provider_api_key = "embedding_model_provider_api_key" # Not needed for providers that don't need authentication (ollama, sentence-transformers). - ``` - See [here](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb#configure-the-destination) for more information and for a list of available models and model providers. - - -4. **Write the pipeline code inside `rest_api_pipeline.py`:** - - The following code shows how you can configure dlt's REST API source to connect to the [OMDb API](https://www.omdbapi.com/), fetch all movies with the word "godzilla" in the title, and load it into a LanceDB table. The REST API source allows you to pull data from any API with minimal code, to learn more read the [dlt docs](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api). - - - {PyPlatformsDltPipeline} - - - The script above will ingest the data into LanceDB as it is, i.e. without creating any embeddings. If we want to embed one of the fields (for example, `"Title"` that contains the movie titles), then we will use dlt's `lancedb_adapter` and modify the script as follows: - - - Add the following import statement: - - {PyPlatformsDltAdapterImport} - - - Modify the pipeline run like this: - - {PyPlatformsDltAdapterUsage} - - This will use the embedding model specified inside `.dlt/secrets.toml` to embed the field `"Title"`. - -5. **Install necessary dependencies:** - ```sh - pip install -r requirements.txt - ``` - - Note: You may need to install the dependencies for your embedding models separately. - ```sh - pip install sentence-transformers - ``` - -6. **Run the pipeline:** - Finally, running the following command will ingest the data into your LanceDB instance. - ```sh - python custom_source.py - ``` - -For more information and advanced usage of dlt's LanceDB integration, read [the dlt documentation](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb). diff --git a/docs/integrations/data/duckdb.mdx b/docs/integrations/data/duckdb.mdx deleted file mode 100644 index f5e3114..0000000 --- a/docs/integrations/data/duckdb.mdx +++ /dev/null @@ -1,119 +0,0 @@ ---- -title: "DuckDB" -sidebarTitle: "DuckDB" -description: "Learn how to use the DuckDB-Lance extension to query Lance tables with SQL." ---- - -LanceDB integrates with [DuckDB](https://duckdb.org/) through the [Lance extension](https://github.com/lance-format/lance-duckdb) for DuckDB. In this page, we'll show how LanceDB manages table lifecycle, and DuckDB provides SQL analytics (including joins) and search over those tables. - -Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer. - - -## Install {#install} - -Install the DuckDB CLI as per [their docs](https://duckdb.org/install) and alternatively, their Python package with `pip install duckdb`. - -Then, open the DuckDB CLI and install and load the Lance extension as follows: - -```sql SQL icon="database" -INSTALL lance; -LOAD lance; -``` - -## Attach the directory namespace in DuckDB {#attach-the-directory-namespace-in-duckdb} - -Attach the LanceDB root directory as a Lance namespace: - -```sql SQL icon="database" -ATTACH './local_lancedb' AS lance_ns (TYPE LANCE); -``` - -In this page, tables are referenced using `lance_ns.main.`, so the table path is `lance_ns.main.lance_duck`. - -## Write Lance table {#write-lance-table} - -Create the `lance_duck` table using SQL and populate it with sample data: - -```sql SQL icon="database" -CREATE OR REPLACE TABLE lance_ns.main.lance_duck AS -SELECT * -FROM ( - VALUES - ('duck', 'quack', [0.9, 0.7, 0.1]::FLOAT[]), - ('horse', 'neigh', [0.3, 0.1, 0.5]::FLOAT[]), - ('dragon', 'roar', [0.5, 0.2, 0.7]::FLOAT[]) -) AS t(animal, noise, vector); -``` - -This table is the source of truth for all DuckDB queries below. - - -The examples below show SQL entered in the DuckDB CLI. You can run the same SQL from -Python as well, using LanceDB and DuckDB's Python clients in your application code. - - -## Query the table with SQL {#query-the-table-with-sql} - -```sql SQL icon="database" -SELECT * - FROM lance_ns.main.lance_duck - LIMIT 5; -``` - -## Vector search {#vector-search} - -```sql SQL icon="database" -SELECT animal, noise, vector, _distance - FROM lance_vector_search( - 'lance_ns.main.lance_duck', - 'vector', - [0.8, 0.7, 0.2]::FLOAT[], - k = 1, - prefilter = true - ) - ORDER BY _distance ASC; -``` - -## Full-text search {#full-text-search} - -```sql SQL icon="database" -SELECT animal, noise, vector, _score - FROM lance_fts( - 'lance_ns.main.lance_duck', - 'animal', - 'the brave knight faced the dragon', - k = 1, - prefilter = true - ) - ORDER BY _score DESC; -``` - -## Hybrid search {#hybrid-search} - -```sql SQL icon="database" -SELECT animal, noise, vector, _hybrid_score, _distance, _score - FROM lance_hybrid_search( - 'lance_ns.main.lance_duck', - 'vector', - [0.8, 0.7, 0.2]::FLOAT[], - 'animal', - 'the duck surprised the dragon', - k = 2, - prefilter = false, - alpha = 0.5, - oversample_factor = 4 - ) - ORDER BY _hybrid_score DESC; -``` - -## Directory namespace model {#directory-namespace-model} - -A directory namespace maps a LanceDB catalog root to namespace-qualified table identifiers in DuckDB. This keeps table discovery and table naming stable as your project grows. - -To learn more about the catalog and namespace model, see [Namespaces and the Catalog Model](/namespaces). - - -## Advanced usage {#advanced-usage} - -See the [docs](https://github.com/lance-format/lance-duckdb) directory in the Lance-DuckDB extension repo -for more advanced usage on SQL and REST API clients. diff --git a/docs/integrations/data/pandas_and_pyarrow.mdx b/docs/integrations/data/pandas_and_pyarrow.mdx deleted file mode 100644 index 170ccf7..0000000 --- a/docs/integrations/data/pandas_and_pyarrow.mdx +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: "Pandas and PyArrow" -sidebarTitle: "Pandas & PyArrow" - ---- - -import { - PyPlatformsPandasAsyncExample, - PyPlatformsPandasCreateTable, - PyPlatformsPandasImports, - PyPlatformsPandasVectorSearch, -} from '/snippets/integrations.mdx'; - -Because Lance is built on top of [Apache Arrow](https://arrow.apache.org/), -LanceDB fits naturally into Pandas-first workflows. You can ingest a `DataFrame`, -query it with LanceDB's vector operators, and keep working in Pandas without any glue code. - -## Create a dataset {#create-a-dataset} - -Start by importing LanceDB alongside your usual Pandas utilities and connect to a temporary database. - - - {PyPlatformsPandasImports} - - -Use the familiar `pd.DataFrame` API to prepare your rows, then pass the entire frame to `db.create_table`. - - - {PyPlatformsPandasCreateTable} - - -## Vector search {#vector-search} - -Queries can return Pandas frames as well, so you can immediately inspect the results or pipe them into downstream analytics. - - - {PyPlatformsPandasVectorSearch} - - -## Async API {#async-api} - -For web services or background jobs that already rely on `asyncio`, use the asynchronous helpers to keep everything non-blocking. - - - {PyPlatformsPandasAsyncExample} - diff --git a/docs/integrations/data/polars_arrow.mdx b/docs/integrations/data/polars_arrow.mdx deleted file mode 100644 index 4cb8573..0000000 --- a/docs/integrations/data/polars_arrow.mdx +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: "Polars" -sidebarTitle: "Polars & Arrow" - ---- - -import { - PyPlatformsPolarsCreateTable, - PyPlatformsPolarsImports, - PyPlatformsPolarsLazyframe, - PyPlatformsPolarsPydantic, - PyPlatformsPolarsVectorSearch, -} from '/snippets/integrations.mdx'; - -LanceDB supports [Polars](https://github.com/pola-rs/polars), a blazingly fast DataFrame library for Python written in Rust. Under the hood, both Lance and Polars speak Arrow, so passing data back and forth stays zero-copy and ergonomic. - -## Create and Query a Table {#create-and-query-a-table} - -Import the required libraries, including the optional Pydantic helpers if you plan to define schemas. - - - {PyPlatformsPolarsImports} - - -Build a Polars `DataFrame`, convert it to Arrow, and use it directly when creating a LanceDB table. - - - {PyPlatformsPolarsCreateTable} - - -Run vector search and keep the results as a Polars `DataFrame` for further processing or visualization. - - - {PyPlatformsPolarsVectorSearch} - - -## Work with LazyFrames {#work-with-lazyframes} - -When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. - - - {PyPlatformsPolarsLazyframe} - - -## Define Schemas with Pydantic {#define-schemas-with-pydantic} - -You can also describe your table via `LanceModel` and continue ingesting data from Polars. This is useful when multiple teams share a schema or when you want validation. - - - {PyPlatformsPolarsPydantic} - diff --git a/docs/integrations/data/pydantic.mdx b/docs/integrations/data/pydantic.mdx deleted file mode 100644 index b6f80ac..0000000 --- a/docs/integrations/data/pydantic.mdx +++ /dev/null @@ -1,90 +0,0 @@ ---- -title: "Pydantic" -sidebarTitle: "Pydantic" - ---- - -import { - PyFrameworksPydanticBaseExample, - PyFrameworksPydanticBaseModel, - PyFrameworksPydanticImports, - PyFrameworksPydanticSetUrl, - PyFrameworksPydanticTypeConversion, - PyFrameworksPydanticVectorField, -} from '/snippets/integrations.mdx'; - -[Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python. -LanceDB integrates with Pydantic for schema inference, data ingestion, and query result casting. -Using `lancedb.pydantic.LanceModel`, users can seamlessly -integrate Pydantic with the rest of the LanceDB APIs. - -First, import the necessary LanceDB and Pydantic modules: - - - {PyFrameworksPydanticImports} - - -Next, define your Pydantic model by inheriting from `LanceModel` and specifying your fields including a vector field: - - - {PyFrameworksPydanticBaseModel} - - -Set the database connection URL: - - - {PyFrameworksPydanticSetUrl} - - -Now you can create a table, add data, and perform vector search operations: - - - {PyFrameworksPydanticBaseExample} - - - -## Vector Field {#vector-field} - -LanceDB provides a `lancedb.pydantic.Vector` method to define a -vector Field in a Pydantic Model. - - - {PyFrameworksPydanticVectorField} - - -This example demonstrates how LanceDB automatically converts Pydantic field types to their corresponding Apache Arrow data types. The `pydantic_to_schema()` function takes a Pydantic model and generates an Arrow schema where: -- `int` fields become `pa.int64()` (64-bit integers) -- `str` fields become `pa.utf8()` (UTF-8 encoded strings) -- `Vector(768)` becomes `pa.list_(pa.float32(), 768)` (fixed-size list of 768 float32 values) -- The `False` parameter indicates that the fields are not nullable - -## Type Conversion {#type-conversion} - -LanceDB automatically convert Pydantic fields to -[Apache Arrow DataType](https://arrow.apache.org/docs/python/generated/pyarrow.DataType.html#pyarrow.DataType). - -Current supported type conversions: - -| Pydantic Field Type | PyArrow Data Type | -| ------------------- | ----------------- | -| `int` | `pyarrow.int64` | -| `float` | `pyarrow.float64` | -| `bool` | `pyarrow.bool` | -| `str` | `pyarrow.utf8()` | -| `list` | `pyarrow.List` | -| `BaseModel` | `pyarrow.Struct` | -| `Vector(n)` | `pyarrow.FixedSizeList(float32, n)` | - -LanceDB supports to create Apache Arrow Schema from a -`pydantic.BaseModel` -via `lancedb.pydantic.pydantic_to_schema` method. - - - {PyFrameworksPydanticTypeConversion} - - -This example shows a more complex Pydantic model with various field types and demonstrates how LanceDB handles: -- Basic types: `int` and `str` fields -- Vector fields: `Vector(1536)` creates a fixed-size list of 1536 float32 values -- List fields: `List[int]` becomes a variable-length list of int64 values -- Schema generation: The `pydantic_to_schema()` function automatically converts all these types to their Arrow equivalents \ No newline at end of file diff --git a/docs/integrations/data/voxel51.mdx b/docs/integrations/data/voxel51.mdx deleted file mode 100644 index 1ee7823..0000000 --- a/docs/integrations/data/voxel51.mdx +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: "Voxel51" -sidebarTitle: "Voxel51" ---- - -import { - PyPlatformsVoxel51BackendFlag, - PyPlatformsVoxel51BackendParams, - PyPlatformsVoxel51BrainConfig, - PyPlatformsVoxel51Cleanup, - PyPlatformsVoxel51ComputeSimilarity, - PyPlatformsVoxel51LoadDataset, - PyPlatformsVoxel51SortBySimilarity, -} from '/snippets/integrations.mdx'; - -# FiftyOne - -[FiftyOne](https://docs.voxel51.com/) is an open source toolkit that enables users to curate better data and build better models. It includes tools for data exploration, visualization, and management, as well as features for collaboration and sharing. - -Any developers, data scientists, and researchers who work with computer vision and machine learning can use FiftyOne to improve the quality of their datasets and deliver insights about their models. - -![example](/static/assets/images/integrations/voxel.gif) - -**FiftyOne** provides an API to create LanceDB tables and run similarity queries, both **programmatically in Python** and via **point-and-click in the App**. - -Let's get started and see how to use **LanceDB** to create a **similarity index** on your FiftyOne datasets. - -## Overview {#overview} - -[Embeddings](/embedding/) are foundational to all of the **vector search** features. In FiftyOne, embeddings are managed by the [**FiftyOne Brain**](https://docs.voxel51.com/user_guide/brain.html) that provides powerful machine learning techniques designed to transform how you curate your data from an art into a measurable science. - -> _Have you ever wanted to find the images most similar to an image in your dataset?_ - -The **FiftyOne Brain** makes computing **visual similarity** really easy. You can compute the similarity of samples in your dataset using an embedding model and store the results in the **brain key**. -You can then sort your samples by similarity or use this information to find potential duplicate images. - - -We'll be doing the following : - -1. **Create Index** - In order to run similarity queries against our media, we need to **index** the data. We can do this via the `compute_similarity()` function. - - In the function, specify the **model** you want to use to generate the embedding vectors, and what **vector search engine** you want to use on the **backend** (here LanceDB). - - You can also give the similarity index a name(`brain_key`), which is useful if you want to run vector searches against multiple indexes. - - -2. **Query** - Once you have generated your similarity index, you can query your dataset with `sort_by_similarity()`. The query can be any of the following: - - - An ID (sample or patch) - - A query vector of same dimension as the index - - A list of IDs (samples or patches) - - A text prompt (search semantically) - -## Prerequisites: install necessary dependencies {#prerequisites-install-necessary-dependencies} - -1. **Create and activate a virtual environment** - -Install virtualenv package and run the following command in your project directory. - -python -m venv fiftyone_ - -From inside the project directory run the following to activate the virtual environment. - - - - source fiftyone_/Scripts/activate - - - fiftyone_/Scripts/activate - - - -2. **Install the following packages in the virtual environment** - - To install FiftyOne, ensure you have activated any virtual environment that you are using, then run - -pip install fiftyone - - - -## Understand basic workflow {#understand-basic-workflow} - -The basic workflow shown below uses LanceDB to create a similarity index on your FiftyOne datasets: - -1. Load a dataset into FiftyOne. - -2. Compute embedding vectors for samples or patches in your dataset, or select a model to use to generate embeddings. - -3. Use the `compute_similarity()` method to generate a LanceDB table for the samples or object patches embeddings in a dataset by setting the parameter `backend="lancedb"` and specifying a `brain_key` of your choice. - -4. Use this LanceDB table to query your data with `sort_by_similarity()`. - -5. If desired, delete the table. - -## Quick Example {#quick-example} - -Let's jump on a quick example that demonstrates this workflow. - - - - {PyPlatformsVoxel51LoadDataset} - -Make sure you install torch ([guide here](https://pytorch.org/get-started/locally/)) before proceeding. - - - {PyPlatformsVoxel51ComputeSimilarity} - - -!!! note - Running the code above will download the clip model (2.6Gb) - -Once the similarity index has been generated, we can query our data in FiftyOne by specifying the `brain_key`: - - - {PyPlatformsVoxel51SortBySimilarity} - - -The returned result are of type - `DatasetView`. - - -`DatasetView` does not hold its contents in-memory. Views simply store the rule(s) that are applied to extract the content of interest from the underlying Dataset when the view is iterated/aggregated on. - -This means, for example, that the contents of a `DatasetView` may change as the underlying Dataset is modified. - - -> _Can you query a view instead of dataset?_ - -Yes, you can also query a view. - -Performing a similarity search on a `DatasetView` will only return results from the view; if the view contains samples that were not included in the index, they will never be included in the result. - -This means that you can index an entire Dataset once and then perform searches on subsets of the dataset by constructing views that contain the images of interest. - - - {PyPlatformsVoxel51Cleanup} - - - -## Using LanceDB backend {#using-lancedb-backend} -By default, calling `compute_similarity()` or `sort_by_similarity()` will use an sklearn backend. - -To use the LanceDB backend, simply set the optional `backend` parameter of `compute_similarity()` to `"lancedb"`: - - - {PyPlatformsVoxel51BackendFlag} - - -Alternatively, you can configure FiftyOne to use the LanceDB backend by setting the following environment variable. - -In your terminal, set the environment variable using: - - - - export FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND=lancedb - - - - $Env:FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND="lancedb" //powershell - - set FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND=lancedb //cmd - - - - -This will only run during the terminal session. Once terminal is closed, environment variable is deleted. - - -Alternatively, you can **permanently** configure FiftyOne to use the LanceDB backend creating a `brain_config.json` at `~/.fiftyone/brain_config.json`. The JSON file may contain any desired subset of config fields that you wish to customize. - -```json -{ - "default_similarity_backend": "lancedb" -} -``` -This will override the default `brain_config` and will set it according to your customization. You can check the configuration by running the following code : - - - {PyPlatformsVoxel51BrainConfig} - - -## LanceDB config parameters {#lancedb-config-parameters} - -The LanceDB backend supports query parameters that can be used to customize your similarity queries. These parameters include: - -| Name | Purpose | Default | -| :------------- | :--------------------------------------------------------------------------------------------------------------- | :--------------- | -| **table_name** | The name of the LanceDB table to use. If none is provided, a new table will be created | `None` | -| **metric** | The embedding distance metric to use when creating a new table. The supported values are ("cosine", "euclidean") | `"cosine"` | -| **uri** | The database URI to use. In this Database URI, tables will be created. | `"/tmp/lancedb"` | - -There are two ways to specify/customize the parameters: - -1. **Using `brain_config.json` file** - -```json -{ - "similarity_backends": { - "lancedb": { - "table_name": "your-table", - "metric": "euclidean", - "uri": "/tmp/lancedb" - } - } -} -``` - -2. **Directly passing to `compute_similarity()` to configure a specific new index** : - - - {PyPlatformsVoxel51BackendParams} - - -For a much more in depth walkthrough of the integration, visit the LanceDB x Voxel51 [docs page](https://docs.voxel51.com/integrations/lancedb.html). diff --git a/docs/integrations/embedding/aws.mdx b/docs/integrations/embedding/aws.mdx deleted file mode 100644 index 4e1c4fb..0000000 --- a/docs/integrations/embedding/aws.mdx +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: AWS Bedrock -sidebarTitle: AWS ---- - -import { PyEmbeddingAwsUsage } from '/snippets/integrations.mdx'; - -AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function. -You can do so by using `awscli` and also add your session_token: -```shell -aws configure -aws configure set aws_session_token "" -``` -to ensure that the credentials are set up correctly, you can run the following command: -```shell -aws sts get-caller-identity -``` - -Supported Embedding modelIDs are: -* `amazon.titan-embed-text-v1` -* `cohere.embed-english-v3` -* `cohere.embed-multilingual-v3` - -Supported parameters (to be passed in `create` method) are: - -| Parameter | Type | Default Value | Description | -|---|---|---|---| -| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 | -| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). | -| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. | -| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. | -| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. | -| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. | -| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. | - -Usage Example: - - - - {PyEmbeddingAwsUsage} - - diff --git a/docs/integrations/embedding/cohere.mdx b/docs/integrations/embedding/cohere.mdx deleted file mode 100644 index 8051462..0000000 --- a/docs/integrations/embedding/cohere.mdx +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Cohere -sidebarTitle: Cohere ---- - -import { PyEmbeddingCohereUsage } from '/snippets/integrations.mdx'; - -Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. -You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API. - -Supported models are: - -- embed-english-v3.0 -- embed-multilingual-v3.0 -- embed-english-light-v3.0 -- embed-multilingual-light-v3.0 -- embed-english-v2.0 -- embed-english-light-v2.0 -- embed-multilingual-v2.0 - - -Supported parameters (to be passed in `create` method) are: - -| Parameter | Type | Default Value | Description | -|---|---|--------|---------| -| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 | -| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. | -| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. | - -Cohere supports following input types: - -| Input Type | Description | -|-------------------------|---------------------------------------| -| "`search_document`" | Used for embeddings stored in a vector| -| | database for search use-cases. | -| "`search_query`" | Used for embeddings of search queries | -| | run against a vector DB | -| "`semantic_similarity`" | Specifies the given text will be used | -| | for Semantic Textual Similarity (STS) | -| "`classification`" | Used for embeddings passed through a | -| | text classifier. | -| "`clustering`" | Used for the embeddings run through a | -| | clustering algorithm | - -Usage Example: - - - - {PyEmbeddingCohereUsage} - - \ No newline at end of file diff --git a/docs/integrations/embedding/colpali.mdx b/docs/integrations/embedding/colpali.mdx deleted file mode 100644 index dcd65a0..0000000 --- a/docs/integrations/embedding/colpali.mdx +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: ColPali -sidebarTitle: ColPali ---- - -import { - PyEmbeddingColpaliSetup, - PyEmbeddingColpaliTextSearch, -} from '/snippets/integrations.mdx'; - -We support [ColPali](https://github.com/illuin-tech/colpali) model embeddings for multimodal multi-vector retrieval. ColPali produces multiple embedding vectors per input (multi-vector), enabling more nuanced similarity matching between text queries and image documents. - -Using ColPali requires the colpali-engine package, which can be installed using `pip install colpali-engine`. - - -ColPali produces **multi-vector** embeddings, meaning each input generates multiple embedding vectors rather than a single vector. Use `MultiVector(func.ndims())` instead of `Vector(func.ndims())` when defining your schema. - - -Supported models are: - -- Metric-AI/ColQwen2.5-3b-multilingual-v1.0 (default) -- vidore/colpali-v1.3 -- vidore/colqwen2-v1.0 -- vidore/colSmol-256M - -Supported parameters (to be passed in `create` method) are: - -| Parameter | Type | Default Value | Description | -|---|---|---|---| -| `model_name` | `str` | `"Metric-AI/ColQwen2.5-3b-multilingual-v1.0"` | The name of the model to use. | -| `device` | `str` | `"auto"` | The device for inference. Can be `"auto"`, `"cpu"`, `"cuda"`, or `"mps"`. | -| `dtype` | `str` | `"bfloat16"` | Data type for model weights (bfloat16, float16, float32, float64). | -| `pooling_strategy` | `str` | `"hierarchical"` | Token pooling strategy: `"hierarchical"`, `"lambda"`, or `None`. | -| `pool_factor` | `int` | `2` | Factor to reduce sequence length when pooling is enabled. | -| `batch_size` | `int` | `2` | Batch size for processing inputs. | -| `quantization_config` | `Optional[BitsAndBytesConfig]` | `None` | Quantization configuration for the model (requires bitsandbytes). | - -This embedding function supports ingesting images as both bytes and URLs. You can query them using text. - - - - {PyEmbeddingColpaliSetup} - - - -Now we can search using text queries: - - - - {PyEmbeddingColpaliTextSearch} - - - diff --git a/docs/integrations/embedding/gemini.mdx b/docs/integrations/embedding/gemini.mdx deleted file mode 100644 index e3359ec..0000000 --- a/docs/integrations/embedding/gemini.mdx +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Gemini -sidebarTitle: Gemini ---- - -import { PyEmbeddingGeminiUsage } from '/snippets/integrations.mdx'; - -With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide. -The Gemini Embedding Model API supports various task types: - -| Task Type | Description | -|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| "`retrieval_query`" | Specifies the given text is a query in a search/retrieval setting. | -| "`retrieval_document`" | Specifies the given text is a document in a search/retrieval setting. Using this task type requires a title but is automatically provided by Embeddings API | -| "`semantic_similarity`" | Specifies the given text will be used for Semantic Textual Similarity (STS). | -| "`classification`" | Specifies that the embeddings will be used for classification. | -| "`clustering`" | Specifies that the embeddings will be used for clustering. | - - -Usage Example: - - - - {PyEmbeddingGeminiUsage} - - \ No newline at end of file diff --git a/docs/integrations/embedding/huggingface.mdx b/docs/integrations/embedding/huggingface.mdx deleted file mode 100644 index 69cf855..0000000 --- a/docs/integrations/embedding/huggingface.mdx +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: Hugging Face -sidebarTitle: Hugging Face ---- - -import { PyEmbeddingHuggingfaceUsage } from '/snippets/integrations.mdx'; - -We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. - -Example usage: - - - - {PyEmbeddingHuggingfaceUsage} - - \ No newline at end of file diff --git a/docs/integrations/embedding/ibm.mdx b/docs/integrations/embedding/ibm.mdx deleted file mode 100644 index cec6c32..0000000 --- a/docs/integrations/embedding/ibm.mdx +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: IBM watsonx -sidebarTitle: IBM WatsonX ---- - -import { PyEmbeddingIbmUsage } from '/snippets/integrations.mdx'; - -Generate text embeddings using IBM's watsonx.ai platform. - -## Supported Models {#supported-models} - -You can find a list of supported models at [IBM watsonx.ai Documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). The currently supported model names are: - -| Model ID | Dimensions | -|----------------------------------------------|------------| -| `ibm/granite-embedding-278m-multilingual` | 768 | -| `ibm/slate-125m-english-rtrvr-v2` | 768 | -| `ibm/slate-30m-english-rtrvr-v2` | 384 | -| `intfloat/multilingual-e5-large` | 1024 | -| `sentence-transformers/all-minilm-l6-v2` | 384 | - - -For new tables, `ibm/granite-embedding-278m-multilingual` is the recommended default. Older model IDs (such as `ibm/slate-125m-english-rtrvr` and `sentence-transformers/all-minilm-l12-v2`) remain resolvable for tables whose stored metadata references them, but they are no longer advertised for new use. - - -## Parameters {#parameters} - -The following parameters can be passed to the `create` method: - -| Parameter | Type | Default Value | Description | -|------------|----------|----------------------------------|-----------------------------------------------------------| -| name | str | `"ibm/slate-125m-english-rtrvr"` | The model ID of the watsonx.ai model to use. Pass one of the current supported IDs above (e.g. `"ibm/granite-embedding-278m-multilingual"`) when creating new tables. | -| api_key | str | None | Optional IBM Cloud API key (or set `WATSONX_API_KEY`) | -| project_id | str | None | Optional watsonx project ID (or set `WATSONX_PROJECT_ID`). Mutually exclusive with `space_id`. | -| space_id | str | None | Optional watsonx deployment space ID (or set `WATSONX_SPACE_ID`). Mutually exclusive with `project_id`. | -| url | str | None | Optional custom URL for the watsonx.ai instance | -| params | dict | None | Optional additional parameters for the embedding model (e.g. `{"truncate_input_tokens": 512}`) | - - -You must supply exactly one of `project_id` or `space_id` (either as an argument or via its environment variable). Setting both, or neither, raises a `ValueError`. - - -## Usage Example {#usage-example} - -First, the watsonx.ai library is an optional dependency, so must be installed separately: - -``` -pip install ibm-watsonx-ai -``` - -Optionally set environment variables (if not passing credentials to `create` directly): - -```sh -export WATSONX_API_KEY="YOUR_WATSONX_API_KEY" -# Provide exactly one of the following: -export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID" -export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID" -``` - - - - {PyEmbeddingIbmUsage} - - diff --git a/docs/integrations/embedding/imagebind.mdx b/docs/integrations/embedding/imagebind.mdx deleted file mode 100644 index 4010de6..0000000 --- a/docs/integrations/embedding/imagebind.mdx +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: ImageBind -sidebarTitle: ImageBind ---- - -import { - PyEmbeddingImagebindSetup, - PyEmbeddingImagebindImageSearch, - PyEmbeddingImagebindAudioSearch, - PyEmbeddingImagebindTextSearch, -} from '/snippets/integrations.mdx'; - -We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`. - -This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data): - -| Parameter | Type | Default Value | Description | -|---|---|---|---| -| `name` | `str` | `"imagebind_huge"` | Name of the model. | -| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. | -| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. | - -Below is an example demonstrating how the API works: - - - - {PyEmbeddingImagebindSetup} - - - -Now, we can search using any modality: - -#### image search {#image-search} - - - {PyEmbeddingImagebindImageSearch} - - -#### audio search {#audio-search} - - - - {PyEmbeddingImagebindAudioSearch} - - -#### Text search {#text-search} -You can add any input query and fetch the result as follows: - - - {PyEmbeddingImagebindTextSearch} - - - -If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues). diff --git a/docs/integrations/embedding/instructor.mdx b/docs/integrations/embedding/instructor.mdx deleted file mode 100644 index aea4d44..0000000 --- a/docs/integrations/embedding/instructor.mdx +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Instructor -sidebarTitle: Instructor ---- - -import { PyEmbeddingInstructorUsage } from '/snippets/integrations.mdx'; - -[Instructor](https://instructor-embedding.github.io/) is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning. - -If you want to calculate customized embeddings for specific sentences, you can follow the unified template to write instructions. - - -Represent the `domain` `text_type` for `task_objective`: - -- `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc. -- `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc. -- `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc. - - -More information about the model can be found at the [source URL](https://github.com/xlang-ai/instructor-embedding). - -| Argument | Type | Default | Description | -|---|---|---|---| -| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use | -| `batch_size` | `int` | `32` | The batch size to use when generating embeddings | -| `device` | `str` | `"cpu"` | The device to use when generating embeddings | -| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings | -| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings | -| `quantize` | `bool` | `False` | Whether to quantize the model | -| `source_instruction` | `str` | `"represent the document for retrieval"` | The instruction for the source column | -| `query_instruction` | `str` | `"represent the document for retrieving the most similar documents"` | The instruction for the query | - - - - - - {PyEmbeddingInstructorUsage} - - \ No newline at end of file diff --git a/docs/integrations/embedding/jina.mdx b/docs/integrations/embedding/jina.mdx deleted file mode 100644 index 7045718..0000000 --- a/docs/integrations/embedding/jina.mdx +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Jina -sidebarTitle: Jina ---- - -import { - PyEmbeddingJinaText, - PyEmbeddingJinaMultimodal, -} from '/snippets/integrations.mdx'; - -## Text Embedding Models {#text-embedding-models} - -Jina embeddings are used to generate embeddings for text and image data. -You also need to set the `JINA_API_KEY` environment variable to use the Jina API. - -You can find a list of supported models under [https://jina.ai/embeddings/](https://jina.ai/embeddings/) - -Supported parameters (to be passed in `create` method) are: - -| Parameter | Type | Default Value | Description | -|---|---|---|---| -| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use | - -Usage Example: - - - - {PyEmbeddingJinaText} - - - -## Multimodal Embedding Models {#multimodal-embedding-models} - -Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list -under [https://jina.ai/embeddings/](https://jina.ai/embeddings/) - -Supported parameters (to be passed in `create` method) are: - -| Parameter | Type | Default Value | Description | -|---|---|---|---| -| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use | - -Usage Example: - - - - {PyEmbeddingJinaMultimodal} - - diff --git a/docs/integrations/embedding/ollama.mdx b/docs/integrations/embedding/ollama.mdx deleted file mode 100644 index 120cde8..0000000 --- a/docs/integrations/embedding/ollama.mdx +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: Ollama -sidebarTitle: Ollama ---- - -import { PyEmbeddingOllamaUsage } from '/snippets/integrations.mdx'; - -Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details: - -- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings) -- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models) - -| Parameter | Type | Default Value | Description | -|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| `name` | `str` | `nomic-embed-text` | The name of the model. | -| `host` | `str` | `http://localhost:11434` | The Ollama host to connect to. | -| `options` | `ollama.Options` or `dict` | `None` | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. | -| `keep_alive` | `float` or `str` | `"5m"` | Controls how long the model will stay loaded into memory following the request. | -| `ollama_client_kwargs` | `dict` | `{}` | kwargs that can be past to the `ollama.Client`. | - - - - {PyEmbeddingOllamaUsage} - - diff --git a/docs/integrations/embedding/openai.mdx b/docs/integrations/embedding/openai.mdx deleted file mode 100644 index cdd60d7..0000000 --- a/docs/integrations/embedding/openai.mdx +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: OpenAI -sidebarTitle: OpenAI ---- - -import { PyEmbeddingOpenaiBasic } from '/snippets/integrations.mdx'; - -LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances: - -| Parameter | Type | Default Value | Description | -|---|---|---|---| -| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. | -| `dim` | `int` | Model default | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it | -| `use_azure` | bool | `False` | Set true to use Azure OpenAI SDK | - - - - {PyEmbeddingOpenaiBasic} - - \ No newline at end of file diff --git a/docs/integrations/embedding/openclip.mdx b/docs/integrations/embedding/openclip.mdx deleted file mode 100644 index 118bb26..0000000 --- a/docs/integrations/embedding/openclip.mdx +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: OpenCLIP -sidebarTitle: OpenCLIP ---- - -import { - PyEmbeddingOpenclipSetup, - PyEmbeddingOpenclipTextSearch, - PyEmbeddingOpenclipImageSearch, -} from '/snippets/integrations.mdx'; - -We support CLIP model embeddings using the open source alternative, [open-clip](https://github.com/mlfoundations/open_clip) which supports various customizations. It is registered as `open-clip` and supports the following customizations: - -| Parameter | Type | Default Value | Description | -|---|---|---|---| -| `name` | `str` | `"ViT-B-32"` | The name of the model. | -| `pretrained` | `str` | `"laion2b_s34b_b79k"` | The name of the pretrained model to load. | -| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. | -| `batch_size` | `int` | `64` | The number of images to process in a batch. | -| `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. | - -This embedding function supports ingesting images as both bytes and urls. You can query them using both text and other images. - - -LanceDB supports ingesting images directly from accessible links. - - - - - {PyEmbeddingOpenclipSetup} - - -Now we can search using text from both the default vector column and the custom vector column - - - {PyEmbeddingOpenclipTextSearch} - - - -Because we're using a multimodal embedding function, we can also search using images - - - - {PyEmbeddingOpenclipImageSearch} - - diff --git a/docs/integrations/embedding/sentence-transformers.mdx b/docs/integrations/embedding/sentence-transformers.mdx deleted file mode 100644 index aa0c9d1..0000000 --- a/docs/integrations/embedding/sentence-transformers.mdx +++ /dev/null @@ -1,169 +0,0 @@ ---- -title: Sentence Transformers -sidebarTitle: Sentence Transformers ---- - -import { PyEmbeddingSentenceTransformersBaai } from '/snippets/integrations.mdx'; - -Allows you to set parameters when registering a `sentence-transformers` object. - - -Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search. - - - -The `trust_remote_code` parameter defaults to `True`, which allows models to execute arbitrary code from their Hugging Face repository during loading. If you are loading untrusted models, set `trust_remote_code=False` to prevent remote code execution. - - -| Parameter | Type | Default Value | Description | -|---|---|---|---| -| `name` | `str` | `all-MiniLM-L6-v2` | The name of the model | -| `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) | -| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model | -| `trust_remote_code` | `bool` | `True` | Whether to trust and execute remote code from the model's Huggingface repository | - - -
-```markdown - - sentence-transformers/all-MiniLM-L12-v2 - - sentence-transformers/paraphrase-mpnet-base-v2 - - sentence-transformers/gtr-t5-base - - sentence-transformers/LaBSE - - sentence-transformers/all-MiniLM-L6-v2 - - sentence-transformers/bert-base-nli-max-tokens - - sentence-transformers/bert-base-nli-mean-tokens - - sentence-transformers/bert-base-nli-stsb-mean-tokens - - sentence-transformers/bert-base-wikipedia-sections-mean-tokens - - sentence-transformers/bert-large-nli-cls-token - - sentence-transformers/bert-large-nli-max-tokens - - sentence-transformers/bert-large-nli-mean-tokens - - sentence-transformers/bert-large-nli-stsb-mean-tokens - - sentence-transformers/distilbert-base-nli-max-tokens - - sentence-transformers/distilbert-base-nli-mean-tokens - - sentence-transformers/distilbert-base-nli-stsb-mean-tokens - - sentence-transformers/distilroberta-base-msmarco-v1 - - sentence-transformers/distilroberta-base-msmarco-v2 - - sentence-transformers/nli-bert-base-cls-pooling - - sentence-transformers/nli-bert-base-max-pooling - - sentence-transformers/nli-bert-base - - sentence-transformers/nli-bert-large-cls-pooling - - sentence-transformers/nli-bert-large-max-pooling - - sentence-transformers/nli-bert-large - - sentence-transformers/nli-distilbert-base-max-pooling - - sentence-transformers/nli-distilbert-base - - sentence-transformers/nli-roberta-base - - sentence-transformers/nli-roberta-large - - sentence-transformers/roberta-base-nli-mean-tokens - - sentence-transformers/roberta-base-nli-stsb-mean-tokens - - sentence-transformers/roberta-large-nli-mean-tokens - - sentence-transformers/roberta-large-nli-stsb-mean-tokens - - sentence-transformers/stsb-bert-base - - sentence-transformers/stsb-bert-large - - sentence-transformers/stsb-distilbert-base - - sentence-transformers/stsb-roberta-base - - sentence-transformers/stsb-roberta-large - - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens - - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens - - sentence-transformers/xlm-r-base-en-ko-nli-ststb - - sentence-transformers/xlm-r-bert-base-nli-mean-tokens - - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens - - sentence-transformers/xlm-r-large-en-ko-nli-ststb - - sentence-transformers/bert-base-nli-cls-token - - sentence-transformers/all-distilroberta-v1 - - sentence-transformers/multi-qa-MiniLM-L6-dot-v1 - - sentence-transformers/multi-qa-distilbert-cos-v1 - - sentence-transformers/multi-qa-distilbert-dot-v1 - - sentence-transformers/multi-qa-mpnet-base-cos-v1 - - sentence-transformers/multi-qa-mpnet-base-dot-v1 - - sentence-transformers/nli-distilroberta-base-v2 - - sentence-transformers/all-MiniLM-L6-v1 - - sentence-transformers/all-mpnet-base-v1 - - sentence-transformers/all-mpnet-base-v2 - - sentence-transformers/all-roberta-large-v1 - - sentence-transformers/allenai-specter - - sentence-transformers/average_word_embeddings_glove.6B.300d - - sentence-transformers/average_word_embeddings_glove.840B.300d - - sentence-transformers/average_word_embeddings_komninos - - sentence-transformers/average_word_embeddings_levy_dependency - - sentence-transformers/clip-ViT-B-32-multilingual-v1 - - sentence-transformers/clip-ViT-B-32 - - sentence-transformers/distilbert-base-nli-stsb-quora-ranking - - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking - - sentence-transformers/distilroberta-base-paraphrase-v1 - - sentence-transformers/distiluse-base-multilingual-cased-v1 - - sentence-transformers/distiluse-base-multilingual-cased-v2 - - sentence-transformers/distiluse-base-multilingual-cased - - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base - - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base - - sentence-transformers/facebook-dpr-question_encoder-multiset-base - - sentence-transformers/facebook-dpr-question_encoder-single-nq-base - - sentence-transformers/gtr-t5-large - - sentence-transformers/gtr-t5-xl - - sentence-transformers/gtr-t5-xxl - - sentence-transformers/msmarco-MiniLM-L-12-v3 - - sentence-transformers/msmarco-MiniLM-L-6-v3 - - sentence-transformers/msmarco-MiniLM-L12-cos-v5 - - sentence-transformers/msmarco-MiniLM-L6-cos-v5 - - sentence-transformers/msmarco-bert-base-dot-v5 - - sentence-transformers/msmarco-bert-co-condensor - - sentence-transformers/msmarco-distilbert-base-dot-prod-v3 - - sentence-transformers/msmarco-distilbert-base-tas-b - - sentence-transformers/msmarco-distilbert-base-v2 - - sentence-transformers/msmarco-distilbert-base-v3 - - sentence-transformers/msmarco-distilbert-base-v4 - - sentence-transformers/msmarco-distilbert-cos-v5 - - sentence-transformers/msmarco-distilbert-dot-v5 - - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned - - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch - - sentence-transformers/msmarco-distilroberta-base-v2 - - sentence-transformers/msmarco-roberta-base-ance-firstp - - sentence-transformers/msmarco-roberta-base-v2 - - sentence-transformers/msmarco-roberta-base-v3 - - sentence-transformers/multi-qa-MiniLM-L6-cos-v1 - - sentence-transformers/nli-mpnet-base-v2 - - sentence-transformers/nli-roberta-base-v2 - - sentence-transformers/nq-distilbert-base-v1 - - sentence-transformers/paraphrase-MiniLM-L12-v2 - - sentence-transformers/paraphrase-MiniLM-L3-v2 - - sentence-transformers/paraphrase-MiniLM-L6-v2 - - sentence-transformers/paraphrase-TinyBERT-L6-v2 - - sentence-transformers/paraphrase-albert-base-v2 - - sentence-transformers/paraphrase-albert-small-v2 - - sentence-transformers/paraphrase-distilroberta-base-v1 - - sentence-transformers/paraphrase-distilroberta-base-v2 - - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 - - sentence-transformers/paraphrase-multilingual-mpnet-base-v2 - - sentence-transformers/paraphrase-xlm-r-multilingual-v1 - - sentence-transformers/quora-distilbert-base - - sentence-transformers/quora-distilbert-multilingual - - sentence-transformers/sentence-t5-base - - sentence-transformers/sentence-t5-large - - sentence-transformers/sentence-t5-xxl - - sentence-transformers/sentence-t5-xl - - sentence-transformers/stsb-distilroberta-base-v2 - - sentence-transformers/stsb-mpnet-base-v2 - - sentence-transformers/stsb-roberta-base-v2 - - sentence-transformers/stsb-xlm-r-multilingual - - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1 - - sentence-transformers/clip-ViT-L-14 - - sentence-transformers/clip-ViT-B-16 - - sentence-transformers/use-cmlm-multilingual - - sentence-transformers/all-MiniLM-L12-v1 -``` -
- - -You can also load many other model architectures from the library. For example models from sources such as BAAI, Nomic, Salesforce Research, etc. See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers). - - - -Here is an example that uses the BAAI embedding model from the Hugging Face Hub [supported models](https://huggingface.co/models?library=sentence-transformers). - - - - {PyEmbeddingSentenceTransformersBaai} - - - -Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models. - diff --git a/docs/integrations/embedding/superlinked.mdx b/docs/integrations/embedding/superlinked.mdx deleted file mode 100644 index f357962..0000000 --- a/docs/integrations/embedding/superlinked.mdx +++ /dev/null @@ -1,140 +0,0 @@ ---- -title: Superlinked -sidebarTitle: Superlinked ---- - -[Superlinked](https://superlinked.com) is a self-hosted inference engine (SIE) for embedding, reranking, and extraction. The `sie-lancedb` package registers SIE as a first-class embedding function in LanceDB's embeddings registry, so embeddings are computed automatically on insert and search. You need a running SIE instance - see the [Superlinked quickstart](https://superlinked.com/docs) for deployment options. - -## Installation {#installation} - - -```bash Python icon=Python -pip install sie-lancedb -``` - -```bash TypeScript icon=js -npm install @superlinked/sie-lancedb @lancedb/lancedb -``` - - -## Registered functions {#registered-functions} - -Importing `sie_lancedb` registers two embedding functions in LanceDB's registry: - -| Name | Purpose | -|---|---| -| `"sie"` | Dense text embeddings | -| `"sie-multivector"` | ColBERT-style late interaction with MaxSim scoring | - -Supported parameters on `.create()`: - -| Parameter | Type | Description | -|---|---|---| -| `model` | `str` | Any of 85+ SIE-supported models (e.g. `BAAI/bge-m3`, `NovaSearch/stella_en_400M_v5`, `jinaai/jina-colbert-v2`) | -| `base_url` | `str` | URL of the SIE endpoint (e.g. `http://localhost:8080`) | - -## Usage {#usage} - -```py Python icon=Python -import lancedb -from lancedb.embeddings import get_registry -from lancedb.pydantic import LanceModel, Vector -import sie_lancedb # registers "sie" and "sie-multivector" - -sie = get_registry().get("sie").create( - model="BAAI/bge-m3", - base_url="http://localhost:8080", -) - -class Documents(LanceModel): - text: str = sie.SourceField() - vector: Vector(sie.ndims()) = sie.VectorField() - -db = lancedb.connect("~/.lancedb") -table = db.create_table("docs", schema=Documents, mode="overwrite") - -table.add([ - {"text": "Machine learning is a subset of AI."}, - {"text": "Neural networks use multiple layers."}, - {"text": "Python is popular for ML development."}, -]) - -results = table.search("What is deep learning?").limit(3).to_list() -``` - -LanceDB handles embedding generation for both inserts and queries automatically, based on the `SourceField` / `VectorField` declarations on the schema. - -## Hybrid search with reranker {#hybrid-search-with-reranker} - -`SIEReranker` plugs into LanceDB's hybrid search pipeline. It uses SIE's cross-encoder `score()` to rerank combined vector + full-text search results. You need a full-text search index on the column first: - -```py Python icon=Python -from sie_lancedb import SIEReranker - -# Create FTS index for hybrid search -table.create_fts_index("text", replace=True) - -results = ( - table.search("What is deep learning?", query_type="hybrid") - .rerank(SIEReranker(model="jinaai/jina-reranker-v2-base-multilingual")) - .limit(5) - .to_list() -) - -for r in results: - print(f"{r['_relevance_score']:.3f} {r['text']}") -``` - -The reranker also works with pure vector or pure FTS search via `.rerank()`. - -## ColBERT and multivector {#colbert-and-multivector} -`SIEMultiVectorEmbeddingFunction` (registered as `"sie-multivector"`) works with LanceDB's native `MultiVector` type and MaxSim scoring for ColBERT and ColPali models: - -```py Python icon=Python -from lancedb.pydantic import MultiVector - -sie_colbert = get_registry().get("sie-multivector").create( - model="jinaai/jina-colbert-v2", - base_url="http://localhost:8080", -) - -class ColBERTDocs(LanceModel): - text: str = sie_colbert.SourceField() - vector: MultiVector(sie_colbert.ndims()) = sie_colbert.VectorField() - -table = db.create_table("colbert_docs", schema=ColBERTDocs, mode="overwrite") -table.add([{"text": "Machine learning is a subset of AI."}]) - -# MaxSim search - query and document multivectors compared token-by-token -results = table.search("What is ML?").limit(5).to_list() -``` - -## Entity extraction {#entity-extraction} - -`SIEExtractor` adds entity extraction to LanceDB's data-enrichment workflows. Extract entities from a text column and merge the results back as a structured Arrow column - enabling filtered search on extracted entities: - -```py Python icon=Python -from sie_lancedb import SIEExtractor - -extractor = SIEExtractor( - base_url="http://localhost:8080", - model="urchade/gliner_multi-v2.1", -) - -extractor.enrich_table( - table, - source_column="text", - target_column="entities", - labels=["person", "technology", "organization"], - id_column="id", -) -``` - -The `entities` column stores structured Arrow data (`list>`), so you can filter on extracted entities in queries. - -## Links {#links} - -- [`sie-lancedb` on PyPI](https://pypi.org/project/sie-lancedb/) -- [`@superlinked/sie-lancedb` on npm](https://www.npmjs.com/package/@superlinked/sie-lancedb) -- [Superlinked on GitHub](https://github.com/superlinked/sie) -- [Superlinked docs](https://superlinked.com/docs) diff --git a/docs/integrations/embedding/voyageai.mdx b/docs/integrations/embedding/voyageai.mdx deleted file mode 100644 index 54d155d..0000000 --- a/docs/integrations/embedding/voyageai.mdx +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: VoyageAI -sidebarTitle: VoyageAI ---- - -import { PyEmbeddingVoyageaiUsage, PyEmbeddingVoyageaiMultimodal } from '/snippets/integrations.mdx'; - -Voyage AI provides cutting-edge embedding and rerankers. - - -Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. -You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API. - -Supported models are: - -- voyage-4-large (best retrieval quality, 1024 default dimensions, supports 256/512/1024/2048) -- voyage-4 (balanced general-purpose, 1024 default dimensions, supports 256/512/1024/2048) -- voyage-4-lite (optimized for latency/cost, 1024 default dimensions, supports 256/512/1024/2048) -- voyage-context-3 -- voyage-3.5 -- voyage-3.5-lite -- voyage-3 -- voyage-3-lite -- voyage-finance-2 -- voyage-multilingual-2 -- voyage-law-2 -- voyage-code-2 -- voyage-multimodal-3.5 (multimodal - supports text, images, and video) - - -**Multimodal Model:** `voyage-multimodal-3.5` supports text, images, and video inputs. It outputs 1024-dimensional embeddings by default, configurable via the `output_dimension` parameter (256, 512, 1024, 2048). See the [VoyageAI multimodal embeddings documentation](https://docs.voyageai.com/docs/multimodal-embeddings) for more details. - - -Supported parameters (to be passed in `create` method) are: - -| Parameter | Type | Default Value | Description | -|---|---|--------|---------| -| `name` | `str` | `None` | The model ID of the model to use. Supported models: voyage-4-large, voyage-4, voyage-4-lite, voyage-3, voyage-3-lite, voyage-3.5, voyage-3.5-lite, voyage-context-3, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2, voyage-multimodal-3.5 | -| `input_type` | `str` | `None` | Type of the input text. Default to None. Other options: query, document. | -| `truncation` | `bool` | `True` | Whether to truncate the input texts to fit within the context length. | -| `output_dimension` | `int` | `None` | Output embedding dimension. Only supported by `voyage-multimodal-3.5`. Valid options: 256, 512, 1024 (default), 2048. | - - -Usage Example: - - - - {PyEmbeddingVoyageaiUsage} - - - -### Multimodal Example {#multimodal-example} - -The `voyage-multimodal-3.5` model can embed text alongside images. You can use image URLs, file paths, or PIL Image objects: - - - - {PyEmbeddingVoyageaiMultimodal} - - \ No newline at end of file diff --git a/docs/integrations/index.mdx b/docs/integrations/index.mdx deleted file mode 100644 index 6406286..0000000 --- a/docs/integrations/index.mdx +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: Integrations -sidebarTitle: Overview -description: Connect LanceDB with popular AI providers, frameworks, and data platforms ---- - -LanceDB seamlessly plugs into the rest of your AI and data engineering stack. Use the sections -below to jump straight into the guides that matter for your workflow. - - -| Group | Description | -|:----------------|:-------------| -| [Embedding models](/integrations/embedding/) | Connect with popular embedding model providers including OpenAI, Cohere, Hugging Face, and more | -| [Reranking models](/integrations/reranking/) | Enhance search results with advanced reranking models and techniques | -| [AI platforms & frameworks](/integrations/ai/) | Integrate with LangChain, LlamaIndex, Kiln, and other AI development frameworks | -| [Data platforms & frameworks](/integrations/data/) | Integrate LanceDB with popular data tools and platforms like DuckDB, Pydantic and dlt | - diff --git a/docs/integrations/lerobotdataset.mdx b/docs/integrations/lerobotdataset.mdx deleted file mode 100644 index 33c28cd..0000000 --- a/docs/integrations/lerobotdataset.mdx +++ /dev/null @@ -1,94 +0,0 @@ ---- -title: "LeRobotDataset" -sidebarTitle: "LeRobotDataset" -description: "Use Lance-backed LeRobotDataset loaders and LanceDB to inspect, filter, and train on robotics datasets from the Hugging Face Hub." ---- - -import { - PyFrameworksLerobotFilterFrames, - PyFrameworksLerobotLancedbImageDataset, - PyFrameworksLerobotLancedbVideoDataset, - PyFrameworksLerobotOpenLanceTables, -} from '/snippets/integrations.mdx'; - -[LeRobot](https://huggingface.co/docs/lerobot/index) is Hugging Face's open-source robotics stack for collecting data, training policies, running simulations, and sharing robotics datasets and models on the Hub. - -[LeRobotDataset v3.0](https://huggingface.co/docs/lerobot/lerobot-dataset-v3) standardizes robot learning data across sensorimotor time series, actions, multi-camera video, and task metadata. Its v3 layout stores high-frequency tabular signals in Parquet, visual streams as MP4 shards, and metadata that reconstructs episode-level views from larger files. - -Lance pairs well with LeRobot when you need high-performance random access, lazy multimodal blob reads, and a single table interface for curation, search, and training data preparation. The `lerobot-lancedb` package ships Lance-backed `LeRobotDataset` subclasses, and LanceDB can open Lance-formatted LeRobot datasets on the Hub directly through `hf://` URIs. - -## Install {#install} - -```bash -pip install lancedb lance lerobot-lancedb -``` - -## Use Lance-backed LeRobotDataset loaders {#use-lance-backed-lerobotdataset-loaders} - -`LeRobotLanceDataset` is useful when your Lance-backed dataset stores decoded image observations. It's a drop-in replacement for `LeRobotDataset`, so existing policy training code keeps working with the usual PyTorch dataset and dataloader patterns. - - - {PyFrameworksLerobotLancedbImageDataset} - - -For datasets that store camera observations as MP4 video segments, use `LeRobotLanceVideoDataset` instead. - - - {PyFrameworksLerobotLancedbVideoDataset} - - - -Use the image loader for Lance-backed repos that store image frames. Use the video loader for MP4-backed LeRobot datasets such as `lance-format/lerobot-pusht-lance`. - - -## Open LeRobot Lance tables with LanceDB {#open-lerobot-lance-tables-with-lancedb} - -Lance-formatted LeRobot datasets published by `lance-format` expose each `.lance` file under `data/` as a LanceDB table. The PushT dataset, for example, has `frames`, `episodes`, and `videos` tables. - - - {PyFrameworksLerobotOpenLanceTables} - - -Opening the tables directly is handy for inspecting schemas, counting rows, sampling metadata, or building curation workflows before any data reaches the training loop. - -## Filter a frame window {#filter-a-frame-window} - -Most robotics workflows want a deterministic slice by `episode_index`, `frame_index`, or task metadata long before training begins. LanceDB filters those rows without touching the video blobs. - - - {PyFrameworksLerobotFilterFrames} - - -With the filtered set in hand, you can materialize a smaller local LanceDB database, add derived columns, attach embeddings, or build vector and scalar indexes for faster repeated access. - -## Example Lance-formatted LeRobot datasets {#example-lance-formatted-lerobot-datasets} - - - - A Lance-formatted version of `lerobot/pusht` with frame, episode, and video tables. - - - A multi-camera robotics dataset packaged as Lance tables for frame-level and episode-level access. - - - -## More resources {#more-resources} - - - - Hugging Face's guide to the v3 dataset layout, streaming, transforms, and migration. - - - API documentation for the Lance-backed LeRobotDataset implementations. - - - -## When to use each interface {#when-to-use-each-interface} - -| Interface | Best for | -|:---|:---| -| `LeRobotDataset` | Standard LeRobot training loops and policy code | -| `LeRobotLanceDataset` | Drop-in training on Lance-backed image datasets | -| `LeRobotLanceVideoDataset` | Drop-in training on Lance-backed video datasets | -| LanceDB | Interactive inspection, filtering, curation, search, indexing, and materializing subsets | -| `lance.dataset(...)` | Lower-level schema, fragment, index, and blob access | diff --git a/docs/integrations/reranking/answerdotai.mdx b/docs/integrations/reranking/answerdotai.mdx deleted file mode 100644 index e3f500d..0000000 --- a/docs/integrations/reranking/answerdotai.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Answer.AI Rerankers -sidebarTitle: "Answer.AI" -description: Use AnswerDotAI's lightweight reranking library with LanceDB. Features unified API for common reranking models, configurable model selection, and comprehensive scoring options. - ---- - -import { PyRerankingAnswerdotaiUsage } from '/snippets/integrations.mdx'; - -# Answer.AI Rerankers - -This integration uses [AnswersDotAI's rerankers](https://github.com/AnswerDotAI/rerankers) to rerank the search results, providing a lightweight, low-dependency, unified API to use all common reranking and cross-encoder models. - -> **Note:** Supported query types – Hybrid, Vector, and FTS. - - - - - {PyRerankingAnswerdotaiUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `model_type` | `str` | `"colbert"` | The type of model to use. Supported model types can be found here: https://github.com/AnswerDotAI/rerankers. | -| `model_name` | `str` | `"answerdotai/answerai-colbert-small-v1"` | The name of the reranker model to use. | -| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. | - - - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | - -### Vector Search {#vector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | - -### FTS Search {#fts-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | diff --git a/docs/integrations/reranking/cohere.mdx b/docs/integrations/reranking/cohere.mdx deleted file mode 100644 index e484f3e..0000000 --- a/docs/integrations/reranking/cohere.mdx +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: Cohere Reranker -sidebarTitle: "Cohere" -description: Integrate Cohere's powerful reranking API with LanceDB for enhanced search results. Supports English and multilingual models with configurable scoring options for vector, FTS, and hybrid search. - ---- - -import { PyRerankingCohereUsage } from '/snippets/integrations.mdx'; - -# Cohere Reranker - -This reranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this reranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this reranker. - - -> **Note:** Supported query types – Hybrid, Vector, and FTS. - -```shell -pip install cohere -``` - - - - {PyRerankingCohereUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `model_name` | `str` | `"rerank-english-v2.0"` | The name of the reranker model to use. Available cohere models are: rerank-english-v2.0, rerank-multilingual-v2.0 | -| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. | -| `top_n` | `str` | `None` | The number of results to return. If None, will return all results. | -| `api_key` | `str` | `None` | The API key for the Cohere API. If not provided, the `COHERE_API_KEY` environment variable is used. | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type | - - - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) | - -### Vector Search {#vector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column | -| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) | - -### FTS Search {#fts-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`) | diff --git a/docs/integrations/reranking/colbert.mdx b/docs/integrations/reranking/colbert.mdx deleted file mode 100644 index 463b3d7..0000000 --- a/docs/integrations/reranking/colbert.mdx +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: ColBERT Reranker -sidebarTitle: "ColBERT" -description: Enhance search results with ColBERT's contextual reranking in LanceDB. Features efficient model deployment, device optimization, and flexible scoring options for vector, FTS, and hybrid search. - ---- - -import { PyRerankingColbertUsage } from '/snippets/integrations.mdx'; - -# ColBERT Reranker - -This reranker uses ColBERT model to rerank the search results. You can use this reranker by passing `ColbertReranker()` to the `rerank()` method. -> **Note:** Supported query types – Hybrid, Vector, and FTS. - - - - - {PyRerankingColbertUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `model_name` | `str` | `"colbert-ir/colbertv2.0"` | The name of the reranker model to use.| -| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. | -| `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. | - - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | - -### Vector Search {#vector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | - -### FTS Search {#fts-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | diff --git a/docs/integrations/reranking/jina.mdx b/docs/integrations/reranking/jina.mdx deleted file mode 100644 index 4624149..0000000 --- a/docs/integrations/reranking/jina.mdx +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: Jina Reranker -sidebarTitle: "Jina AI" -description: Integrate Jina's multilingual reranking API with LanceDB for improved search results. Features model selection, API key management, and flexible scoring options for all search types. - ---- - -import { PyRerankingJinaUsage } from '/snippets/integrations.mdx'; - -# Jina Reranker - -This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this reranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this reranker. - - -> **Note:** Supported query types – Hybrid, Vector, and FTS. - - - - - {PyRerankingJinaUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `model_name` | `str` | `"jina-reranker-v2-base-multilingual"` | The name of the reranker model to use. You can find the list of available models in https://jina.ai/reranker. | -| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. | -| `top_n` | `str` | `None` | The number of results to return. If None, will return all results. | -| `api_key` | `str` | `None` | The API key for the Jina API. If not provided, the `JINA_API_KEY` environment variable is used. | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. | - - - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | - -### Vector Search {#vector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | - -### FTS Search {#fts-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | diff --git a/docs/integrations/reranking/openai.mdx b/docs/integrations/reranking/openai.mdx deleted file mode 100644 index 413f95e..0000000 --- a/docs/integrations/reranking/openai.mdx +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: OpenAI Reranker (Experimental) -sidebarTitle: "OpenAI" -description: Explore experimental search reranking using OpenAI's GPT models in LanceDB. Features configurable model selection, API key management, and comprehensive scoring options for all search types. - ---- - -import { PyRerankingOpenaiUsage } from '/snippets/integrations.mdx'; - -# OpenAI Reranker (Experimental) - -This reranker uses OpenAI chat model to rerank the search results. You can use this reranker by passing `OpenAI()` to the `rerank()` method. -> **Note:** Supported query types – Hybrid, Vector, and FTS. - -> **Warning:** This reranker is experimental. OpenAI does not have a dedicated reranking model, so it uses a chat model under the hood. - - - - {PyRerankingOpenaiUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `model_name` | `str` | `"gpt-4-turbo-preview"` | The name of the reranker model to use.| -| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. | -| `api_key` | `str` | `None` | The API key to use. If None, will use the OPENAI_API_KEY environment variable. - - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | - -### Vector Search {#vector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | - -### FTS Search {#fts-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | diff --git a/docs/integrations/reranking/voyageai.mdx b/docs/integrations/reranking/voyageai.mdx deleted file mode 100644 index 62e9a16..0000000 --- a/docs/integrations/reranking/voyageai.mdx +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: VoyageAI Reranker -sidebarTitle: "Voyage AI" -description: Integrate VoyageAI's cutting-edge reranking models with LanceDB. Features model selection, API key management, and comprehensive scoring options for all search types. - ---- - -import { PyRerankingVoyageaiUsage } from '/snippets/integrations.mdx'; - -# VoyageAI Reranker - -Voyage AI provides cutting-edge embedding and rerankers. - -This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this reranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this reranker. - - -> **Note:** Supported query types – Hybrid, Vector, and FTS. - - - - - {PyRerankingVoyageaiUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `model_name` | `str` | `None` | The name of the reranker model to use. Available models are: rerank-2, rerank-2-lite | -| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. | -| `top_n` | `str` | `None` | The number of results to return. If None, will return all results. | -| `api_key` | `str` | `None` | The API key for the Voyage AI API. If not provided, the `VOYAGE_API_KEY` environment variable is used. | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type | -| `truncation` | `bool` | `None` | Whether to truncate the input to satisfy the "context length limit" on the query and the documents. | - - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column | -| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) | - -### Vector Search {#vector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column | -| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) | - -### FTS Search {#fts-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column | -| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) | diff --git a/docs/integrations/reranking/watsonx.mdx b/docs/integrations/reranking/watsonx.mdx deleted file mode 100644 index 775af4f..0000000 --- a/docs/integrations/reranking/watsonx.mdx +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: Watsonx Reranker -sidebarTitle: "IBM watsonx.ai" -description: Rerank LanceDB search results with the IBM watsonx.ai text rerank API. Supports vector, FTS, and hybrid search with configurable models, projects, and spaces. - ---- - -import { PyRerankingWatsonxUsage } from '/snippets/integrations.mdx'; - -# Watsonx Reranker - -This reranker uses the [IBM watsonx.ai](https://cloud.ibm.com/docs/apis/watsonx-ai#text-rerank) text rerank API to reorder search results. Pass `WatsonxReranker()` to the `rerank()` method on a query. Credentials come from the `WATSONX_API_KEY` and `WATSONX_PROJECT_ID` (or `WATSONX_SPACE_ID`) environment variables, or can be passed explicitly as arguments. - -> **Note:** Supported query types – Hybrid, Vector, and FTS. - -```shell -pip install ibm-watsonx-ai -``` - - - - {PyRerankingWatsonxUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `model_name` | `str` | `"cross-encoder/ms-marco-minilm-l-12-v2"` | The rerank model ID. See [supported rerank models](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx#rerank). | -| `column` | `str` | `"text"` | The name of the table column to use as document input. | -| `top_n` | `int` | `None` | The number of results to return. If `None`, all results are returned. | -| `return_score` | `str` | `"relevance"` | Options are `"relevance"` or `"all"`. Controls which score columns are kept on the result. | -| `api_key` | `str` | `None` | IBM Cloud API key. Falls back to the `WATSONX_API_KEY` environment variable. | -| `project_id` | `str` | `None` | watsonx.ai project ID. Falls back to `WATSONX_PROJECT_ID`. Mutually exclusive with `space_id`. | -| `space_id` | `str` | `None` | watsonx.ai deployment space ID. Falls back to `WATSONX_SPACE_ID`. Mutually exclusive with `project_id`. | -| `url` | `str` | `"https://us-south.ml.cloud.ibm.com"` | watsonx.ai service URL. | -| `truncate_input_tokens` | `int` | `None` | Truncate each document to this many tokens before scoring. | - - -You must supply exactly one of `project_id` or `space_id` (either as an argument or via its environment variable). Setting both, or neither, raises a `ValueError`. - - -## Supported scores for each query type {#supported-scores-for-each-query-type} - -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | - -### Vector Search {#vector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | - -### FTS Search {#fts-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | diff --git a/docs/integrations/stable-worldmodel.mdx b/docs/integrations/stable-worldmodel.mdx deleted file mode 100644 index ee35e21..0000000 --- a/docs/integrations/stable-worldmodel.mdx +++ /dev/null @@ -1,107 +0,0 @@ ---- -title: "Stable World Model" -sidebarTitle: "Stable World Model" -description: "Use Stable World Model with LanceDB-backed datasets for reproducible world model research, fast data loading, and compact storage." ---- - -import { - PyFrameworksStableWorldmodelCollectLance, - PyFrameworksStableWorldmodelConvert, - PyFrameworksStableWorldmodelEvaluate, - PyFrameworksStableWorldmodelLoadLance, -} from '/snippets/integrations.mdx'; - -[Stable World Model](https://github.com/galilai-group/stable-worldmodel) is a research platform for collecting data, training world models, and evaluating policies with model-predictive control across standardized environments. - -The LanceDB integration is built into Stable World Model's data format registry. Lance is the default backend for collected datasets, so a path ending in `.lance` gives you an append-friendly LanceDB table with episode-contiguous rows and fast indexed reads. - -Random access speed is the bottleneck for world model training, since the loop repeatedly samples temporal windows from high-dimensional observations, actions, and rewards. The faster those windows arrive, the more GPU time goes into training rather than waiting on the data loader. - -## Install {#install} - -```bash -pip install stable-worldmodel -``` - -Datasets and checkpoints are stored under `$STABLEWM_HOME`, which defaults to `~/.stable_worldmodel/`. - -## Collect data into Lance {#collect-data-into-lance} - -Stable World Model uses Lance by default when you collect to a `.lance` path. -Replace `your_expert_policy` with the expert or scripted policy you use to collect demonstrations. - - - {PyFrameworksStableWorldmodelCollectLance} - - -Every writer accepts a `mode` argument such as `append`, `overwrite`, or `error`. The default is append, so re-running collection extends the existing dataset. - -## Load a Lance dataset for training {#load-a-lance-dataset-for-training} - -The dataset loader autodetects the Lance format from the path. - - - {PyFrameworksStableWorldmodelLoadLance} - - -Your model code stays focused on the world model objective while LanceDB handles the storage layout and read path. - -## Evaluate with model-predictive control {#evaluate-with-model-predictive-control} - -After training a world model on the Lance-backed dataset, Stable World Model can evaluate it with planning solvers such as CEM. -Replace `world_model` with the trained model object from your training loop. - - - {PyFrameworksStableWorldmodelEvaluate} - - -## Convert between formats {#convert-between-formats} - -Stable World Model can convert between registered dataset formats. A common workflow is to collect in Lance for fast training reads, then export to the video layout for compact inspection artifacts. - - - {PyFrameworksStableWorldmodelConvert} - - -## Throughput {#throughput} - -The Stable World Model README reports the following PushT benchmark results from `scripts/benchmark/compare_h5_lance.py`: - -| Format | Source | Cache | samples/s | ms/step | -|:---|:---|:---|---:|---:| -| HDF5 | local | no-cache | 1,416.1 | 45.2 | -| HDF5 | local | cached | 1,474.0 | 43.4 | -| LanceDB | local | no-cache | 4,814.8 | 13.3 | -| LanceDB | local | cached | 4,431.3 | 14.4 | -| Video | local | - | 1,330.6 | 48.1 | -| LanceDB | s3 | no-cache | 3,183.7 | 20.1 | -| LanceDB | s3 | cached | 3,253.2 | 19.7 | -| HDF5 | s3 | no-cache | 9.1 | 7,032.5 | -| HDF5 | s3 | cached | 756.5 | 84.6 | - -In that benchmark, local LanceDB reached about **3.4x** the no-cache throughput of local HDF5, while S3-backed LanceDB reached about **350x** the no-cache throughput of S3-backed HDF5. Even with cache enabled, S3-backed LanceDB was about **4.3x** faster than S3-backed HDF5. - -These numbers come from the Stable World Model project's own benchmark setup, so they're best read as a reproducible directional baseline that may shift across environments, models, and storage configurations. - -## Storage {#storage} - -The same README reports these local storage sizes for the benchmark dataset: - -| Format | Local size | -|:---|---:| -| HDF5 | 43.12 GB | -| LanceDB | 13.31 GB | -| Video | 496.29 MB | - -LanceDB used about **69% less local storage than HDF5** in the reported benchmark, while preserving a table interface built for fast training reads and append-heavy collection. - -## More resources {#more-resources} - - - - Installation, quick start, supported formats, benchmarks, environments, solvers, and citation. - - - Full upstream documentation with tutorials, API references, and guides. - - diff --git a/docs/lance.mdx b/docs/lance.mdx deleted file mode 100644 index d6db5f1..0000000 --- a/docs/lance.mdx +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: "Lance format" -sidebarTitle: "Lance format" -description: "Open-source lakehouse format for multimodal AI." -icon: "/static/assets/logo/lance-logo-gray.svg" ---- - -[Lance](https://lance.org/) is an open-source, columnar lakehouse format for multimodal AI. -It provides a file format, table format, and lightweight catalog spec, allowing developers -to build a complete open lakehouse on top of object storage. - -Building on top of open foundations and optimizing the format for random access -(without compromising scan performance) enables -high-performance vector search, full-text search, indexing, and feature engineering capabilities. -[LanceDB](/enterprise) builds on these capabilities so teams can work with one multimodal data layer -instead of moving data across separate storage, search, feature, and training systems. - - - Visit the Lance format documentation to learn more about its design, features, and how it enables the multimodal lakehouse. - - -## Capabilities of the Lance format {#capabilities-of-the-lance-format} - -Capability | What it enables ---- | --- -Multimodal storage | Store images, video, audio, text, embeddings, annotations, metadata, features, and more, all in one table. -First-class blob API | Store large binary objects such as images, video, audio, and model artifacts in blob columns with lazy reads and streaming byte access. -Fast random access and scans | Sample, shuffle, and retrieve individual rows efficiently without giving up high-throughput sequential reads. -Flexible data evolution | Add, drop, rename, or alter columns as datasets change, often without rewriting existing data files. -Versioned tables | Reproduce experiments, restore previous states, and tie downstream artifacts to the exact table version they used. -Hybrid search and indexing | Combine vector search, full-text search, and scalar filters on the same dataset with Lance indexes. -Open lakehouse interoperability | Build on object storage and connect Lance tables to open engines such as PyTorch, Ray, Spark, Trino, DuckDB and Polars. - -## Key concepts {#key-concepts} - -The following concepts are core to the Lance format: - - - - **Arrow-native, columnar storage** and **interoperability** with the open lakehouse ecosystem (including other file formats and compute engines). - - - **Zero-copy** data evolution, meaning you can easily add derived columns (like features or embeddings) at a later time, **without full table rewrites**. Only new data is written; expensive existing data (like images/videos) remain untouched. - - - Data is **versioned**, with each insert operation creating a new version of the dataset and an update to the manifest that tracks versions via metadata - - - - -### Data versioning {#data-versioning} - -Data in Lance tables are versioned -- this helps keep LanceDB scalable and consistent. -We do not immediately blow away old versions when creating new ones because other clients might be -in the middle of querying the old version. It's important to retain older versions for as long as they -might be queried. - -Each version contains metadata and just the new/updated data in your transaction. So if you have 100 -versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead -of a single version, which can result in slower queries. - -### Data compaction {#data-compaction} - -As you insert more data, your dataset will grow and you'll need to perform compaction to maintain query -throughput (i.e., keep latencies down to a minimum). Compaction is the process of merging fragments -together to reduce the amount of metadata that needs to be managed, and to reduce the number of files -that need to be opened while scanning the dataset. - -Running compaction on a Lance dataset will do the following: - -- Remove deleted rows from fragments -- Remove dropped columns from fragments -- Merge small fragments into larger ones - -Compaction focuses on read performance, not immediate disk reclamation. During compaction, Lance writes -new compacted files while older files are still referenced by previous table versions. This means disk -usage can increase temporarily until old versions are cleaned up. - -### Data deletion and recovery {#data-deletion-and-recovery} - -Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. -It simply marks the row as deleted in the `DataFile` that represents a fragment. - -For a given version of the dataset, each fragment can have up to one deletion file (if no rows were ever -deleted from that fragment, it will not have a deletion file). This is important to keep in mind because -it means that the data is still there, and can be recovered if needed, as long as that version still -exists based on your backup policy. - - - Lance is a separate open source project. Check out its documentation to learn more. - diff --git a/docs/namespaces/index.mdx b/docs/namespaces/index.mdx deleted file mode 100644 index e7f4adc..0000000 --- a/docs/namespaces/index.mdx +++ /dev/null @@ -1,226 +0,0 @@ ---- -title: "Namespaces and the Catalog Model" -sidebarTitle: "Overview" -description: "Understand LanceDB as a catalog-level abstraction over Lance's table format, and learn how namespaces help organize Lance tables." -icon: "sitemap" -keywords: ["namespace", "catalog", "lance format", "table format", "lancedb"] ---- - -Despite its name, LanceDB is not a "database" in the traditional sense -- it is a **Multimodal Lakehouse** that builds on the table abstraction, -similar to many other lakehouse projects. LanceDB exposes a catalog-level abstraction over the Lance table format, via a *namespace spec*. -If you're coming from traditional databases or lakehouses, you can think of a namespace as the catalog path that says where a table name lives. - -Lance provides the **file** and **table** formats to store and manage your data and indexes. -LanceDB operates at the catalog layer (used to organize, discover, and operate on many Lance tables) -and provides a compute engine on top of the Lance format. - -This is why many SDK methods in LanceDB, like `create_table`, `open_table`, `drop_table`, and -`rename_table`, accept namespace input. The SDK methods expose that input in the idiom of each -language: Python uses `namespace_path`, Rust uses builder methods like `.namespace(...)`, and -TypeScript uses `namespacePath` arguments. - -## Namespace hierarchy {#namespace-hierarchy} - -Namespaces are generalizations of catalog specs that give platform developers a clean way to present Lance tables in the structures users expect. The diagram below shows how the hierarchy can go beyond a single level. -A namespace can contain a collection of tables, and it can also contain namespaces recursively. - -![](/static/assets/images/namespaces/lance-namespace.png) - -Before diving into examples, it helps to keep two terms in mind: the **namespace client** is the abstraction that presents a consistent namespace API, while the **namespace implementation** is the concrete backend that resolves namespaces and table locations (for example, a local directory or an external catalog). -If you want to go deeper, see the Lance format [namespace documentation](https://lance.org/format/namespace/). - -## Namespace paths and names {#namespace-paths-and-names} - -A namespace path is a list of components. For example, `["prod", "search"]` means the `search` -namespace inside the `prod` namespace. The empty path, `[]`, means the root namespace. - -Each component is a name, not a filesystem path segment. Namespace names can't be empty, and each -component can contain only letters, numbers, underscores, hyphens, and periods. That keeps the same -identifier usable across local directory namespaces and REST namespace identifiers. - -## Directory namespaces {#directory-namespaces} - -The simplest namespace model in LanceDB is a single root namespace, often represented by one -directory: - -```bash -./local_lancedb (root) -└── prod - └── search - └── user (table) - └── data (table) -``` - -As a user of LanceDB OSS, you might never notice namespaces at first, because LanceDB exposes the single-level hierarchy shown above, with the data stored in the `data/` directory, where the root namespace is implicit. Connecting to this namespace is as simple as connecting to the catalog root: - - -```python Python icon="python" -import lancedb - -# Connect to the directory namespace root -db = lancedb.connect("./local_lancedb") -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; - -// Connect to the directory namespace root -const db = await lancedb.connect("./local_lancedb"); -``` - -```rust Rust icon="rust" -use lancedb::connect; - -// Connect to the directory namespace root -let db = connect("./local_lancedb").execute().await?; -``` - - -This creates the default namespace directory (`data/`) under the specified root path. - -You can also explicitly connect to a namespace using `lancedb.connect_namespace(...)` with the directory namespace implementation: - - -```python Python icon="python" -import lancedb - -# Local namespace-backed catalog root (DirectoryNamespace) -# See https://lance.org/format/namespace/dir/catalog-spec/ -db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"}) - -table_name = "user" -data = [{"id": 1, "vector": [0.1, 0.2], "name": "alice"}] - -table = db.create_table(table_name, data=data, mode="create") -print(f"Created table: {table.name}") -# Created table: user -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; - -// Local namespace-backed catalog root (DirectoryNamespace) -// See https://lance.org/format/namespace/dir/catalog-spec/ -const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" }); - -const table = await db.createTable( - "user", - [{ id: 1, vector: [0.1, 0.2], name: "alice" }], - { mode: "create" }, -); -console.log(`Created table: ${table.name}`); -// Created table: user -``` - -```rust Rust icon="rust" -use std::collections::HashMap; -use std::sync::Arc; -use arrow_schema::{DataType, Field, Schema}; - -// Local namespace-backed catalog root (DirectoryNamespace) -// See https://lance.org/format/namespace/dir/catalog-spec/ -let mut properties = HashMap::new(); -properties.insert("root".to_string(), "./local_lancedb".to_string()); -let db = lancedb::connect_namespace("dir", properties) - .execute() - .await?; - -let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), -])); -let table = db - .create_empty_table("user", schema) - .execute() - .await?; -println!("Created table: {}", table.name()); -// Created table: user -``` - - - -- For simple use cases in LanceDB OSS, you don't need to go too deep into namespaces. -- To integrate LanceDB with external catalogs and to use it as a true **multimodal lakehouse**, it's useful to understand the different namespace implementations and how to use them in your organization's setup. - - -## Remote or external catalog namespaces {#remote-or-external-catalog-namespaces} - -The example above showed local directory-based namespaces. LanceDB also supports namespaces backed by remote object stores and external catalogs, via the REST namespace implementation. - -For remote object stores with central metadata/catalog services (either commercial or open source), -use the REST namespace implementation. It is backed by REST routes -(for example `POST /v1/namespace/{id}/create` and `GET /v1/namespace/{id}/list`) and server-provided table locations. - -For authentication, any property prefixed with `headers` is forwarded as an HTTP header -(for example `headers.Authorization` becomes `Authorization`, and `headers.X-API-Key` becomes `X-API-Key`). -LanceDB Enterprise REST requests use the `x-api-key` header for API-key authentication. Deployments -that route multiple databases through the same endpoint can also use headers such as -`x-lancedb-database` or `x-lancedb-database-prefix` for database context. - - -```python Python icon="python" -import os -import lancedb - -# Remote namespace-backed catalog root (RestNamespace) -# See https://lance.org/format/namespace/rest/catalog-spec/ -db = lancedb.connect_namespace( - "rest", - { - "uri": "https://.internal..com", - "headers.x-api-key": os.environ["API_KEY"], - # or: - # "headers.Authorization": f"Bearer {os.environ['REST_AUTH_TOKEN']}", - }, -) -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; - -// Remote namespace-backed catalog root (RestNamespace) -// See https://lance.org/format/namespace/rest/catalog-spec/ -const db = await lancedb.connectNamespace("rest", { - uri: "https://.internal..com", - headers: { - "x-api-key": process.env.API_KEY ?? "", - // or: - // Authorization: `Bearer ${process.env.REST_AUTH_TOKEN}`, - }, -}); -``` - -```rust Rust icon="rust" -use std::collections::HashMap; - -// Remote namespace-backed catalog root (RestNamespace) -// See https://lance.org/format/namespace/rest/catalog-spec/ -let mut properties = HashMap::new(); -properties.insert( - "uri".to_string(), - "https://.internal..com".to_string(), -); -properties.insert( - "headers.x-api-key".to_string(), - std::env::var("API_KEY")?, -); -// or: -// properties.insert( -// "headers.Authorization".to_string(), -// format!("Bearer {}", std::env::var("REST_AUTH_TOKEN")?), -// ); -let db = lancedb::connect_namespace("rest", properties) - .execute() - .await?; -``` - -[LanceDB Enterprise](/enterprise) operates a REST namespace server on top of the Lance format, so any REST client that can speak the REST namespace API -contract can be used to interact with it. For authentication examples in LanceDB Enterprise, visit -the [Namespaces in SDKs](/namespaces/usage#namespaces-in-lancedb-enterprise) page. - -## Best practices {#best-practices} - -Below, we list some best practices for working with namespaces: -- For simple use cases and single, stand-alone applications, the directory-based root namespace is sufficient and requires no special configuration. -- For remote storage locations, introduce explicit namespaces when multiple teams, environments, or domains share the same catalog. -- Treat namespace paths as stable identifiers (for example `"prod/search"`, `"staging/recs"`). -- For maintainability reasons, avoid hard-coding object-store table paths in application code -- instead, prefer catalog identifiers + namespaces. diff --git a/docs/namespaces/usage.mdx b/docs/namespaces/usage.mdx deleted file mode 100644 index 5c9096c..0000000 --- a/docs/namespaces/usage.mdx +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: "Using Namespaces" -sidebarTitle: "Using namespaces" -description: "Use LanceDB's namespace-aware table and catalog APIs in Python, TypeScript, and Rust." -icon: "folder-tree" -keywords: ["namespace", "create_table", "open_table", "list_tables", "catalog"] ---- - -import { - PyNamespaceTableOps, - PyNamespaceAdminOps, - TsNamespaceTableOps, - TsNamespaceAdminOps, - RsNamespaceAdminOps, - RsNamespaceTableOps, -} from '/snippets/connection.mdx'; - -As your table organization needs grow over time and your projects become more complex, you can use namespaces to organize your tables in a way that reflects your business domains, teams, or environments. - -As described in the [Namespaces and Catalog Model](/namespaces) section, namespaces are LanceDB's way of generalizing catalog specs, providing developers a clean way to manage hierarchical organization of tables in the catalog. The SDKs treat a namespace as a path and can use it for table resolution when you use LanceDB outside the root namespace. - -## Table operations with namespace paths {#table-operations-with-namespace-paths} - -Let's imagine a scenario where your table management needs have evolved, and you now have the following multi-level structure to organize your tables outside the root namespace. -``` -./local_lancedb (root) -└── prod - └── search - └── user (table) - └── data (table) - └── recommendations - └── user (table) - └── data (data) -``` - -Below, we show how you would express table operations within that namespace. Each item in the namespace -list (`["prod", "search"]`) represents a level in the namespace hierarchy, and the table name is -specified when you create, open, list, or drop it. - -The SDK methods expose the namespace path in the idiom of each language: - -- Python: pass `namespace_path=["prod", "search"]` to table operations. -- Rust: call builder methods such as `.namespace(vec!["prod".to_string(), "search".to_string()])`. -- TypeScript: pass a `namespacePath` array, for example `await db.openTable("user", ["prod", "search"])`. - - - - {PyNamespaceTableOps} - - - - {TsNamespaceTableOps} - - - - {RsNamespaceTableOps} - - - - -Using namespaces is **optional** in LanceDB, and most basic use cases do not require to work with them. -An empty namespace (`[]`), which is the default, means "root namespace", and the data will be stored in -the `data/` directory under the specified root path. - - -## Namespace management APIs {#namespace-management-apis} - -You can open/create/drop tables inside a namespace path (like `["prod", "search"]`). -All three SDKs expose namespace lifecycle operations directly. -In Python, use `lancedb.connect_namespace(...)` when calling namespace lifecycle methods such as -`create_namespace`, `list_namespaces`, `describe_namespace`, and `drop_namespace`. -In TypeScript, use `lancedb.connectNamespace(...)` and call `createNamespace`, `listNamespaces`, -`describeNamespace`, and `dropNamespace` on the returned `Connection`. -In Rust, use `lancedb::connect_namespace(...)` and call `create_namespace`, `list_namespaces`, -and `drop_namespace`. - - - - {PyNamespaceAdminOps} - - - - {TsNamespaceAdminOps} - - - - {RsNamespaceAdminOps} - - - -Namespace creation and deletion have modes that control what happens when the target already exists, -doesn't exist, or contains data: - -- Create mode: `create` fails if the namespace already exists, `exist_ok` keeps the existing namespace, and `overwrite` replaces it. -- Drop mode: `fail` reports an error when the namespace doesn't exist, and `skip` treats a missing namespace as a successful no-op. -- Drop behavior: `restrict` keeps non-empty namespaces from being dropped, and `cascade` drops child namespaces and tables first. - -Namespace path components can't be empty. Each component can contain only letters, numbers, -underscores, hyphens, and periods. - -Listing APIs return the immediate children of the requested namespace path. Use `limit` with the -returned `page_token` to page through large catalogs; pass an empty namespace path (`[]`) when you -want to list from the root namespace. - -## Namespaces in LanceDB Enterprise {#namespaces-in-lancedb-enterprise} - -In LanceDB Enterprise deployments, configure namespace-backed federated databases in a TOML file under your deployment's `config` directory. -LanceDB Enterprise supports both directory-based (`ns_impl = "dir"`) and REST-based (`ns_impl = "rest"`) namespace implementations. -The example below shows how to configure a directory-based namespace implementation in LanceDB Enterprise. -```toml -# Federated database configuration for DirectoryNamespace -# This example uses minio storage -[federated_dbs.federated_dir_test] -ns_impl = "dir" -root = "s3:///" -"storage.region" = "us-east-1" -"storage.endpoint" = "http://localhost:9000" -"storage.access_key_id" = "minioadmin" -"storage.secret_access_key" = "minioadmin" -"storage.allow_http" = "true" -# Far future expiration (year 2100) -"storage.expires_at_millis" = "4102444800000" -``` - -The example above uses MinIO, but the same approach applies to other cloud object storage platforms based on your deployment. - -For REST-based namespace servers, you can specify the namespace implementation as `"rest"` with forwarding prefixed headers -for authentication and context propagation. - -```toml -[federated_dbs.federated_rest_test] -ns_impl = "rest" -uri = "http://.internal.catalog.com" -forward_header_prefixes = ["X-forward"] -``` - -With `forward_header_prefixes = ["X-forward"]`, any incoming header starting with `X-forward` is forwarded to -`http://.internal.catalog.com`. This is useful for auth propagation, for example sending -`X-forward-authorization: Bearer xxxx`. - -For the LanceDB REST API itself, requests use `x-api-key` for API-key authentication. If your endpoint -serves more than one database, LanceDB can also use headers such as `x-lancedb-database` or -`x-lancedb-database-prefix` to route the request to the right database context. - -## Related references {#related-references} - -- [Client SDK API references](/api-reference) -- [REST API Reference](/api-reference/rest) -- [Namespaces and the Catalog Model](/namespaces) diff --git a/docs/performance.mdx b/docs/performance.mdx deleted file mode 100644 index 0d54f7e..0000000 --- a/docs/performance.mdx +++ /dev/null @@ -1,241 +0,0 @@ ---- -title: "Performance Tips and Best Practices" -sidebarTitle: "Performance Tips" -description: "Optimize LanceDB for your workload across ingestion, indexing, querying, and maintenance." -icon: "gauge-high" -keywords: ["performance", "tuning", "best practices", "optimization", "ingestion", "indexing", "vector search", "filtering", "compaction", "oss", "enterprise"] ---- - -LanceDB is performant by default. This page covers performance best practices that matter when you want to ensure you get the right performance for a specific workload. Use the table below to jump to the area relevant to what you're working on. - -| When you're working on... | Read | -|---------------------------|------| -| Loading data into a table | [Ingestion](#ingestion) | -| Running filtered or vector queries at scale | [Indexing](#indexing) | -| Iterating over large result sets (training, export, migration) | [Querying](#querying) | -| Keeping a long-lived dataset healthy | [Maintenance](#maintenance) | -| Inspecting query plans | [Diagnostics](#diagnostics) | - - -When using Python with multiprocessing, use `spawn` rather than `fork`. LanceDB is multi-threaded internally, and `fork` plus a multi-threaded process is unsafe. - - -## Ingestion {#ingestion} - -If ingestion is taking longer than expected on a large dataset, the cause is almost always how `add()` is called: each call commits a new version and a new fragment, so a per-row loop pays that per-call overhead at every row. The best practice is to pick the ingestion mode that matches your data shape — bulk ingestion when the data is already materialized, or iterator ingestion when it's streamed or computed on the fly. - - -**Why `merge_insert()` is significantly slower than `add()`** - -A merge has to scan existing data to find matches on the join key (or look them up via a scalar index, if one exists), and then delete-and-reinsert any updated rows in a single transaction; `add()` simply appends new fragments. - -Use `add()` for pure appends, and reach for `merge_insert()` only when you need upsert or conditional-insert logic. When you do use it, build a scalar index on the join column first — otherwise the match step falls back to a full column scan, which is the dominant cost at scale. - - -### Bulk ingestion: for data you already have {#bulk-ingestion-for-data-you-already-have} - -For materialized inputs (Arrow Tables, DataFrames) and file-backed sources (`pyarrow.dataset(...)`), LanceDB auto-parallelizes the write across workers, estimating the partition count from the data size — more partitions means more concurrent writes and higher throughput, up to the CPU core count. - -```python Python icon="python" -table.add(arrow_table) # in-memory -table.add(df) # pandas -table.add(ds.dataset("data/", format="parquet")) # streams from disk, still parallelized -``` - -Pass `progress=True` to watch it happen: LanceDB shows a live tqdm bar with rows written, throughput in MB/s, and active worker count. - -```python Python icon="python" -table.add(ds.dataset("data/", format="parquet"), progress=True) -``` - -```text -71%|███████▏ | 710000/1000000 [00:12<00:05, 58.8kit/s, 42.3 MB/s | 8/8 workers] -``` - -For larger-than-memory data, prefer scanning a file-backed dataset (`ds.dataset(...)`) over a hand-built `pyarrow.RecordBatchReader`: a `Dataset` can be counted and rescanned, so LanceDB knows the row count upfront (better auto-parallelism) and can retry a failed write from the start — a reader can only be consumed once, so neither is possible. Reach for the iterator path below only when the data genuinely can't be backed by files, e.g. to apply custom data transformations as you ingest. - -For very large initial loads, create the table empty first; passing data directly to `create_table(name, data)` skips the auto-parallel path. - -### Iterator ingestion: for data you transform on the fly {#iterator-ingestion-for-data-you-transform-on-the-fly} - -If each row needs work before it can be written — applying a custom transformation as you ingest, for example — the data doesn't exist as a file you can point `ds.dataset(...)` at. The best practice is to pass an iterator of `pyarrow.RecordBatch` instead; LanceDB consumes one batch at a time as you produce them. - -```python Python icon="python" -def stream(): - for raw in source: - vectors = model.encode(raw["text"]) - yield pa.RecordBatch.from_pydict({**raw, "vector": vectors}) - -table.add(stream()) -``` - -Use decently large chunks of several thousand rows or more, rather than yielding single-row batches. - - -**Set `write_parallelism` yourself for large inputs** - -A reader can't be counted or rescanned the way a `Dataset` can (see above), so LanceDB can't auto-size parallelism for it — set `write_parallelism` explicitly for large inputs: - -```python Python icon="python" -table.add(stream(), write_parallelism=4) -``` - -Each partition becomes its own fragment, so don't over-allocate on a small input — budget one unit of parallelism per ~100K rows or ~1 GB of data as a rule of thumb. - - -### Bulk ingests into a remote table {#bulk-ingests-into-a-remote-table} - -Enterprise - -This section applies only to remote tables (`db://` connections). Embedded (local) writes are unaffected. - -When you write to a remote LanceDB Enterprise table, the client splits each write partition into one or more HTTP `insert` parts and uploads them under a single upload id. Each part is streamed rather than buffered, so peak memory stays bounded. But each part still has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as: - -```text -lancedb.remote.errors.HttpError: operation timed out -``` - -Two environment variables control how parts are cut. Both are picked up automatically by the Python and TypeScript clients, and are the only way to tune this from those SDKs: - -| Variable | Default | What it controls | -|----------|---------|------------------| -| `LANCE_CLIENT_MAX_BYTES_PER_REQUEST` | 8 GiB | Maximum size of a single insert part, in LZ4-compressed Arrow IPC bytes. Set to `0` to disable splitting and send one request per partition. | -| `LANCE_CLIENT_MAX_REQUEST_DURATION` | Half the read timeout | Maximum wall-clock time, in whole seconds, that any one insert part may stay open. Set to `0` to disable the time-based cut. | - -A part is cut when it hits either limit, whichever comes first. Lower the byte budget when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor. - -```bash -export LANCE_CLIENT_MAX_BYTES_PER_REQUEST=1073741824 # 1 GiB -export LANCE_CLIENT_MAX_REQUEST_DURATION=120 # 2 minutes -``` - -Splitting into more parts does not change the final table. The server stages every part under the shared upload id and merges them atomically when the write completes. - -## Indexing {#indexing} - -### Vector indexes {#vector-indexes} - -If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), the cause is the default brute-force scan over every vector. That works fine below ~100K vectors, but past that you should build a dedicated vector index. Pick the type by your data shape: - -| Index | When to use | -|-------|-------------| -| `IVF_PQ` | General-purpose default; what Enterprise builds automatically. Higher accuracy than RQ at small dimensions (≤ 256). | -| `IVF_RQ` | Maximum compression on high-dim vectors, faster builds than PQ. | -| `IVF_HNSW_SQ` | Best recall/latency for unfiltered search; higher latency variance under selective filters. | -| `IVF_FLAT` | Required for binary vectors with `hamming`. | - -The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). For parameter tuning, see [Vector Indexing](/indexing/vector-index). - -### Scalar indexes {#scalar-indexes} - -If filtered queries slow down as the table grows — even when the filter is selective — the cause is a full column scan: without a scalar index, LanceDB evaluates the `where(...)` predicate on every row, and the same applies to `merge_insert` join keys. The best practice is to build a scalar index on every column you filter or join on, picking the type by the column's shape: - -| Index | Best for | -|-------|----------| -| `BTREE` (default) | Numeric, string, temporal columns with mostly distinct values | -| `BITMAP` | Boolean and low-cardinality columns (< ~1,000 distinct values) | -| `LABEL_LIST` | `List` columns queried with `array_has_any` / `array_has_all` | - -See [Scalar Indexing](/indexing/scalar-index). - -### Full-text search {#full-text-search} - -If your full-text index is much larger than expected, or takes longer than expected to build, the cause is usually phrase-query flags being enabled when they aren't needed: `with_position=True` and `remove_stop_words=False` both significantly inflate index size and build time. The best practice is to keep the defaults for most workloads, and only enable those flags when you actually need to search for phrases. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options. - -## Compaction and cleanup {#compaction-and-cleanup} - -Two things accumulate on a long-lived table as more and more data gets added to it: - -- **Many small fragments build up as you write**, slowing down queries that have to scan across more files. **Compaction** merges them back into larger fragments. -- **Old versions build up as the table changes**, growing disk usage beyond the live data size (LanceDB retains them for time-travel and rollback). **Cleanup** prunes versions older than a retention window. - -In OSS, you run them yourself via `optimize()`, which bundles both into a single call: - -```python Python icon="python" -from datetime import timedelta -table.optimize() # 7-day default retention -table.optimize(cleanup_older_than=timedelta(days=1)) # reclaim space sooner -``` - -The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates — see [Reindexing](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed — rebuild the index after large update batches. - -LanceDB Enterprise handles both compaction and cleanup automatically. - -## Querying {#querying} - -Three knobs materially affect query latency, memory use, and recall. Be deliberate about each one on every query: - - - - If queries return more data than you need or take longer than expected, the cause is usually projecting more columns than necessary, or letting the result count run unbounded. The best practice is to always pass both `select()` and `limit()`: - - ```python Python icon="python" - table.search(emb).select(["id", "title"]).limit(20) - ``` - - - If a query with `prefilter=False` returns fewer than `limit` results — sometimes zero — the cause is that post-filter applies the predicate after the top-k is selected, so only candidates that already passed the search are filtered. Pre-filter is the default and guarantees every result satisfies the predicate. Switch to `prefilter=False` only when fewer-than-`limit` results are acceptable. See [Filtering](/search/filtering). - - - If recall is lower than expected, the cause is that the search-time knobs limit how many candidates the index considers. Adjust based on your index type: - - - Quantized indexes (PQ, RQ, SQ): `refine_factor` pulls extra candidates and re-scores them on full vectors. - - HNSW-backed indexes: `ef` at search time. Start at `1.5 × k`, raise toward `10 × k` if recall is short. - - IVF candidate breadth: `nprobes` is auto-tuned; override only when a selective pre-filter leaves too few neighbors. - - - -For hybrid search, the default `RRFReranker()` combines vector and FTS results into a single ranking via reciprocal rank fusion. See [Hybrid Search](/search/hybrid-search). - -### Avoid materializing the whole table {#avoid-materializing-the-whole-table} - -If you need every row in the table (for training, export, or migration), calling `to_pandas()` or `to_arrow()` will run you out of memory on any non-trivial dataset — both materialize the full table at once. The best practice is to iterate via `table.search(...)` or `table.query(...)`, which work the same way in both LanceDB OSS and Enterprise. - -In OSS, you can also stream batches through the underlying Lance dataset directly — useful when you need filter pushdown or fragment-level parallelism via `ds.scanner(...)`: - -```python Python icon="python" -ds = table.to_lance() -for batch in ds.to_batches(columns=["id", "text"], batch_size=10000): - process(batch) -``` - - -**API disparity between OSS and Enterprise.** - -LanceDB OSS exposes the `to_pandas()`, `to_arrow()`, and `table.to_lance()` methods for direct Lance dataset access. Enterprise's `RemoteTable` exposes none of these. Only `table.search(...)` and `table.query(...)`. In general, it's always best to go through `search()` and `query()` to keep your code portable across both OSS and Enterprise. - - -### Diagnostics {#diagnostics} - -To analyze a slow query, inspect what the query engine actually did and the state of the indexes it touched. These two tools surface that information — use them in this order: - -```python Python icon="python" -print(table.search(emb).where("year > 2000").limit(10).analyze_plan()) -print(table.index_stats("vector_idx")) # num_unindexed_rows should be ~0 -``` - -`analyze_plan()` returns the execution plan with per-stage timings, so you can see where the query actually spent its time. `index_stats()` shows how many rows are still unindexed — you want `num_unindexed_rows` to be ~0. In `analyze_plan`, look for: - -| Plan pattern | Fix | -|--------------|-----| -| `LanceScan` with high `bytes_read` / `iops` | Add a missing index, project columns with `select()`, or check that the dataset has been compacted | -| Multiple sequential filters | Reorder filter conditions | - -[Optimize Query Performance](/search/optimize-queries) walks through a fully worked before/after example, including how `KNNVectorDistance` and `output_batches` change once indexes are in place. - -## Where to go next {#where-to-go-next} - - - - Read execution plans, find the bottleneck. - - - Index types, parameters, and tuning in depth. - - - Pre- vs post-filter, scalar indexes, list columns. - - - Benchmark methodology and reference latency numbers. - - diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx deleted file mode 100644 index a60a347..0000000 --- a/docs/quickstart.mdx +++ /dev/null @@ -1,472 +0,0 @@ ---- -title: Quickstart -sidebarTitle: "Quickstart" -description: "Get started with LanceDB in minutes." -icon: rocket ---- -import { - PyConnect, - PyConnectAsync, - PyConnectEnterpriseQuickstart, - PyConnectObjectStorage, - PyConnectObjectStorageAsync, - RsConnect, - RsConnectEnterpriseQuickstart, - RsConnectObjectStorage, - TsConnect, - TsConnectEnterpriseQuickstart, - TsConnectObjectStorage, -} from '/snippets/connection.mdx'; -import { - PyQuickstartData, - PyQuickstartCreateTable, - PyQuickstartCreateTableAsync, - PyQuickstartAddFeature, - PyQuickstartCurateWithMetadata, - PyQuickstartMultimodalBytes, - PyQuickstartQueryFeature, - PyQuickstartVectorSearch1, - PyQuickstartVectorSearch1Async, - PyQuickstartOutputPandas, - RsQuickstartAddFeature, - RsQuickstartCurateWithMetadata, - RsQuickstartCreateTable, - RsQuickstartData, - RsQuickstartDefineStruct, - RsQuickstartMultimodalBytes, - RsQuickstartQueryFeature, - RsQuickstartVectorSearch1, - TsQuickstartAddFeature, - TsQuickstartCurateWithMetadata, - TsQuickstartCreateTable, - TsQuickstartData, - TsQuickstartMultimodalBytes, - TsQuickstartQueryFeature, - TsQuickstartVectorSearch1, -} from '/snippets/quickstart.mdx'; - -As described in [the landing page](/), LanceDB provides one data layer for -curation, feature engineering, search and retrieval, and model training. Whether you are preparing -training data, building a RAG or agentic retrieval system, reviewing examples, or adding model-generated -features, you'll work with the same underlying table and search primitives. - -Let's get started in just a few steps! - -## 1\. Install LanceDB {#1-install-lancedb} - -Install LanceDB in your client SDK. - - -```bash pip icon="terminal" -pip install lancedb -``` - -```bash uv icon="terminal" -uv add lancedb - -# Or, in an existing virtual environment: -uv pip install lancedb -``` - -```bash TypeScript icon=js -npm install @lancedb/lancedb -``` - -```bash Rust icon=Rust -cargo add lancedb -``` - - -### Python pre-release builds {#python-pre-release-builds} - -To pick up the latest features and bug fixes -before the next stable release, install a pre-release from LanceDB's Fury index. - - -```bash pip icon="terminal" -pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb -``` - -```bash uv icon="terminal" -uv venv -uv pip install --prerelease allow --index https://pypi.fury.io/lancedb/ lancedb - -# To add to pyproject.toml, use: -uv add --prerelease allow --index https://pypi.fury.io/lancedb/ lancedb -``` - - - -Pre-release builds receive the same level of testing as stable releases, but their availability is not guaranteed -for more than 6 months after release. For real-world workloads, we recommend you use the latest stable release -as far as possible. - - - -The default `lancedb` wheel targets `x86-64-haswell` (AVX2 + FMA + F16C). On older x86_64 CPUs without AVX2 -(Intel Sandy Bridge, Ivy Bridge, Westmere; AMD Bulldozer, Piledriver, Steamroller), `import lancedb` crashes -with `Illegal instruction`. - -Install `lancedb-compat` instead. It exposes the same API (`import lancedb` still works), is built at the -`x86-64-v2` baseline, and uses runtime SIMD dispatch to still leverage AVX2, FMA, or AVX-512 when available: - -```bash pip icon="terminal" -pip install lancedb-compat -``` - -The two packages share the same `lancedb/` namespace and conflict at install time, so pick one. To switch, -uninstall the other first (`pip uninstall lancedb && pip install lancedb-compat`). - - -## 2\. Connect to a LanceDB database {#2-connect-to-a-lancedb-database} - -LanceDB supports several URI patterns to connect to a database. - -- A local filesystem path (when using it as an embedded library) -- A `db://...` URI (when using LanceDB Enterprise) -- An object storage URI: `s3://...`, `gs://...`, or `az://...` (when connecting directly from the client SDK) - -### Connect via local directory path {#connect-via-local-directory-path} - -The simplest way to begin is to use LanceDB as an embedded library. Import LanceDB in your -client SDK of choice and point to a local directory path. - - - - {PyConnect} - - - - {PyConnectAsync} - - - - {TsConnect} - - - - { "use lancedb::connect;\n" } - { "\n" } - {RsConnect} - - - - -### Connect via object storage URIs {#connect-via-object-storage-uris} - -You can also connect directly to object storage from the client SDK: - - - - {PyConnectObjectStorage} - - - - {PyConnectObjectStorageAsync} - - - - {TsConnectObjectStorage} - - - - {RsConnectObjectStorage} - - - -For credentials, endpoints, and provider-specific options, see -[Configuring storage](/storage/configuration). - -### Connect to LanceDB Enterprise {#connect-to-lancedb-enterprise} - -If you're using LanceDB Enterprise, you can connect to the remote database using the -`db://` URI along with the API key, region, and cluster endpoint you received from the -LanceDB team. Pass the cluster endpoint via `host_override` so the client routes -requests to your deployment. - - - - { "import lancedb\n\n" } - {PyConnectEnterpriseQuickstart} - - - - { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } - {TsConnectEnterpriseQuickstart} - - - - { "use lancedb::connect;\n\n" } - {RsConnectEnterpriseQuickstart} - - - - -`host_override` is the full URL of your cluster endpoint, including the scheme -(`https://`) and a port if your deployment listens on a non-default one -(e.g. `https://your-enterprise-endpoint.com:443`). If you don't have the -endpoint, [contact the LanceDB team](mailto:contact@lancedb.com). - - -To learn more about `RemoteTable` semantics and how Enterprise differs operationally from -embedded LanceDB, see the [Enterprise overview](/enterprise). - -## 3\. Create a new table {#3-create-a-new-table} - -Let's create a small table of characters from the kingdom of Camelot. Each row stores source text, -metadata, structured fields, and a vector embedding in the same LanceDB table. - - -The embeddings we use in this example are synthetic and for demonstration purposes only. In a real AI -data workflow, you would generate them from text, images, audio, or video using an embedding model of choice. - - -Each row has source text, metadata, structured fields, and a vector: - -```json -{ - "id": "2", - "name": "Merlin", - "role": "Wizard", - "description": "Advisor and prophet with deep magical knowledge.", - "stats": { "strength": 2, "magic": 5, "leadership": 4, "wisdom": 5 }, - "vector": [0.2, 0.9, 0.4, 0.9] -} -``` - -The full raw records are included below: - - - - - {PyQuickstartData} - - - - {TsQuickstartData} - - - - {RsQuickstartDefineStruct} - {RsQuickstartData} - - - - -You can now create a LanceDB table from those records. The code below creates a LanceDB table -with the appropriate schema and ingests the data. - - - - {PyQuickstartCreateTable} - - - - {PyQuickstartCreateTableAsync} - - - - {TsQuickstartCreateTable} - - - - {RsQuickstartCreateTable} - - - -## 4\. Semantic search {#4-semantic-search} - -Search is a useful capability for all kinds of AI data pipelines. Below, we do a vector similarity -search for samples similar to a "_wise magical advisor_" (transforming the natural language query to -an embedding), and project only the columns needed by the next step. - -Search (which requires random access) is a ubiquitous access pattern that appears in many workloads: -whether you're building a RAG or recommendation system, serving agent memory, or curating a training -dataset. - - - - {PyQuickstartVectorSearch1} - - - - {PyQuickstartVectorSearch1Async} - - - - {TsQuickstartVectorSearch1} - - - - {RsQuickstartVectorSearch1} - - - -The example for Python above shows how to convert results to a Polars DataFrame. -Depending on your language, you can collect query results as a list/array of objects or DataFrames -to be used downstream in your application. - - - Use the `to_pandas()` method to convert query results into a Pandas DataFrame. - - {PyQuickstartOutputPandas} - - - -## 5\. Curation {#5-curation} - -Searching for relevant results can be more useful when combined with metadata filters. -In this tiny example, we filter to examples with high `magic` stats. - - - - {PyQuickstartCurateWithMetadata} - - - - {TsQuickstartCurateWithMetadata} - - - - {RsQuickstartCurateWithMetadata} - - - -When working with large datasets, it's common to use the same pattern to filter on quality labels, -train/eval splits, numeric fields, categorical values, timestamp windows, or generated tags and labels. - -## 6\. Add a derived feature {#6-add-a-derived-feature} - -Feature engineering is the process of cleaning up your data and creating new signals that -help your model learn, make better predictions, or your agent retrieve more useful information. -In the example below, we add a `power_score` column from the structured `stats` fields. -Lance supports data evolution, so you can add new columns without rewriting the entire table. - - - - {PyQuickstartAddFeature} - - - - {TsQuickstartAddFeature} - - - - { "use lancedb::table::NewColumnTransform;\n\n" } - {RsQuickstartAddFeature} - - - -Next, you can query a compact view of the new feature: - - - - {PyQuickstartQueryFeature} - - - - {TsQuickstartQueryFeature} - - - - {RsQuickstartQueryFeature} - - - -| name | role | power_score | -| --- | --- | --- | -| King Arthur | King | 3.5 | -| Merlin | Wizard | 4.0 | -| Sir Lancelot | Knight | 3.0 | - -The same workflow is used for data preparation tasks when adding derived features, cached model signals, review scores, or dataset -quality indicators. - -## 7\. Store multimodal data {#7-store-multimodal-data} - -Multimodal data is a first-class citizen in LanceDB. Binary data (image, audio, video, etc.) is -stored as blobs or inline Arrow binary types in a LanceDB column, and they benefit from the same -table operations and data versioning semantics as other data types. All the data is governed -in the same table, so you can search, filter, and retrieve multimodal records together with structured -fields, metadata, and embeddings. - -In this example, the -[`lancedb/magical_kingdom`](https://huggingface.co/datasets/lancedb/magical_kingdom) dataset stores -character images, descriptions, structured stats, image embeddings, and text embeddings together. - -Say we downloaded the image for Sir Lancelot from that dataset locally. You can read the image bytes -in your client SDK and store them in a LanceDB column. The image bytes can be used for downstream tasks -like retrieval, evaluation, or training. - -
- Sir Lancelot from the lancedb/magical_kingdom dataset -
- -These snippets load the local image file and store the bytes in an `image` column: - - - - {PyQuickstartMultimodalBytes} - - - - {TsQuickstartMultimodalBytes} - - - - {RsQuickstartMultimodalBytes} - - - -For more examples, see the [multimodal data](/tables/multimodal) section. - -## Code {#code} - -See the full code for these examples (including helper functions) in the -`quickstart` file for the appropriate client language in the -[files provided in the repo](https://github.com/lancedb/docs/tree/main/tests). - - -## Next steps {#next-steps} -You've learned how to install LanceDB, connect, create one table for AI data, retrieve related -examples, curate with metadata, add a derived feature, and represent multimodal records. These same -primitives apply across the AI data lifecycle, from data preparation and feature engineering to -retrieval, evaluation, and training. - -Continue to the table and search guides to build on this example with schema options, appends, -updates, versioning, indexing, full-text search, hybrid search, and reranking. - - - - Build on this quickstart with table creation, updates, and schema tips. - - - Learn how to build Retrieval-Augmented Generation (RAG) applications using LanceDB. - - - Create vector, full-text, and scalar indexes to speed up queries on larger datasets. - - - Use LanceDB for projected, shuffled, random-access reads in training workflows. - - diff --git a/docs/reranking/cross_encoder.mdx b/docs/reranking/cross_encoder.mdx deleted file mode 100644 index 1bb9660..0000000 --- a/docs/reranking/cross_encoder.mdx +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: Cross Encoder Reranker -sidebarTitle: "Cross Encoder" -description: Implement semantic search reranking in LanceDB using Cross Encoder models. Features configurable model selection, device optimization, and comprehensive scoring options for all search types. - ---- - -import { PyRerankingCrossEncoderUsage } from '/snippets/integrations.mdx'; - -# Cross Encoder Reranker - -This reranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this reranker by passing `CrossEncoderReranker()` to the `rerank()` method. -> **Note:** Supported query types – Hybrid, Vector, and FTS. - - - - - {PyRerankingCrossEncoderUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `model_name` | `str` | `"cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.| -| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. | -| `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". | -| `trust_remote_code` | `bool` | `True` | Passed to Sentence Transformers model loading. Set this to `False` when you only want to load models that do not require custom repository code. | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", returns only `_relevance_score`. If "all" is supported, returns relevance score along with the vector and/or FTS scores depending on query type. | - -The reranker loads the model locally through `sentence-transformers`, so install the local model -runtime dependencies you need, such as PyTorch and any device-specific acceleration packages. - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | - -### Vector Search {#vector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | - -### FTS Search {#fts-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | diff --git a/docs/reranking/custom-reranker.mdx b/docs/reranking/custom-reranker.mdx deleted file mode 100644 index f19108b..0000000 --- a/docs/reranking/custom-reranker.mdx +++ /dev/null @@ -1,134 +0,0 @@ ---- -title: Building Custom Rerankers -sidebarTitle: "Custom rerankers" -description: Learn how to create custom rerankers in LanceDB by extending the base Reranker class. -icon: "code" ---- - -You can build your own custom reranker in LanceDB by subclassing the base `Reranker` class. At a -minimum, you need to implement `rerank_hybrid()`, which is the logic that combines vector and -full-text search results. Beyond that, you can optionally implement `rerank_vector()` and -`rerank_fts()` if you want your reranker to also handle pure vector or pure full-text searches. - -Decide up front which surfaces — hybrid, pure vector, or pure full-text — your reranker should -cover, and only override the ones you need. The base class leaves `rerank_vector()` and -`rerank_fts()` unimplemented, so calling `.rerank(...)` on a single-modality search you haven't -overridden raises `NotImplementedError` rather than silently returning unsorted results. That's a -useful guard, but worth knowing about before you wire up a query path you didn't plan for. - -The Python base class exposes hybrid, vector-only, and FTS-only rerank hooks. TypeScript and Rust -currently expose the custom reranker interface for hybrid reranking. In Rust, a custom reranker must -also satisfy the trait bounds `Debug + Send + Sync`. - -## Interface {#interface} - -The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the -results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates -the results and removes the duplicates without taking the scores into consideration. It only keeps the -first copy of the row encountered. This works well in cases that don't require the scores of semantic -and full-text search to combine the results. If you want to use the scores or want to support -`return_score="all"`, you'll need to implement your own merging algorithm. The base -`return_score` option accepts only `"relevance"` and `"all"`. - -Whichever methods you override, your reranker has one job on the way out: attach a -`_relevance_score` column with the most relevant rows at the top. LanceDB will reject the result -if that column is missing, and downstream `.limit(...)` calls trust the order you return, so -sort descending before handing the table back. - -For vector-only reranking in Python, pass a text query to `.rerank(..., query_string="...")`. -The vector query itself may be numeric, but `rerank_vector(query, vector_results)` still receives -a string query so your reranker can score each candidate against the user's text intent. - -Below, we show the pseudocode of a custom reranker that combines the results of semantic and full-text -search using a linear combination of the scores: - - -```python Python icon="python" -from lancedb.rerankers import Reranker -import pyarrow as pa - -class MyReranker(Reranker): - def __init__(self, param1, param2, ..., return_score="relevance"): - super().__init__(return_score) - self.param1 = param1 - self.param2 = param2 - - def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table): - # Use the built-in merging function - combined_result = self.merge_results(vector_results, fts_results) - - # Do something with the combined results - # ... - - # Return the combined results - return combined_result - - def rerank_vector(self, query: str, vector_results: pa.Table): - # Do something with the vector results - # ... - - # Return the vector results - return vector_results - - def rerank_fts(self, query: str, fts_results: pa.Table): - # Do something with the FTS results - # ... - - # Return the FTS results - return fts_results -``` - - -## Example {#example} - -As an example, let's build custom reranker that enhances the Cohere Reranker by accepting a filter -query, and accepts any other `CohereReranker` params as `kwargs`. - - -```python Python icon="python" -from typing import List, Union -import pandas as pd -from lancedb.rerankers import CohereReranker - -class ModifiedCohereReranker(CohereReranker): - def __init__(self, filters: Union[str, List[str]], **kwargs): - super().__init__(**kwargs) - filters = filters if isinstance(filters, list) else [filters] - self.filters = filters - - def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table: - combined_result = super().rerank_hybrid(query, vector_results, fts_results) - df = combined_result.to_pandas() - for filter in self.filters: - df = df.query("not text.str.contains(@filter)") - - return pa.Table.from_pandas(df) - - def rerank_vector(self, query: str, vector_results: pa.Table)-> pa.Table: - vector_results = super().rerank_vector(query, vector_results) - df = vector_results.to_pandas() - for filter in self.filters: - df = df.query("not text.str.contains(@filter)") - - return pa.Table.from_pandas(df) - - def rerank_fts(self, query: str, fts_results: pa.Table)-> pa.Table: - fts_results = super().rerank_fts(query, fts_results) - df = fts_results.to_pandas() - for filter in self.filters: - df = df.query("not text.str.contains(@filter)") - - return pa.Table.from_pandas(df) -``` - - - -Under the hood, `vector_results` and `fts_results` are PyArrow tables. You can learn more about -PyArrow tables [here](https://arrow.apache.org/docs/python). The advantage of PyArrow tables is their -interoperability -- you can easily convert them to Pandas/Polars DataFrames, `PyDict`, `PyList`, etc. - -The benefits are also bidirectional -- just as you can easily convert PyArrow tables _to_ Pandas -DataFrames using the `to_pandas()` method -- you can perform DataFrame transformations -and just as easily convert the DataFrame back to PyArrow tables using `pa.Table.from_pandas()` method -as shown in the example above. - diff --git a/docs/reranking/eval.mdx b/docs/reranking/eval.mdx deleted file mode 100644 index ab4c1b7..0000000 --- a/docs/reranking/eval.mdx +++ /dev/null @@ -1,103 +0,0 @@ ---- -title: "Evaluating Hybrid Search Performance" -sidebarTitle: "Evaluation" -description: Learn about evaluating hybrid search performance in LanceDB. -icon: "chart-bar" ---- - -Hybrid search is an often misused and/or misunderstood term. In this section, we're using -the definition of "hybrid search" to mean using a combination of keyword-based and vector search. -Because the vector search operates in a dense embedding space and keyword-based search operate -in a sparse embedding space, their relevance scores cannot be directly compared. -Combining results from multiple searches thus requires a reranking step. - -Before evaluating hybrid search, build an FTS index on the text column and use a table with embedding -metadata or explicit query vectors for the vector side. Otherwise the evaluation is measuring setup -fallbacks or errors rather than the reranker. - -## Reranking strategies {#reranking-strategies} - -There are two common approaches for reranking search results from multiple sources. - -- **Score-based**: Calculate final relevance scores from the individual search algorithm scores. Examples: Reciprocal Rank Fusion (the default in LanceDB), mean reciprocal rank fusion, and weighted linear combination of semantic and keyword-based search scores. - -- **Relevance-based**: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models - - -If you call `.rerank()` on a hybrid query without passing a reranker, LanceDB defaults to -`RRFReranker()` — a score-based reranker that uses Reciprocal Rank Fusion. This is the -score-based path most readers encounter first; `LinearCombinationReranker` is an alternative -score-based strategy you opt into explicitly. - - -By default, rerankers return `_relevance_score`. Pass `return_score="all"` when a reranker -supports it, and you also need the original vector or FTS scores for debugging. -Evaluation code can rely on returned rows being ordered by descending `_relevance_score`. Empty -reranked result sets still include the `_relevance_score` column. - -The hybrid `rerank(...)` method also accepts a `normalize` argument that controls how the raw -vector and FTS scores are made comparable before reranking: - -- `normalize="score"` (the default) — normalizes the raw vector and FTS scores directly. -- `normalize="rank"` — converts each result list to ranks first, then normalizes. - -This choice materially affects score-based rerankers (such as `LinearCombinationReranker`), so -when you evaluate score-based strategies, treat `normalize` as a tunable hyperparameter -alongside the reranker itself. - -For score-based evaluation, the main built-in knobs are: - -- `RRFReranker(K=60)`: rank fusion with a positive smoothing constant. -- `MRRReranker(weight_vector=0.5, weight_fts=0.5)`: weighted reciprocal rank fusion where the weights must sum to `1.0`. -- `LinearCombinationReranker(weight=0.7, fill=1.0)`: blends vector and FTS scores directly; `fill` controls how strongly to penalize a result missing from one side. - -Even though there are many more strategies for reranking, there are no "universally best" -ones that work well for all cases, because reranking quality is dataset and application specific. -Evaluating whether a reranking strategy is a good fit is also a challenge. In the next -section, we discuss an example evaluation of different reranking strategies on a sample dataset. - -## Example evaluation {#example-evaluation} - -The table below shows our evaluation results from an experiment comparing multiple rerankers on -~800 hybrid search queries. This is a modified version of an evaluation script by -[LlamaIndex](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures -hit-rate \@ top-k. - -### Using OpenAI `text-embedding-ada-002` {#using-openai-text-embedding-ada-002} - -Vector Search baseline: **0.64** - -| Reranker | Top-3 | Top-5 | Top-10 | -| --- | --- | --- | --- | -| Linear Combination | `0.73` | `0.74` | `0.85` | -| Cross Encoder | `0.71` | `0.70` | `0.77` | -| Cohere | `0.81` | `0.81` | `0.85` | -| ColBERT | `0.68` | `0.68` | `0.73` | - - - - -### Using OpenAI `text-embedding-3-small` {#using-openai-text-embedding-3-small} - -Vector Search baseline: **0.59** - -| Reranker | Top-3 | Top-5 | Top-10 | -| --- | --- | --- | --- | -| Linear Combination | `0.68` | `0.70` | `0.84` | -| Cross Encoder | `0.72` | `0.72` | `0.79` | -| Cohere | `0.79` | `0.79` | `0.84` | -| ColBERT | `0.70` | `0.70` | `0.76` | - - - -## Conclusion {#conclusion} - -The results show that the reranking methods can significantly improve the search relevance. However, -the improvement we saw was not consistent across all rerankers. In reality, the choice of reranker -likely depends on the dataset and the application. - -It's also important to note that the reranking methods are not a -replacement for the search methods they supplement. They are complementary and it's likely that you'd -have to tune them together to get the best results. The latency vs. recall tradeoff is also an -important factor to consider when choosing the reranker. Hopefully this evaluation -gives you a starting point for your own experiments with hybrid search in LanceDB! diff --git a/docs/reranking/index.mdx b/docs/reranking/index.mdx deleted file mode 100644 index eb12449..0000000 --- a/docs/reranking/index.mdx +++ /dev/null @@ -1,133 +0,0 @@ ---- -title: "Reranking Search Results" -sidebarTitle: "Overview" -description: "Use a reranker to improve search relevance." -icon: "sort-amount-down" -keywords: ["re-ranking", "reranker", "rerank"] ---- - -import { PyRerankingLinearCombinationUsage, PyRerankingRrfUsage, PyRerankingCohereUsage } from '/snippets/integrations.mdx'; - -Reranking is the process of re-ordering search results to improve relevance, often using a -different model than the one used for the initial search. LanceDB has built-in support for reranking -with models from Cohere, Sentence-Transformers, and more. - -### Quickstart {#quickstart} - -To use a reranker, you run a search and pass the results to the `rerank()` method. The examples below -move from the simplest, model-free rerankers to a model-based one. Each is a complete, runnable script. - -**1. Linear combination (simplest).** `LinearCombinationReranker` normalizes the vector and full-text -scores and blends them with a single `weight` (default `0.7`, favoring the vector score). It runs no -model, and reranks [hybrid search](/search/hybrid-search) results. - - - - {PyRerankingLinearCombinationUsage} - - - -**2. Reciprocal Rank Fusion.** `RRFReranker` fuses results by rank position instead of raw score, so it -sidesteps having to make vector and full-text scores comparable. It loads no model either, and it's the -default reranker for hybrid search. - - - - {PyRerankingRrfUsage} - - - -**3. Cohere (model-based).** For higher relevance, a model-based reranker scores each result against the -query with a trained model. `CohereReranker` works with vector, full-text, or hybrid search, and needs -the `cohere` package plus either `COHERE_API_KEY` in the environment or an `api_key` argument. - - - - {PyRerankingCohereUsage} - - - -Reach for the model-free rerankers (`LinearCombinationReranker`, `RRFReranker`) when cost and latency -matter most; reach for a model-based one like `CohereReranker` or `CrossEncoderReranker` when you need -higher relevance and can afford to score every query and document pair with a model. - -### Supported Rerankers {#supported-rerankers} - -LanceDB supports the following rerankers out of the box. The first three are score-based and run no -model; the rest are model-based. The built-in rerankers are documented in this section; the hosted -providers that need an API key live under [integrations](/integrations/reranking). - -| Reranker | Default model | -| --------------------------- | -------------------------------------- | -| `RRFReranker` | None (reciprocal rank fusion) | -| `LinearCombinationReranker` | None (weighted score blend) | -| `MRRReranker` | None (weighted reciprocal rank) | -| `CohereReranker` | `rerank-english-v3.0` | -| `CrossEncoderReranker` | `cross-encoder/ms-marco-TinyBERT-L-6` | -| `ColbertReranker` | `colbert-ir/colbertv2.0` | -| `AnswerdotaiRerankers` | `answerdotai/answerai-colbert-small-v1`| -| `JinaReranker` | `jina-reranker-v2-base-multilingual` | -| `OpenaiReranker` | `gpt-4-turbo-preview` | -| `VoyageAIReranker` | No default (model name required) | -| `WatsonxReranker` | `cross-encoder/ms-marco-minilm-l-12-v2`| - -The model-based rerankers need their provider package installed, and the hosted ones -(`CohereReranker`, `JinaReranker`, `OpenaiReranker`, `VoyageAIReranker`, `WatsonxReranker`) also need an API key, passed -as an `api_key` argument or set in the provider-specific environment variable. - -Rerankers add `_relevance_score` and return rows ordered by descending relevance. Python rerankers -accept `return_score="relevance"` or `return_score="all"`; use `"all"` when you want to keep the -original vector distance or FTS score columns for debugging. Model-based rerankers read from -`column="text"` by default, so either return that column in the search results or pass a different -column. - -Use `refine_factor` on vector or hybrid queries when you're reranking approximate IVF-PQ results and -want a larger candidate pool before the final ranking step. A value of `3` asks LanceDB to fetch -`limit * 3` candidates, refine them with the full vectors, and keep the requested `limit`. Higher -values can improve recall, but they also increase query latency. - - -**SDK coverage differs across languages** - -The provider-specific rerankers in the table above -(`CohereReranker`, `CrossEncoderReranker`, `ColbertReranker`, and others under `lancedb.rerankers`) -are currently **Python-only**. The TypeScript and Rust SDKs currently expose hybrid reranking through -the generic `Reranker` interface (`rerankHybrid` / `rerank_hybrid`) and the built-in `RRFReranker`. -In TypeScript, create the built-in RRF reranker with `await RRFReranker.create(k)`. To use a -model-based reranker from TypeScript or Rust, you must implement the hybrid reranker interface -yourself. - - - -### Multi-vector reranking {#multi-vector-reranking} -Most rerankers support reranking based on multiple vectors. To rerank based on multiple vectors, you can pass a list of vectors to the `rerank` method. Here's an example of how to rerank based on multiple vector columns using the `CrossEncoderReranker`: - - -```python Python icon="python" -from lancedb.rerankers import CrossEncoderReranker - -reranker = CrossEncoderReranker() - -query = "hello" - -# `deduplicate=True` requires `_rowid` on every input result set, -# so call `.with_row_id(True)` on each search before passing it in. -res1 = table.search(query, vector_column_name="vector").limit(3).with_row_id(True) -res2 = table.search(query, vector_column_name="text_vector").limit(3).with_row_id(True) -res3 = table.search(query, vector_column_name="meta_vector").limit(3).with_row_id(True) - -reranked = reranker.rerank_multivector([res1, res2, res3], deduplicate=True) -``` - - -- Passing `deduplicate=True` to `rerank_multivector(...)` raises a `ValueError` if any of the -input result sets is missing the `_rowid` column. Therefore, it's recommended to add `.with_row_id(True)` to every -`table.search(...)` call before reranking, or omit `deduplicate=True` if you don't need it. -- `RRFReranker.rerank_multivector(...)` always requires `_rowid` on its inputs, regardless of -the `deduplicate` flag. - -## Creating Custom Rerankers {#creating-custom-rerankers} - -LanceDB also allows you to create custom rerankers by extending the base `Reranker` class. The custom reranker -should implement the `rerank` method that takes a list of search results and returns a reranked list of -search results. This is covered in more detail in the [creating custom rerankers](/reranking/custom-reranker/) section. diff --git a/docs/reranking/linear_combination.mdx b/docs/reranking/linear_combination.mdx deleted file mode 100644 index 8917815..0000000 --- a/docs/reranking/linear_combination.mdx +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: Linear Combination Reranker -sidebarTitle: "Linear Combination" -description: Learn about LanceDB's deprecated Linear Combination Reranker for combining semantic and full-text search scores. - ---- - -import { PyRerankingLinearCombinationUsage } from '/snippets/integrations.mdx'; - -# Linear Combination Reranker - -> **Note:** This reranker is deprecated. Use the `RRFReranker` if you need a score-based reranker. - -The Linear Combination Reranker combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified, and defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search. - -> **Note:** Supported query type – Hybrid search only. - - - - - {PyRerankingLinearCombinationUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `weight` | `float` | `0.7` | The weight to use for the semantic search score. The weight for the full-text search score is `1 - weight`. | -| `fill` | `float` | `1.0` | Score used when a result is missing from one side of the hybrid query. The default strongly penalizes missing vector or FTS matches. | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", returns only `_relevance_score`. If "all", returns all scores from the vector and FTS search along with the relevance score. | - -`weight` must be between `0` and `1`. If either the vector or FTS side returns no rows, the reranker -returns the non-empty side with `_relevance_score` attached rather than failing. - - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column | -| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) | diff --git a/docs/reranking/mrr.mdx b/docs/reranking/mrr.mdx deleted file mode 100644 index b517f8b..0000000 --- a/docs/reranking/mrr.mdx +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: MRR Reranker -sidebarTitle: "MRR Algorithm" -description: Combine and rerank search results using Mean Reciprocal Rank (MRR) algorithm in LanceDB. Supports weighted scoring for hybrid and multivector search. - ---- - -import { PyRerankingMrrUsage } from '/snippets/integrations.mdx'; - -# MRR Reranker - -This reranker uses the Mean Reciprocal Rank (MRR) algorithm to combine and rerank search results from vector and full-text search. You can use this reranker by passing `MRRReranker()` to the `rerank()` method. The MRR algorithm calculates the average of reciprocal ranks across different search results, providing a balanced way to merge results from multiple ranking systems. - -> **Note:** Supported query types – Hybrid and Multivector search. - - - - {PyRerankingMrrUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `weight_vector` | `float` | `0.5` | Weight for vector search results (0.0 to 1.0). | -| `weight_fts` | `float` | `0.5` | Weight for FTS search results (0.0 to 1.0). | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score`. If "all", will return all scores from the vector and FTS search along with the relevance score. | - -**Note:** `weight_vector` + `weight_fts` must equal 1.0. - -For multivector reranking, input result sets need `_rowid` so LanceDB can identify the same row -across the ranked lists. Add `.with_row_id(True)` to each vector search before passing the results -to the reranker. - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | - -### Multivector Search {#multivector-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector distances from all searches along with `_relevance_score`. | diff --git a/docs/reranking/rrf.mdx b/docs/reranking/rrf.mdx deleted file mode 100644 index 3a58cec..0000000 --- a/docs/reranking/rrf.mdx +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: Reciprocal Rank Fusion Reranker -sidebarTitle: "RRF Algorithm" -description: Learn about LanceDB's default Reciprocal Rank Fusion (RRF) reranker for hybrid search. Implements the Cormack et al. algorithm for optimal search result ranking. - ---- - -import { PyRerankingRrfUsage } from '/snippets/integrations.mdx'; - -# Reciprocal Rank Fusion Reranker - -**Reciprocal Rank Fusion (RRF)** is a model-free way to merge several ranked result lists into a -single ordering. Rather than comparing raw similarity scores (which aren't directly comparable -across, say, a vector search and a full-text search), RRF looks only at each document's *rank -position* in each list. It scores every document with the formula `1 / (rank + K)`, sums those -contributions across the lists, and re-sorts by the total. Documents that rank highly in more than -one search rise to the top. Because there's no model to load or call, it's fast and cheap, which is -why it's the default reranker for LanceDB hybrid search. The implementation follows the -[Cormack et al. paper](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf). - -> **Supported query types:** hybrid search and [multi-vector reranking](/reranking#multi-vector-reranking). -> Because RRF fuses two or more ranked lists, it can't rerank a single vector or full-text result set -> on its own. Calling `rerank_vector` or `rerank_fts` on an `RRFReranker` raises `NotImplementedError`. - - - - - {PyRerankingRrfUsage} - - - -## Accepted Arguments {#accepted-arguments} -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical. | -| `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score`. If "all", will return all scores from the vector and FTS search along with the relevance score. | - -`K` must be greater than `0`. In TypeScript, construct the built-in reranker with -`await RRFReranker.create(k)` before passing it to `.rerank(...)`. - -## Multi-vector reranking {#multi-vector-reranking} - -`RRFReranker` can also fuse the results of several vector searches with `rerank_multivector`, applying -the same rank-fusion algorithm across more than two lists. Every input result set must include the -`_rowid` column, so add `.with_row_id(True)` to each `table.search(...)` call before reranking, -otherwise the call raises a `ValueError`. See [multi-vector reranking](/reranking#multi-vector-reranking) -for a full example. - -## Supported Scores for each query type {#supported-scores-for-each-query-type} -You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type: - -### Hybrid Search {#hybrid-search} -|`return_score`| Status | Description | -| --- | --- | --- | -| `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column. | -| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | diff --git a/docs/search/filtering.mdx b/docs/search/filtering.mdx deleted file mode 100644 index b8ebf37..0000000 --- a/docs/search/filtering.mdx +++ /dev/null @@ -1,266 +0,0 @@ ---- -title: "Metadata Filtering in LanceDB" -sidebarTitle: Filtering -description: Filter search results in LanceDB based on metadata fields. -icon: "filter" ---- - -LanceDB supports filtering features of query results based on metadata fields. -While joint vector and metadata search at scale presents a significant challenge, -LanceDB achieves sub-100ms latency at thousands of QPS, enabling efficient vector search -with filtering capabilities even on datasets containing billions of records. - -**Pre-filtering** means LanceDB applies the metadata `where(...)` condition before running vector search, so the search only considers rows that already match the filter. **Post-filtering** means LanceDB runs vector search first and only then filters the returned candidates. Pre-filtering is enabled by default. In practice, pre-filtering is better when the filter is part of the result contract; post-filtering can be lower-latency for expensive or non-indexable filters, but it can return fewer than `limit` rows, or even zero, if the nearest neighbors do not pass the filter. - -On hybrid queries, the same `where(...)` filter is applied to both the vector and full-text halves of the query. The prefilter or postfilter choice controls whether that happens before each subquery scores candidates or after the subquery top-k is produced. - -## Chaining `where` clauses {#chaining-where-clauses} - -In more recent LanceDB SDK versions (see the callout box below for the exact version numbers), you can call `where(...)` (Python and TypeScript) or `only_if(...)` (Rust) more than once on the same query builder. Each additional filter is combined with the previous one using logical `AND`, so `where("a > 0").where("b < 10")` is equivalent to `where("(a > 0) AND (b < 10)")`. - -For a fixed predicate, writing one `where(...)` clause with `AND` is just as valid and often clearer. Chaining is mainly useful when code composes filters incrementally, such as applying a shared base predicate in a helper and then adding a per-call predicate at the query site. - - -```python Python icon="python" -# Both filters apply: item is in the list AND price is above 15. -result = ( - table.search([100, 102]) - .where("item IN ('foo', 'bar', 'baz')") - .where("price > 15.0") - .limit(3) - .to_arrow() -) -``` - -```typescript TypeScript icon="square-js" -// Both filters apply: item is in the list AND price is above 15. -const result = await table - .search([100, 102]) - .where("item IN ('foo', 'bar', 'baz')") - .where("price > 15.0") - .limit(3) - .toArray(); -``` - - - -In Python SDK versions before `0.34.0` and Rust/TypeScript SDK versions before `0.31.0`, a second `where(...)` or `only_if(...)` call replaced the first filter, so only the last predicate was applied. If your code needs to run on those older versions, write a single predicate with `AND` instead of chaining calls. When upgrading to the latest SDKs, review any existing chained filters and drop earlier calls you no longer want to apply. - - -## Example: Metadata Filtering {#example-metadata-filtering} - -To illustrate filtering capabilities, let's try four data points with combinations of vectors and metadata: - - -```python Python icon="python" -data = [ - {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, - {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, - {"vector": [10.2, 100.8], "item": "baz", "price": 30.0}, - {"vector": [1.4, 9.5], "item": "fred", "price": 40.0}, -] -table = db.create_table("metadata_filter_example", data=data, mode="overwrite") -``` - -```typescript TypeScript icon="square-js" -const data = [ - { vector: [3.1, 4.1], item: "foo", price: 10.0 }, - { vector: [5.9, 26.5], item: "bar", price: 20.0 }, - { vector: [10.2, 100.8], item: "baz", price: 30.0 }, - { vector: [1.4, 9.5], item: "fred", price: 40.0 }, -]; - -const tableName = "metadata_filter_example"; -const table = await db.createTable(tableName, data, { - mode: "overwrite", -}); -``` - - -### Filtering Without Vector Search {#filtering-without-vector-search} - -You can always filter your data without search. This is useful when you need to query based on metadata: - - -```python Python icon="python" -filtered_no_search_result = ( - table.search() - .where("(item IN ('foo', 'bar', 'baz')) AND (price > 15.0)") - .limit(3) - .to_arrow() -) -``` - -```typescript TypeScript icon="square-js" -const filteredResult = await table - .query() - .where("(item IN ('foo', 'bar', 'baz')) AND (price > 15.0)") - .limit(3) - .toArray(); -``` - - - -If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set. - - -### Pre-Filtering with Vector Search {#pre-filtering-with-vector-search} - - -```python Python icon="python" -filtered_result = ( - table.search([100, 102]) - .where("(item IN ('foo', 'bar')) AND (price > 15.0)") - .limit(3) - .to_arrow() -) -``` - -```typescript TypeScript icon="square-js" -const results = await table - .search([100, 102]) - .where("(item IN ('foo', 'bar')) AND (price > 15.0)") - .toArray(); -``` - - -### Post-Filtering with Vector Search {#post-filtering-with-vector-search} - - -```python Python icon="python" -post_filtered_result = ( - table.search([100, 102]) - .where("(item IN ('foo', 'bar')) AND (price > 15.0)", prefilter=False) - .limit(3) - .to_arrow() -) -``` - -```typescript TypeScript icon="square-js" -const postFilteredResult = await (table.search([100, 102]) as VectorQuery) - .where("(item IN ('foo', 'bar')) AND (price > 15.0)") - .postfilter() - .limit(3) - .toArray(); -``` - - - -When querying large tables, omitting a `limit` clause may overwhelm resources and return excessive data. It's always recommended -to be mindful of the potential impact on performance and costs when working with really large tables. - - -## Filtering with SQL {#filtering-with-sql} - -Because it's built on top of DataFusion, LanceDB embraces the utilization of standard SQL expressions as predicates for filtering operations. SQL can be used during vector search, update, and deletion operations. - -LanceDB supports a growing list of SQL expressions: - -| SQL Expression | Description | -|:---------------|:-------------| -| `>, >=, <, <=, =` | Comparison operators | -| `AND`, `OR`, `NOT` | Logical operators | -| `IS NULL`, `IS NOT NULL` | Null checks | -| `IS TRUE`, `IS NOT TRUE`, `IS FALSE`, `IS NOT FALSE` | Boolean checks | -| `IN` | Value matching from a set | -| `LIKE`, `NOT LIKE` | Pattern matching | -| `CAST` | Type conversion | -| `regexp_match(column, pattern)` | Regular expression matching | -| [DataFusion Functions](https://datafusion.apache.org/user-guide/sql/scalar_functions.html) | Additional SQL functions | - -### Simple SQL Filters {#simple-sql-filters} - -For example, the following filter string is acceptable: - - -```python Python icon="python" -tbl.search([100, 102]).where( - "(item IN ('foo', 'baz')) AND (price > 20.0)" -).to_arrow() -``` - -```typescript TypeScript icon="square-js" -await table - .search([100, 102]) - .where("(item IN ('foo', 'baz')) AND (price > 20.0)") - .toArray(); -``` - - -### Advanced SQL Filters {#advanced-sql-filters} - -If your column name contains special characters, upper-case characters, or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html), -you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the -path must be wrapped in backticks. - -```sql -`CUBE` = 10 AND `UpperCaseName` = '3' AND `column name with space` IS NOT NULL -AND `nested with space`.`inner with space` < 2 -``` - - -Field names containing periods (.) are NOT supported. - - -### Dates, Timestamps, Decimals {#dates-timestamps-decimals} - -Literals for dates, timestamps, and decimals can be written by writing the string -value after the type name. For example: - - -```sql SQL icon="SQL" -date_col = date '2021-01-01' -and timestamp_col = timestamp '2021-01-01 00:00:00' -and decimal_col = decimal(8,3) '1.000' -``` - - -For timestamp columns, the precision can be specified as a number in the type -parameter. Microsecond precision (6) is the default. - -| SQL | Time unit | -|:-------------- |:------------ | -| `timestamp(0)` | Seconds | -| `timestamp(3)` | Milliseconds | -| `timestamp(6)` | Microseconds | -| `timestamp(9)` | Nanoseconds | - -## Apache Arrow Mapping {#apache-arrow-mapping} - -LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format. -The mapping from SQL types to Arrow types is: - -| SQL type | Arrow type | -|:--------------------------------------------------------- |:------------------ | -| `boolean` | `Boolean` | -| `tinyint` / `tinyint unsigned` | `Int8` / `UInt8` | -| `smallint` / `smallint unsigned` | `Int16` / `UInt16` | -| `int` or `integer` / `int unsigned` or `integer unsigned` | `Int32` / `UInt32` | -| `bigint` / `bigint unsigned` | `Int64` / `UInt64` | -| `float` | `Float32` | -| `double` | `Float64` | -| `decimal(precision, scale)` | `Decimal128` | -| `date` | `Date32` | -| `timestamp` | `Timestamp` [^1] | -| `string` | `Utf8` | -| `binary` | `Binary` | - - -## Best Practices {#best-practices} - -**Scalar Indexes**: We strongly recommend creating scalar indices on columns used for filtering, whether combined with a search operation or applied independently (e.g., for updates or deletions). - -For best performance with large tables or high query volumes: - -- Build a scalar index on frequently filtered columns -- Use exact column names in filters (e.g., `user_id` instead of `USER_ID`) -- Avoid complex transformations in filter expressions (keep them simple) -- When running concurrent queries, use connection pooling for better throughput - -For a column of type LIST(T), you can use `LABEL_LIST` to create a scalar index. Then you should leverage DataFusion's [array functions](https://datafusion.apache.org/user-guide/sql/scalar_functions.html#array-functions) like `array_has_any` or `array_has_all` for optimized filtering. - -## Limitations {#limitations} - -Both **pre-filtering** and **post-filtering** can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to call `bypass_vector_index()` if you know that the filter is highly selective. - -Similarly, a highly selective post-filter can lead to false positives. Increasing both `nprobes` and `refine_factor` can mitigate this issue. When deciding between pre-filtering and post-filtering, pre-filtering is generally the safer choice if you're uncertain. diff --git a/docs/search/fts-examples.mdx b/docs/search/fts-examples.mdx deleted file mode 100644 index dff4426..0000000 --- a/docs/search/fts-examples.mdx +++ /dev/null @@ -1,561 +0,0 @@ ---- -title: Full-Text Search Examples -sidebarTitle: FTS Examples -description: Worked examples of fuzzy search, boosting, boolean queries, and substring search with LanceDB full-text search. -icon: "book-open" ---- - -These worked examples build on the concepts from the [Full-Text Search guide](/search/full-text-search). They walk through creating sample tables, building FTS indices, and running fuzzy, phrase, boosted, boolean, and substring queries. - -## Fuzzy Search and Boosting Example {#fuzzy-search-and-boosting-example} - -### Generate Data {#generate-data} - -First, let's create a table with sample text data for testing fuzzy search: - - -```python Python icon="python" -import lancedb -import numpy as np -import pandas as pd -import random - -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) - -# Generate 100 rows of random " " text -table_name = "fts-fuzzy-boosting-test" -vectors = [np.random.randn(128) for _ in range(100)] -verbs = ("runs", "hits", "jumps", "drives", "barfs") -adv = ("crazily.", "dutifully.", "foolishly.", "merrily.", "occasionally.") -adj = ("adorable", "clueless", "dirty", "odd", "stupid") - -def sentence(nouns): - return " ".join(random.choice(words) for words in (nouns, verbs, adv, adj)) - -text = [sentence(("puppy", "car")) for _ in range(100)] -text2 = [sentence(("rabbit", "girl", "monkey")) for _ in range(100)] -count = [random.randint(1, 10000) for _ in range(100)] -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb" - -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); - -// Generate 100 rows of random " " text -const tableName = "fts-fuzzy-boosting-test-ts"; -const n = 100; -const verbs = ["runs", "hits", "jumps", "drives", "barfs"]; -const adverbs = ["crazily", "dutifully", "foolishly", "merrily", "occasionally"]; -const adjectives = ["adorable", "clueless", "dirty", "odd", "stupid"]; - -const pick = (words: string[]) => words[Math.floor(Math.random() * words.length)]; -const sentence = (nouns: string[]) => - [nouns, verbs, adverbs, adjectives].map(pick).join(" "); - -const vectors = Array.from({ length: n }, () => - Array.from({ length: 128 }, () => Math.random() * 2 - 1) -); -const text = Array.from({ length: n }, () => sentence(["puppy", "car"])); -const text2 = Array.from({ length: n }, () => sentence(["rabbit", "girl", "monkey"])); -const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) + 1); -``` - - -### Create Table {#create-table} - - -```python Python icon="python" -# Create table with sample data -table = db.create_table( - table_name, - data=pd.DataFrame({ - "vector": vectors, - "id": [i % 2 for i in range(100)], - "text": text, - "text2": text2, - "count": count, - }), - mode="overwrite" -) -``` - -```typescript TypeScript icon="square-js" -// Create table with sample data -const data = makeArrowTable( - vectors.map((vector, i) => ({ - vector, - id: i % 2, - text: text[i], - text2: text2[i], - count: count[i], - })) -); - -const table = await db.createTable(tableName, data, { mode: "overwrite" }); -``` - - -### Construct FTS Index {#construct-fts-index} - -Create a full-text search index on the first text column: - - -```python Python icon="python" -# Create FTS index on first text column -table.create_fts_index("text") -wait_for_index(table, "text_idx") -``` - -```typescript TypeScript icon="square-js" -// Create FTS index on first text column -await table.createIndex("text", { config: Index.fts() }); -await waitForIndex(table, "text_idx"); -``` - - -Then, create an index on the second text column: - - -```python Python icon="python" -# Create FTS index on second text column -table.create_fts_index("text2") -wait_for_index(table, "text2_idx") -``` - -```typescript TypeScript icon="square-js" -// Create FTS index on second text column -await table.createIndex("text2", { config: Index.fts() }); -await waitForIndex(table, "text2_idx"); -``` - - -### Basic and Fuzzy Search {#basic-and-fuzzy-search} - -Now we can perform basic, fuzzy, and prefix match searches: - -#### Basic Exact Search {#basic-exact-search} - - -```python Python icon="python" -from lancedb.query import MatchQuery - -# Basic match (exact search) -basic_match_results = ( - table.search(MatchQuery("crazily", "text")) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -import { MatchQuery } from "@lancedb/lancedb"; - -// Basic match (exact search) -const basicMatchResults = await table.query() - .fullTextSearch(new MatchQuery("crazily", "text")) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -#### Fuzzy Search with Typos {#fuzzy-search-with-typos} - - -```python Python icon="python" -# Fuzzy match (allows typos) -fuzzy_results = ( - table.search(MatchQuery("craziou", "text", fuzziness=2)) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -// Fuzzy match (allows typos) -const fuzzyResults = await table.query() - .fullTextSearch(new MatchQuery("craziou", "text", { - fuzziness: 2, - })) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -#### Prefix based Match {#prefix-based-match} - -Prefix-based match allows you to search for documents containing words that start with a specific prefix. - - -```python Python icon="python" -# Fuzzy match (allows typos) -fuzzy_results = ( - table.search(MatchQuery("cra", "text", prefix_length=3)) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -// Fuzzy match (allows typos) -const fuzzyResults = await table.query() - .fullTextSearch(new MatchQuery("cra", "text", { - prefixLength: 3, - })) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -### Phrase Match {#phrase-match} - -Phrase matching enables you to search for exact sequences of words. Unlike regular text search -which matches individual terms independently, phrase matching requires words to appear in the -specified order with no intervening terms. - - -Phrase queries are supported but only for a single column; providing multiple columns with a quoted phrase raises an error. - - -Phrase matching is particularly useful for: - -- Searching for specific multi-word expressions -- Matching exact titles or quotes -- Finding precise word combinations in a specific order - - -```python Python icon="python" -# Exact phrase match -from lancedb.query import PhraseQuery - -phrase_results = ( - table.search(PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - -// Exact phrase match -const phraseResults = await table.query() - .fullTextSearch(new PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -#### Flexible Phrase Match {#flexible-phrase-match} -To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, even if they are not directly adjacent or in the exact order, as long as they are within the specified `slop` value. - -For example, the phrase query "puppy merrily" would not return any results by default. However, if you set `slop=1`, it will match phrases like "puppy jumps merrily", "puppy runs merrily", and similar variations where one word appears between "puppy" and "merrily". - - -```python Python icon="python" -# Flexible phrase match with slop=1 for 'puppy merrily' -from lancedb.query import PhraseQuery - -phrase_results = ( - table.search(PhraseQuery("puppy merrily", "text", slop=1)) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - -// Flexible phrase match with slop=1 for 'puppy runs' -const phraseResults = await table.query() - .fullTextSearch(new PhraseQuery("puppy runs", "text", { slop: 1 })) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - -### Search with Boosting {#search-with-boosting} - -Boosting allows you to control the relative importance of different search terms or fields -in your queries. This feature is particularly useful when you need to: - -* Prioritize matches in certain columns -* Promote specific terms while demoting others -* Fine-tune relevance scoring for better search results - -| Parameter | Type | Default | Description | -| -------------- | ----- | -------- | ------------------------------------------------------------------ | -| positive | Query | required | The primary query terms to match and promote in results | -| negative | Query | required | Terms to demote in the search results | -| negative_boost | float | 0.5 | Multiplier for negative matches (lower values = stronger demotion) | - - -```python Python icon="python" -from lancedb.query import MatchQuery, BoostQuery, MultiMatchQuery - -# Boost data with 'runs' in text more than 'puppy' in text -boosting_results = ( - table.search( - BoostQuery( - MatchQuery("runs", "text"), - MatchQuery("puppy", "text"), - negative_boost=0.2, - ), - ) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) - -# Search across both text and text2 -multi_match_results = ( - table.search(MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) - .limit(100) - .to_pandas() -) - -# Search with field boosting -multi_match_boosting_results = ( - table.search( - MultiMatchQuery("crazily", ["text", "text2"], boosts=[1.0, 2.0]), - ) - .select(["id", "text", "text2"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -import { MatchQuery, BoostQuery, MultiMatchQuery } from "@lancedb/lancedb"; - -// Boosting Example -const boostingResults = await table.query() - .fullTextSearch(new BoostQuery(new MatchQuery("runs", "text"), new MatchQuery("puppy", "text"), { - negativeBoost: 0.2, - })) - .select(["id", "text"]) - .limit(100) - .toArray(); - -// Search across both text fields -const multiMatchResults = await table.query() - .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) - .limit(100) - .toArray(); - -// Search with field boosting -const multiMatchBoostingResults = await table.query() - .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"], { - boosts: [1.0, 2.0], - })) - .select(["id", "text", "text2"]) - .limit(100) - .toArray(); -``` - - - -- Use fuzzy search when handling user input that may contain typos or variations -- Apply field boosting to prioritize matches in more important columns -- Combine fuzzy search with boosting for robust and precise search results - -**Recommendations for optimal FTS performance:** - -- Create full-text search indices on text columns that will be frequently searched -- For hybrid search combining text and vectors, see our [hybrid search guide](/search/hybrid-search/) -- For performance benchmarks, check our [benchmark results](/enterprise/benchmarks/) -- For complex queries, use SQL to combine FTS with other filter conditions - - -### Boolean Queries {#boolean-queries} -LanceDB supports boolean logic in full-text search, allowing you to combine multiple queries using `and` and `or` operators. This is useful when you want to match documents that satisfy multiple conditions (intersection) or at least one of several conditions (union). - -#### Combining Two Match Queries {#combining-two-match-queries} - -In Python, you can combine two MatchQuery objects using either the `and` function or the `&` operator (e.g., `MatchQuery("puppy", "text") and MatchQuery("merrily", "text")`); both methods are supported and yield the same result. Similarly, you can use either the `or` function or the `|` operator to perform an or query. - -In TypeScript, boolean queries are constructed using the `BooleanQuery` class with a list of [Occur, subquery] pairs. For example, to perform an AND query: - -```sql SQL icon="code" -BooleanQuery([ -[Occur.Must, new MatchQuery("puppy", "text")], -[Occur.Must, new MatchQuery("merrily", "text")], -]) -``` - -This approach allows you to specify complex boolean logic by combining multiple subqueries with different Occur values (such as `Must`, `Should`, or `MustNot`). - - -**Which queries are allowed?** - -A boolean query must include at least one `SHOULD` or `MUST` clause. Queries that contain only a `MUST_NOT` clause are not allowed. - - - -```python Python icon="python" -from lancedb.query import MatchQuery - -# Example: Find documents containing both "puppy" and "merrily" -and_query = MatchQuery("puppy", "text") & MatchQuery("merrily", "text") -and_results = ( - table.search(and_query) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) - -# Example: Find documents containing either "puppy" or "merrily" -or_query = MatchQuery("puppy", "text") | MatchQuery("merrily", "text") -or_results = ( - table.search(or_query) - .select(["id", "text"]) - .limit(100) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" expandable=true -import { MatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; - -// Flexible boolean queries with MatchQuery - -// Find documents containing both "puppy" and "merrily" -const mustResults = await table - .search( - new BooleanQuery([ - [Occur.Must, new MatchQuery("puppy", "text")], - [Occur.Must, new MatchQuery("merrily", "text")], - ]), - ) - .select(["id", "text"]) - .limit(100) - .toArray(); - -// Find documents containing either "puppy" or "merrily" -const shouldResults = await table - .search( - new BooleanQuery([ - [Occur.Should, new MatchQuery("puppy", "text")], - [Occur.Should, new MatchQuery("merrily", "text")], - ]), - ) - .select(["id", "text"]) - .limit(100) - .toArray(); -``` - - - -**How to use booleans?** - -- Use `and`/`&`(Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). -- Use `or`/`|`(Python), `Occur.Should`(Typescript) for union (documents must match at least one query). - - -## Substring Search Example {#substring-search-example} - -LanceDB supports searching for substrings in text columns using n-gram tokenization. This is useful for finding partial matches within text content. - -### Setting Up the Table {#setting-up-the-table} - -First, create a table with sample text data and configure n-gram tokenization: - - -```python Python icon="python" -import pyarrow as pa -import lancedb - -db = lancedb.connect(":memory:") - -data = pa.table({"text": ["hello world", "lance database", "lance is cool"]}) -table = db.create_table("test", data=data) -table.create_fts_index("text", base_tokenizer="ngram") -``` - - -### Basic Substring Search {#basic-substring-search} - -With the default n-gram settings (minimum length of 3), you can search for substrings of length 3 or more: - - -```python Python icon="python" -results = table.search("lan", query_type="fts").limit(10).to_list() -assert len(results) == 2 -assert set(r["text"] for r in results) == {"lance database", "lance is cool"} - -results = ( - table.search("nce", query_type="fts").limit(10).to_list() -) # spellchecker:disable-line -assert len(results) == 2 -assert set(r["text"] for r in results) == {"lance database", "lance is cool"} -``` - - -### Handling Short Substrings {#handling-short-substrings} - -By default, the minimum n-gram length is 3, so shorter substrings like "la" won't match: - - -```python Python icon="python" -results = table.search("la", query_type="fts").limit(10).to_list() -assert len(results) == 0 -``` - - -### Customizing N-gram Parameters {#customizing-n-gram-parameters} - -You can customize the n-gram behavior by adjusting the minimum length and using prefix-only matching: - - -```python Python icon="python" -table.create_fts_index( - "text", - base_tokenizer="ngram", - replace=True, - ngram_min_length=2, - prefix_only=True, -) -``` - - -### Testing Custom N-gram Settings {#testing-custom-n-gram-settings} - -With the new settings, you can now search for shorter substrings and use prefix-only matching: - - -```python Python icon="python" -results = table.search("lan", query_type="fts").limit(10).to_list() -assert len(results) == 2 -assert set(r["text"] for r in results) == {"lance database", "lance is cool"} - -results = ( - table.search("nce", query_type="fts").limit(10).to_list() -) # spellchecker:disable-line -assert len(results) == 0 - -results = table.search("la", query_type="fts").limit(10).to_list() -assert len(results) == 2 -assert set(r["text"] for r in results) == {"lance database", "lance is cool"} -``` - diff --git a/docs/search/full-text-search.mdx b/docs/search/full-text-search.mdx deleted file mode 100644 index 5063374..0000000 --- a/docs/search/full-text-search.mdx +++ /dev/null @@ -1,565 +0,0 @@ ---- -title: Full-Text Search (FTS) -sidebarTitle: Full-Text Search (FTS) -description: Learn how to implement full-text search in LanceDB using BM25 for keyword-based retrieval. -icon: "book" ---- - -import { - PyFtsPrefiltering, - PyFtsPostfiltering, - PyFtsIncrementalIndex, -} from '/snippets/search.mdx' - -LanceDB provides support for Full-Text Search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. - -## Basic Usage {#basic-usage} - -Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords. - -### Table Setup {#table-setup} - -First, open or create the table you want to search: - - -```python Python icon="python" -import lancedb -from lancedb.index import FTS - -uri = "data/sample-lancedb" -db = lancedb.connect(uri) - -table = db.create_table( - "my_table_fts", - data=[ - {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}, - {"vector": [5.9, 26.5], "text": "There are several kittens playing"}, - ], -) -``` - -```ts TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -const uri = "data/sample-lancedb" -const db = await lancedb.connect(uri); - -const data = [ - { vector: [3.1, 4.1], text: "Frodo was a happy puppy" }, - { vector: [5.9, 26.5], text: "There are several kittens playing" }, -]; -const tbl = await db.createTable("my_table", data, { mode: "overwrite" }); -``` - -```rust Rust icon="rust" -let uri = "data/sample-lancedb"; -let db = connect(uri).execute().await?; -let initial_data: Box = create_some_records()?; -let tbl = db - .create_table("my_table", initial_data) - .execute() - .await?; -``` - - -### Construct FTS Index {#construct-fts-index} - -Create a full-text search index on your text column: - - -In Python, this page shows the synchronous `create_fts_index(...)` form. For the -asynchronous equivalent (`await table.create_index("text", config=FTS(...))`), see -[FTS index](/indexing/fts-index). - - - -```python Python icon="python" -table.create_fts_index("text") -``` - -```typescript TypeScript icon="square-js" -await tbl.createIndex("text", { - config: lancedb.Index.fts(), -}); -``` - -```rust Rust icon="rust" -tbl - .create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) - .execute() - .await?; -``` - - -### Full-text Search {#full-text-search} - -Perform full-text search and retrieve results: - - -```python Python icon="python" -results = table.search("puppy") - .limit(10) - .select(["text"]) - .to_list() -# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}] -``` - -```typescript TypeScript icon="square-js" -const results = await tbl - .search("puppy", "fts") - .select(["text"]) - .limit(10) - .toArray(); -``` - -```rust Rust icon="Rust" -let results = tbl - .query() - .full_text_search(FullTextSearchQuery::new("puppy".to_owned())) - .select(lancedb::query::Select::Columns(vec!["text".to_owned()])) - .limit(10) - .execute() - .await?; -``` - - -The search is conducted on all indexed columns by default, so it's useful when there are multiple indexed columns. - -If you want to specify which columns to search use `fts_columns="text"` - - -LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead. - - -If a table has more than one FTS index, specify the indexed text column in the query. In Python you can use `fts_columns` or the query builder's `nearest_to_text(..., columns=...)`; in TypeScript, use `query().nearestToText(..., columns)`. The newer Lance-native FTS does not accept legacy Tantivy-only index parameters. - -### Keeping the index up to date {#keeping-the-index-up-to-date} - -Rows you add after building an FTS index aren't part of the index until you optimize the table. Until then, queries fall back to a flat scan over the unindexed fragments to keep results complete, which slows them down as the unindexed tail grows. Call `table.optimize()` to fold new rows into the existing index — it's the same operation used for vector indexes: - - - -{PyFtsIncrementalIndex} - - -```typescript TypeScript icon="square-js" -await tbl.add([{ vector: [3.1, 4.1], text: "Frodo was a happy puppy" }]); -await tbl.optimize(); -``` - -```rust Rust icon="rust" -tbl.add(new_data).execute().await?; -tbl.optimize(OptimizeAction::All).await?; -``` - - -A useful rule of thumb is to call `optimize()` after roughly 100,000 row changes or 20 data-modification operations, whichever comes first. For tables with continuous ingest, schedule it on a cadence that keeps `num_unindexed_rows` (from `table.index_stats(...)`) close to zero. If you want to skip the flat scan over unindexed rows entirely — for example, on a hot read path where stale results are acceptable — call `.fast_search()` on the query so the search returns only indexed results. - -## Advanced Usage {#advanced-usage} - -### Tokenize Table Data {#tokenize-table-data} - -By default, the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are longer than 40 characters. All words are converted to lowercase. - -Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. You should set the `base_tokenizer` parameter rather than `tokenizer_name` because you cannot customize the tokenizer if `tokenizer_name` is specified. - -Tokenization and language filters are separate settings. `base_tokenizer` controls how text is split into searchable tokens. `language` controls stemming and stop-word removal when `stem=True` or `remove_stop_words=True`; choose the tokenizer for CJK or mixed-language segmentation. - -For example, to enable stemming for English: - - -```python Python icon="python" -table.create_fts_index("text", language="English", replace=True) -``` - - -The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc. - -**Default index parameters:** -- `base_tokenizer`: `"simple"` -- `language`: English -- `with_position`: false -- `max_token_length`: 40 -- `lower_case`: true -- `stem`: true -- `remove_stop_words`: true -- `ascii_folding`: true -- `custom_stop_words`: `None` — pass a `list[str]` to drop additional words beyond the language defaults. Requires `remove_stop_words=True`. - -For multilingual use cases, use `base_tokenizer="icu"` for unicode-aware word segmentation on mixed-language text. ICU stands for [International Components for Unicode](https://icu.unicode.org/). The ICU tokenizer uses bundled ICU4X segmenter data, so it does not require external tokenizer model files. It is a good default when documents mix languages or include scripts where the simple tokenizer would keep an unspaced span as one large token. - -The Python API also supports tokenizer implementations that load language model files. Use `base_tokenizer="jieba/default"` for Jieba tokenization, which segments Chinese text into searchable word tokens when the text is written without spaces between words. Use Lindera-backed tokenizers for dictionary-based East Asian morphological segmentation, such as `base_tokenizer="lindera/ipadic"` for Japanese or `base_tokenizer="lindera/ko-dic"` for Korean when you have installed and compiled that Lindera model. These are language-specific tokenizers; ICU is the broader mixed-language option. - - -```python Python icon="python" -table.create_fts_index( - "text", - base_tokenizer="jieba/default", - stem=False, - remove_stop_words=False, - ascii_folding=False, - replace=True, -) -``` - - -Model-backed tokenizers require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to a different model root: - -```bash -export LANCE_LANGUAGE_MODEL_HOME=/path/to/lance/language_models -``` - -For example, `jieba/default` is resolved under `/jieba/default/...`, `lindera/ipadic` under `/lindera/ipadic/...`, and `lindera/ko-dic` under `/lindera/ko-dic/...`. - - -Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. If you use another stemming language, such as Arabic, Greek, Romanian, Tamil, or Turkish, set `remove_stop_words=False` or pass `custom_stop_words`. - - -For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e': - - -```python Python icon="python" -table.create_fts_index( - "text", - language="French", - stem=True, - ascii_folding=True, - replace=True, - ) -``` - - -### Filtering Options {#filtering-options} - -LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported. - -This can be invoked via the familiar `where` syntax. - -With pre-filtering: - - - - -{PyFtsPrefiltering} - - -```typescript TypeScript icon="square-js" -await tbl -.search("puppy") -.select(["id", "doc"]) -.limit(10) -.where("meta='foo'") -.prefilter(true) -.toArray(); -``` - -```rust Rust icon="Rust" -table - .query() - .full_text_search(FullTextSearchQuery::new("puppy".to_owned())) - .select(lancedb::query::Select::Columns(vec!["doc".to_owned()])) - .limit(10) - .only_if("meta='foo'") - .execute() - .await?; -``` - - -With post-filtering: - - - -{PyFtsPostfiltering} - - -```typescript TypeScript icon="square-js" -await tbl -.search("apple") -.select(["id", "doc"]) -.limit(10) -.where("meta='foo'") -.prefilter(false) -.toArray(); -``` - -```rust Rust icon="Rust" -table - .query() - .full_text_search(FullTextSearchQuery::new(words[0].to_owned())) - .select(lancedb::query::Select::Columns(vec!["doc".to_owned()])) - .postfilter() - .limit(10) - .only_if("meta='foo'") - .execute() - .await?; -``` - - -### Phrase vs. Terms Queries {#phrase-vs-terms-queries} - - -Lance-based FTS doesn't support queries using boolean operators `OR`, `AND` in the search string. - - -For full-text search you can specify either a **phrase** query like `"the old man and the sea"`, -or a **terms** search query like `old man sea`. - -To search for a phrase, the index must be created with `with_position=True` and `remove_stop_words=False`: - - -```python Python icon="python" -table.create_fts_index("text", with_position=True, replace=True) -``` - - -This will allow you to search for phrases, but it will also significantly increase the index size and indexing time. - -### Fuzzy Search {#fuzzy-search} - -Fuzzy search allows you to find matches even when the search terms contain typos or slight variations. -LanceDB uses the classic [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) -to find similar terms within a specified edit distance. - -| Parameter | Type | Default | Description | -| -------------- | ---- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| fuzziness | int | 0 | Maximum edit distance allowed for each term. If not specified, automatically set based on term length: 0 for length ≤ 2, 1 for length ≤ 5, 2 for length > 5 | -| max_expansions | int | 50 | Maximum number of terms to consider for fuzzy matching. Higher values may improve recall but increase search time | - -For a complete walkthrough that creates a sample table and demonstrates fuzzy search and relevance boosting, see the [fuzzy search example](/search/fts-examples#fuzzy-search-and-boosting-example). - -### Search for Substring {#search-for-substring} - -LanceDB supports searching for substrings in the text column, you can set the `base_tokenizer` parameter to `"ngram"` to enable this feature, and use the parameters `ngram_min_length` and `ngram_max_length` to control the length of the substrings: - -| Parameter | Type | Default | Description | -| ---------------- | ---- | ------- | -------------------------------------------------- | -| ngram_min_length | int | 3 | Minimum length of the n-grams to search for | -| ngram_max_length | int | 3 | Maximum length of the n-grams to search for | -| prefix_only | bool | false | Whether to only search for prefixes of the n-grams | - - -## More Examples {#more-examples} - -For complete worked examples of fuzzy search, prefix matching, phrase matching, boosting, boolean queries, and substring search — including sample data generation and index setup — see [Full-Text Search Examples](/search/fts-examples). - -## Full-Text Search on Array Fields {#full-text-search-on-array-fields} - -LanceDB supports full-text search on string array columns, enabling efficient keyword-based search across multiple values within a single field (e.g., tags, keywords). - -### Setting Up the Connection {#setting-up-the-connection} - -Connect to your LanceDB instance: - - -```python Python icon="python" -import lancedb - -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) -``` - -```typescript TypeScript icon="square-js" expandable=true -import * as lancedb from "@lancedb/lancedb" - -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); -``` - - -### Defining the Schema {#defining-the-schema} - -Create a schema that includes an array field for tags: - - -```python Python icon="python" -table_name = "fts-array-field-test" -schema = pa.schema([ - pa.field("id", pa.string()), - pa.field("tags", pa.list_(pa.string())), - pa.field("description", pa.string()) -]) -``` - -```typescript TypeScript icon="square-js" expandable=true -const tableName = "fts-array-field-test-ts"; - -// Create schema -const schema = new Schema([ - new Field("id", new Utf8(), false), - new Field("tags", new List(new Field("item", new Utf8()))), - new Field("description", new Utf8(), false) -]); -``` - - -### Creating Sample Data {#creating-sample-data} - -Generate sample data with array fields containing tags: - - -```python Python icon="python" expandable=true -# Generate sample data -data = { - "id": [f"doc_{i}" for i in range(10)], - "tags": [ - ["python", "machine learning", "data science"], - ["deep learning", "neural networks", "AI"], - ["database", "indexing", "search"], - ["vector search", "embeddings", "AI"], - ["full text search", "indexing", "database"], - ["python", "web development", "flask"], - ["machine learning", "deep learning", "pytorch"], - ["database", "SQL", "postgresql"], - ["search engine", "elasticsearch", "indexing"], - ["AI", "transformers", "NLP"] - ], - "description": [ - "Python for data science projects", - "Deep learning fundamentals", - "Database indexing techniques", - "Vector search implementations", - "Full-text search guide", - "Web development with Python", - "Machine learning with PyTorch", - "Database management systems", - "Search engine optimization", - "AI and NLP applications" - ] -} -``` - -```typescript TypeScript icon="square-js" expandable=true -// Generate sample data -const data = makeArrowTable( - Array(10).fill(0).map((_, i) => ({ - id: `doc_${i}`, - tags: [ - ["python", "machine learning", "data science"], - ["deep learning", "neural networks", "AI"], - ["database", "indexing", "search"], - ["vector search", "embeddings", "AI"], - ["full text search", "indexing", "database"], - ["python", "web development", "flask"], - ["machine learning", "deep learning", "pytorch"], - ["database", "SQL", "postgresql"], - ["search engine", "elasticsearch", "indexing"], - ["AI", "transformers", "NLP"] - ][i], - description: [ - "Python for data science projects", - "Deep learning fundamentals", - "Database indexing techniques", - "Vector search implementations", - "Full-text search guide", - "Web development with Python", - "Machine learning with PyTorch", - "Database management systems", - "Search engine optimization", - "AI and NLP applications" - ][i] - })), - { schema } -); -``` - - -### Creating the Table and Adding Data {#creating-the-table-and-adding-data} - -Create the table and populate it with the sample data: - - -```python Python icon="python" -# Create table and add data -table = db.create_table(table_name, schema=schema, mode="overwrite") -table_data = pa.Table.from_pydict(data, schema=schema) -table.add(table_data) -``` - -```typescript TypeScript icon="square-js" -// Create table -const table = await db.createTable(tableName, data, { mode: "overwrite" }); -console.log(`Created table: ${tableName}`); -``` - - -### Building the Full-Text Search Index {#building-the-full-text-search-index} - -Create an FTS index on the tags column to enable efficient text search: - - -```python Python icon="python" -# Create FTS index -table.create_fts_index("tags") -wait_for_index(table, "tags_idx") -``` - -```typescript TypeScript icon="square-js" -// Create FTS index -console.log("Creating FTS index on 'tags' column..."); -await table.createIndex("tags", { - config: Index.fts() -}); - -// Wait for index -const ftsIndexName = "tags_idx"; -await waitForIndex(table, ftsIndexName); -``` - - -### Performing Fuzzy Search {#performing-fuzzy-search} - -Search for terms with typos using fuzzy matching: - - -```python Python icon="python" -# Search examples -print("\nSearching for 'learning' in tags with a typo:") -result = ( - table.search(MatchQuery("learnin", column="tags", fuzziness=1)) - .select(['id', 'tags', 'description']) - .to_arrow() -) -``` - -```typescript TypeScript icon="square-js"> -// Search examples -console.log("\nSearching for 'learning' in tags with a typo:"); -const fuzzyResults = await table.query() - .fullTextSearch(new MatchQuery("learnin", "tags", { - fuzziness: 2, - })) - .select(["id", "tags", "description"]) - .toArray(); -console.log(fuzzyResults); -``` - - -### Performing Phrase Search {#performing-phrase-search} - -Search for exact phrases within the array fields: - - -```python Python icon="python" -print("\nSearching for 'machine learning' in tags:") -result = ( - table.search(PhraseQuery("machine learning", column="tags")) - .select(['id', 'tags', 'description']) - .to_arrow() -) -``` - -```typescript TypeScript icon="square-js" -console.log("\nSearching for 'machine learning' in tags:"); -const phraseResults = await table.query() - .fullTextSearch(new PhraseQuery("machine learning", "tags")) - .select(["id", "tags", "description"]) - .toArray(); -console.log(phraseResults); -``` - diff --git a/docs/search/hybrid-search.mdx b/docs/search/hybrid-search.mdx deleted file mode 100644 index d4109c6..0000000 --- a/docs/search/hybrid-search.mdx +++ /dev/null @@ -1,355 +0,0 @@ ---- -title: Hybrid Search -sidebarTitle: Hybrid search -description: Learn how to perform hybrid search in LanceDB by combining vector and full-text search techniques with reranking. -icon: "search" ---- - -In certain cases, you may want to retrieve documents that are semantically similar to a given query, -but also prioritize specific keywords. This is an example of **hybrid search**, a query method that combines -multiple search techniques. - -For detailed examples, look at this [Python Notebook](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/saas_examples/python_notebook/Hybrid_search.ipynb) or the [**TypeScript Example**](https://github.com/lancedb/vectordb-recipes/tree/main/examples/saas_examples/ts_example/hybrid-search) - -## Example: Hybrid Search {#example-hybrid-search} - -### 1\. Setup {#1-setup} -Import the necessary libraries and dependencies for working with LanceDB, OpenAI embeddings, and reranking. - - -```python Python icon="python" -import os -import lancedb -import openai -from lancedb.embeddings import get_registry -from lancedb.pydantic import LanceModel, Vector -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import "@lancedb/lancedb/embedding/openai"; -import { Utf8 } from "apache-arrow"; -``` - - -### 2\. Connect to LanceDB {#2-connect-to-lancedb} -Establish a connection to your LanceDB instance, with different options for Enterprise setups or open source. - -OSS - - -```python Python icon="python" -uri = "data/sample-lancedb" -db = lancedb.connect(uri) -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; - -const databaseDir = "data/sample-lancedb"; -const db = await lancedb.connect(databaseDir); -``` - - -Enterprise - -For LanceDB Enterprise, set the `db://` URI, region and the host override to your private cloud endpoint: - - -```python Python icon="python" -host_override = os.environ.get("LANCEDB_HOST_OVERRIDE") - -db = lancedb.connect( - uri=uri, - api_key=api_key, - region=region, - host_override=host_override -) -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; - -const uri = "db://my-lancedb-instance/my-database"; -const apiKey = process.env.LANCEDB_API_KEY; -const region = process.env.LANCEDB_REGION; -const hostOverride = process.env.LANCEDB_HOST_OVERRIDE; - -const db = await lancedb.connect(uri, { - apiKey, - region - hostOverride, -}); -``` - - - - -### 3\. Configure Embedding Model {#3-configure-embedding-model} -Set up the any embedding model that will convert text into vector representations for semantic search. - - -```python Python icon="python" -embeddings = get_registry().get("sentence-transformers").create() -``` - -```typescript TypeScript icon="square-js" -const embedFunc = lancedb.embedding.getRegistry().get("openai")?.create({ - model: "text-embedding-ada-002", -}) as lancedb.embedding.EmbeddingFunction; -``` - - -### 4\. Create Table and Schema {#4-create-table-and-schema} -Define the data structure for your documents, including both the text content and its vector representation. - - -```python Python icon="python" -class Documents(LanceModel): - text: str = embeddings.SourceField() - vector: Vector(embeddings.ndims()) = embeddings.VectorField() - -table_name = "hybrid_search_example" -table = db.create_table(table_name, schema=Documents, mode="overwrite") -``` - -```typescript TypeScript icon="square-js" -const documentSchema = lancedb.embedding.LanceSchema({ - text: embedFunc.sourceField(new Utf8()), - vector: embedFunc.vectorField(), -}); - -const tableName = "hybrid_search_example"; -const table = await db.createEmptyTable(tableName, documentSchema, { - mode: "overwrite", -}); -``` - - -### 5\. Add Data {#5-add-data} -Insert sample documents into your table, which will be used for both semantic and keyword search. - - -```python Python icon="python" -data = [ - {"text": "rebel spaceships striking from a hidden base"}, - {"text": "have won their first victory against the evil Galactic Empire"}, - {"text": "during the battle rebel spies managed to steal secret plans"}, - {"text": "to the Empire's ultimate weapon the Death Star"}, -] -table.add(data=data) -``` - -```typescript TypeScript icon="square-js" -const data = [ - { text: "rebel spaceships striking from a hidden base" }, - { text: "have won their first victory against the evil Galactic Empire" }, - { text: "during the battle rebel spies managed to steal secret plans" }, - { text: "to the Empire's ultimate weapon the Death Star" }, -]; -await table.add(data); -console.log(`Created table: ${tableName} with ${data.length} rows`); -``` - - -### 6\. Build Full Text Index {#6-build-full-text-index} -Create a full-text search index on the text column to enable keyword-based search capabilities. - - -```python Python icon="python" -table.create_fts_index("text") -wait_for_index(table, "text_idx") -``` - -```typescript TypeScript icon="square-js" -console.log("Creating full-text search index..."); -await table.createIndex("text", { - config: lancedb.Index.fts(), -}); -await waitForIndex(table as any, "text_idx"); -``` - - -### 7\. Set Reranker [Optional] {#7-set-reranker-optional} -Initialize the reranker that will combine and rank results from both semantic and keyword search. By default, lancedb uses RRF reranker, but you can choose other rerankers like `Cohere`, `CrossEncoder`, or others lister in integrations section. - - -```python Python icon="python" -reranker = RRFReranker() -``` - -```typescript TypeScript icon="square-js" -const reranker = await lancedb.rerankers.RRFReranker.create(); -``` - - -### 8\. Hybrid Search {#8-hybrid-search} -Perform a hybrid search query that combines semantic similarity with keyword matching, using the specified reranker to merge and rank the results. - - -```python Python icon="python" -results = ( - table.search( - "flower moon", - query_type="hybrid", - vector_column_name="vector", - fts_columns="text", - ) - .rerank(reranker) - .limit(10) - .to_pandas() -) - -print("Hybrid search results:") -print(results) -``` - -```typescript TypeScript icon="square-js" -console.log("Performing hybrid search..."); -const queryVector = await embedFunc.computeQueryEmbeddings("full moon in May"); -const hybridResults = await table - .query() - .fullTextSearch("flower moon") - .nearestTo(queryVector) - .rerank(reranker) - .select(["text"]) - .limit(10) - .toArray(); - -console.log("Hybrid search results:"); -console.log(hybridResults); -``` - - -### 9\. Hybrid Search - Explicit Vector and Text Query pattern {#9-hybrid-search-explicit-vector-and-text-query-pattern} -You can also pass the vector and text query explicitly. This is useful if you're not using the embedding API or if you're using a separate embedder service. - - -```python Python icon="python" -vector_query = [0.1, 0.2, 0.3, 0.4, 0.5] -text_query = "flower moon" -( - table.search(query_type="hybrid") - .vector(vector_query) - .text(text_query) - .limit(5) - .to_pandas() -) -``` - - -## Query controls {#query-controls} - -Hybrid queries inherit the same builder API as vector and FTS queries, so the same knobs for filtering, distance bounds, and row identity apply. These compose with `.rerank(...)` and the explicit `.vector()` / `.text()` form shown above. - - -Always set `.limit(...)` on production hybrid queries. LanceDB's default search limit is 10, but an -explicit cap gives you a clear top-k contract to tune before reranking. - - -### Returning row IDs {#returning-row-ids} - -Pass `with_row_id(True)` (Python) or `withRowId()` (TypeScript) to include the internal `_rowid` column in the results. This is useful for joining hybrid results back to a primary table, or for deduping across multiple queries: - - -```python Python icon="python" -results = ( - table.search("flower moon", query_type="hybrid") - .with_row_id(True) - .limit(10) - .to_pandas() -) -# results now contains a `_rowid` column alongside `_relevance_score` -``` - -```typescript TypeScript icon="square-js" -const results = await table - .query() - .fullTextSearch("flower moon") - .nearestTo(queryVector) - .withRowId() - .limit(10) - .toArray(); -``` - - -### Bounding vector distance {#bounding-vector-distance} - -`distance_range(lower, upper)` (Python) and `distanceRange(lower, upper)` (TypeScript) constrain the vector half of the hybrid query to the half-open interval `[lower, upper)`. This is helpful when you want to cap how far semantic candidates can drift from the query vector before reranking: - - -```python Python icon="python" -results = ( - table.search("flower moon", query_type="hybrid") - .distance_range(lower_bound=0.0, upper_bound=0.4) - .limit(10) - .to_pandas() -) -``` - -```typescript TypeScript icon="square-js" -const results = await table - .query() - .fullTextSearch("flower moon") - .nearestTo(queryVector) - .distanceRange(0.0, 0.4) - .limit(10) - .toArray(); -``` - - -Either bound can be omitted to leave that side unbounded. - -### Prefilter vs. postfilter {#prefilter-vs-postfilter} - -When the query carries a metadata filter via `where(...)`, you can choose whether the filter runs before or after the vector and FTS sub-queries. **Prefiltering** (the default) applies `where` to the candidate set before scoring, which is usually what you want — it shrinks the working set and benefits from any scalar indexes on the filter columns. **Postfiltering** runs the filter on the already-ranked top-k from each sub-query; this can be faster when the filter is non-selective or unindexed, but it may return fewer than `limit` rows because some of the top-k may be filtered out. - - -```python Python icon="python" -# Prefilter (default): filter applied before scoring -table.search("flower moon", query_type="hybrid") \ - .where("category = 'film'", prefilter=True) \ - .limit(10) \ - .to_pandas() - -# Postfilter: filter applied after the sub-queries return top-k -table.search("flower moon", query_type="hybrid") \ - .where("category = 'film'", prefilter=False) \ - .limit(10) \ - .to_pandas() -``` - -```typescript TypeScript icon="square-js" -// Prefilter (default): just call .where(...) -await table.query() - .fullTextSearch("flower moon") - .nearestTo(queryVector) - .where("category = 'film'") - .limit(10) - .toArray(); - -// Postfilter: chain .postfilter() after .where(...) -await table.query() - .fullTextSearch("flower moon") - .nearestTo(queryVector) - .where("category = 'film'") - .postfilter() - .limit(10) - .toArray(); -``` - - -The choice gets baked into both sub-queries, so the vector and FTS halves see the filter applied the same way. Use [`explain_plan`](/search/optimize-queries#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. - -## More on Reranking {#more-on-reranking} - -You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB comes with [**built-in rerankers**](https://docs.lancedb.com/reranking) and you can implement your own **custom reranker** as well. - -By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers: - -| Argument | Type | Default | Description | -|:---------|:-----|:--------|:------------| -| `normalize` | `str` | `"score"` | The method to normalize the scores. Can be `rank` or `score`. If `rank`, the scores are converted to ranks and then normalized. If `score`, the scores are normalized directly. | -| `reranker` | `Reranker` | `RRF()` | The reranker to use. If not specified, the default reranker is used. | diff --git a/docs/search/index.mdx b/docs/search/index.mdx deleted file mode 100644 index b6f168d..0000000 --- a/docs/search/index.mdx +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Search -sidebarTitle: Overview -description: "Comprehensive guide to all search capabilities in LanceDB including vector search, full-text search, hybrid search, and more." -icon: "list" ---- - -| Feature | Description | -|:---------------|:------------| -| [Vector Search](/search/vector-search/) | Semantic similarity search with multiple distance metrics | -| [Multivector Search](/search/multivector-search/) | Search using multiple vector embeddings per document | -| [Full-Text Search](/search/full-text-search/) | Keyword-based search with BM25 and pre-filtering | -| [Hybrid Search](/search/hybrid-search/) | Combines vector and full-text search with reranking | -| [Filtering](/search/filtering/) | Filter results based on metadata fields | -| [SQL Queries](/search/sql/index) | SQL query capabilities for data exploration and analytics | - -## Before you search {#before-you-search} - -- Vector search can run without an ANN index as an exhaustive scan. That's useful while prototyping, but build a vector index before relying on low-latency searches over larger tables. -- Full-text and hybrid text search require an FTS index on the text column you query. If a table has multiple FTS indexes, specify the target column. FTS also supports phrase, boolean, boosted, multi-match, and fuzzy query forms when you need more than plain terms. -- Phrase FTS queries require an index created with token positions enabled. -- Multivector search currently uses cosine similarity and accepts either one query vector or a matrix of query vectors; every query vector must match the inner dimension of the multivector column. -- Set an explicit `.limit(...)` for production queries. The default top-k is 10 for search builders, - but spelling it out makes latency and result-count assumptions visible. Query builders also - support controls such as prefilter/postfilter, distance ranges, row-id inclusion, offset - pagination, and Arrow/Pandas/list result materialization. diff --git a/docs/search/multivector-search.mdx b/docs/search/multivector-search.mdx deleted file mode 100644 index a11ca48..0000000 --- a/docs/search/multivector-search.mdx +++ /dev/null @@ -1,246 +0,0 @@ ---- -title: "Multivector Search" -sidebarTitle: Multivector search -description: Learn how to perform multivector search in LanceDB to handle multiple vector embeddings per document, which is ideal for late-interaction models like ColBERT and ColPaLi. -icon: "braille" ---- - -LanceDB's multivector support enables you to store and search multiple vector embeddings for a single item. - -This capability is particularly valuable when working with late-interaction models like ColBERT and ColPaLi, which generate multiple embeddings per document. - -In this tutorial, you'll create a table with multiple vector embeddings per document and learn how to perform multivector search. For more end-to-end examples, see the [VectorDB recipes repository](https://github.com/lancedb/vectordb-recipes/tree/main/examples). - -## Multivector Support {#multivector-support} - -Each item in your dataset can have a column containing multiple vectors, which LanceDB can efficiently index and search. When performing a search, you can query with either a single vector embedding or multiple vector embeddings. - - -Currently, only the `cosine` metric is supported for multivector search. The vector value type can be `float16`, `float32`, or `float64`. - - -Each query vector must match the inner vector dimension in the multivector column. This applies to both single-vector queries and multi-vector query matrices. - -## Computing Similarity {#computing-similarity} - -MaxSim (Maximum Similarity) is a key concept in late-interaction models that: - -- Computes the maximum similarity between each query embedding and all document embeddings -- Sums these maximum similarities to get the final relevance score -- Effectively captures fine-grained semantic matches between query and document tokens - -The MaxSim calculation can be expressed as: - -$$ -\text{MaxSim}(Q, D) = \sum_{i=1}^{|Q|} \max_{j=1}^{|D|} \text{sim}(q_i, d_j) -$$ - -Where $sim$ is the similarity function (e.g., cosine similarity). - -$$ -Q = \{q_1, q_2, ..., q_{|Q|}\} -$$ - -$Q$ represents the query embeddings, and $D = \{d_1, d_2, ..., d_{|D|}\}$ represents the document embeddings. - -## Using Multivector Search {#using-multivector-search} - -### 1\. Setup {#1-setup} - -Connect to LanceDB and import the required libraries. - - -```python Python icon="python" -import lancedb -import numpy as np -import pyarrow as pa - -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="your-region" -) -``` - - -### 2\. Define Schema {#2-define-schema} - -Define a schema that specifies a multivector field. A multivector field is a nested list structure in which each document contains multiple vectors. In this case, we'll create a schema with: - -1. An ID field as an integer (int64) -2. A vector field that is a list of lists of float32 values - - The outer list represents multiple vectors per document - - Each inner list is a 256-dimensional vector - - Using float32 for memory efficiency while maintaining precision - - -```python Python icon="python" -db = lancedb.connect("data/multivector_demo") -schema = pa.schema( - [ - pa.field("id", pa.int64()), - # float16, float32, and float64 are supported - pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))), - ] -) -``` - - -### 3\. Generate Multivectors {#3-generate-multivectors} - -Generate sample data where each document contains multiple vector embeddings, which can represent different aspects or views of the same document. - -In this example, we create **1024 documents** where each document has **2 random vectors** of **dimension 256**, simulating a real-world scenario where you might have multiple embeddings per item. - - -```python Python icon="python" -data = [ - { - "id": i, - "vector": np.random.random(size=(2, 256)).tolist(), # Each document has 2 vectors - } - for i in range(1024) -] -``` - - -### 4\. Create a Table {#4-create-a-table} - -Create a table with the defined schema and sample data, which will store multiple vectors per document for similarity search. - - -```python Python icon="python" -tbl = db.create_table("multivector_example", data=data, schema=schema) -``` - - -### 5\. Build an Index {#5-build-an-index} - -Only cosine similarity is supported for multivector search operations. -For faster search, build the standard `IVF_PQ` index over your vectors: - - -```python Python icon="python" -tbl.create_index(metric="cosine", vector_column_name="vector") -``` - - - -**Indexing matters more for multivector tables than for single-vector ones.** - -A brute-force scan over a multivector column has to compare every query vector to every document vector in every row, so the cost grows with both the row count and the number of vectors per row. -In LanceDB OSS, the query will just run, so a large unindexed multivector table can stall a process for a long time before returning results. - -On LanceDB Enterprise, the brute-force KNN safety check applies a stricter row threshold to multivector columns — roughly 10× lower than for single-vector columns. So an unindexed multivector table will start being rejected with a "vector search would use brute-force KNN" error well before a comparable single-vector table would. Build the index before you start hitting it from production traffic, even if the dataset is small enough that you'd skip indexing for a single-vector workload. - - -### 6\. Query a Single Vector {#6-query-a-single-vector} - -When searching with a single query vector, it will be compared against all vectors in each document, and the similarity scores will be aggregated to find the most relevant documents. - - -```python Python icon="python" -query = np.random.random(256) -results_single = tbl.search(query).limit(5).to_pandas() -``` - - -### 7\. Query Multiple Vectors {#7-query-multiple-vectors} - -With multiple query vectors, LanceDB calculates similarity using late interaction, a late-interaction technique that computes relevance by finding the best-matching pairs between query and document vectors. This approach provides more nuanced matching while maintaining fast retrieval speeds. - - -```python Python icon="python" -query_multi = np.random.random(size=(2, 256)) -results_multi = tbl.search(query_multi).limit(5).to_pandas() -``` - - - -Visit the [Hugging Face embedding integration](/integrations/embedding/huggingface/) page for information on embedding models. - -## Simple Example: ColBERT Embeddings {#simple-example-colbert-embeddings} - -[ColBERT](https://arxiv.org/abs/2004.12832) is the most well-known late-interaction retrieval model that -represents each document and query as multiple token embeddings and scores matches by taking the best -token-to-token similarities (MaxSim) across them. - -Install the dependencies before running this example: - -```bash -pip install pylate lancedb pandas -``` - - -```python Python icon="python" -import numpy as np -import pyarrow as pa -import lancedb -from pylate import models - -# 1) Load a late-interaction model via PyLate -# PyLate docs show ColBERT() + encode(..., is_query=...) :contentReference[oaicite:2]{index=2} -model = models.ColBERT(model_name_or_path="lightonai/GTE-ModernColBERT-v1") - -# You can discover dim from one embedding (avoid guessing) -dim = model.encode(["hello"], is_query=True)[0].shape[1] - -# 2) Create a LanceDB table with a multivector column -db = lancedb.connect("./pylate_lancedb") -schema = pa.schema([ - pa.field("doc_id", pa.string()), - pa.field("text", pa.string()), - # multivector: list> :contentReference[oaicite:3]{index=3} - pa.field("mv", pa.list_(pa.list_(pa.float32(), dim))), -]) - -docs = [ - {"doc_id": "1", "text": "The train to Tokyo leaves at 5pm."}, - {"doc_id": "2", "text": "That Pho restaurant in Hanoi is highly rated."}, - {"doc_id": "3", "text": "This is a noodle bar in Osaka, Japan."}, -] - -# 3) Encode documents with PyLate (token vectors per doc) -doc_texts = [d["text"] for d in docs] -doc_embs = model.encode(doc_texts, is_query=False) # list/array of (T, dim) per doc :contentReference[oaicite:4]{index=4} - -rows = [] -for d, emb in zip(docs, doc_embs): - emb = np.asarray(emb, dtype=np.float32) - rows.append({**d, "mv": emb.tolist()}) - -tbl = db.create_table("docs", data=rows, schema=schema, mode="overwrite") - -# 4) Build an index + query using a query matrix. -# Multivector brute-force scales with rows × vectors-per-row, so build the index -# at much smaller dataset sizes than you would for single-vector search — and -# always before exposing the table to remote traffic. -tbl.create_index(vector_column_name="mv", metric="cosine") - -query = "Tell me about ramen in Japan" -q_emb = np.asarray(model.encode([query], is_query=True)[0], dtype=np.float32) # (Tq, dim) :contentReference[oaicite:5]{index=5} - -out = tbl.search(q_emb).limit(5).to_pandas() # multivector search accepts a matrix :contentReference[oaicite:6]{index=6} -print(out[["doc_id", "text"]]) -``` - - -Late-interaction model implementations evolve rapidly, so it's a good idea to check the latest popular models -when trying multivector search. - -## Advanced Example: XTR Embeddings {#advanced-example-xtr-embeddings} - -[ConteXtualized Token Retriever (XTR)](https://arxiv.org/abs/2304.01982) is a late-interaction retrieval model that represents text as token-level vectors instead of a single embedding. -This lets search score token-to-token matches (MaxSim), which can improve fine-grained relevance. - -The notebook linked below shows how to integrate XTR, which prioritizes critical document -tokens during the initial retrieval stage and removes the gathering stage to improve performance significantly. -By focusing on the most semantically salient tokens early in the process, XTR reduces computational complexity -while improving recall and ensuring rapid identification of candidate documents. - - - diff --git a/docs/search/optimize-queries.mdx b/docs/search/optimize-queries.mdx deleted file mode 100644 index 244e876..0000000 --- a/docs/search/optimize-queries.mdx +++ /dev/null @@ -1,483 +0,0 @@ ---- -title: "Optimize Query Performance" -sidebarTitle: Query optimization -description: "Analyze and optimize query performance in LanceDB." -icon: "gauge" ---- - -LanceDB provides two powerful tools for query analysis and optimization: `explain_plan` and `analyze_plan`. Let's take a better look at how they work: - -| Method | Purpose | Description | -| :------------- | :----------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `explain_plan` | Query Analysis | Print the resolved query plan to understand how the query will be executed. Helpful for identifying slow queries or unexpected query results. | -| `analyze_plan` | Performance Tuning | Execute the query and return a physical execution plan annotated with runtime metrics including execution time, number of rows processed, and I/O stats. Essential for performance tuning and debugging. | - -## Query Analysis Tools {#query-analysis-tools} - -### Inspect a query plan {#inspect-a-query-plan} - -Reveals the logical query plan before execution, helping you identify potential issues with query structure and index usage. This tool is useful for: - -- Verifying query optimization strategies -- Validating index selection -- Understanding query execution order -- Detecting missing indices - -### Analyze a query plan {#analyze-a-query-plan} -Executes the query and provides detailed runtime metrics, including: - -- Operation duration (`_elapsed_compute_`) -- Data processing statistics (`_output_rows_`, `_bytes_read_`) -- Index effectiveness (`_index_comparisons_`, `_indices_loaded_`) -- Resource utilization (`_iops_`, `_requests_`) - -Together, these tools offer a comprehensive view of query performance, from planning to execution. Use `explain_plan` to verify your query structure and `analyze_plan` to measure and optimize actual performance. - -Metadata filters are prefiltered by default, which usually shows the filter pushed into the -`LanceScan` or index scan. If you set `prefilter=False`, expect a separate `FilterExec` after -search instead; that can be useful for some expensive filters, but it changes both latency and -the number of rows available after filtering. - -## Reading the Execution Plan {#reading-the-execution-plan} - -To demonstrate query performance analysis, we'll use a table containing 1.2M rows sampled from the [Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). Initially, the table has no indices, allowing us to observe the impact of optimization. - -Let's examine a vector search query that: - -- Filters rows where `identifier` is between 0 and 1,000,000 -- Returns the top 100 matches -- Projects specific columns: `chunk_index`, `title`, and `identifier` - - -```python Python icon="python" -# explain_plan -query_explain_plan = ( - table.search(query_embed) - .where("identifier > 0 AND identifier < 1000000") - .select(["chunk_index", "title", "identifier"]) - .limit(100) - .explain_plan(True) -) -``` - -```typescript TypeScript icon="square-js" -// explain_plan -const explainPlan = await table - .search(queryEmbed) - .where("identifier > 0 AND identifier < 1000000") - .select(["chunk_index", "title", "identifier"]) - .limit(100) - .explainPlan(true); -``` - - -### Execution Plan Components {#execution-plan-components} - -The execution plan reveals the sequence of operations performed to execute your query. Let's examine each component: - -``` -ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] - RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" - CoalesceBatchesExec: target_batch_size=1024 - GlobalLimitExec: skip=0, fetch=100 - FilterExec: _distance@3 IS NOT NULL - SortExec: TopK(fetch=100), expr=[_distance@3 ASC NULLS LAST], preserve_partitioning=[false] - KNNVectorDistance: metric=l2 - FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 - LanceScan: uri=***, projection=[vector, identifier], row_id=true, row_addr=false, ordered=false -``` - -#### 1\. Base Layer (LanceScan) {#1-base-layer-lancescan} - -- Initial data scan loading only specified columns to minimize I/O -- Unordered scan enabling parallel processing - -``` -LanceScan: -- projection=[vector, identifier] -- row_id=true, row_addr=false, ordered=false -``` - -#### 2\. First Filter {#2-first-filter} - -- Apply requested filter on `identifier` column -- Reduces the number of vectors that need KNN computation - -``` -FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 -``` - -#### 3\. Vector Search {#3-vector-search} - -- Computes L2 (Euclidean) distances between query vector and all vectors that passed the filter - -``` -KNNVectorDistance: metric=l2 -``` - -#### 4\. Results Processing {#4-results-processing} - -- Filters out null distance results -- Sorts by distance and takes top 100 results -- Processes in batches of 1024 for optimal memory usage - -``` -SortExec: TopK(fetch=100) -- expr=[_distance@3 ASC NULLS LAST] -- preserve_partitioning=[false] -FilterExec: _distance@3 IS NOT NULL -GlobalLimitExec: skip=0, fetch=100 -CoalesceBatchesExec: target_batch_size=1024 -``` - -#### 5\. Data Retrieval {#5-data-retrieval} - -- `RemoteTake` is a key component of Lance's I/O cache -- Handles efficient data retrieval from remote storage locations -- Fetches specific rows and columns needed for the final output -- Optimizes network bandwidth by only retrieving required data - -``` -RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" -``` - -#### 6\. Final Output {#6-final-output} - -- Returns only requested columns and maintains column ordering - -```python -ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] -``` - -This plan demonstrates a basic search without index optimizations: it performs a full scan and filter before vector search. - -## Performance Analysis {#performance-analysis} - -Let's use `analyze_plan` to run the query and analyze the query performance, which will help us identify potential bottlenecks: - - -```python Python icon="python" -# analyze_plan -query_analyze_plan = ( - table.search(query_embed) - .where("identifier > 0 AND identifier < 1000000") - .select(["chunk_index", "title", "identifier"]) - .limit(100) - .analyze_plan() -) -``` - -```typescript TypeScript icon="square-js" -// analyze_plan -const analyzePlan = await table - .search(queryEmbed) - .where("identifier > 0 AND identifier < 1000000") - .select(["chunk_index", "title", "identifier"]) - .limit(100) - .analyzePlan(); -``` - - -### Performance Metrics Analysis {#performance-metrics-analysis} - -``` -ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance], metrics=[output_rows=100, elapsed_compute=1.424µs] - RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title", metrics=[output_rows=100, elapsed_compute=175.53097ms, output_batches=1, remote_takes=100] - CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=100, elapsed_compute=2.748µs] - GlobalLimitExec: skip=0, fetch=100, metrics=[output_rows=100, elapsed_compute=1.819µs] - FilterExec: _distance@3 IS NOT NULL, metrics=[output_rows=100, elapsed_compute=10.275µs] - SortExec: TopK(fetch=100), expr=[_distance@3 ASC NULLS LAST], preserve_partitioning=[false], metrics=[output_rows=100, elapsed_compute=39.259451ms, row_replacements=546] - KNNVectorDistance: metric=l2, metrics=[output_rows=1099508, elapsed_compute=56.783526ms, output_batches=1076] - FilterExec: identifier@1 > 0 AND identifier@1 < 1000000, metrics=[output_rows=1099508, elapsed_compute=17.136819ms] - LanceScan: uri=***, projection=[vector, identifier], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1200000, elapsed_compute=21.348178ms, bytes_read=1852931072, iops=78, requests=78] -``` - -#### 1\. Data Loading (LanceScan) {#1-data-loading-lancescan} - -- Scanned 1,200,000 rows from the LanceDB table -- Read 1.86GB of data in 78 I/O operations -- Only loaded necessary columns (`vector` and `identifier`) -- Unordered scan for parallel processing - -#### 2\. Filtering and Search {#2-filtering-and-search} - -- Applied prefilter condition (`identifier > 0 AND identifier < 1000000`) -- Reduced dataset from 1.2M to 1,099,508 rows -- KNN search used L2 (Euclidean) distance metric -- Vector comparisons processed in 1076 batches - -#### 3\. Results Processing {#3-results-processing} - -- KNN results sorted by distance (TopK with fetch=100) -- Null distances filtered out -- Batches coalesced to target size of 1024 rows -- Additional columns fetched for final results -- Remote take operation for 100 results -- Final projection of required columns - -### Distributed metrics on remote tables {#distributed-metrics-on-remote-tables} - -Enterprise - -When you call `analyze_plan` against a LanceDB Enterprise table, the query runs across a pool of workers. By default the returned plan aggregates each operator's metrics into a single value, matching the local single-node output. Pass a `distributed_metrics` mode when you need to see how work was split across workers: - -| Mode | What it shows | When to use it | -| :------------ | :---------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ | -| `aggregate` | One summary value per operator (default). Same shape as local `analyze_plan`. | Everyday performance checks and comparisons against a local plan. | -| `per_worker` | Metrics reported separately for each worker that participated in the query. | Diagnosing stragglers or uneven work distribution across the cluster. | -| `full` | Both the aggregate summary and the per-worker breakdown. | Deep investigations where you need the totals *and* the per-worker detail in a single output. | - -The parameter only affects remote plans — local queries always return the aggregate output regardless of what you pass. - - -```python Python icon="python" -# Show a per-worker breakdown for a remote query -plan = ( - table.search(query_embed) - .where("identifier > 0 AND identifier < 1000000") - .limit(100) - .analyze_plan(distributed_metrics="per_worker") -) -print(plan) -``` - -```typescript TypeScript icon="square-js" -// Show a per-worker breakdown for a remote query -const plan = await table - .search(queryEmbed) - .where("identifier > 0 AND identifier < 1000000") - .limit(100) - .analyzePlan("per_worker"); -``` - - - -Use `per_worker` or `full` sparingly — the plan output grows with the size of your worker pool. Stick with the default `aggregate` mode for routine tuning. - - -### Key Observations {#key-observations} - -- Vector search is the primary bottleneck (1,099,508 vector comparisons) -- Significant I/O overhead (1.86GB data read) -- Full table scan due to lack of indices -- Substantial optimization potential through proper index implementation - -## Optimized Query Execution {#optimized-query-execution} - -After creating vector and scalar indices, the execution plan shows: - -``` -ProjectionExec: expr=[chunk_index@3 as chunk_index, title@4 as title, identifier@2 as identifier, _distance@0 as _distance] - RemoteTake: columns="_distance, _rowid, identifier, chunk_index, title" - CoalesceBatchesExec: target_batch_size=1024 - GlobalLimitExec: skip=0, fetch=100 - SortExec: TopK(fetch=100), expr=[_distance@0 ASC NULLS LAST], preserve_partitioning=[false] - ANNSubIndex: name=vector_idx, k=100, deltas=1 - ANNIvfPartition: uuid=83916fd5-fc45-4977-bad9-1f0737539bb9, nprobes=20, deltas=1 - ScalarIndexQuery: query=AND(identifier > 0,identifier < 1000000) -``` - -### Optimized Plan Analysis {#optimized-plan-analysis} - -#### 1\. Scalar Index Query {#1-scalar-index-query} - -``` -ScalarIndexQuery: query=AND(identifier > 0,identifier < 1000000) -metrics=[ - output_rows=2 - index_comparisons=2,301,824 - indices_loaded=2 - output_batches=1 - parts_loaded=562 - elapsed_compute=86.979354ms -] -``` - -- Range filter using scalar index -- Only 2 index files and 562 scalar index parts loaded -- 2.3M index comparisons for matches - -#### 2\. Vector Search {#2-vector-search} - -``` -ANNSubIndex: name=vector_idx, k=100, deltas=1 -metrics=[ - output_rows=2,000 - index_comparisons=25,893 - indices_loaded=0 - output_batches=20 - parts_loaded=20 - elapsed_compute=111.849043ms -] -``` - -- IVF index with 20 probes -- Only 20 index parts loaded -- 25,893 vector comparisons -- 2,000 matching vectors - -#### 3\. Results Processing {#3-results-processing-2} - -``` -SortExec: TopK(fetch=100), expr=[_distance@0 ASC NULLS LAST], preserve_partitioning=[false] -GlobalLimitExec: skip=0, fetch=100 -CoalesceBatchesExec: target_batch_size=1024 -``` - -- Sorts by distance -- Limits to top 100 results -- Batches into groups of 1024 - -#### 4\. Data Fetching {#4-data-fetching} - -``` -RemoteTake: columns="_distance, _rowid, identifier, chunk_index, title" -metrics=[output_rows=100, elapsed_compute=113.491859ms, output_batches=1, remote_takes=100] -``` - -- Single output batch -- One remote take per row - -#### 5\. Final Projection {#5-final-projection} - -``` -ProjectionExec: expr=[chunk_index@3 as chunk_index, title@4 as title, identifier@2 as identifier, _distance@0 as _distance] -``` - -- Returns specified columns: chunk_index, title, identifier, and distance - -### Performance Improvements {#performance-improvements} - -#### 1\. Initial Data Access {#1-initial-data-access} - -``` -ScalarIndexQuery metrics: -- indices_loaded=2 -- parts_loaded=562 -- output_batches=1 -``` - -- Before: Full table scan of 1.2M rows, 1.86GB data -- After: Only 2 indices and 562 scalar index parts loaded -- Benefit: Eliminated table scans for prefilter - -#### 2\. Vector Search Efficiency {#2-vector-search-efficiency} - -``` -ANNSubIndex: -- index_comparisons=25,893 -- indices_loaded=0 -- parts_loaded=20 -- output_batches=20 -``` - -- Before: L2 calculations on 1,099,508 vectors -- After: - - 99.8% reduction in vector comparisons - - Decreased output batches from 1,076 to 20 - -#### 3\. Data Retrieval Optimization {#3-data-retrieval-optimization} - -``` -RemoteTake: -- remote_takes=100 -- output_batches=1 -``` - -- RemoteTake operation remains consistent - -## Performance Optimization Guide {#performance-optimization-guide} - -### 1\. Index Implementation {#1-index-implementation} - -#### When to Create Indices {#when-to-create-indices} - -- Columns used in WHERE clauses -- Vector columns for similarity searches -- Join columns used in `merge_insert` - -#### Index Type Selection {#index-type-selection} - -| Data Type | Recommended Index | Use Case | -| ----------- | ------------------ | ---------------------------------------- | -| Vector | IVF_PQ/IVF_HNSW_SQ/IVF_HNSW_FLAT | Approximate nearest neighbor search | -| Scalar | B-Tree | Range queries and sorting | -| Categorical | Bitmap | Multi-value filters and set operations | -| `List` | Label_list | Multi-label classification and filtering | - - -Use `table.index_stats()` to monitor index coverage. -A well-optimized table should have `num_unindexed_rows ~ 0`. - - -### 2\. Query Plan Optimization {#2-query-plan-optimization} - -#### Common Patterns and Fixes {#common-patterns-and-fixes} - -| Plan Pattern | Optimization | -| ------------------------------------------ | -------------------------------------------- | -| LanceScan with high _bytes_read_ or _iops_ | Add missing index | -| | Use `select()` to limit returned columns | -| | Check whether the dataset has been compacted | -| Multiple sequential filters | Reorder filter conditions | - -!!! note "Regular Performance Analysis" - Regularly analyze your query plans to identify and address performance bottlenecks. - The `analyze_plan` output provides detailed metrics to guide optimization efforts. - -### 3\. Getting Started with Optimization {#3-getting-started-with-optimization} - -For vector search performance: - -- Create ANN index on your vector column(s) as described in the [index guide](/indexing/vector-index/) -- If you often filter by metadata, create [scalar indices](/indexing/scalar-index/) on those columns - -## Analyzing non-vector queries {#analyzing-non-vector-queries} - -`explain_plan` and `analyze_plan` aren't vector-specific — they're available on every query builder, including FTS and hybrid. The most common reason to look at the plan for a non-vector query is to confirm whether your `where` clause pushed into the scan (good) or ran as a separate `FilterExec` step on top of the search results (often slower, and a hint that the filter column needs a scalar index). - -### FTS queries {#fts-queries} - - -```python Python icon="python" -plan = ( - table.search("puppy", query_type="fts") - .where("category = 'animals'", prefilter=True) - .limit(10) - .explain_plan(True) -) -print(plan) -``` - -```typescript TypeScript icon="square-js" -const plan = await table - .query() - .fullTextSearch("puppy") - .where("category = 'animals'") - .limit(10) - .explainPlan(true); -``` - - -In an indexed FTS plan you should see a `MatchQuery` (or other FTS execution node) reading from the inverted index, with the metadata filter pushed down. If the plan shows a `LanceScan` followed by `FilterExec` over the entire text column, the FTS index either isn't covering the column or the filter isn't using a scalar index — both worth investigating. - -### Hybrid queries {#hybrid-queries} - -For hybrid queries, `explain_plan` returns the reranker label followed by the vector and FTS sub-plans, indented for readability: - - -```python Python icon="python" -plan = ( - table.search("flower moon", query_type="hybrid") - .where("category = 'film'", prefilter=True) - .limit(10) - .explain_plan(True) -) -print(plan) -# RRFReranker(...) -# -# -``` - - -`analyze_plan` does the same, but executes both sub-queries and labels them as `Vector Search Plan:` and `FTS Search Plan:` in the output. This is the easiest way to see whether the filter pushed into both halves uniformly, and which half is dominating latency. diff --git a/docs/search/vector-search.mdx b/docs/search/vector-search.mdx deleted file mode 100644 index 185761f..0000000 --- a/docs/search/vector-search.mdx +++ /dev/null @@ -1,453 +0,0 @@ ---- -title: "Vector Search" -sidebarTitle: Vector search -description: "Learn how to run vector search queries in LanceDB. Includes best practices, tips and examples." -icon: "arrow-up-right-dots" ---- -import { - PyConfigureDistanceMetric as ConfigureDistanceMetric, - PySelectVectorColumn as SelectVectorColumn, - PyIndexNestedColumn as IndexNestedColumn, - PyExactVsApproximateDistances as ExactVsApproximateDistances, - PyVectorSearchPrefilter as VectorSearchPrefilter, - PyVectorSearchPostfilter as VectorSearchPostfilter, - PyMultivectorSearch as MultivectorSearch, - PySearchDistanceRange as SearchDistanceRange, - PySearchBinaryVectors as SearchBinaryVectors, - PyBatchSearch as BatchSearch, - PyFastSearch as FastSearch, - PyBruteForceSearch as BruteForceSearch, - PyBypassVectorIndex as BypassVectorIndex, - TsSearch2, - TsSelectVectorColumn, - TsIndexNestedColumn, - TsExactVsApproximate, - TsVectorSearchPrefilter, - TsVectorSearchPostfilter, - TsDistanceRange, - TsBinarySearch, - TsBatchSearch, - TsFastSearch, - TsBruteForceSearch, - TsBypassVectorIndex, - RsConfigureDistanceMetric, - RsExactVsApproximate, - RsVectorSearchPrefilter, - RsVectorSearchPostfilter, - RsSearchDistanceRange, - RsFastSearch, - RsBruteForceSearch, - RsBypassVectorIndex, - RsBatchSearch, - RsBinarySearch, -} from '/snippets/search.mdx'; - -Vector search is a technique used to search for similar items based on their vector representations, called embeddings. It is also known as similarity search, nearest neighbor search, or approximate nearest neighbor search. - -![](/static/assets/images/search/vector-db-basics.png) - -Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a multimodal lakehouse like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups. - -## Supported distance metrics {#supported-distance-metrics} - -Distance metrics determine how LanceDB compares vectors to find similar matches. Euclidean or `l2` is the default, and used for general-purpose similarity, `cosine` for unnormalized embeddings, `dot` for normalized embeddings (best performance), or `hamming` for binary vectors. - - -Ensure you always use the same distance metric that your embedding model was trained with. Most modern embedding models use cosine similarity, so `cosine` is often the best choice. However, if your vectors are normalized, you should use `dot` for best performance. - - -The right metric improves both search accuracy and query performance. Currently, LanceDB supports the following metrics: - -| Distance metric | Mathematical form | Notes | -|---|---|---| -| `l2` | $\|x-y\|_2=\sqrt{\sum_i (x_i-y_i)^2}$ | Measures the straight-line distance between two points in vector space. Calculated as the square root of the sum of squared differences between corresponding vector components. | -| `cosine` | $1-\frac{x\cdot y}{\|x\|_2\|y\|_2}$ | Measures directional difference between vectors. Computed as 1 minus cosine similarity (the dot product normalized by both vector magnitudes), so vector length does not affect the score. Use for unnormalized vectors. | -| `dot` | $x\cdot y=\sum_i x_i y_i$ | Calculates the sum of products of corresponding vector components. Provides raw similarity scores without normalization, sensitive to vector magnitudes. Use for normalized vectors for best performance. | -| `hamming` | $\sum_i \mathbf{1}[x_i\neq y_i]$ | Counts the number of positions where corresponding bits differ between binary vectors. Only applicable to binary vectors stored as packed uint8 arrays. | - -For indexed search, supported distance metrics vary by index type: - -| Index type | Supported distance metrics | -|---|---| -| `IVF_FLAT` | `["l2", "cosine", "dot", "hamming"]` | -| `IVF_PQ` | `["l2", "cosine", "dot"]` | -| `IVF_SQ` | `["l2", "cosine", "dot"]` | -| `IVF_RQ` | `["l2", "cosine", "dot"]` | -| `IVF_HNSW_FLAT` | `["l2", "cosine", "dot"]` | -| `IVF_HNSW_PQ` | `["l2", "cosine", "dot"]` | -| `IVF_HNSW_SQ` | `["l2", "cosine", "dot"]` | - -### Configure Distance Metric {#configure-distance-metric} - -By default, `l2` will be used as metric type. You can specify the metric type as -`cosine` or `dot` if required (`hamming` is supported for `IVF_FLAT` index only). - -**Note:** You can configure the distance metric during search only if there's no vector index. If a vector index exists, the distance metric will always be the one you specified when creating the index. - - - -{ConfigureDistanceMetric} - - - -{TsSearch2} - - - -{RsConfigureDistanceMetric} - - - -Here you can see the same search but using `cosine` similarity instead of `l2` distance. The result focuses on vector direction rather than absolute distance, which works better for normalized embeddings. - -Set `.limit(...)` on vector searches you run in applications. You can page through results with `.offset(...)` and include LanceDB's internal row id with `.with_row_id()` / `.withRowId()` when you need a stable handle for follow-up operations. - -## Selecting the vector column {#selecting-the-vector-column} - -If your table has exactly one vector column, you can omit the column name and LanceDB will pick it for you. This works for both top-level columns (such as `vector`) and vector fields nested inside a struct (such as `image.embedding`). - -When LanceDB can't infer a single column, it raises a `ValueError` (Python) or rejects the query (Node/Rust). Two cases trigger this: - -- No vector column: the schema has no `fixed_size_list` or `list` of floats. -- Multiple candidates: more than one column matches the query's dimension. The error lists every candidate path so you can pick one explicitly. - -To disambiguate, pass the field path with dot notation. Wrap any segment that contains characters outside `[A-Za-z0-9_]` in backticks (for example, `` `image-meta`.`embedding.v1` ``). - - - -{SelectVectorColumn} - - - -{TsSelectVectorColumn} - - - -The same field-path syntax works when creating an index on a nested vector column: - - - -{IndexNestedColumn} - - - -{TsIndexNestedColumn} - - - -When several columns share a name across structs (for example, `image.embedding` and `text.embedding`), LanceDB still picks the one whose dimension matches your query vector. If two candidates have the same dimension, you must pass the column name explicitly. - -## Vector Search With ANN Index {#vector-search-with-ann-index} - -Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. - -The trade-off is that the results are not guaranteed to be the true nearest neighbors of the query, but are usually "good enough" for most use cases. - -Use ANN search for large-scale applications where speed matters more than perfect recall. LanceDB uses approximate nearest neighbor algorithms to deliver fast results without examining every vector in your dataset. - - -When a vector index is used, `_distance` is not always the true distance between full vectors. On quantized ANN indexes, LanceDB may compute `_distance` from the compressed representation for speed. Use `refine_factor` when you want reranking on full vectors. - - -### Exact vs Approximate Distances {#exact-vs-approximate-distances} - -When doing vector search, the meaning of "distance" depends on whether you are using an index and whether `refine_factor` is specified as part of your query. -`nprobes` controls how many partitions are searched to find candidates, `approx_mode` controls the query-time speed/recall trade-off for RQ-quantized indexes, and `refine_factor` controls how many candidates are rescored on full vectors for better distance fidelity and reranking quality. - -The table below summarizes the behavior of `_distance` in search results based on your query configuration: - -| Query mode | Neighbor quality | `_distance` in results | -| :--- | :--- | :--- | -| No index or `.bypass_vector_index()` | Exact kNN (100% recall) | True distance on full vectors | -| Indexed ANN, no `refine_factor` | Approximate neighbors | Distance on the index representation: exact for flat indexes, approximate for quantized indexes | -| Indexed ANN + `refine_factor(1)` | Approximate neighbors (same candidate set) | Distances recomputed on full vectors for reranked candidates | -| Indexed ANN + `refine_factor(>1)` | Better recall than no refine (usually) | Distances recomputed on full vectors for reranked candidates | - - - -{ExactVsApproximateDistances} - - - -{TsExactVsApproximate} - - - -{RsExactVsApproximate} - - - -For deeper tuning guidance on indexing and performance estimation, see the [vector indexes](/indexing/vector-index/#search-configuration) page, -For tuning `nprobes`, see below. - -### Tuning approximate mode {#tuning-approximate-mode} - -Use `approx_mode` when you want to adjust the speed/recall trade-off for approximate vector search at query time. This setting currently applies only to RQ-quantized indexes, such as `IVF_RQ`; other index types ignore it. - -The supported values are: - -| Value | Behavior | -| :--- | :--- | -| `fast` | Prefer lower query latency, which can reduce recall. | -| `normal` | Use the default balance between query latency and recall. This only has an effect for RQ indexes built with `num_bits > 1`. | -| `accurate` | Prefer higher recall, which can increase query latency. | - -You can change `approx_mode` per query without rebuilding the index. For RQ indexes built with `num_bits=1`, `normal` uses the same one-bit scoring path as `fast`. If you also set `refine_factor`, LanceDB first uses `approx_mode` while finding candidates, then reranks the selected candidates on full vectors. - -### Tuning `nprobes` {#tuning-nprobes} - -- `nprobes` controls how many partitions are searched at query time. -- `nprobes` improves candidate recall, but does not by itself make `_distance` exact. -- By default, LanceDB automatically tunes `nprobes` to achieve the best performance without noticeably sacrificing accuracy. -- In most cases, leave `nprobes` unset and use the auto-tuned value. -- Only tune `nprobes` manually when recall is below your target, or when you need even higher performance for your workload. -- If recall is too low, increase `nprobes` gradually, but after a certain threshold, increasing `nprobes` yields only marginal accuracy gains. -- If you need higher performance and have recall headroom, decrease `nprobes` gradually. - -For filtered ANN searches, you can also set `minimum_nprobes` and `maximum_nprobes`. LanceDB starts -with the minimum and can scan more partitions up to the maximum if the filter leaves too few -candidates. Calling `nprobes(n)` fixes both values to `n`, which disables that adaptive behavior. - -### Vector Search with Prefiltering {#vector-search-with-prefiltering} - -This is the default vector search setting. You can use prefiltering to boost query performance by reducing the search space before vector calculations begin. The system first applies your filter criteria to the dataset, then conducts vector search operations only on the remaining relevant subset. - - - -{VectorSearchPrefilter} - - - -{TsVectorSearchPrefilter} - - - -{RsVectorSearchPrefilter} - - - -This filters out rows where label ≤ 2 before doing vector search, then picks specific columns from the top 5 matches. - -The `.where("label > 2")` applies a filter before vector search, `.select(["text", "keywords", "label"])` chooses specific columns to return, and `.limit(5)` restricts results to the top `5` most similar vectors. - -As a result, you'll see a result with just the data you want from the most similar vectors. - -### Vector Search with Postfiltering {#vector-search-with-postfiltering} - -Use postfiltering to prioritize vector similarity by searching the full dataset first, then applying metadata filters to the top results. This approach ensures you get the most similar vectors before filtering, which can be crucial when similarity is more important than metadata constraints. - - - -{VectorSearchPostfilter} - - - -{TsVectorSearchPostfilter} - - - -{RsVectorSearchPostfilter} - - - -Here you can see how to do vector search first to get the most similar vectors, then filter by label > 1 on those results. - -The `prefilter=False` parameter tells LanceDB to apply the filter after vector search instead of before, `.where("label > 1")` filters the top results by metadata, and `.select()` chooses which columns to include. - -In the end, you receive a query result with the best matches that also meet your metadata requirements. - - -[Post-filtering](/search/filtering/#post-filtering-with-vector-search) in LanceDB applies -the filter condition after obtaining the nearest neighbors based on vector similarity. - - -## Multivector Search {#multivector-search} - -Use multivector search when your documents contain multiple embeddings and you need sophisticated matching between query and document vector pairs. The late interaction approach finds the most relevant combinations across all available embeddings and provides nuanced similarity scoring. - -Only `cosine` similarity is supported as the distance metric for multivector search operations. -Every query vector must match the inner dimension of the multivector column; LanceDB rejects mismatched query dimensions rather than guessing how to reshape them. - - - -{MultivectorSearch} - - - -Here you can see how to take 2 query vectors and find the best matching pairs between them and document vectors using late interaction. The `np.random.random(size=(2, 256))` creates a 2×256 array with two random query vectors, `.limit(5)` returns the top 5 best document-query combinations, and `.to_pandas()` provides results in a DataFrame format. - -**Read more:** [Multivector search](/search/multivector-search/) - -## Advanced Search Scenarios {#advanced-search-scenarios} - -### Search With Distance Range {#search-with-distance-range} - -Use `distance_range` search when you need vectors within particular similarity bounds rather than just the closest neighbors. The system filters results to only include vectors that fall within your specified distance thresholds from the query. - - - -{SearchDistanceRange} - - - -{TsDistanceRange} - - - -{RsSearchDistanceRange} - - - -This shows three ways to search within distance ranges: bounded, upper bound only, and lower bound only. - -The `distance_range()` method filters results by similarity thresholds - the first example finds vectors with distance between `0.1` and `0.5`, the second finds vectors closer than `0.5`, and the third finds vectors farther than `0.1`. - - Each approach returns Arrow tables with vectors that fall within your specified distance thresholds. - -### Search With Binary Vectors {#search-with-binary-vectors} - -Use binary vector search for scenarios involving binary embeddings, such as those produced by hashing algorithms. The system stores these efficiently as packed uint8 arrays and uses Hamming distance calculations to determine vector similarity. - - -The number of dimensions of the binary vector must be a multiple of 8. A vector of dimensionality 128 will be stored as a `uint8` array of size 16. - - - - -{SearchBinaryVectors} - - - -{TsBinarySearch} - - - -{RsBinarySearch} - - - -Here you can see how to set up a table for binary vectors, pack them efficiently into bytes, and search using Hamming distance. - -The schema defines a 32-byte vector field (256 bits ÷ 8), `np.random.randint(0, 2, size=256)` creates binary vectors, `np.packbits()` compresses them to bytes, and `.distance_type("hamming")` specifies `hamming` distance for similarity calculation. - -The search produces an Arrow table with binary vectors ranked by how many bits differ from the query. - -## Scaling Vector Search {#scaling-vector-search} - -### Batch Search {#batch-search} - -Use batch search to handle multiple query vectors simultaneously. This gives you significant efficiency gains over individual queries. LanceDB processes all vectors in parallel and organizes results with a `query_index` field that maps each result set back to its originating query. - - - -{BatchSearch} - - - -{TsBatchSearch} - - - -{RsBatchSearch} - - - -This takes 5 query embeddings and finds the top 5 matches for each one in a single batch operation. - -The `load_dataset()` loads embeddings from a Hugging Face dataset, `query_embeds` contains `5` query vectors, and `.search(query_embeds)` processes all queries simultaneously. - -The final query result contains all results, including a `query_index` to tell you which query each result came from. - - -When processing batch queries, the results include a `query_index` field -to explicitly associate each result set with its corresponding query in -the input batch. - - -### Search With Asynchronous Indexing {#search-with-asynchronous-indexing} - -To optimize for speed over completeness, enable the `fast_search` flag in your query to skip searching unindexed data. - -While vector indexing occurs asynchronously, newly added vectors are immediately -searchable through a fallback brute-force search mechanism. This ensures zero -latency between data insertion and searchability, though it may temporarily -increase query response times. - - - -{FastSearch} - - - -{TsFastSearch} - - - -{RsFastSearch} - - - -Here you can see how to turn on fast search mode to skip unindexed vectors and only look through indexed data for speed. - -The `fast_search=True` parameter tells LanceDB to only search indexed vectors, skipping any recently added data that hasn't been indexed yet. - -You'll obtain a query result with the top `5` matches from indexed vectors, but might miss data that was just added. - -## Brute Force Search {#brute-force-search} - -### Search With No Index {#search-with-no-index} - -The simplest way to perform vector search is to perform a brute force search, without an index, where the distance between the query vector and all the vectors in the database are computed, with the top-k closest vectors returned. - -This is equivalent to a k-nearest neighbours (kNN) search in vector space. - -Choose brute force search when you need guaranteed 100% recall, typically with smaller datasets where query speed isn't the primary concern. The system scans every vector in the table and calculates precise distances to find the exact nearest neighbors. - - - -{BruteForceSearch} - - - -{TsBruteForceSearch} - - - -{RsBruteForceSearch} - - - -This carries out a brute force search through every vector in the table to find the 3 closest matches to a random 1536-dimensional query. You'll get back a list of the most similar vectors with exact distances. - -![](/static/assets/images/search/knn_search.png) - -As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in. - -### Bypass the Vector Index {#bypass-the-vector-index} - -Use `bypass_vector_index` to get exact, ground-truth results by performing exhaustive searches across all vectors. Instead of relying on approximate methods, the system directly compares your query against every vector in the table, ensuring 100% recall at the cost of increased query time. - - - -{BypassVectorIndex} - - - -{TsBypassVectorIndex} - - - -{RsBypassVectorIndex} - - - -This skips the approximate index and checks every single vector for exact, ground-truth results. - -The `.bypass_vector_index()` method forces LanceDB to perform an exhaustive search through all vectors instead of using the approximate nearest neighbor index, ensuring exact results but at the cost of slower performance. - -The output is a query result with the top 5 exact matches, guaranteeing 100% recall but taking longer to run. - -This approach is particularly useful when: -- Evaluating ANN index quality -- Calculating recall metrics to tune index parameters -- Ensuring exact results for critical applications diff --git a/docs/snippets/ann_indexes.mdx b/docs/snippets/ann_indexes.mdx deleted file mode 100644 index ef4021e..0000000 --- a/docs/snippets/ann_indexes.mdx +++ /dev/null @@ -1,12 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const TsImport = "import * as lancedb from \"@lancedb/lancedb\";\nimport type { VectorQuery } from \"@lancedb/lancedb\";\n"; - -export const TsIngest = "const db = await lancedb.connect(databaseDir);\n\nconst data = Array.from({ length: 5_000 }, (_, i) => ({\n vector: Array(128).fill(i),\n id: `${i}`,\n content: \"\",\n longId: `${i}`,\n}));\n\nconst table = await db.createTable(\"my_vectors\", data, {\n mode: \"overwrite\",\n});\nawait table.createIndex(\"vector\", {\n config: lancedb.Index.ivfPq({\n numPartitions: 10,\n numSubVectors: 16,\n }),\n});\n"; - -export const TsSearch1 = "const search = table.search(Array(128).fill(1.2)).limit(2) as VectorQuery;\nconst results1 = await search.nprobes(20).refineFactor(10).toArray();\n"; - -export const TsSearch2 = "const results2 = await table\n .search(Array(128).fill(1.2))\n .where(\"id != '1141'\")\n .limit(2)\n .toArray();\n"; - -export const TsSearch3 = "const results3 = await table\n .search(Array(128).fill(1.2))\n .select([\"id\"])\n .limit(2)\n .toArray();\n"; - diff --git a/docs/snippets/basic_usage.mdx b/docs/snippets/basic_usage.mdx deleted file mode 100644 index d377efa..0000000 --- a/docs/snippets/basic_usage.mdx +++ /dev/null @@ -1,100 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyBasicAddColumns = "table.add_columns(\n {\n \"power\": \"cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)\"\n }\n)\n"; - -export const PyBasicAddData = "magical_characters = [\n {\n \"id\": 9,\n \"name\": \"Morgan le Fay\",\n \"role\": \"Sorceress\",\n \"description\": \"A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.\",\n \"vector\": [0.10, 0.84, 0.25, 0.70],\n \"stats\": { \"strength\": 2, \"courage\": 3, \"magic\": 5, \"wisdom\": 4 }\n },\n {\n \"id\": 10,\n \"name\": \"The Lady of the Lake\",\n \"role\": \"Mystical Guardian\",\n \"description\": \"A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.\",\n \"vector\": [0.00, 0.90, 0.58, 0.88],\n \"stats\": { \"strength\": 2, \"courage\": 3, \"magic\": 5, \"wisdom\": 5 }\n }\n]\ntable.add(magical_characters)\n"; - -export const PyBasicAsyncApi = "import lancedb\n\nasync_db = await lancedb.connect_async(uri)\nasync_table = await async_db.create_table(\n \"camelot_async\",\n data=data,\n mode=\"overwrite\",\n)\n\nquery_vector = [0.03, 0.85, 0.61, 0.90]\nasync_results = await (\n await async_table.search(query_vector)\n).limit(5).select([\"name\", \"role\", \"description\"]).to_polars()\nprint(async_results)\n"; - -export const PyBasicCreateEmptyTable = "schema = pa.schema(\n [\n pa.field(\"id\", pa.uint16()),\n pa.field(\"name\", pa.string()),\n pa.field(\"role\", pa.string()),\n pa.field(\"description\", pa.string()),\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\n \"stats\",\n pa.struct(\n [\n pa.field(\"strength\", pa.int8()),\n pa.field(\"courage\", pa.int8()),\n pa.field(\"magic\", pa.int8()),\n pa.field(\"wisdom\", pa.int8()),\n ]\n ),\n ),\n ]\n)\ndb.create_table(\"camelot_pa\", schema=schema, mode=\"overwrite\")\n"; - -export const PyBasicCreateTable = "table = db.create_table(\"camelot\", data=data, mode=\"overwrite\")\n"; - -export const PyBasicCreateTablePandas = "pandas_df = pd.DataFrame(data)\ntable_pd = db.create_table(\"camelot_pd\", data=pandas_df, mode=\"overwrite\")\n"; - -export const PyBasicCreateTablePolars = "polars_df = pl.DataFrame(data)\ntable_pl = db.create_table(\"camelot_pl\", data=polars_df, mode=\"overwrite\")\n"; - -export const PyBasicDeleteRows = "table.delete('role = \"Traitor Knight\"')\n"; - -export const PyBasicDropColumns = "table.drop_columns([\"power\"])\n"; - -export const PyBasicDropTable = "db.drop_table(\"camelot\")\n"; - -export const PyBasicImports = "import json\n\nimport lancedb\nimport pandas as pd\nimport polars as pl\nimport pyarrow as pa\n"; - -export const PyBasicOpenTable = "table = db.open_table(\"camelot\")\n"; - -export const PyBasicSortPolars = "# Sort Polars DataFrame by power in descending order\nprint(r4.sort(\"power\", descending=True).limit(5))\n"; - -export const PyBasicVectorSearch = "query_vector = [0.03, 0.85, 0.61, 0.90]\nresult = table.search(query_vector).limit(5).to_polars()\nprint(result)\n"; - -export const PyBasicVectorSearchQ1 = "# Who are the characters similar to \"wizard\"?\nquery_vector_1 = [0.03, 0.85, 0.61, 0.90]\nr1 = (\n table.search(query_vector_1)\n .limit(5)\n .select([\"name\", \"role\", \"description\"])\n .to_polars()\n)\nprint(r1)\n"; - -export const PyBasicVectorSearchQ2 = "# Who are the characters with high magic stats?\nquery_vector_2 = [0.03, 0.85, 0.61, 0.90]\nr2 = (\n table.search(query_vector_2)\n .where(\"stats.magic > 3\")\n .select([\"name\", \"role\", \"description\"])\n .limit(5)\n .to_polars()\n)\nprint(r2)\n"; - -export const PyBasicVectorSearchQ3 = "# Who are the strongest characters?\nr3 = (\n table.search()\n .where(\"stats.strength > 3\")\n .select([\"name\", \"role\", \"description\"])\n .limit(5)\n .to_polars()\n)\nprint(r3)\n"; - -export const PyBasicVectorSearchQ4 = "# Who are the strongest characters?\nr4 = (\n table.search()\n .select([\"name\", \"role\", \"description\", \"power\"])\n .to_polars()\n)\nprint(r4)\n"; - -export const PyDataLoad = "with open(data_path, \"r\") as f:\n data = json.load(f)\n"; - -export const TsBasicAddColumns = "await table.addColumns([\n {\n name: \"power\",\n valueSql:\n \"cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)\",\n },\n]);\n"; - -export const TsBasicAddData = "const magicalCharacters = [\n {\n id: 9,\n name: \"Morgan le Fay\",\n role: \"Sorceress\",\n description:\n \"A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.\",\n vector: [0.1, 0.84, 0.25, 0.7],\n stats: { strength: 2, courage: 3, magic: 5, wisdom: 4 },\n },\n {\n id: 10,\n name: \"The Lady of the Lake\",\n role: \"Mystical Guardian\",\n description:\n \"A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.\",\n vector: [0.0, 0.9, 0.58, 0.88],\n stats: { strength: 2, courage: 3, magic: 5, wisdom: 5 },\n },\n];\nawait table.add(magicalCharacters);\n"; - -export const TsBasicCreateEmptyTable = "const schema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int16()),\n new arrow.Field(\"name\", new arrow.Utf8()),\n new arrow.Field(\"role\", new arrow.Utf8()),\n new arrow.Field(\"description\", new arrow.Utf8()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n new arrow.Field(\n \"stats\",\n new arrow.Struct([\n new arrow.Field(\"strength\", new arrow.Int8()),\n new arrow.Field(\"courage\", new arrow.Int8()),\n new arrow.Field(\"magic\", new arrow.Int8()),\n new arrow.Field(\"wisdom\", new arrow.Int8()),\n ]),\n ),\n]);\nawait db.createEmptyTable(\"camelot_empty\", schema, { mode: \"overwrite\" });\n"; - -export const TsBasicCreateTable = "let table = await db.createTable(\"camelot\", data, {\n mode: \"overwrite\",\n});\n"; - -export const TsBasicDeleteRows = "await table.delete('role = \"Traitor Knight\"');\n"; - -export const TsBasicDropColumns = "await table.dropColumns([\"power\"]);\n"; - -export const TsBasicDropTable = "await db.dropTable(\"camelot\");\n"; - -export const TsBasicImports = "import * as lancedb from \"@lancedb/lancedb\";\nimport * as arrow from \"apache-arrow\";\n"; - -export const TsBasicOpenTable = "table = await db.openTable(\"camelot\");\n"; - -export const TsBasicVectorSearch = "const queryVector = [0.03, 0.85, 0.61, 0.9];\nconst result = await table.search(queryVector).limit(5).toArray();\nconsole.log(result);\n"; - -export const TsBasicVectorSearchQ1 = "// Who are the characters similar to \"wizard\"?\nconst queryVector1 = [0.03, 0.85, 0.61, 0.9];\nconst r1 = await table\n .search(queryVector1)\n .limit(5)\n .select([\"name\", \"role\", \"description\"])\n .toArray();\nconsole.log(r1);\n"; - -export const TsBasicVectorSearchQ2 = "// Who are the characters similar to \"wizard\" with high magic stats?\nconst queryVector2 = [0.03, 0.85, 0.61, 0.9];\nconst r2 = await table\n .search(queryVector2)\n .where(\"stats.magic > 3\")\n .select([\"name\", \"role\", \"description\"])\n .limit(5)\n .toArray();\nconsole.log(r2);\n"; - -export const TsBasicVectorSearchQ3 = "// Who are the strongest characters?\nconst r3 = await table\n .query()\n .where(\"stats.strength > 3\")\n .select([\"name\", \"role\", \"description\"])\n .limit(5)\n .toArray();\nconsole.log(r3);\n"; - -export const TsBasicVectorSearchQ4 = "// Who are the strongest characters?\nconst r4 = await table\n .query()\n .select([\"name\", \"role\", \"description\", \"power\"])\n .toArray();\nconsole.log(r4);\n"; - -export const TsDataLoad = "const data = JSON.parse(fs.readFileSync(dataPath, \"utf-8\"));\n"; - -export const RsBasicAddColumns = "table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"power\".to_string(),\n \"cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)\"\n .to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\n"; - -export const RsBasicAddData = "let magical_characters = vec![\n Character {\n id: 9,\n name: \"Morgan le Fay\".to_string(),\n role: \"Sorceress\".to_string(),\n description: \"A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.\".to_string(),\n vector: [0.10, 0.84, 0.25, 0.70],\n stats: Stats {\n strength: 2,\n courage: 3,\n magic: 5,\n wisdom: 4,\n },\n },\n Character {\n id: 10,\n name: \"The Lady of the Lake\".to_string(),\n role: \"Mystical Guardian\".to_string(),\n description: \"A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.\".to_string(),\n vector: [0.00, 0.90, 0.58, 0.88],\n stats: Stats {\n strength: 2,\n courage: 3,\n magic: 5,\n wisdom: 5,\n },\n },\n];\ntable\n .add(characters_to_reader(camelot_schema(), &magical_characters))\n .execute()\n .await\n .unwrap();\n"; - -export const RsBasicCreateEmptyTable = "let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int16, false),\n Field::new(\"name\", DataType::Utf8, false),\n Field::new(\"role\", DataType::Utf8, false),\n Field::new(\"description\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n Field::new(\n \"stats\",\n DataType::Struct(arrow_schema::Fields::from(vec![\n Arc::new(Field::new(\"strength\", DataType::Int8, false)),\n Arc::new(Field::new(\"courage\", DataType::Int8, false)),\n Arc::new(Field::new(\"magic\", DataType::Int8, false)),\n Arc::new(Field::new(\"wisdom\", DataType::Int8, false)),\n ])),\n false,\n ),\n]));\ndb.create_empty_table(\"camelot_empty\", schema)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsBasicCreateTable = "let mut table = db\n .create_table(\"camelot\", characters_to_reader(schema.clone(), &data))\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsBasicDeleteRows = "table.delete(\"role = 'Traitor Knight'\").await.unwrap();\n"; - -export const RsBasicDropColumns = "table.drop_columns(&[\"power\"]).await.unwrap();\n"; - -export const RsBasicDropTable = "db.drop_table(\"camelot\", &[]).await.unwrap();\n"; - -export const RsBasicImports = "use arrow_array::types::Float32Type;\nuse arrow_array::{\n FixedSizeListArray, Int8Array, Int16Array, RecordBatch, RecordBatchIterator, StringArray,\n StructArray,\n};\nuse arrow_schema::{DataType, Field, FieldRef, Schema};\nuse futures_util::TryStreamExt;\nuse lancedb::database::CreateTableMode;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse lancedb::{connect, table::NewColumnTransform};\n"; - -export const RsBasicOpenTable = "table = db.open_table(\"camelot\").execute().await.unwrap();\n"; - -export const RsBasicVectorSearch = "let query_vector = [0.03, 0.85, 0.61, 0.90];\nlet result = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .limit(5)\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; - -export const RsBasicVectorSearchQ1 = "// Who are the characters similar to \"wizard\"?\nlet query_vector_1 = [0.03, 0.85, 0.61, 0.90];\nlet r1 = table\n .query()\n .nearest_to(&query_vector_1)\n .unwrap()\n .limit(5)\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n ]))\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\nprintln!(\"{r1:?}\");\n"; - -export const RsBasicVectorSearchQ2 = "// Who are the characters similar to \"wizard\" with high magic stats?\nlet query_vector_2 = [0.03, 0.85, 0.61, 0.90];\nlet r2 = table\n .query()\n .nearest_to(&query_vector_2)\n .unwrap()\n .only_if(\"stats.magic > 3\")\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n ]))\n .limit(5)\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\nprintln!(\"{r2:?}\");\n"; - -export const RsBasicVectorSearchQ3 = "// Who are the strongest characters?\nlet r3 = table\n .query()\n .only_if(\"stats.strength > 3\")\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n ]))\n .limit(5)\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\nprintln!(\"{r3:?}\");\n"; - -export const RsBasicVectorSearchQ4 = "// Who are the strongest characters?\nlet r4 = table\n .query()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"power\".to_string(),\n ]))\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\nprintln!(\"{r4:?}\");\n"; - -export const RsDataLoad = "let data: Vec =\n serde_json::from_str(&fs::read_to_string(camelot_json_path()).unwrap()).unwrap();\n"; - diff --git a/docs/snippets/build_with_ai_agents.mdx b/docs/snippets/build_with_ai_agents.mdx deleted file mode 100644 index 1a82d6f..0000000 --- a/docs/snippets/build_with_ai_agents.mdx +++ /dev/null @@ -1,7 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyCamelotBatches = "import json\nfrom collections.abc import Iterator\nfrom pathlib import Path\n\n\ndef validated_batches(\n input_path: Path, batch_size: int\n) -> Iterator[list[dict]]:\n raw_records = json.loads(input_path.read_text())\n asset_root = input_path.parent.parent\n batch: list[dict] = []\n\n for raw in raw_records:\n payload = dict(raw)\n image_path = asset_root / payload.pop(\"img\")\n payload[\"image_filename\"] = image_path.name\n payload[\"image\"] = image_path.read_bytes()\n\n character = Character.model_validate(payload)\n batch.append(character.model_dump(mode=\"python\"))\n\n if len(batch) == batch_size:\n yield batch\n batch = []\n\n if batch:\n yield batch\n"; - -export const PyCamelotOssIngestion = "import lancedb\n\n\ndef ingest_oss(\n input_path: Path,\n uri: str = \"data/camelot.lancedb\",\n table_name: str = \"camelot_multimodal\",\n batch_size: int = 64,\n):\n db = lancedb.connect(uri)\n if table_name in db.list_tables():\n raise ValueError(\n f\"Table {table_name!r} already exists. Choose a fresh table name.\"\n )\n\n table = db.create_table(table_name, schema=Character)\n for batch in validated_batches(input_path, batch_size):\n table.add(batch)\n\n table.optimize()\n return table\n\n\nif __name__ == \"__main__\":\n ingest_oss(Path(\"data/camelot.json\"))\n"; - -export const PyCamelotSchema = "from lancedb.pydantic import LanceModel\nfrom pydantic import ConfigDict\n\n\nclass Stats(LanceModel):\n model_config = ConfigDict(strict=True, extra=\"forbid\")\n\n strength: int\n courage: int\n magic: int\n wisdom: int\n\n\nclass Character(LanceModel):\n model_config = ConfigDict(strict=True, extra=\"forbid\")\n\n id: int\n name: str\n role: str\n description: str\n stats: Stats\n image_filename: str\n image: bytes\n"; diff --git a/docs/snippets/connection.mdx b/docs/snippets/connection.mdx deleted file mode 100644 index c7d8a18..0000000 --- a/docs/snippets/connection.mdx +++ /dev/null @@ -1,36 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyConnect = "import lancedb\n\nuri = \"ex_lancedb\"\ndb = lancedb.connect(uri)\n"; - -export const PyConnectAsync = "import lancedb\n\nuri = \"ex_lancedb\"\nasync_db = await lancedb.connect_async(uri)\n"; - -export const PyConnectEnterpriseQuickstart = "uri = \"db://your-database-uri\"\napi_key = \"your-api-key\"\nregion = \"us-east-1\"\nhost_override = \"https://your-enterprise-endpoint.com\"\n\ndb = lancedb.connect(\n uri=uri,\n api_key=api_key,\n region=region,\n host_override=host_override,\n)\n"; - -export const PyConnectObjectStorage = "import lancedb\n\nuri = \"s3://your-bucket/path\"\n# You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\ndb = lancedb.connect(uri)\n"; - -export const PyConnectObjectStorageAsync = "import lancedb\n\nuri = \"s3://your-bucket/path\"\n# You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\nasync_db = await lancedb.connect_async(uri)\n"; - -export const PyNamespaceAdminOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\nnamespace = [\"prod\", \"search\"]\n\ndb.create_namespace([\"prod\"])\ndb.create_namespace([\"prod\", \"search\"])\n\nchild_namespaces = db.list_namespaces(namespace_path=[\"prod\"]).namespaces\nprint(f\"Child namespaces under {namespace}: {child_namespaces}\")\n# Child namespaces under ['prod', 'search']: ['search']\n\nmetadata = db.describe_namespace([\"prod\", \"search\"])\nprint(f\"Metadata for namespace {namespace}: {metadata}\")\n# Metadata for namespace ['prod', 'search']: properties=None\n\ndb.drop_namespace([\"prod\", \"search\"], mode=\"skip\")\ndb.drop_namespace([\"prod\"], mode=\"skip\")\n"; - -export const PyNamespaceTableOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\n\n# Create namespace tree: prod/search\ndb.create_namespace([\"prod\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"search\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"recommendations\"], mode=\"exist_ok\")\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 1, \"vector\": [0.1, 0.2], \"name\": \"alice\"}],\n namespace_path=[\"prod\", \"search\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 2, \"vector\": [0.3, 0.4], \"name\": \"bob\"}],\n namespace_path=[\"prod\", \"recommendations\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\n# Verify\nprint(db.list_namespaces()) # ['prod']\nprint(db.list_namespaces(namespace_path=[\"prod\"])) # ['recommendations', 'search']\nprint(db.list_tables(namespace_path=[\"prod\", \"search\"])) # ['user']\nprint(db.list_tables(namespace_path=[\"prod\", \"recommendations\"])) # ['user']\n"; - -export const TsConnect = "import * as lancedb from \"@lancedb/lancedb\";\n\nasync function connectExample(uri: string) {\n const db = await lancedb.connect(uri);\n return db;\n}\n"; - -export const TsConnectEnterpriseQuickstart = "const uri = \"db://your-database-uri\";\nconst apiKey = \"your-api-key\";\nconst region = \"us-east-1\";\nconst hostOverride = \"https://your-enterprise-endpoint.com\";\n\nconst db = await lancedb.connect(uri, {\n apiKey,\n region,\n hostOverride,\n});\n"; - -export const TsConnectObjectStorage = "async function connectObjectStorageExample() {\n const uri = \"s3://your-bucket/path\";\n // You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\n const db = await lancedb.connect(uri);\n return db;\n}\n"; - -export const TsNamespaceAdminOps = "const db = await lancedb.connectNamespace(\"dir\", { root: \"./local_lancedb\" });\nconst namespace = [\"prod\", \"search\"];\n\nawait db.createNamespace([\"prod\"]);\nawait db.createNamespace([\"prod\", \"search\"]);\n\nconst childNamespaces = (await db.listNamespaces([\"prod\"])).namespaces;\nconsole.log(`Child namespaces under ${JSON.stringify(namespace)}:`, childNamespaces);\n// Child namespaces under [\"prod\",\"search\"]: [ 'search' ]\n\nconst metadata = await db.describeNamespace([\"prod\", \"search\"]);\nconsole.log(`Metadata for namespace ${JSON.stringify(namespace)}:`, metadata);\n\nawait db.dropNamespace([\"prod\", \"search\"], { mode: \"skip\" });\nawait db.dropNamespace([\"prod\"], { mode: \"skip\" });\n"; - -export const TsNamespaceTableOps = "const db = await lancedb.connectNamespace(\"dir\", { root: \"./local_lancedb\" });\n\n// Create namespace tree: prod/search and prod/recommendations\nawait db.createNamespace([\"prod\"], { mode: \"exist_ok\" });\nawait db.createNamespace([\"prod\", \"search\"], { mode: \"exist_ok\" });\nawait db.createNamespace([\"prod\", \"recommendations\"], { mode: \"exist_ok\" });\n\nawait db.createTable(\n \"user\",\n [{ id: 1, vector: [0.1, 0.2], name: \"alice\" }],\n [\"prod\", \"search\"],\n { mode: \"create\" }, // use \"overwrite\" only if you want to replace existing table\n);\n\nawait db.createTable(\n \"user\",\n [{ id: 2, vector: [0.3, 0.4], name: \"bob\" }],\n [\"prod\", \"recommendations\"],\n { mode: \"create\" },\n);\n\n// Verify\nconsole.log((await db.listNamespaces()).namespaces); // [\"prod\"]\nconsole.log((await db.listNamespaces([\"prod\"])).namespaces); // [\"recommendations\", \"search\"]\nconsole.log(await db.tableNames([\"prod\", \"search\"])); // [\"user\"]\nconsole.log(await db.tableNames([\"prod\", \"recommendations\"])); // [\"user\"]\n"; - -export const RsConnect = "async fn connect_example(uri: &str) {\n let db = connect(uri).execute().await.unwrap();\n let _ = db;\n}\n"; - -export const RsConnectEnterpriseQuickstart = "let uri = \"db://your-database-uri\";\nlet api_key = \"your-api-key\";\nlet region = \"us-east-1\";\nlet host_override = \"https://your-enterprise-endpoint.com\";\n"; - -export const RsConnectObjectStorage = "let uri = \"s3://your-bucket/path\";\n// You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\n"; - -export const RsNamespaceAdminOps = "let mut properties = std::collections::HashMap::new();\nproperties.insert(\"root\".to_string(), \"./local_lancedb\".to_string());\nlet db = lancedb::connect_namespace(\"dir\", properties).execute().await?;\nlet namespace = vec![\"prod\".to_string(), \"search\".to_string()];\n\ndb.create_namespace(lance_namespace::models::CreateNamespaceRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n})\n.await?;\ndb.create_namespace(lance_namespace::models::CreateNamespaceRequest {\n id: Some(namespace.clone()),\n ..Default::default()\n})\n.await?;\n\nlet child_namespaces = db\n .list_namespaces(lance_namespace::models::ListNamespacesRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n })\n .await?;\nprintln!(\n \"Child namespaces under {:?}: {:?}\",\n namespace, child_namespaces\n);\n// Child namespaces under [\"prod\", \"search\"]: [\"search\"]\n\ndb.drop_namespace(lance_namespace::models::DropNamespaceRequest {\n id: Some(namespace.clone()),\n ..Default::default()\n})\n.await?;\ndb.drop_namespace(lance_namespace::models::DropNamespaceRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n})\n.await?;\n"; - -export const RsNamespaceTableOps = "let conn = connect(uri).execute().await?;\nlet search_namespace = vec![\"prod\".to_string(), \"search\".to_string()];\nlet recommendations_namespace = vec![\"prod\".to_string(), \"recommendations\".to_string()];\n\nlet schema = std::sync::Arc::new(arrow_schema::Schema::new(vec![\n arrow_schema::Field::new(\"id\", arrow_schema::DataType::Int64, false),\n]));\n\nconn.create_empty_table(\"user\", schema.clone())\n .namespace(search_namespace.clone())\n .execute()\n .await?;\n\nconn.create_empty_table(\"user\", schema)\n .namespace(recommendations_namespace.clone())\n .execute()\n .await?;\n\nlet search_table_names = conn\n .table_names()\n .namespace(search_namespace)\n .execute()\n .await?;\nlet recommendation_table_names = conn\n .table_names()\n .namespace(recommendations_namespace)\n .execute()\n .await?;\n\nprintln!(\"{search_table_names:?}\"); // [\"user\"]\nprintln!(\"{recommendation_table_names:?}\"); // [\"user\"]\n"; - diff --git a/docs/snippets/custom_embedding_function.mdx b/docs/snippets/custom_embedding_function.mdx deleted file mode 100644 index bdbb591..0000000 --- a/docs/snippets/custom_embedding_function.mdx +++ /dev/null @@ -1,8 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const TsCallCustomFunction = "const registry = getRegistry();\n\nconst sentenceTransformer = await registry\n .get(\"sentence-transformers\")!\n .create();\n\nconst schema = LanceSchema({\n vector: sentenceTransformer.vectorField(),\n text: sentenceTransformer.sourceField(),\n});\n\nconst db = await lancedb.connect(databaseDir);\nconst table = await db.createEmptyTable(\"table\", schema, {\n mode: \"overwrite\",\n});\n\nawait table.add([{ text: \"hello\" }, { text: \"world\" }]);\n\nconst results = await table.search(\"greeting\").limit(1).toArray();\n"; - -export const TsEmbeddingImpl = "@register(\"sentence-transformers\")\nclass SentenceTransformersEmbeddings extends TextEmbeddingFunction {\n name = \"Xenova/all-miniLM-L6-v2\";\n #ndims!: number;\n extractor!: FeatureExtractionPipeline;\n\n async init() {\n this.extractor = await pipeline(\"feature-extraction\", this.name, {\n dtype: \"fp32\",\n });\n this.#ndims = await this.generateEmbeddings([\"hello\"]).then(\n (e) => e[0].length,\n );\n }\n\n ndims() {\n return this.#ndims;\n }\n\n toJSON() {\n return {\n name: this.name,\n };\n }\n async generateEmbeddings(texts: string[]) {\n const output = await this.extractor(texts, {\n pooling: \"mean\",\n normalize: true,\n });\n return output.tolist();\n }\n}\n"; - -export const TsImports = "import * as lancedb from \"@lancedb/lancedb\";\nimport {\n LanceSchema,\n TextEmbeddingFunction,\n getRegistry,\n register,\n} from \"@lancedb/lancedb/embedding\";\n"; - diff --git a/docs/snippets/embedding.mdx b/docs/snippets/embedding.mdx deleted file mode 100644 index 8db4472..0000000 --- a/docs/snippets/embedding.mdx +++ /dev/null @@ -1,46 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyAsyncOpenaiEmbeddings = "db = await lancedb.connect_async(uri)\nfunc = get_registry().get(\"openai\").create(name=\"text-embedding-ada-002\")\n\nclass Words(LanceModel):\n text: str = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n\ntable = await db.create_table(\"words\", schema=Words, mode=\"overwrite\")\nawait table.add([{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}])\n\nquery = \"greetings\"\nactual = await (await table.search(query)).limit(1).to_pydantic(Words)[0]\nprint(actual.text)\n"; - -export const PyCreateEmbeddingFunction = "func = get_registry().get(\"openai\").create(\n name=\"text-embedding-3-small\",\n max_retries=7,\n)\n"; - -export const PyEmbeddingFunction = "from functools import cached_property\n\nfrom lancedb.embeddings import TextEmbeddingFunction, register\n\nclass MyEmbeddingModel:\n def __init__(self, model_name: str):\n self.model_name = model_name\n\n def encode(self, texts: list[str]) -> list[list[float]]:\n return [[1.0, 2.0, 3.0] for _ in texts]\n\n@register(\"my-embedder\")\nclass MyTextEmbedder(TextEmbeddingFunction):\n model_name: str = \"my-model\"\n\n def generate_embeddings(self, texts: list[str]) -> list[list[float]]:\n # Your embedding logic here\n return self._model.encode(texts)\n\n def ndims(self) -> int:\n # Return the dimensionality of the embeddings\n return len(self.generate_embeddings([\"test\"])[0])\n\n @cached_property\n def _model(self) -> MyEmbeddingModel:\n # Initialize your model once\n return MyEmbeddingModel(self.model_name)\n"; - -export const PyImports = "from lancedb.pydantic import LanceModel, Vector\nfrom lancedb.embeddings import get_registry\n"; - -export const PyManualQueryEmbeddings = "db = lancedb.connect(\"/tmp/db\")\nfunc = get_registry().get(\"openai\").create(name=\"text-embedding-ada-002\")\n\nclass Words(LanceModel):\n text: str = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"words\", schema=Words, mode=\"overwrite\")\ntable.add([{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}])\n\nquery_vector = func.generate_embeddings([\"greetings\"])[0]\n# --8<-- [start:manual_query_search]\n# query_vector is assumed to already be generated by your embedding function\nactual = table.search(query_vector).limit(1).to_pydantic(Words)[0]\nprint(actual.text)\n# --8<-- [end:manual_query_search]\n"; - -export const PyManualQuerySearch = "# query_vector is assumed to already be generated by your embedding function\nactual = table.search(query_vector).limit(1).to_pydantic(Words)[0]\nprint(actual.text)\n"; - -export const PyOpenaiEmbeddings = "db = lancedb.connect(\"/tmp/db\")\nfunc = get_registry().get(\"openai\").create(name=\"text-embedding-ada-002\")\n\nclass Words(LanceModel):\n text: str = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"words\", schema=Words, mode=\"overwrite\")\ntable.add([{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}])\n\nquery = \"greetings\"\nactual = table.search(query).limit(1).to_pydantic(Words)[0]\nprint(actual.text)\n"; - -export const PyRegisterDevice = "import torch\n\nregistry = get_registry()\nif torch.cuda.is_available():\n registry.set_var(\"device\", \"cuda\")\n\nfunc = registry.get(\"huggingface\").create(device=\"$var:device:cpu\")\n"; - -export const PyRegisterSecret = "registry = get_registry()\nregistry.set_var(\"api_key\", \"sk-...\")\n\nfunc = registry.get(\"openai\").create(api_key=\"$var:api_key\")\n"; - -export const TsCreateEmbeddingFunction = "const func = getRegistry().get(\"openai\")!.create({\n model: \"text-embedding-3-small\",\n});\n"; - -export const TsEmbeddingFunction = "const db = await lancedb.connect(databaseDir);\n\n@register(\"my_embedding\")\nclass MyEmbeddingFunction extends EmbeddingFunction {\n constructor(optionsRaw = {}) {\n super();\n const options = this.resolveVariables(optionsRaw);\n // Initialize using options\n }\n ndims() {\n return 3;\n }\n protected getSensitiveKeys(): string[] {\n return [];\n }\n embeddingDataType(): Float {\n return new Float32();\n }\n async computeQueryEmbeddings(_data: string) {\n // This is a placeholder for a real embedding function\n return [1, 2, 3];\n }\n async computeSourceEmbeddings(data: string[]) {\n // This is a placeholder for a real embedding function\n return Array.from({ length: data.length }).fill([\n 1, 2, 3,\n ]) as number[][];\n }\n}\n\nconst func = new MyEmbeddingFunction();\n\nconst data = [{ text: \"pepperoni\" }, { text: \"pineapple\" }];\n\n// Option 1: manually specify the embedding function\nconst table = await db.createTable(\"vectors\", data, {\n embeddingFunction: {\n function: func,\n sourceColumn: \"text\",\n vectorColumn: \"vector\",\n },\n mode: \"overwrite\",\n});\n\n// Option 2: provide the embedding function through a schema\n\nconst schema = LanceSchema({\n text: func.sourceField(new Utf8()),\n vector: func.vectorField(),\n});\n\nconst table2 = await db.createTable(\"vectors2\", data, {\n schema,\n mode: \"overwrite\",\n});\n"; - -export const TsImports = "import * as lancedb from \"@lancedb/lancedb\";\nimport \"@lancedb/lancedb/embedding/openai\";\nimport { LanceSchema, getRegistry, register } from \"@lancedb/lancedb/embedding\";\nimport { EmbeddingFunction } from \"@lancedb/lancedb/embedding\";\nimport { type Float, Float32, Utf8 } from \"apache-arrow\";\n"; - -export const TsManualQueryEmbeddings = "const db = await lancedb.connect(databaseDir);\nconst func = getRegistry()\n .get(\"openai\")\n ?.create({ model: \"text-embedding-ada-002\" }) as EmbeddingFunction;\n\nconst wordsSchema = LanceSchema({\n text: func.sourceField(new Utf8()),\n vector: func.vectorField(),\n});\nconst tbl = await db.createEmptyTable(\"words\", wordsSchema, {\n mode: \"overwrite\",\n});\nawait tbl.add([{ text: \"hello world\" }, { text: \"goodbye world\" }]);\n\nconst queryVector = await func.computeQueryEmbeddings(\"greetings\");\n// --8<-- [start:manual_query_search]\n// queryVector is assumed to already be generated by your embedding function\nconst actual = (await tbl.search(queryVector).limit(1).toArray())[0];\n// --8<-- [end:manual_query_search]\n"; - -export const TsManualQuerySearch = "// queryVector is assumed to already be generated by your embedding function\nconst actual = (await tbl.search(queryVector).limit(1).toArray())[0];\n"; - -export const TsOpenaiEmbeddings = "const db = await lancedb.connect(databaseDir);\nconst func = getRegistry()\n .get(\"openai\")\n ?.create({ model: \"text-embedding-ada-002\" }) as EmbeddingFunction;\n\nconst wordsSchema = LanceSchema({\n text: func.sourceField(new Utf8()),\n vector: func.vectorField(),\n});\nconst tbl = await db.createEmptyTable(\"words\", wordsSchema, {\n mode: \"overwrite\",\n});\nawait tbl.add([{ text: \"hello world\" }, { text: \"goodbye world\" }]);\n\nconst query = \"greetings\";\nconst actual = (await tbl.search(query).limit(1).toArray())[0];\n"; - -export const TsRegisterModelFallback = "const registry = getRegistry();\nregistry.setVar(\"openai_model\", \"text-embedding-3-large\");\n\nconst func = registry.get(\"openai\")!.create({\n model: \"$var:openai_model:text-embedding-3-small\",\n});\n"; - -export const TsRegisterSecret = "const registry = getRegistry();\nregistry.setVar(\"api_key\", \"sk-...\");\n\nconst func = registry.get(\"openai\")!.create({\n apiKey: \"$var:api_key\",\n});\n"; - -export const RsCreateEmbeddingFunction = "use std::sync::Arc;\n\nuse lancedb::embeddings::openai::OpenAIEmbeddingFunction;\n\nlet api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\nlet embedding = Arc::new(\n OpenAIEmbeddingFunction::new_with_model(api_key, \"text-embedding-3-small\")\n .expect(\"failed to create OpenAI embedding function\"),\n);\n"; - -export const RsEmbeddingFunction = "use std::{borrow::Cow, sync::Arc};\n\nuse arrow_array::{Array, FixedSizeListArray, Float32Array};\nuse arrow_schema::{DataType, Field, Schema};\nuse lancedb::{\n connect,\n embeddings::{EmbeddingDefinition, EmbeddingFunction},\n Result,\n};\n\n#[derive(Debug, Clone)]\nstruct MyTextEmbedder {\n dim: usize,\n}\n\nimpl EmbeddingFunction for MyTextEmbedder {\n fn name(&self) -> &str {\n \"my-embedder\"\n }\n\n fn source_type(&self) -> Result> {\n Ok(Cow::Owned(DataType::Utf8))\n }\n\n fn dest_type(&self) -> Result> {\n Ok(Cow::Owned(DataType::new_fixed_size_list(\n DataType::Float32,\n self.dim as i32,\n true,\n )))\n }\n\n fn compute_source_embeddings(&self, source: Arc) -> Result> {\n let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim]));\n let field = Arc::new(Field::new(\"item\", DataType::Float32, true));\n Ok(Arc::new(FixedSizeListArray::new(\n field,\n self.dim as i32,\n values,\n None,\n )))\n }\n\n fn compute_query_embeddings(&self, _input: Arc) -> Result> {\n unimplemented!()\n }\n}\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n db.embedding_registry()\n .register(\"my-embedder\", Arc::new(MyTextEmbedder { dim: 3 }))?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n db.create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\n \"text\",\n \"my-embedder\",\n Some(\"vector\"),\n ))?\n .execute()\n .await?;\n\n Ok(())\n}\n"; - -export const RsManualQueryEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n // Manually generate embeddings for the query (Enterprise path)\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n // --8<-- [start:manual_query_search]\n // query_vector is assumed to already be generated by your embedding function\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n // --8<-- [end:manual_query_search]\n\n Ok(())\n}\n"; - -export const RsManualQuerySearch = "// query_vector is assumed to already be generated by your embedding function\nlet mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\nwhile let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n}\n"; - -export const RsOpenaiEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n\n Ok(())\n}\n"; - diff --git a/docs/snippets/filtering.mdx b/docs/snippets/filtering.mdx deleted file mode 100644 index f4fc0ca..0000000 --- a/docs/snippets/filtering.mdx +++ /dev/null @@ -1,8 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const TsSearch = "const _result = await tbl\n .search(Array(1536).fill(0.5))\n .limit(1)\n .where(\"id = 10\")\n .toArray();\n"; - -export const TsSqlSearch = "await tbl.query().where(\"id = 10\").limit(10).toArray();\n"; - -export const TsVecSearch = "const result = await (\n tbl.search(Array(1536).fill(0)) as lancedb.VectorQuery\n)\n .where(\"(item IN ('item 0', 'item 2')) AND (id > 10)\")\n .postfilter()\n .toArray();\n"; - diff --git a/docs/snippets/full_text_search.mdx b/docs/snippets/full_text_search.mdx deleted file mode 100644 index 7d71409..0000000 --- a/docs/snippets/full_text_search.mdx +++ /dev/null @@ -1,4 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const TsFullTextSearch = "const result = await tbl\n .query()\n .nearestToText(\"apple\")\n .select([\"id\", \"doc\"])\n .limit(10)\n .toArray();\nexpect(result.length).toBe(10);\n"; - diff --git a/docs/snippets/geneva_dependency_verification.mdx b/docs/snippets/geneva_dependency_verification.mdx deleted file mode 100644 index 5afae69..0000000 --- a/docs/snippets/geneva_dependency_verification.mdx +++ /dev/null @@ -1,12 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyCondaClusterInline = "from geneva.cluster.builder import KubeRayClusterBuilder\n\ncluster = (\n KubeRayClusterBuilder.create(\"my-cluster\")\n .ray_init_kwargs({\n \"runtime_env\": {\n \"conda\": {\n \"channels\": [\"conda-forge\"],\n \"dependencies\": [\n \"python=3.10\",\n \"ffmpeg<8\",\n \"torchvision=0.22.1\",\n ],\n },\n \"config\": {\"eager_install\": True},\n }\n })\n .build()\n)\n"; - -export const PyCondaClusterPath = "from geneva.cluster.builder import KubeRayClusterBuilder\n\ncluster = (\n KubeRayClusterBuilder.create(\"my-cluster\")\n .ray_init_kwargs({\n \"runtime_env\": {\"conda\": \"environment.yml\"}\n })\n .build()\n)\n"; - -export const PyEnvVarsViaCluster = "from geneva.cluster.builder import KubeRayClusterBuilder\nimport os\n\ncluster = (\n KubeRayClusterBuilder.create(\"my-cluster\")\n .ray_init_kwargs({\n \"runtime_env\": {\n \"env_vars\": {\n \"AWS_ACCESS_KEY_ID\": os.environ[\"AWS_ACCESS_KEY_ID\"],\n \"AWS_SECRET_ACCESS_KEY\": os.environ[\"AWS_SECRET_ACCESS_KEY\"],\n }\n }\n })\n .build()\n)\n"; - -export const PyPipManifest = "import geneva\nfrom geneva.manifest.builder import PipManifestBuilder\n\nmanifest = (\n PipManifestBuilder.create(\"my-manifest\")\n .pip([\n \"numpy==1.26.4\",\n \"torch==2.0.1\",\n \"attrs==23.2.0\",\n ])\n .build()\n)\n\nconn = geneva.connect(\"s3://my-bucket/my-db\")\nconn.define_manifest(\"my-manifest\", manifest)\nwith conn.context(cluster=\"my-cluster\", manifest=\"my-manifest\"):\n conn.open_table(\"my-table\").backfill(\"my-column\")\n"; - -export const PyQuickFixManifest = "from geneva.manifest.builder import PipManifestBuilder\n\nmanifest = PipManifestBuilder.create(\"fix\").pip([\"numpy==1.26.4\"]).build()\n"; - diff --git a/docs/snippets/geneva_profiling_memory.mdx b/docs/snippets/geneva_profiling_memory.mdx deleted file mode 100644 index f6addc4..0000000 --- a/docs/snippets/geneva_profiling_memory.mdx +++ /dev/null @@ -1,22 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyBoundedCache = "from functools import lru_cache\n\nclass GoodEmbedding:\n def __init__(self):\n self._embed = None\n\n def setup(self):\n model = load_model()\n self._embed = lru_cache(maxsize=1024)(model.embed)\n\n def __call__(self, text: str) -> list[float]:\n if self._embed is None:\n self.setup()\n return self._embed(text)\n"; - -export const PyConfidenceCheck = "def __call__(self, x):\n scratch = bytearray(8 * 1024 * 1024) # 8 MiB\n self._scratches.append(scratch) # <-- deliberate leak\n return ...\n"; - -export const PyLeakyAggregator = "class BadAggregator:\n def __init__(self):\n self.history = []\n\n def __call__(self, batch: pa.RecordBatch) -> pa.Array:\n self.history.append(batch) # holds every batch ever processed\n ...\n"; - -export const PyLeakyCache = "class BadEmbedding:\n def __init__(self):\n self.cache: dict[str, list[float]] = {}\n\n def __call__(self, text: str) -> list[float]:\n if text not in self.cache:\n self.cache[text] = self.model.embed(text)\n return self.cache[text]\n"; - -export const PyLeakyClosure = "class BadDeferred:\n def __init__(self):\n self.work_queue = []\n\n def __call__(self, x: pa.Array) -> pa.Array:\n # Lambda captures `x` by reference — the whole Array stays alive\n self.work_queue.append(lambda: expensive(x))\n ...\n"; - -export const PyLogMemory = "import resource, pyarrow as pa\n\ndef log_memory(seq: int) -> None:\n rss_bytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss\n # ru_maxrss is bytes on macOS, KiB on Linux:\n import sys\n if sys.platform != \"darwin\":\n rss_bytes *= 1024\n arrow_live = pa.total_allocated_bytes()\n print(\n f\"seq={seq} \"\n f\"rss_mb={rss_bytes // 1024**2} \"\n f\"arrow_live_mb={arrow_live // 1024**2} \"\n f\"gap_mb={(rss_bytes - arrow_live) // 1024**2}\",\n flush=True,\n )\n"; - -export const PyMemrayTrackerUdf = "import os, pathlib, uuid\nfrom typing import Any\nimport memray\nimport geneva\nimport pyarrow as pa\n\n_MEMRAY_OUT_DIR_ENV = \"MY_UDF_MEMRAY_OUT_DIR\"\n\n\n@geneva.udf(data_type=pa.list_(pa.float32(), 512))\nclass MyEmbedding:\n def __init__(self):\n self.model = None\n self._tracker: Any = None # memray.Tracker, when profiling is on\n\n def setup(self):\n # Open a memray tracker per worker process, if requested. Each\n # worker writes its own .bin file so traces don't collide.\n out_dir = os.environ.get(_MEMRAY_OUT_DIR_ENV)\n if out_dir:\n pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)\n bin_path = pathlib.Path(out_dir) / (\n f\"memray-{os.getpid()}-{uuid.uuid4().hex}.bin\"\n )\n self._tracker = memray.Tracker(\n str(bin_path), native_traces=False, follow_fork=False\n )\n self._tracker.__enter__()\n self.model = load_model()\n\n def __call__(self, text: str) -> list[float]:\n if self.model is None:\n self.setup()\n return self.model.embed(text)\n"; - -export const PyRayClusterProfile = "from geneva.runners.ray._mgr import ray_cluster\n\nwith ray_cluster(\n local=True,\n extra_env={\"MY_UDF_MEMRAY_OUT_DIR\": \"/tmp/my-udf-profile\"},\n):\n table.backfill(\"embedding\", concurrency=1)\n"; - -export const PyStatefulUdfClass = "@geneva.udf(data_type=pa.list_(pa.float32(), 512))\nclass MyEmbedding:\n def __init__(self):\n self.model = None\n\n def setup(self):\n self.model = load_model() # allocated once per actor\n\n def __call__(self, text: str) -> list[float]:\n if self.model is None:\n self.setup()\n return self.model.embed(text)\n"; - -export const PyTorchInferenceMode = "def __call__(self, text: str) -> list[float]:\n with torch.inference_mode(): # <-- prevents autograd graph retention\n return self.model.encode(text)\n"; - diff --git a/docs/snippets/geneva_scalar_udtfs.mdx b/docs/snippets/geneva_scalar_udtfs.mdx deleted file mode 100644 index a51feb5..0000000 --- a/docs/snippets/geneva_scalar_udtfs.mdx +++ /dev/null @@ -1,20 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyAddColumnsScalarUdtf = "@udf(data_type=pa.list_(pa.float32(), 512))\ndef clip_embedding(clip_bytes: bytes) -> list[float]:\n return embed_model.encode(clip_bytes)\n\n# Add an embedding column to the clips table\nclips.add_columns({\"embedding\": clip_embedding})\n\n# Backfill computes embeddings for all existing clips\nclips.backfill(\"embedding\")\n"; - -export const PyChainingUdtfViews = "# videos → clips (1:N)\nclips = db.create_udtf_view(\n \"clips\", source=videos.search(None), udtf=extract_clips\n)\n\n# clips → frames (1:N)\nframes = db.create_udtf_view(\n \"frames\", source=clips.search(None), udtf=extract_frames\n)\n"; - -export const PyCreateScalarUdtfView = "import geneva\n\ndb = geneva.connect(\"/data/mydb\")\nvideos = db.open_table(\"videos\")\n\n# Create the 1:N materialized view\nclips = db.create_udtf_view(\n \"clips\",\n source=videos.search(None).select([\"video_path\", \"metadata\"]),\n udtf=extract_clips,\n)\n\n# Populate — runs the UDTF on every source row\nclips.refresh()\n"; - -export const PyDocumentChunkingFull = "from geneva import connect, chunker, udf\nfrom typing import Iterator, NamedTuple\nimport pyarrow as pa\n\nclass Chunk(NamedTuple):\n chunk_index: int\n chunk_text: str\n\n@chunker\ndef chunk_document(text: str) -> Iterator[Chunk]:\n \"\"\"Split a document into overlapping chunks.\"\"\"\n words = text.split()\n chunk_size = 500\n overlap = 50\n for i, start in enumerate(range(0, len(words), chunk_size - overlap)):\n chunk_words = words[start:start + chunk_size]\n yield Chunk(chunk_index=i, chunk_text=\" \".join(chunk_words))\n\ndb = connect(\"/data/mydb\")\ndocs = db.open_table(\"documents\")\n\n# Create chunked view — inherits doc_id, title, etc. from source\nchunks = db.create_udtf_view(\n \"doc_chunks\",\n source=docs.search(None).select([\"doc_id\", \"title\", \"text\"]),\n udtf=chunk_document,\n)\nchunks.refresh()\n\n# Add embeddings to chunks for semantic search\n@udf(data_type=pa.list_(pa.float32(), 1536))\ndef embed_text(chunk_text: str) -> list[float]:\n return embedding_model.encode(chunk_text)\n\nchunks.add_columns({\"embedding\": embed_text})\nchunks.backfill(\"embedding\") # Backfills embeddings on all existing chunks\n\n# Query — parent columns available alongside chunk columns\nchunks.search(None).select([\"doc_id\", \"title\", \"chunk_text\", \"embedding\"]).to_pandas()\n"; - -export const PyDocumentChunkingUdtf = "from geneva import chunker\nfrom typing import Iterator, NamedTuple\n\nclass Chunk(NamedTuple):\n chunk_index: int\n chunk_text: str\n\n@chunker\ndef chunk_document(text: str) -> Iterator[Chunk]:\n \"\"\"Split a document into overlapping chunks.\"\"\"\n words = text.split()\n chunk_size = 500\n overlap = 50\n for i, start in enumerate(range(0, len(words), chunk_size - overlap)):\n chunk_words = words[start:start + chunk_size]\n yield Chunk(chunk_index=i, chunk_text=\" \".join(chunk_words))\n"; - -export const PyIncrementalRefresh = "# Add new videos to the source table\nvideos.add(new_video_data)\n\n# Incremental refresh — only processes the new videos\nclips.refresh()\n"; - -export const PyScalarUdtfBatch = "@chunker(batch=True, output_schema=clip_schema)\ndef extract_clips(batch: pa.RecordBatch) -> pa.RecordBatch:\n \"\"\"Process rows in batches. Same 1:N semantic per row.\"\"\"\n ...\n"; - -export const PyScalarUdtfIterator = "from geneva import chunker\nfrom typing import Iterator, NamedTuple\n\nclass Clip(NamedTuple):\n clip_start: float\n clip_end: float\n clip_bytes: bytes\n\n@chunker\ndef extract_clips(video_path: str, duration: float) -> Iterator[Clip]:\n \"\"\"Yields multiple clips per video.\"\"\"\n clip_length = 10.0\n for start in range(0, int(duration), int(clip_length)):\n end = min(start + clip_length, duration)\n clip_data = extract_video_segment(video_path, start, end)\n yield Clip(clip_start=start, clip_end=end, clip_bytes=clip_data)\n"; - -export const PyScalarUdtfList = "@chunker\ndef extract_clips(video_path: str, duration: float) -> list[Clip]:\n clips = []\n for start in range(0, int(duration), 10):\n end = min(start + 10, duration)\n clips.append(Clip(clip_start=start, clip_end=end, clip_bytes=b\"...\"))\n return clips\n"; - diff --git a/docs/snippets/geneva_udfs_index.mdx b/docs/snippets/geneva_udfs_index.mdx deleted file mode 100644 index 923102a..0000000 --- a/docs/snippets/geneva_udfs_index.mdx +++ /dev/null @@ -1,8 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyRegistrationScalarUdtf = "db = geneva.connect(\"/data/mydb\")\ndb.create_udtf_view(\"my_view\", source=my_source, udtf=my_chunker)\n"; - -export const PyRegistrationUdf = "mock_table.add_columns({\"col\": my_udf})\n"; - -export const PyRegistrationUdtf = "db = geneva.connect(\"/data/mydb\")\ndb.create_udtf_view(\"my_view\", source=my_source, udtf=my_udtf)\n"; - diff --git a/docs/snippets/indexing.mdx b/docs/snippets/indexing.mdx deleted file mode 100644 index 278a097..0000000 --- a/docs/snippets/indexing.mdx +++ /dev/null @@ -1,72 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyFtsIndexAsync = "import asyncio\n\nimport lancedb\nimport polars as pl\nfrom lancedb.index import FTS\n\ndata = pl.DataFrame(\n {\n \"id\": [1, 2],\n \"text\": [\n \"His first language is spanish\",\n \"Her first language is english\",\n ],\n }\n)\n\nasync def main(data: pl.DataFrame):\n uri = \"ex_lancedb\"\n db = await lancedb.connect_async(uri)\n tbl = await db.create_table(\"my_text\", data=data, mode=\"overwrite\")\n\n await tbl.create_index(\"text\", config=FTS(language=\"English\"))\n\n response = await tbl.search(\"spanish\", query_type=\"fts\")\n result = await response.limit(1).to_polars()\n print(result)\n return result\n\nif __name__ == \"__main__\":\n asyncio.run(main(data))\n"; - -export const PyFtsIndexCreate = "table_name = \"fts-index-create\"\ntable = db.open_table(table_name)\ntable.create_fts_index(\"text\")\n"; - -export const PyFtsIndexNested = "from lancedb.query import MatchQuery, PhraseQuery\n\ntable = db.open_table(\"fts-index-nested\")\n\n# Index a text leaf inside a struct column using a dotted path.\ntable.create_fts_index(\"payload.text\", with_position=True)\n\n# The same dotted path works in MatchQuery and PhraseQuery.\nmatches = (\n table.search(MatchQuery(\"puppy\", \"payload.text\")).limit(5).to_list()\n)\nphrases = (\n table.search(PhraseQuery(\"puppy runs\", \"payload.text\"))\n .limit(5)\n .to_list()\n)\n"; - -export const PyFtsIndexWait = "table_name = \"fts-index-wait\"\n\ntable = db.open_table(table_name)\ntable.create_fts_index(\"text\")\n\nindex_name = \"text_idx\"\ntable.wait_for_index([index_name])\n"; - -export const PyGpuIndexCuda = "table.create_index(\n num_partitions=256,\n num_sub_vectors=96,\n accelerator=\"cuda\",\n)\n"; - -export const PyGpuIndexMps = "table.create_index(\n num_partitions=256,\n num_sub_vectors=96,\n accelerator=\"mps\",\n)\n"; - -export const PyReindexingIncremental = "table = db.open_table(\"reindexing_incremental\")\ntable.add([{\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"}])\ntable.optimize()\n"; - -export const PyScalarIndexBuild = "tbl = db.open_table(\"scalar_index_build\")\ntbl.create_scalar_index(\"book_id\")\ntbl.create_scalar_index(\"publisher\", index_type=\"BITMAP\")\n"; - -export const PyScalarIndexFilter = "table = db.open_table(\"books\")\nresult = table.search().where(\"book_id = 2\").limit(10).to_pandas()\n"; - -export const PyScalarIndexNestedFields = "import pyarrow as pa\nfrom lancedb.index import BTree\n\nmetadata_type = pa.struct(\n [\n pa.field(\"user_id\", pa.int32()),\n pa.field(\"user.id\", pa.int32()),\n ]\n)\ndata = pa.Table.from_arrays(\n [\n pa.array([1, 2, 3], type=pa.int32()),\n pa.array(\n [\n {\"user_id\": 10, \"user.id\": 100},\n {\"user_id\": 20, \"user.id\": 200},\n {\"user_id\": 30, \"user.id\": 300},\n ],\n type=metadata_type,\n ),\n ],\n names=[\"user_id\", \"metadata\"],\n)\ntable = await db.create_table(\"nested_scalar_index\", data)\n\n# Index a nested struct field.\nawait table.create_index(\n \"metadata.user_id\", config=BTree(), name=\"nested_user_id_idx\"\n)\n\n# Escape literal dots inside a segment with backticks.\nawait table.create_index(\n \"metadata.`user.id`\", config=BTree(), name=\"escaped_user_id_idx\"\n)\n\n# `columns` is returned as the canonical path you passed in.\nfor index in await table.list_indices():\n print(index.name, index.columns)\n# nested_user_id_idx ['metadata.user_id']\n# escaped_user_id_idx ['metadata.`user.id`']\n"; - -export const PyScalarIndexOptimize = "table.add([{\"vector\": [7, 8], \"book_id\": 4}])\ntable.optimize()\n"; - -export const PyScalarIndexPrefilter = "table = db.open_table(\"book_with_embeddings\")\ntable.search([1.2] * 2).where(\"book_id != 3\").limit(10).to_pandas()\n"; - -export const PyScalarIndexUuidData = "def generate_random_names():\n base_names = [\"Alice\", \"Bob\", \"Carla\", \"David\", \"Eve\", \"Frank\", \"Grace\"]\n letter = random.choice(string.ascii_uppercase)\n return f\"{random.choice(base_names)} {letter}.\"\n\ndef generate_uuids(num_items):\n return [uuid.uuid4().bytes for _ in range(num_items)]\n\n# Generate some UUIDs and random names\nn = 7\nuuids = generate_uuids(n)\nnames = [generate_random_names() for _ in range(n)]\n"; - -export const PyScalarIndexUuidTable = "table_name = \"index-on-uuid\"\n\nuuid_array = pa.array(uuids, pa.uuid())\nname_array = pa.array(names, pa.string())\nschema = pa.schema(\n [\n pa.field(\"id\", pa.uuid()),\n pa.field(\"name\", pa.string()),\n ]\n)\ndata_table = pa.Table.from_arrays([uuid_array, name_array], schema=schema)\ntable = db.create_table(table_name, data=data_table, mode=\"overwrite\")\n"; - -export const PyScalarIndexUuidType = "import pyarrow as pa\n"; - -export const PyScalarIndexUuidUpsert = "new_users = [\n {\"id\": uuid.uuid4().bytes, \"name\": \"Hannah D.\"},\n {\"id\": uuid.uuid4().bytes, \"name\": \"Ian B.\"},\n]\n# Insert or update using the UUID index\ntable.merge_insert(\n \"id\"\n).when_matched_update_all().when_not_matched_insert_all().execute(new_users)\n"; - -export const PyScalarIndexUuidWait = "index_name = \"id_idx\"\ntable.create_scalar_index(\"id\")\ntable.wait_for_index([index_name])\n"; - -export const PyScalarIndexWait = "index_name = \"label_idx\"\ntable.wait_for_index([index_name])\n"; - -export const PyVectorIndexAsyncConfig = "import lancedb\nimport numpy as np\nfrom lancedb.index import IvfPq\n\nasync def main():\n data = [\n {\"id\": i, \"vector\": np.random.random(8).astype(np.float32).tolist()}\n for i in range(512)\n ]\n\n db = await lancedb.connect_async(\"ex_lancedb\")\n table = await db.create_table(\n \"vector_index_async\", data=data, mode=\"overwrite\"\n )\n\n await table.create_index(\n \"vector\",\n config=IvfPq(\n distance_type=\"cosine\",\n num_partitions=16,\n num_sub_vectors=4,\n ),\n )\n return await table.list_indices()\n"; - -export const PyVectorIndexBinaryAddData = "table.add(data)\n"; - -export const PyVectorIndexBinaryBuildIndex = "table.create_index(\n metric=\"hamming\",\n vector_column_name=\"vector\",\n index_type=\"IVF_FLAT\",\n)\n"; - -export const PyVectorIndexBinarySchema = "table = tmp_db.create_table(table_name, schema=schema, mode=\"overwrite\")\n"; - -export const PyVectorIndexBinarySearch = "query = np.random.randint(0, 2, size=ndim)\nquery = np.packbits(query)\ndf = table.search(query).metric(\"hamming\").limit(10).to_pandas()\ndf.vector = df.vector.apply(np.unpackbits)\n"; - -export const PyVectorIndexBuildHnsw = "table.create_index(index_type=\"IVF_HNSW_SQ\")\n"; - -export const PyVectorIndexBuildIvf = "table_name = \"vector-index-build-ivf\"\ntable = db.open_table(table_name)\ntable.create_index(\n metric=\"cosine\",\n vector_column_name=\"keywords_embeddings\",\n)\n"; - -export const PyVectorIndexBypassRecall = "query = np.random.random(128)\nk = 10\n\n# Ground truth: flat (exhaustive) scan, ignoring the ANN index.\ntruth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()[\"id\"])\n\n# ANN results with the current nprobes setting.\nann = set(table.search(query).nprobes(20).limit(k).to_pandas()[\"id\"])\n\nrecall_at_k = len(truth & ann) / k\n"; - -export const PyVectorIndexCheckStatus = "index_name = \"keywords_embeddings_idx\"\ntable.wait_for_index([index_name])\nprint(table.index_stats(index_name))\n"; - -export const PyVectorIndexConfigureIvf = "table.create_index(metric=\"l2\", num_partitions=16, num_sub_vectors=4)\n"; - -export const PyVectorIndexCustomName = "# Override the default `{column}_idx` convention by passing `name=...`.\ntable.create_index(\n metric=\"cosine\",\n vector_column_name=\"keywords_embeddings\",\n name=\"my_custom_index\",\n)\ntable.wait_for_index([\"my_custom_index\"])\nprint(table.index_stats(\"my_custom_index\"))\n"; - -export const PyVectorIndexDistanceRange = "# Only return results whose distance falls within [0.0, 0.5).\n# Useful for near-duplicate detection or thresholded similarity search.\n(\n table.search(np.random.random(128))\n .distance_range(lower_bound=0.0, upper_bound=0.5)\n .limit(10)\n .to_pandas()\n)\n"; - -export const PyVectorIndexNestedField = "# The vector column `embedding` is nested inside the `image` struct.\n# Pass its full dotted path as `vector_column_name`; the same path is used\n# at query time and is what `list_indices()` reports under `columns`.\ntable.create_index(\n vector_column_name=\"image.embedding\",\n num_partitions=1,\n num_sub_vectors=1,\n name=\"image_embedding_idx\",\n)\n\nresults = (\n table.search([0.0, 1.0], vector_column_name=\"image.embedding\")\n .limit(1)\n .to_list()\n)\n"; - -export const PyVectorIndexNprobes = "# Always scan 10 partitions; scan up to 50 only if the initial pass\n# returns fewer than `limit` results (common with narrow filters).\n(\n table.search(np.random.random(128))\n .minimum_nprobes(10)\n .maximum_nprobes(50)\n .where(\"id > 100\")\n .limit(5)\n .to_pandas()\n)\n"; - -export const PyVectorIndexQueryHnsw = "tbl = table\ntbl.search(np.random.random((16))).limit(2).to_pandas()\n"; - -export const PyVectorIndexQueryIvf = "tbl = table\ntbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor(\n 10\n).to_pandas()\n"; - -export const PyVectorIndexSetup = "table_name = \"vector-index-tbl\"\ntable = db.open_table(table_name)\n"; - diff --git a/docs/snippets/integrations.mdx b/docs/snippets/integrations.mdx deleted file mode 100644 index de11f10..0000000 --- a/docs/snippets/integrations.mdx +++ /dev/null @@ -1,182 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyEmbeddingAwsUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\nmodel = get_registry().get(\"bedrock-text\").create()\n\nclass TextModel(LanceModel):\n text: str = model.SourceField()\n vector: Vector(model.ndims()) = model.VectorField()\n\ndf = pd.DataFrame({\"text\": [\"hello world\", \"goodbye world\"]})\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"bedrock-demo\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\n\ntbl.add(df)\nrs = tbl.search(\"hello\").limit(1).to_pandas()\nprint(rs.head())\n"; - -export const PyEmbeddingCohereUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\ncohere = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"cohere\")\n .create(name=\"embed-multilingual-v2.0\")\n)\n\nclass TextModel(LanceModel):\n text: str = cohere.SourceField()\n vector: Vector(cohere.ndims()) = cohere.VectorField()\n\ndata = [{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cohere-demo\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\ntbl.add(data)\n"; - -export const PyEmbeddingColpaliSetup = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nimport requests\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, MultiVector\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"colpali-demo\"))\nfunc = get_registry().get(\"colpali\").create()\n\nclass Images(LanceModel):\n label: str\n image_uri: str = func.SourceField()\n image_bytes: bytes = func.SourceField()\n vector: MultiVector(func.ndims()) = func.VectorField()\n vec_from_bytes: MultiVector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"images\", schema=Images)\nlabels = [\"cat\", \"dog\", \"horse\"]\nuris = [\n \"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg\",\n \"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg\",\n \"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg\",\n]\nimage_bytes = [requests.get(uri).content for uri in uris]\ntable.add(\n pd.DataFrame({\"label\": labels, \"image_uri\": uris, \"image_bytes\": image_bytes})\n)\n"; - -export const PyEmbeddingColpaliTextSearch = "actual = (\n table.search(\"a furry pet\", vector_column_name=\"vector\")\n .limit(1)\n .to_pydantic(Images)[0]\n)\nprint(actual.label)\n"; - -export const PyEmbeddingGeminiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\nmodel = get_registry().get(\"gemini-text\").create()\n\nclass TextModel(LanceModel):\n text: str = model.SourceField()\n vector: Vector(model.ndims()) = model.VectorField()\n\ndf = pd.DataFrame({\"text\": [\"hello world\", \"goodbye world\"]})\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"gemini-demo\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\n\ntbl.add(df)\nrs = tbl.search(\"hello\").limit(1).to_pandas()\nprint(rs.head())\n"; - -export const PyEmbeddingHuggingfaceUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"huggingface-demo\"))\nmodel = get_registry().get(\"huggingface\").create(name=\"facebook/bart-base\")\n\nclass Words(LanceModel):\n text: str = model.SourceField()\n vector: Vector(model.ndims()) = model.VectorField()\n\ndf = pd.DataFrame({\"text\": [\"hi hello sayonara\", \"goodbye world\"]})\ntable = db.create_table(\"greets\", schema=Words)\ntable.add(df)\nquery = \"old greeting\"\nactual = table.search(query).limit(1).to_pydantic(Words)[0]\nprint(actual.text)\n"; - -export const PyEmbeddingIbmUsage = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nwatsonx_embed = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"watsonx\")\n .create(\n name=\"ibm/slate-125m-english-rtrvr\",\n api_key=os.environ.get(\"WATSONX_API_KEY\"),\n project_id=os.environ.get(\"WATSONX_PROJECT_ID\"),\n )\n)\n\nclass TextModel(LanceModel):\n text: str = watsonx_embed.SourceField()\n vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"watsonx-demo\"))\ntbl = db.create_table(\"watsonx_test\", schema=TextModel, mode=\"overwrite\")\ntbl.add(data)\n\nrs = tbl.search(\"hello\").limit(1).to_pandas()\nprint(rs.head())\n"; - -export const PyEmbeddingImagebindAudioSearch = "query_audio = \"./assets/car_audio2.wav\"\nactual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]\nprint(actual.text == \"car\")\n"; - -export const PyEmbeddingImagebindImageSearch = "query_image = \"./assets/dog_image2.jpg\"\nactual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]\nprint(actual.text == \"dog\")\n"; - -export const PyEmbeddingImagebindSetup = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\ndb = lancedb.connect(\"/tmp/imagebind-db\")\nfunc = get_registry().get(\"imagebind\").create()\n\nclass ImageBindModel(LanceModel):\n text: str\n image_uri: str = func.SourceField()\n audio_path: str\n vector: Vector(func.ndims()) = func.VectorField()\n\ntext_list = [\"A dog.\", \"A car\", \"A bird\"]\nimage_paths = [\n \"./assets/dog_image.jpg\",\n \"./assets/car_image.jpg\",\n \"./assets/bird_image.jpg\",\n]\naudio_paths = [\n \"./assets/dog_audio.wav\",\n \"./assets/car_audio.wav\",\n \"./assets/bird_audio.wav\",\n]\n\ninputs = [\n {\"text\": a, \"audio_path\": b, \"image_uri\": c}\n for a, b, c in zip(text_list, audio_paths, image_paths)\n]\n\ntable = db.create_table(\"img_bind\", schema=ImageBindModel)\ntable.add(inputs)\n"; - -export const PyEmbeddingImagebindTextSearch = "query = \"an animal which flies and tweets\"\nactual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]\nprint(actual.text == \"bird\")\n"; - -export const PyEmbeddingInstructorUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\ninstructor = (\n get_registry()\n .get(\"instructor\")\n .create(\n source_instruction=\"represent the document for retrieval\",\n query_instruction=\"represent the document for retrieving the most similar documents\",\n )\n)\n\nclass Schema(LanceModel):\n vector: Vector(instructor.ndims()) = instructor.VectorField()\n text: str = instructor.SourceField()\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"instructor-demo\"))\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\n\ntexts = [\n {\n \"text\": \"Capitalism has been dominant in the Western world since the end of feudalism.\"\n },\n {\n \"text\": \"The disparate impact theory is especially controversial under the Fair Housing Act.\"\n },\n {\n \"text\": \"Disparate impact in United States labor law refers to practices in employment.\"\n },\n]\n\ntbl.add(texts)\n"; - -export const PyEmbeddingJinaMultimodal = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nimport requests\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\nos.environ[\"JINA_API_KEY\"] = os.environ.get(\"JINA_API_KEY\", \"jina_*\")\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-images\"))\nfunc = get_registry().get(\"jina\").create()\n\nclass Images(LanceModel):\n label: str\n image_uri: str = func.SourceField()\n image_bytes: bytes = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n vec_from_bytes: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"images\", schema=Images)\nlabels = [\"cat\", \"cat\", \"dog\", \"dog\", \"horse\", \"horse\"]\nuris = [\n \"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg\",\n \"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg\",\n \"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg\",\n \"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg\",\n \"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg\",\n \"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg\",\n]\nimage_bytes = [requests.get(uri).content for uri in uris]\ntable.add(\n pd.DataFrame({\"label\": labels, \"image_uri\": uris, \"image_bytes\": image_bytes})\n)\n"; - -export const PyEmbeddingJinaText = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nos.environ[\"JINA_API_KEY\"] = os.environ[\"JINA_API_KEY\"]\n\njina_embed = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"jina\")\n .create(name=\"jina-embeddings-v2-base-en\")\n)\n\nclass TextModel(LanceModel):\n text: str = jina_embed.SourceField()\n vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()\n\ndata = [{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-text\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\n\ntbl.add(data)\n"; - -export const PyEmbeddingOllamaUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"ollama-demo\"))\nfunc = get_registry().get(\"ollama\").create(name=\"nomic-embed-text\")\n\nclass Words(LanceModel):\n text: str = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"words\", schema=Words, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n ]\n)\n\nquery = \"greetings\"\nactual = table.search(query).limit(1).to_pydantic(Words)[0]\nprint(actual.text)\n"; - -export const PyEmbeddingOpenaiBasic = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\ndb_path = Path(tempfile.mkdtemp()) / \"openai-embeddings\"\ndb = lancedb.connect(str(db_path))\nfunc = get_registry().get(\"openai\").create(name=\"text-embedding-ada-002\")\n\nclass Words(LanceModel):\n text: str = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"words\", schema=Words, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n ]\n)\n\nquery = \"greetings\"\nactual = table.search(query).limit(1).to_pydantic(Words)[0]\nprint(actual.text)\n"; - -export const PyEmbeddingOpenclipImageSearch = "import io\n\nfrom PIL import Image\n\nquery_image_uri = \"http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg\"\nimage_bytes = requests.get(query_image_uri).content\nquery_image = Image.open(io.BytesIO(image_bytes))\nactual = table.search(query_image).limit(1).to_pydantic(Images)[0]\nprint(actual.label == \"dog\")\n\nother = (\n table.search(query_image, vector_column_name=\"vec_from_bytes\")\n .limit(1)\n .to_pydantic(Images)[0]\n)\nprint(other.label)\n"; - -export const PyEmbeddingOpenclipSetup = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nimport requests\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"openclip-demo\"))\nfunc = get_registry().get(\"open-clip\").create()\n\nclass Images(LanceModel):\n label: str\n image_uri: str = func.SourceField()\n image_bytes: bytes = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n vec_from_bytes: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"images\", schema=Images)\nlabels = [\"cat\", \"cat\", \"dog\", \"dog\", \"horse\", \"horse\"]\nuris = [\n \"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg\",\n \"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg\",\n \"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg\",\n \"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg\",\n \"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg\",\n \"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg\",\n]\nimage_bytes = [requests.get(uri).content for uri in uris]\ntable.add(\n pd.DataFrame({\"label\": labels, \"image_uri\": uris, \"image_bytes\": image_bytes})\n)\n"; - -export const PyEmbeddingOpenclipTextSearch = "actual = table.search(\"man's best friend\").limit(1).to_pydantic(Images)[0]\nprint(actual.label)\n\nfrombytes = (\n table.search(\"man's best friend\", vector_column_name=\"vec_from_bytes\")\n .limit(1)\n .to_pydantic(Images)[0]\n)\nprint(frombytes.label)\n"; - -export const PyEmbeddingSentenceTransformersBaai = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"sentence-transformers\"))\nmodel = (\n get_registry()\n .get(\"sentence-transformers\")\n .create(name=\"BAAI/bge-small-en-v1.5\", device=\"cpu\")\n)\n\nclass Words(LanceModel):\n text: str = model.SourceField()\n vector: Vector(model.ndims()) = model.VectorField()\n\ntable = db.create_table(\"words\", schema=Words)\ntable.add(\n [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n ]\n)\n\nquery = \"greetings\"\nactual = table.search(query).limit(1).to_pydantic(Words)[0]\nprint(actual.text)\n"; - -export const PyEmbeddingVoyageaiMultimodal = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\n# Create multimodal embedding function with custom dimension\nvoyageai = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"voyageai\")\n .create(name=\"voyage-multimodal-3.5\", output_dimension=512)\n)\n\nclass ImageModel(LanceModel):\n image_uri: str = voyageai.SourceField()\n vector: Vector(voyageai.ndims()) = voyageai.VectorField()\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"voyageai-multimodal\"))\ntbl = db.create_table(\"images\", schema=ImageModel, mode=\"overwrite\")\n\n# Add images using URLs\ntbl.add(\n [\n {\"image_uri\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/300px-PNG_transparency_demonstration_1.png\"},\n ]\n)\n\n# Search with text query\nresults = tbl.search(\"dice\").limit(1).to_list()\nprint(results)\n"; - -export const PyEmbeddingVoyageaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nvoyageai = (\n EmbeddingFunctionRegistry.get_instance().get(\"voyageai\").create(name=\"voyage-3\")\n)\n\nclass TextModel(LanceModel):\n text: str = voyageai.SourceField()\n vector: Vector(voyageai.ndims()) = voyageai.VectorField()\n\ndata = [{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"voyageai-demo\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\n\ntbl.add(data)\n"; - -export const PyFrameworksAgnoAgent = "agent = Agent(\n model=OpenAIResponses(id=\"gpt-5-mini\"),\n knowledge=knowledge,\n search_knowledge=True,\n instructions=\"Search the transcript and answer only from retrieved context.\",\n markdown=True,\n)\n"; - -export const PyFrameworksAgnoCliChat = "agent.print_response(\n \"Summarize the loaded video transcript in 5 concise bullet points.\",\n stream=True,\n)\nwhile True:\n question = input(\"You: \").strip()\n if question.lower() in {\"exit\", \"quit\", \"bye\"}:\n break\n agent.print_response(question, stream=True)\n"; - -export const PyFrameworksAgnoIngestYoutube = "youtube_url = \"https://www.youtube.com/watch?v=wl6mFyXoxos\"\nvideo_id = extract_video_id(youtube_url)\nytt = YouTubeTranscriptApi()\ntranscript_segments = ytt.fetch(video_id, languages=[\"en\", \"en-US\"]).to_raw_data()\ntranscript_text = \" \".join(segment[\"text\"] for segment in transcript_segments)\n\nknowledge.insert(\n name=f\"YouTube Transcript ({video_id})\",\n text_content=transcript_text,\n metadata={\"source\": \"youtube\", \"video_id\": video_id, \"video_url\": youtube_url},\n)\n"; - -export const PyFrameworksAgnoSetup = "import os\nimport re\n\nfrom agno.agent import Agent\nfrom agno.knowledge.embedder.openai import OpenAIEmbedder\nfrom agno.knowledge.knowledge import Knowledge\nfrom agno.models.openai import OpenAIResponses\nfrom agno.vectordb.lancedb import LanceDb, SearchType\nfrom youtube_transcript_api import YouTubeTranscriptApi\n\nif \"OPENAI_API_KEY\" not in os.environ:\n os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n\ndef extract_video_id(youtube_url: str) -> str:\n match = re.search(r\"(?<=v=)[\\w-]+\", youtube_url) or re.search(\n r\"(?<=be/)[\\w-]+\", youtube_url\n )\n if not match:\n raise ValueError(\"Could not parse YouTube video ID from URL\")\n return match.group(0)\n\nknowledge = Knowledge(\n vector_db=LanceDb(\n uri=\"./tmp/lancedb\",\n table_name=\"youtube_transcripts\",\n search_type=SearchType.hybrid,\n embedder=OpenAIEmbedder(id=\"text-embedding-3-small\"),\n ),\n)\n"; - -export const PyFrameworksLangchainAddImages = "image_uris = [\"./assets/image-1.png\", \"./assets/image-2.png\"]\nvector_store.add_images(uris=image_uris)\n# here image_uris are local fs paths to the images.\n"; - -export const PyFrameworksLangchainAddTexts = "vector_store.add_texts(texts=[\"test_123\"], metadatas=[{\"source\": \"wiki\"}])\n\n# Additionaly, to explore the table you can load it into a df or save it in a csv file:\n\ntbl = vector_store.get_table()\nprint(\"tbl:\", tbl)\npd_df = tbl.to_pandas()\npd_df.to_csv(\"docsearch.csv\", index=False)\n\n# you can also create a new vector store object using an older connection object:\nvector_store = LanceDB(connection=tbl, embedding=embeddings)\n"; - -export const PyFrameworksLangchainCreateIndex = "# for creating vector index\nvector_store.create_index(vector_col=\"vector\", metric=\"cosine\")\n\n# for creating scalar index(for non-vector columns)\nvector_store.create_index(col_name=\"text\")\n"; - -export const PyFrameworksLangchainMaxMarginalRelevance = "result = docsearch.max_marginal_relevance_search(query=\"text\")\nresult_texts = [doc.page_content for doc in result]\nprint(result_texts)\n\n# search by vector :\nresult = docsearch.max_marginal_relevance_search_by_vector(\n embeddings.embed_query(\"text\")\n)\nresult_texts = [doc.page_content for doc in result]\nprint(result_texts)\n"; - -export const PyFrameworksLangchainQuickStart = "import os\n\nfrom langchain.document_loaders import TextLoader\nfrom langchain.vectorstores import LanceDB\nfrom langchain_openai import OpenAIEmbeddings\nfrom langchain_text_splitters import CharacterTextSplitter\n\nos.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n\nloader = TextLoader(\n \"../../modules/state_of_the_union.txt\"\n) # Replace with your data path\ndocuments = loader.load()\n\ndocuments = CharacterTextSplitter().split_documents(documents)\nembeddings = OpenAIEmbeddings()\n\ndocsearch = LanceDB.from_documents(documents, embeddings)\nquery = \"What did the president say about Ketanji Brown Jackson\"\ndocs = docsearch.similarity_search(query)\nprint(docs[0].page_content)\n"; - -export const PyFrameworksLangchainSimilaritySearch = "docs = docsearch.similarity_search(query)\nprint(docs[0].page_content)\n"; - -export const PyFrameworksLangchainSimilaritySearchByVector = "docs = docsearch.similarity_search_by_vector(query)\nprint(docs[0].page_content)\n"; - -export const PyFrameworksLangchainSimilaritySearchByVectorWithScores = "query_embedding = embeddings.embed_query(\"text\")\ndocs = docsearch.similarity_search_by_vector_with_relevance_scores(query_embedding)\nprint(\"relevance score - \", docs[0][1])\nprint(\"text- \", docs[0][0].page_content[:1000])\n"; - -export const PyFrameworksLangchainSimilaritySearchWithScores = "docs = docsearch.similarity_search_with_relevance_scores(query)\nprint(\"relevance score - \", docs[0][1])\nprint(\"text- \", docs[0][0].page_content[:1000])\n"; - -export const PyFrameworksLangchainVectorStoreConfig = "db_url = \"db://lang_test\" # url of db you created\napi_key = \"xxxxx\" # your API key\nregion = \"us-east-1-dev\" # your selected region\n\nvector_store = LanceDB(\n uri=db_url,\n api_key=api_key, # (dont include for local API)\n region=region, # (dont include for local API)\n embedding=embeddings,\n table_name=\"langchain_test\", # Optional\n)\n"; - -export const PyFrameworksLerobotFilterFrames = "frame_rows = (\n frames.search()\n .where(\"episode_index = 0 AND frame_index < 10\", prefilter=True)\n .select([\"episode_index\", \"frame_index\", \"timestamp\", \"action\"])\n .limit(10)\n .to_list()\n)\n\nfor row in frame_rows:\n print(row[\"episode_index\"], row[\"frame_index\"], row[\"timestamp\"])\n"; - -export const PyFrameworksLerobotLancedbImageDataset = "from lerobot_lancedb import LeRobotLanceDataset\n\ndataset = LeRobotLanceDataset(\n repo_id=\"your-org/your-lerobot-lance-images\",\n delta_timestamps={\n \"observation.images.front\": [-0.2, -0.1, 0.0],\n },\n return_uint8=True,\n)\n\nsample = dataset[0]\nprint(sample[\"observation.state\"].shape)\nprint(sample[\"action\"].shape)\n"; - -export const PyFrameworksLerobotLancedbVideoDataset = "from lerobot_lancedb import LeRobotLanceVideoDataset\n\nvideo_dataset = LeRobotLanceVideoDataset(\n repo_id=\"lance-format/lerobot-pusht-lance\",\n delta_timestamps={\n \"observation.images.image\": [-0.2, -0.1, 0.0],\n },\n return_uint8=True,\n)\n\nvideo_sample = video_dataset[0]\nprint(video_sample[\"observation.images.image\"].shape)\n"; - -export const PyFrameworksLerobotOpenLanceTables = "import lancedb\n\ndb = lancedb.connect(\"hf://datasets/lance-format/lerobot-pusht-lance/data\")\nframes = db.open_table(\"frames\")\nepisodes = db.open_table(\"episodes\")\nvideos = db.open_table(\"videos\")\n\nprint(len(frames), len(episodes), len(videos))\nprint(frames.schema)\n"; - -export const PyFrameworksLlamaindexAddReranker = "from lancedb.rerankers import ColbertReranker\n\nreranker = ColbertReranker()\nvector_store._add_reranker(reranker)\n"; - -export const PyFrameworksLlamaindexFiltering = "from llama_index.core.vector_stores import (\n FilterCondition,\n FilterOperator,\n MetadataFilter,\n MetadataFilters,\n)\n\nquery_filters = MetadataFilters(\n filters=[\n MetadataFilter(\n key=\"creation_date\", operator=FilterOperator.EQ, value=\"2024-05-23\"\n ),\n MetadataFilter(key=\"file_size\", value=75040, operator=FilterOperator.GT),\n ],\n condition=FilterCondition.AND,\n)\n"; - -export const PyFrameworksLlamaindexHybridSearch = "from lancedb.rerankers import ColbertReranker\n\nreranker = ColbertReranker()\nvector_store._add_reranker(reranker)\n\nquery_engine = index.as_query_engine(\n filters=query_filters,\n vector_store_kwargs={\n \"query_type\": \"hybrid\",\n },\n)\n\nresponse = query_engine.query(\"How much did Viaweb charge per month?\")\n"; - -export const PyFrameworksLlamaindexQuickStart = "import logging\nimport sys\nimport textwrap\n\nimport openai\n\n# Uncomment to see debug logs\n# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)\n# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\nfrom llama_index.core import (\n Document,\n SimpleDirectoryReader,\n StorageContext,\n VectorStoreIndex,\n)\nfrom llama_index.vector_stores.lancedb import LanceDBVectorStore\n\nopenai.api_key = \"sk-...\"\n\ndocuments = SimpleDirectoryReader(\"./data/your-data-dir/\").load_data()\nprint(\"Document ID:\", documents[0].doc_id, \"Document Hash:\", documents[0].hash)\n\n## For LanceDB Enterprise :\n# vector_store = LanceDBVectorStore(\n# uri=\"db://db_name\", # your remote DB URI\n# api_key=\"sk_..\", # lancedb enterprise api key\n# region=\"your-region\" # the region you configured\n# host_override=\"https://your-host.com\" # if you have a custom host, otherwise omit this\n# )\n\nvector_store = LanceDBVectorStore(\n uri=\"./lancedb\", mode=\"overwrite\", query_type=\"vector\"\n)\nstorage_context = StorageContext.from_defaults(vector_store=vector_store)\n\nindex = VectorStoreIndex.from_documents(documents, storage_context=storage_context)\nlance_filter = \"metadata.file_name = 'paul_graham_essay.txt' \"\nretriever = index.as_retriever(vector_store_kwargs={\"where\": lance_filter})\nresponse = retriever.retrieve(\"What did the author do growing up?\")\n"; - -export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search(\"hello world\").limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n"; - -export const PyFrameworksPydanticBaseModel = "class LanceDocs(LanceModel):\n text: str\n vector: Vector(2)\n"; - -export const PyFrameworksPydanticImports = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.pydantic import LanceModel, Vector\n"; - -export const PyFrameworksPydanticSetUrl = "db = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"pydantic-docs\"))\n"; - -export const PyFrameworksPydanticTypeConversion = "from typing import List, Optional\n\nimport pyarrow as pa\nimport pydantic\nfrom lancedb.pydantic import Vector, pydantic_to_schema\n\nclass FooModel(pydantic.BaseModel):\n id: int\n s: str\n vec: Vector(1536) # fixed_size_list[1536]\n li: List[int]\n\nschema = pydantic_to_schema(FooModel)\nassert schema == pa.schema(\n [\n pa.field(\"id\", pa.int64(), False),\n pa.field(\"s\", pa.utf8(), False),\n pa.field(\"vec\", pa.list_(pa.float32(), 1536)),\n pa.field(\"li\", pa.list_(pa.int64()), False),\n ]\n)\n"; - -export const PyFrameworksPydanticVectorField = "import pyarrow as pa\nimport pydantic\nfrom lancedb.pydantic import Vector, pydantic_to_schema\n\nclass MyModel(pydantic.BaseModel):\n id: int\n url: str\n embeddings: Vector(768)\n\nschema = pydantic_to_schema(MyModel)\nassert schema == pa.schema(\n [\n pa.field(\"id\", pa.int64(), False),\n pa.field(\"url\", pa.utf8(), False),\n pa.field(\"embeddings\", pa.list_(pa.float32(), 768)),\n ]\n)\n"; - -export const PyFrameworksStableWorldmodelCollectLance = "import stable_worldmodel as swm\n\nworld = swm.World(\"swm/PushT-v1\", num_envs=8)\nworld.set_policy(your_expert_policy)\nworld.collect(\"data/pusht_demo.lance\", episodes=100, seed=0)\n"; - -export const PyFrameworksStableWorldmodelConvert = "swm.data.convert(\n \"data/pusht_demo.lance\",\n \"data/pusht_video\",\n dest_format=\"video\",\n fps=30,\n)\n"; - -export const PyFrameworksStableWorldmodelEvaluate = "from stable_worldmodel.policy import PlanConfig, WorldModelPolicy\nfrom stable_worldmodel.solver import CEMSolver\n\nsolver = CEMSolver(model=world_model, num_samples=300)\npolicy = WorldModelPolicy(solver=solver, config=PlanConfig(horizon=10))\n\nworld.set_policy(policy)\nresults = world.evaluate(episodes=50)\nprint(f\"Success Rate: {results['success_rate']:.1f}%\")\n"; - -export const PyFrameworksStableWorldmodelLoadLance = "dataset = swm.data.load_dataset(\"data/pusht_demo.lance\", num_steps=16)\n\nbatch = dataset[0]\nprint(batch.keys())\n"; - -export const PyPlatformsDltAdapterImport = "from dlt.destinations.adapters import lancedb_adapter\n"; - -export const PyPlatformsDltAdapterUsage = "load_info = pipeline.run(\n lancedb_adapter(\n movies_source,\n embed=\"Title\",\n )\n)\n"; - -export const PyPlatformsDltPipeline = "# Import necessary modules\nimport dlt\nfrom rest_api import rest_api_source\n\n# Configure the REST API source\nmovies_source = rest_api_source(\n {\n \"client\": {\n \"base_url\": \"https://www.omdbapi.com/\",\n \"auth\": { # authentication strategy for the OMDb API\n \"type\": \"api_key\",\n \"name\": \"apikey\",\n \"api_key\": dlt.secrets[\n \"sources.rest_api.api_token\"\n ], # read API credentials directly from secrets.toml\n \"location\": \"query\",\n },\n \"paginator\": { # pagination strategy for the OMDb API\n \"type\": \"page_number\",\n \"base_page\": 1,\n \"total_path\": \"totalResults\",\n \"maximum_page\": 5,\n },\n },\n \"resources\": [ # list of API endpoints to request\n {\n \"name\": \"movie_search\",\n \"endpoint\": {\n \"path\": \"/\",\n \"params\": {\n \"s\": \"godzilla\",\n \"type\": \"movie\",\n },\n },\n }\n ],\n }\n)\n\nif __name__ == \"__main__\":\n # Create a pipeline object\n pipeline = dlt.pipeline(\n pipeline_name=\"movies_pipeline\",\n destination=\"lancedb\", # this tells dlt to load the data into LanceDB\n dataset_name=\"movies_data_pipeline\",\n )\n\n # Run the pipeline\n load_info = pipeline.run(movies_source)\n\n # pretty print the information on data that was loaded\n print(load_info)\n"; - -export const PyPlatformsDuckdbCreateTable = "import lancedb\n\ndb = lancedb.connect(\"data/sample-lancedb\")\ndata = [\n {\"vector\": [3.1, 4.1], \"item\": \"foo\", \"price\": 10.0},\n {\"vector\": [5.9, 26.5], \"item\": \"bar\", \"price\": 20.0},\n]\ntable = db.create_table(\"pd_table\", data=data)\n"; - -export const PyPlatformsDuckdbMeanPrice = "duckdb.query(\"SELECT mean(price) FROM arrow_table\")\n"; - -export const PyPlatformsDuckdbQueryTable = "import duckdb\n\narrow_table = table.to_lance()\n\nduckdb.query(\"SELECT * FROM arrow_table\")\n"; - -export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> None:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n async_results = await (\n async_table.search([0.6, 0.4, 0.8])\n .select([\"text\", \"_distance\"])\n .limit(1)\n .to_pandas()\n )\n print(async_results)\n\nasyncio.run(run_pandas_async_example())\n"; - -export const PyPlatformsPandasCreateTable = "pandas_df = pd.DataFrame(\n [\n {\"id\": \"1\", \"text\": \"dragon\", \"vector\": [0.9, 0.1, 0.3]},\n {\"id\": \"2\", \"text\": \"griffin\", \"vector\": [0.4, 0.5, 0.2]},\n {\"id\": \"3\", \"text\": \"phoenix\", \"vector\": [0.7, 0.3, 0.6]},\n ]\n)\npandas_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"pandas-demo\"))\npandas_table = pandas_db.create_table(\"creatures\", data=pandas_df, mode=\"overwrite\")\n"; - -export const PyPlatformsPandasImports = "import asyncio\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\n"; - -export const PyPlatformsPandasVectorSearch = "pandas_results = (\n pandas_table.search([0.9, 0.1, 0.3])\n .select([\"text\", \"_distance\"])\n .limit(1)\n .to_pandas()\n)\nprint(pandas_results)\n"; - -export const PyPlatformsPolarsCreateTable = "birds = pl.DataFrame(\n {\n \"text\": [\"phoenix\", \"sparrow\"],\n \"vector\": [\n [0.1, 0.2, 0.3],\n [0.8, 0.6, 0.5],\n ],\n }\n)\npolars_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"polars-demo\"))\npolars_table = polars_db.create_table(\n \"birds\", data=birds.to_arrow(), mode=\"overwrite\"\n)\n"; - -export const PyPlatformsPolarsImports = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport polars as pl\nfrom lancedb.pydantic import LanceModel, Vector\n"; - -export const PyPlatformsPolarsLazyframe = "lazy_frame = polars_table.to_polars().lazy()\nprint(lazy_frame.select([\"text\"]).collect())\n"; - -export const PyPlatformsPolarsPydantic = "class BirdModel(LanceModel):\n text: str\n vector: Vector(3)\n\nschema_table = polars_db.create_table(\n \"birds_schema\", schema=BirdModel, mode=\"overwrite\"\n)\nschema_table.add(birds.to_dicts())\n"; - -export const PyPlatformsPolarsVectorSearch = "polars_results = (\n polars_table.search([0.1, 0.2, 0.3])\n .select([\"text\", \"_distance\"])\n .limit(1)\n .to_polars()\n)\nprint(polars_results)\n"; - -export const PyPlatformsVoxel51BackendFlag = "import fiftyone.brain as fob\n\n# Re-run similarity creation using the LanceDB backend explicitly\nfob.compute_similarity(\n dataset,\n model=\"clip-vit-base32-torch\",\n brain_key=\"lancedb_index\",\n backend=\"lancedb\",\n)\n"; - -export const PyPlatformsVoxel51BackendParams = "lancedb_index = fob.compute_similarity(\n dataset,\n model=\"clip-vit-base32-torch\",\n backend=\"lancedb\",\n brain_key=\"lancedb_index\",\n table_name=\"your-table\",\n metric=\"euclidean\",\n uri=\"/tmp/lancedb\",\n)\n"; - -export const PyPlatformsVoxel51BrainConfig = "import fiftyone.brain as fob\n\n# Print your current brain config\nprint(fob.brain_config)\n"; - -export const PyPlatformsVoxel51Cleanup = "# Step 5 (optional): Cleanup\n\n# Delete the LanceDB table\nlancedb_index.cleanup()\n\n# Delete run record from FiftyOne\ndataset.delete_brain_run(\"lancedb_index\")\n"; - -export const PyPlatformsVoxel51ComputeSimilarity = "# Steps 2 and 3: Compute embeddings and create a similarity index\nlancedb_index = fob.compute_similarity(\n dataset,\n model=\"clip-vit-base32-torch\",\n brain_key=\"lancedb_index\",\n backend=\"lancedb\",\n)\n"; - -export const PyPlatformsVoxel51LoadDataset = "import fiftyone as fo\nimport fiftyone.brain as fob\nimport fiftyone.zoo as foz\n\n# Step 1: Load your data into FiftyOne\ndataset = foz.load_zoo_dataset(\"quickstart\")\n"; - -export const PyPlatformsVoxel51SortBySimilarity = "# Step 4: Query your data\nquery = dataset.first().id # query by sample ID\nview = dataset.sort_by_similarity(\n query,\n brain_key=\"lancedb_index\",\n k=10, # limit to 10 most similar samples\n)\n"; - -export const PyRerankingAnswerdotaiUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import AnswerdotaiRerankers\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = AnswerdotaiRerankers()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingCohereUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CohereReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = CohereReranker(api_key=os.environ[\"COHERE_API_KEY\"])\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingColbertUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import ColbertReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = ColbertReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingCrossEncoderUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CrossEncoderReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = CrossEncoderReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingJinaUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import JinaReranker\n\nembedder = get_registry().get(\"jina\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = JinaReranker(api_key=os.environ[\"JINA_API_KEY\"])\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingLinearCombinationUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import LinearCombinationReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = LinearCombinationReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingMrrUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import MRRReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n\n# Run multivector search across multiple vector columns\nrs1 = tbl.search(\"hello\").limit(10).with_row_id(True).to_arrow()\nrs2 = tbl.search(\"greeting\").limit(10).with_row_id(True).to_arrow()\ncombined = MRRReranker().rerank_multivector([rs1, rs2])\n"; - -export const PyRerankingOpenaiUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import OpenaiReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = OpenaiReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingRrfUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import RRFReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = RRFReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingVoyageaiUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import VoyageAIReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = VoyageAIReranker(model_name=\"rerank-2\")\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const PyRerankingWatsonxUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import WatsonxReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\n\n# Credentials pulled from WATSONX_API_KEY and WATSONX_PROJECT_ID (or WATSONX_SPACE_ID)\nreranker = WatsonxReranker(\n api_key=os.environ[\"WATSONX_API_KEY\"],\n project_id=os.environ[\"WATSONX_PROJECT_ID\"],\n)\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; - -export const TsFrameworksGenkitCustomIndexer = "export const menuPdfIndexer = lancedbIndexerRef({\n // Using all defaults, for dbUri, tableName, and embedder, etc\n});\n\nconst chunkingConfig = {\n minLength: 1000,\n maxLength: 2000,\n splitter: \"sentence\",\n overlap: 100,\n delimiters: \"\",\n} as any;\n\nasync function extractTextFromPdf(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n\nexport const indexMenu = ai.defineFlow(\n {\n name: \"indexMenu\",\n inputSchema: z.string().describe(\"PDF file path\"),\n outputSchema: z.void(),\n },\n async (filePath: string) => {\n filePath = path.resolve(filePath);\n\n // Read the pdf.\n const pdfTxt = await ai.run(\"extract-text\", () => extractTextFromPdf(filePath));\n\n // Divide the pdf text into segments.\n const chunks = await ai.run(\"chunk-it\", async () => chunk(pdfTxt, chunkingConfig));\n\n // Convert chunks of text into documents to store in the index.\n const documents = chunks.map((text) => {\n return Document.fromText(text, { filePath });\n });\n\n // Add documents to the index.\n await ai.index({\n indexer: menuPdfIndexer,\n documents,\n options: {\n writeMode: WriteMode.Overwrite,\n } as any,\n });\n },\n);\n"; - -export const TsFrameworksGenkitCustomRetriever = "export const menuRetriever = lancedbRetrieverRef({\n tableName: \"table\", // Use the same table name as the indexer.\n displayName: \"Menu\", // Use a custom display name.\n});\n\nexport const menuQAFlow = ai.defineFlow(\n { name: \"Menu\", inputSchema: z.string(), outputSchema: z.string() },\n async (input: string) => {\n // retrieve relevant documents\n const docs = await ai.retrieve({\n retriever: menuRetriever,\n query: input,\n options: {\n k: 3,\n },\n });\n\n const extractedContent = docs.map((doc) => {\n if (doc.content && Array.isArray(doc.content) && doc.content.length > 0) {\n if (doc.content[0].media && doc.content[0].media.url) {\n return doc.content[0].media.url;\n }\n }\n return \"No content found\";\n });\n\n console.log(\"Extracted content:\", extractedContent);\n\n const { text } = await ai.generate({\n model: gemini(\"gemini-2.0-flash\"),\n prompt: `\nYou are acting as a helpful AI assistant that can answer \nquestions about the food available on the menu at Genkit Grub Pub.\n\nUse only the context provided to answer the question.\nIf you don't know, do not make up an answer.\nDo not add or change items on the menu.\n\nContext:\n${extractedContent.join(\"\\n\\n\")}\n\nQuestion: ${input}`,\n docs,\n });\n\n return text;\n },\n);\n"; - -export const TsFrameworksGenkitUsage = "import { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from \"genkitx-lancedb\";\nimport { textEmbedding004, vertexAI } from \"@genkit-ai/vertexai\";\nimport { gemini } from \"@genkit-ai/vertexai\";\nimport { z, genkit } from \"genkit\";\nimport { Document } from \"genkit/retriever\";\nimport { chunk } from \"llm-chunk\";\nimport { readFile } from \"fs/promises\";\nimport path from \"path\";\nimport pdf from \"pdf-parse/lib/pdf-parse\";\n\nconst ai = genkit({\n plugins: [\n // vertexAI provides the textEmbedding004 embedder\n vertexAI(),\n\n // the local vector store requires an embedder to translate from text to vector\n lancedb([\n {\n dbUri: \".db\", // optional lancedb uri, default to .db\n tableName: \"table\", // optional table name, default to table\n embedder: textEmbedding004,\n },\n ]),\n ],\n});\n"; - diff --git a/docs/snippets/ivf_pq.mdx b/docs/snippets/ivf_pq.mdx deleted file mode 100644 index fd1c636..0000000 --- a/docs/snippets/ivf_pq.mdx +++ /dev/null @@ -1,6 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const RsCreateIndex = "// For this example, `table` is a lancedb::Table with a column named\n// \"vector\" that is a vector column with dimension 128.\n\n// By default, if the column \"vector\" appears to be a vector column,\n// then an IVF_PQ index with reasonable defaults is created.\ntable\n .create_index(&[\"vector\"], Index::Auto)\n .execute()\n .await?;\n// For advanced cases, it is also possible to specifically request an\n// IVF_PQ index and provide custom parameters.\ntable\n .create_index(\n &[\"vector\"],\n Index::IvfPq(\n // Here we specify advanced indexing parameters. In this case\n // we are creating an index that my have better recall than the\n // default but is also larger and slower.\n IvfPqIndexBuilder::default()\n // This overrides the default distance type of l2\n .distance_type(DistanceType::Cosine)\n // With 1000 rows this have been ~31 by default\n .num_partitions(50)\n // With dimension 128 this would have been 8 by default\n .num_sub_vectors(16),\n ),\n )\n .execute()\n .await?;\n"; - -export const RsSearch1 = "let query_vector = [1.0; 128];\n// By default the index will find the 10 closest results using default\n// search parameters that give a reasonable tradeoff between accuracy\n// and search latency\nlet mut results = table\n .vector_search(&query_vector)?\n // Note: you should always set the distance_type to match the value used\n // to train the index\n .distance_type(DistanceType::Cosine)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n// We can also provide custom search parameters. Here we perform a\n// slower but more accurate search\nlet mut results = table\n .vector_search(&query_vector)?\n .distance_type(DistanceType::Cosine)\n // Override the default of 10 to get more rows\n .limit(15)\n // Override the default of 20 to search more partitions\n .nprobes(30)\n // Override the default of None to apply a refine step\n .refine_factor(1)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\nOk(())\n"; - diff --git a/docs/snippets/merge_insert.mdx b/docs/snippets/merge_insert.mdx deleted file mode 100644 index bdd55a6..0000000 --- a/docs/snippets/merge_insert.mdx +++ /dev/null @@ -1,8 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const TsInsertIfNotExists = "const table2 = await db.createTable(\"domains\", [\n { domain: \"google.com\", name: \"Google\" },\n { domain: \"github.com\", name: \"GitHub\" },\n]);\n\nconst newDomains = [\n { domain: \"google.com\", name: \"Google\" },\n { domain: \"facebook.com\", name: \"Facebook\" },\n];\nawait table2\n .mergeInsert(\"domain\")\n .whenNotMatchedInsertAll()\n .execute(newDomains);\nawait table2.countRows(); // 3\n"; - -export const TsReplaceRange = "const table3 = await db.createTable(\"chunks\", [\n { doc_id: 0, chunk_id: 0, text: \"Hello\" },\n { doc_id: 0, chunk_id: 1, text: \"World\" },\n { doc_id: 1, chunk_id: 0, text: \"Foo\" },\n { doc_id: 1, chunk_id: 1, text: \"Bar\" },\n]);\n\nconst newChunks = [{ doc_id: 1, chunk_id: 0, text: \"Baz\" }];\n\nawait table3\n .mergeInsert([\"doc_id\", \"chunk_id\"])\n .whenMatchedUpdateAll()\n .whenNotMatchedInsertAll()\n .whenNotMatchedBySourceDelete({ where: \"doc_id = 1\" })\n .execute(newChunks);\n\nawait table3.countRows(\"doc_id = 1\"); // 1\n"; - -export const TsUpsertBasic = "const table = await db.createTable(\"users\", [\n { id: 0, name: \"Alice\" },\n { id: 1, name: \"Bob\" },\n]);\n\nconst newUsers = [\n { id: 1, name: \"Bobby\" },\n { id: 2, name: \"Charlie\" },\n];\nawait table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll()\n .whenNotMatchedInsertAll()\n .execute(newUsers);\n\nawait table.countRows(); // 3\n"; - diff --git a/docs/snippets/multimodal.mdx b/docs/snippets/multimodal.mdx deleted file mode 100644 index 22c3617..0000000 --- a/docs/snippets/multimodal.mdx +++ /dev/null @@ -1,54 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyBlobApiIngest = "import lancedb\n\ndb = lancedb.connect(db_path_factory(\"blob_db\"))\n \n# Create sample data\ndata = [\n {\"id\": 1, \"video\": b\"fake_video_bytes_1\"},\n {\"id\": 2, \"video\": b\"fake_video_bytes_2\"}\n]\n \n# Create the table\ntbl = db.create_table(\"videos\", data=data, schema=schema)\n"; - -export const PyBlobApiSchema = "import pyarrow as pa\n\n# Define schema with Blob API metadata for lazy loading\nschema = pa.schema([\n pa.field(\"id\", pa.int64()),\n pa.field(\n \"video\", \n pa.large_binary(), \n metadata={\"lance-encoding:blob\": \"true\"} # Enable Blob API\n ),\n])\n"; - -export const PyBlobApiToPandas = "# Default: blob columns come back lazily\ndf_lazy = tbl.to_pandas()\n\n# Materialize blob bytes eagerly\ndf_bytes = tbl.to_pandas(blob_mode=\"bytes\")\n\n# Return descriptors instead of payloads\ndf_desc = tbl.to_pandas(blob_mode=\"descriptions\")\n\n# Forward extra kwargs to PyArrow's to_pandas\ndf_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)\n"; - -export const PyCreateDummyData = "# Create some dummy images\ndef create_dummy_image(color):\n img = Image.new('RGB', (100, 100), color=color)\n buf = io.BytesIO()\n img.save(buf, format='PNG')\n return buf.getvalue()\n\n# Create dataset with metadata, vectors, and image blobs\ndata = [\n {\n \"id\": 1,\n \"filename\": \"red_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image('red'),\n \"label\": \"red\"\n },\n {\n \"id\": 2,\n \"filename\": \"blue_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image('blue'),\n \"label\": \"blue\"\n }\n]\n"; - -export const PyDefineSchema = "# Define schema explictly to ensure image_blob is treated as binary\nschema = pa.schema([\n pa.field(\"id\", pa.int32()),\n pa.field(\"filename\", pa.string()),\n pa.field(\"vector\", pa.list_(pa.float32(), 128)),\n pa.field(\"image_blob\", pa.binary()), # Important: Use pa.binary() for blobs\n pa.field(\"label\", pa.string())\n])\n"; - -export const PyIngestData = "tbl = db.create_table(\"images\", data=data, schema=schema, mode=\"overwrite\")\n"; - -export const PyMultimodalImports = "import lancedb\nimport pyarrow as pa\nimport pandas as pd\nimport numpy as np\nimport io\nfrom PIL import Image\n"; - -export const PyProcessResults = "# Convert back to PIL Image\nfor _, row in results.iterrows():\n image_bytes = row['image_blob']\n image = Image.open(io.BytesIO(image_bytes))\n print(f\"Retrieved image: {row['filename']}, Size: {image.size}\")\n # You can now use 'image' with other libraries or display it\n"; - -export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries can't materialize blob columns,\n# so omit them from the projection\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"vector\"])\n .to_pandas(split_blocks=True, self_destruct=True)\n)\n"; - -export const PySearchData = "# Search for similar images\nquery_vector = np.random.rand(128).astype(np.float32)\nresults = tbl.search(query_vector).limit(1).to_pandas()\n"; - -export const TsBlobApiIngest = "const blobData = lancedb.makeArrowTable(\n [\n { id: 1, video: Buffer.from(\"fake_video_bytes_1\") },\n { id: 2, video: Buffer.from(\"fake_video_bytes_2\") },\n ],\n { schema: blobSchema },\n);\nconst blobTable = await db.createTable(\"videos\", blobData, {\n mode: \"overwrite\",\n});\n"; - -export const TsBlobApiSchema = "const blobSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\n \"video\",\n new arrow.LargeBinary(),\n true,\n new Map([[\"lance-encoding:blob\", \"true\"]]),\n ),\n]);\n"; - -export const TsCreateDummyData = "const createDummyImage = (color: string): Uint8Array => {\n const pngHeader = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);\n return Buffer.concat([Buffer.from(pngHeader), Buffer.from(color, \"utf8\")]);\n};\n\nconst data = [\n {\n id: 1,\n filename: \"red_square.png\",\n vector: Array.from({ length: 128 }, (_, i) => (i % 16) / 16),\n image_blob: createDummyImage(\"red\"),\n label: \"red\",\n },\n {\n id: 2,\n filename: \"blue_square.png\",\n vector: Array.from({ length: 128 }, (_, i) => ((i + 8) % 16) / 16),\n image_blob: createDummyImage(\"blue\"),\n label: \"blue\",\n },\n];\n"; - -export const TsDefineSchema = "const schema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int32()),\n new arrow.Field(\"filename\", new arrow.Utf8()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 128,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n new arrow.Field(\"image_blob\", new arrow.Binary()),\n new arrow.Field(\"label\", new arrow.Utf8()),\n]);\n"; - -export const TsIngestData = "const multimodalData = lancedb.makeArrowTable(data, { schema });\nconst tbl = await db.createTable(\"images\", multimodalData, {\n mode: \"overwrite\",\n});\n"; - -export const TsMultimodalImports = "import * as arrow from \"apache-arrow\";\nimport { Buffer } from \"node:buffer\";\nimport * as lancedb from \"@lancedb/lancedb\";\n"; - -export const TsProcessResults = "for (const row of results) {\n const imageBytes = row.image_blob as Uint8Array;\n console.log(\n `Retrieved image: ${row.filename}, Byte length: ${imageBytes.length}`,\n );\n}\n"; - -export const TsSearchData = "const queryVector = Array.from({ length: 128 }, (_, i) => (i % 16) / 16);\nconst results = await tbl.search(queryVector).limit(1).toArray();\n"; - -export const RsBlobApiIngest = "let blob_rows = vec![\n (1_i64, b\"fake_video_bytes_1\".to_vec()),\n (2_i64, b\"fake_video_bytes_2\".to_vec()),\n];\n\nlet blob_schema = Arc::new(blob_schema);\nlet blob_batch = RecordBatch::try_new(\n blob_schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(blob_rows.iter().map(|row| row.0))),\n Arc::new(LargeBinaryArray::from_iter_values(\n blob_rows.iter().map(|row| row.1.as_slice()),\n )),\n ],\n)\n.unwrap();\nlet blob_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(blob_batch)].into_iter(), blob_schema));\nlet blob_table = db\n .create_table(\"videos\", blob_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsBlobApiSchema = "let blob_metadata = HashMap::from([(\n \"lance-encoding:blob\".to_string(),\n \"true\".to_string(),\n)]);\nlet blob_schema = Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"video\", DataType::LargeBinary, true).with_metadata(blob_metadata),\n]);\n"; - -export const RsCreateDummyData = "let create_dummy_image = |color: u8| -> Vec {\n let mut png_like = vec![137, 80, 78, 71, 13, 10, 26, 10];\n png_like.push(color);\n png_like\n};\n\nlet data = vec![\n (\n 1_i32,\n \"red_square.png\",\n vec![0.1_f32; 128],\n create_dummy_image(1),\n \"red\",\n ),\n (\n 2_i32,\n \"blue_square.png\",\n vec![0.2_f32; 128],\n create_dummy_image(2),\n \"blue\",\n ),\n];\n"; - -export const RsDefineSchema = "let schema = Schema::new(vec![\n Field::new(\"id\", DataType::Int32, false),\n Field::new(\"filename\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n false,\n ),\n Field::new(\"image_blob\", DataType::Binary, false),\n Field::new(\"label\", DataType::Utf8, false),\n]);\n"; - -export const RsIngestData = "let schema = Arc::new(schema);\nlet image_batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int32Array::from_iter_values(data.iter().map(|row| row.0))),\n Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.1))),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n data.iter()\n .map(|row| Some(row.2.iter().copied().map(Some).collect::>())),\n 128,\n ),\n ),\n Arc::new(BinaryArray::from_iter_values(\n data.iter().map(|row| row.3.as_slice()),\n )),\n Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.4))),\n ],\n)\n.unwrap();\nlet image_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone()));\nlet table = db\n .create_table(\"images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsMultimodalImports = "use std::collections::HashMap;\nuse std::sync::Arc;\n\nuse arrow_array::types::Float32Type;\nuse arrow_array::{\n BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch,\n RecordBatchIterator, StringArray,\n};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures_util::TryStreamExt;\nuse lancedb::connect;\nuse lancedb::database::CreateTableMode;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n"; - -export const RsProcessResults = "for batch in &results {\n let filenames = batch\n .column_by_name(\"filename\")\n .unwrap()\n .as_any()\n .downcast_ref::()\n .unwrap();\n let images = batch\n .column_by_name(\"image_blob\")\n .unwrap()\n .as_any()\n .downcast_ref::()\n .unwrap();\n\n for row in 0..batch.num_rows() {\n let image_bytes = images.value(row);\n println!(\n \"Retrieved image: {}, Byte length: {}\",\n filenames.value(row),\n image_bytes.len()\n );\n }\n}\n"; - -export const RsSearchData = "let query_vector = vec![0.1_f32; 128];\nlet results = table\n .query()\n .nearest_to(query_vector)\n .unwrap()\n .limit(1)\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\n"; - diff --git a/docs/snippets/openai.mdx b/docs/snippets/openai.mdx deleted file mode 100644 index f279e47..0000000 --- a/docs/snippets/openai.mdx +++ /dev/null @@ -1,6 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const RsImports = "\nuse std::{iter::once, sync::Arc};\n\nuse arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n arrow::IntoArrow,\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n"; - -export const RsOpenaiEmbeddings = "#[tokio::main]\nasync fn main() -> Result<()> {\n let tempdir = tempfile::tempdir().unwrap();\n let tempdir = tempdir.path().to_str().unwrap();\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n\n let db = connect(tempdir).execute().await?;\n db.embedding_registry()\n .register(\"openai\", embedding.clone())?;\n\n let table = db\n .create_table(\"vectors\", make_data())\n .add_embedding(EmbeddingDefinition::new(\n \"text\",\n \"openai\",\n Some(\"embeddings\"),\n ))?\n .execute()\n .await?;\n\n let query = Arc::new(StringArray::from_iter_values(once(\"something warm\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n let mut results = table\n .vector_search(query_vector)?\n .limit(1)\n .execute()\n .await?;\n\n let rb = results.next().await.unwrap()?;\n let out = rb\n .column_by_name(\"text\")\n .unwrap()\n .as_any()\n .downcast_ref::()\n .unwrap();\n let text = out.iter().next().unwrap().unwrap();\n println!(\"Closest match: {}\", text);\n Ok(())\n}\n"; - diff --git a/docs/snippets/quickstart.mdx b/docs/snippets/quickstart.mdx deleted file mode 100644 index 810a0c0..0000000 --- a/docs/snippets/quickstart.mdx +++ /dev/null @@ -1,76 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyQuickstartAddData = "more_data = [\n {\n \"id\": \"4\",\n \"name\": \"Morgana\",\n \"role\": \"Sorceress\",\n \"description\": \"Powerful sorceress of Avalon.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 4},\n \"vector\": [0.3, 0.9, 0.6, 0.8],\n \"power_score\": 3.75,\n },\n]\n\n# Add data to table\ntable.add(more_data)\n"; - -export const PyQuickstartAddFeature = "table.add_columns(\n {\n \"power_score\": \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\"\n }\n)\n"; - -export const PyQuickstartCreateTable = "table = db.create_table(\"characters\", data=data, mode=\"overwrite\")\n"; - -export const PyQuickstartCreateTableAsync = "async_table = await async_db.create_table(\n \"characters\",\n data=data,\n mode=\"overwrite\",\n)\n"; - -export const PyQuickstartCreateTableNoOverwrite = "table = db.create_table(\"characters\", data=data)\n"; - -export const PyQuickstartCurateWithMetadata = "curated = (\n table.search(query_vector)\n .where(\"stats.magic >= 4\")\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .to_polars()\n)\nprint(curated)\n"; - -export const PyQuickstartData = "data = [\n {\n \"id\": \"1\",\n \"name\": \"King Arthur\",\n \"role\": \"King\",\n \"description\": \"Leader of Camelot and wielder of Excalibur.\",\n \"stats\": {\"strength\": 4, \"magic\": 1, \"leadership\": 5, \"wisdom\": 4},\n \"vector\": [0.7, 0.1, 0.9, 0.7],\n },\n {\n \"id\": \"2\",\n \"name\": \"Merlin\",\n \"role\": \"Wizard\",\n \"description\": \"Advisor and prophet with deep magical knowledge.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 5},\n \"vector\": [0.2, 0.9, 0.4, 0.9],\n },\n {\n \"id\": \"3\",\n \"name\": \"Sir Lancelot\",\n \"role\": \"Knight\",\n \"description\": \"Legendary knight known for courage and combat skill.\",\n \"stats\": {\"strength\": 5, \"magic\": 1, \"leadership\": 3, \"wisdom\": 3},\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n },\n]\n"; - -export const PyQuickstartDataAsync = "data = [\n {\n \"id\": \"1\",\n \"name\": \"King Arthur\",\n \"role\": \"King\",\n \"description\": \"Leader of Camelot and wielder of Excalibur.\",\n \"stats\": {\"strength\": 4, \"magic\": 1, \"leadership\": 5, \"wisdom\": 4},\n \"vector\": [0.7, 0.1, 0.9, 0.7],\n },\n {\n \"id\": \"2\",\n \"name\": \"Merlin\",\n \"role\": \"Wizard\",\n \"description\": \"Advisor and prophet with deep magical knowledge.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 5},\n \"vector\": [0.2, 0.9, 0.4, 0.9],\n },\n {\n \"id\": \"3\",\n \"name\": \"Sir Lancelot\",\n \"role\": \"Knight\",\n \"description\": \"Legendary knight known for courage and combat skill.\",\n \"stats\": {\"strength\": 5, \"magic\": 1, \"leadership\": 3, \"wisdom\": 3},\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n },\n]\n"; - -export const PyQuickstartMultimodalBytes = "from pathlib import Path\n\nimage_path = Path(\"docs/static/assets/images/quickstart/sir-lancelot.jpg\")\nimage_bytes = image_path.read_bytes()\n\nmultimodal_table = db.create_table(\n \"character_images\",\n data=[\n {\n \"id\": \"lancelot\",\n \"description\": \"Portrait of Sir Lancelot\",\n \"image\": image_bytes,\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n }\n ],\n mode=\"overwrite\",\n)\n"; - -export const PyQuickstartOpenTable = "table = db.open_table(\"characters\")\n"; - -export const PyQuickstartOutputPandas = "# Ensure you run `pip install pandas` beforehand\nresult = table.search(query_vector).limit(2).to_pandas()\nprint(result)\n"; - -export const PyQuickstartQueryFeature = "features = table.search().select([\"name\", \"role\", \"power_score\"]).to_polars()\nprint(features)\n"; - -export const PyQuickstartVectorSearch1 = "# Search for examples similar to a \"wise magical advisor\"\nquery_vector = [0.2, 0.8, 0.4, 0.9]\n\n# Ensure you run `pip install polars` beforehand\nresult = (\n table.search(query_vector)\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .to_polars()\n)\nprint(result)\n"; - -export const PyQuickstartVectorSearch1Async = "# Search for examples similar to a \"wise magical advisor\"\nquery_vector = [0.2, 0.8, 0.4, 0.9]\n\n# Ensure you run `pip install polars` beforehand\nasync_result = await (\n await async_table.search(query_vector)\n).select([\"name\", \"role\", \"description\", \"_distance\"]).limit(2).to_polars()\nprint(async_result)\n"; - -export const PyQuickstartVectorSearch2 = "# Search for examples similar to a \"powerful sorceress\"\nquery_vector = [0.3, 0.9, 0.6, 0.8]\n\nresults = table.search(query_vector).limit(2).to_polars()\nprint(results)\n"; - -export const TsQuickstartAddData = "const moreData = [\n {\n id: \"4\",\n name: \"Morgana\",\n role: \"Sorceress\",\n description: \"Powerful sorceress of Avalon.\",\n stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },\n vector: [0.3, 0.9, 0.6, 0.8],\n power_score: 3.75,\n },\n];\n\n// Add data to table\nawait table.add(moreData);\n"; - -export const TsQuickstartAddFeature = "await table.addColumns([\n {\n name: \"power_score\",\n valueSql:\n \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\",\n },\n]);\n"; - -export const TsQuickstartCreateTable = "let table = await db.createTable(\"characters\", data, { mode: \"overwrite\" });\n"; - -export const TsQuickstartCreateTableNoOverwrite = "table = await db.createTable(\"characters\", data);\n"; - -export const TsQuickstartCurateWithMetadata = "const curated = await table\n .search(queryVector)\n .where(\"stats.magic >= 4\")\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(curated);\n"; - -export const TsQuickstartData = "const data = [\n {\n id: \"1\",\n name: \"King Arthur\",\n role: \"King\",\n description: \"Leader of Camelot and wielder of Excalibur.\",\n stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },\n vector: [0.7, 0.1, 0.9, 0.7],\n },\n {\n id: \"2\",\n name: \"Merlin\",\n role: \"Wizard\",\n description: \"Advisor and prophet with deep magical knowledge.\",\n stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },\n vector: [0.2, 0.9, 0.4, 0.9],\n },\n {\n id: \"3\",\n name: \"Sir Lancelot\",\n role: \"Knight\",\n description: \"Legendary knight known for courage and combat skill.\",\n stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n];\n"; - -export const TsQuickstartMultimodalBytes = "const arrow = await import(\"apache-arrow\");\nconst path = await import(\"node:path\");\nconst { readFile } = await import(\"node:fs/promises\");\n\nconst imagePath = path.resolve(\n \"../../docs/static/assets/images/quickstart/sir-lancelot.jpg\",\n);\nconst imageBytes = await readFile(imagePath);\nconst imageSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Utf8()),\n new arrow.Field(\"description\", new arrow.Utf8()),\n new arrow.Field(\"image\", new arrow.Binary()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n]);\nconst imageData = lancedb.makeArrowTable(\n [\n {\n id: \"lancelot\",\n description: \"Portrait of Sir Lancelot\",\n image: imageBytes,\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n ],\n { schema: imageSchema },\n);\nconst multimodalTable = await db.createTable(\n \"character_images\",\n imageData,\n { mode: \"overwrite\" },\n);\n"; - -export const TsQuickstartOpenTable = "table = await db.openTable(\"characters\");\n"; - -export const TsQuickstartOutputArray = "result = await table.search(queryVector).limit(2).toArray();\nconsole.table(result);\n"; - -export const TsQuickstartQueryFeature = "const features = await table\n .query()\n .select([\"name\", \"role\", \"power_score\"])\n .toArray();\nconsole.table(features);\n"; - -export const TsQuickstartVectorSearch1 = "// Search for examples similar to a \"wise magical advisor\"\nlet queryVector = [0.2, 0.8, 0.4, 0.9];\n\nlet result = await table\n .search(queryVector)\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(result);\n"; - -export const TsQuickstartVectorSearch2 = "// Search for examples similar to a \"powerful sorceress\"\nqueryVector = [0.3, 0.9, 0.6, 0.8];\n\nconst results = await table.search(queryVector).limit(2).toArray();\nconsole.table(results);\n"; - -export const RsQuickstartAddFeature = "table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"power_score\".to_string(),\n \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\"\n .to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\n"; - -export const RsQuickstartCreateTable = "let schema = characters_schema();\nlet table = db\n .create_table(\"characters\", characters_to_reader(schema.clone(), &data))\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsQuickstartCreateTableNoOverwrite = "let table = db\n .create_table(\"characters\", characters_to_reader(schema.clone(), &data))\n .execute()\n .await\n .unwrap();\n"; - -export const RsQuickstartCurateWithMetadata = "let curated: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .only_if(\"stats.magic >= 4\")\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{curated:?}\");\n"; - -export const RsQuickstartData = "let data = vec![\n Character {\n id: \"1\".to_string(),\n name: \"King Arthur\".to_string(),\n role: \"King\".to_string(),\n description: \"Leader of Camelot and wielder of Excalibur.\".to_string(),\n stats: Stats {\n strength: 4,\n magic: 1,\n leadership: 5,\n wisdom: 4,\n },\n vector: [0.7, 0.1, 0.9, 0.7],\n },\n Character {\n id: \"2\".to_string(),\n name: \"Merlin\".to_string(),\n role: \"Wizard\".to_string(),\n description: \"Advisor and prophet with deep magical knowledge.\".to_string(),\n stats: Stats {\n strength: 2,\n magic: 5,\n leadership: 4,\n wisdom: 5,\n },\n vector: [0.2, 0.9, 0.4, 0.9],\n },\n Character {\n id: \"3\".to_string(),\n name: \"Sir Lancelot\".to_string(),\n role: \"Knight\".to_string(),\n description: \"Legendary knight known for courage and combat skill.\".to_string(),\n stats: Stats {\n strength: 5,\n magic: 1,\n leadership: 3,\n wisdom: 3,\n },\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n];\n"; - -export const RsQuickstartDefineStruct = "// Define structs representing the data schema\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct Stats {\n strength: i8,\n magic: i8,\n leadership: i8,\n wisdom: i8,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct Character {\n id: String,\n name: String,\n role: String,\n description: String,\n stats: Stats,\n vector: [f32; 4],\n}\n\nfn characters_schema() -> Arc {\n Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"name\", DataType::LargeUtf8, false),\n Field::new(\"role\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\n \"stats\",\n DataType::Struct(arrow_schema::Fields::from(vec![\n Arc::new(Field::new(\"strength\", DataType::Int8, false)),\n Arc::new(Field::new(\"magic\", DataType::Int8, false)),\n Arc::new(Field::new(\"leadership\", DataType::Int8, false)),\n Arc::new(Field::new(\"wisdom\", DataType::Int8, false)),\n ])),\n false,\n ),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n ]))\n}\n"; - -export const RsQuickstartMultimodalBytes = "use std::sync::Arc;\n\nuse arrow_array::{\n BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator,\n};\nuse arrow_schema::{DataType, Field, Schema};\n\nlet image_path = std::path::Path::new(env!(\"CARGO_MANIFEST_DIR\"))\n .join(\"../../docs/static/assets/images/quickstart/sir-lancelot.jpg\");\nlet image_bytes = std::fs::read(image_path).unwrap();\n\nlet image_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\"image\", DataType::Binary, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n]));\nlet image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]];\nlet image_batch = RecordBatch::try_new(\n image_schema.clone(),\n vec![\n Arc::new(LargeStringArray::from_iter_values([\"lancelot\"])),\n Arc::new(LargeStringArray::from_iter_values([\n \"Portrait of Sir Lancelot\",\n ])),\n Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n image_vectors\n .iter()\n .map(|vector| Some(vector.iter().copied().map(Some).collect::>())),\n 4,\n ),\n ),\n ],\n)\n.unwrap();\nlet image_reader: Box = Box::new(\n RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema),\n);\nlet multimodal_table = db\n .create_table(\"character_images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsQuickstartOutputArray = "let result: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; - -export const RsQuickstartQueryFeature = "let features: DataFrame = table\n .query()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"power_score\".to_string(),\n ]))\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{features:?}\");\n"; - -export const RsQuickstartVectorSearch1 = "// Search for examples similar to a \"wise magical advisor\"\nlet query_vector = [0.2, 0.8, 0.4, 0.9];\n\nlet result: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; - diff --git a/docs/snippets/search.mdx b/docs/snippets/search.mdx deleted file mode 100644 index 4d9f0ec..0000000 --- a/docs/snippets/search.mdx +++ /dev/null @@ -1,172 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyBasicFts = "uri = \"data/sample-lancedb\"\ndb = lancedb.connect(uri)\n\ntable = db.create_table(\n \"my_table_fts\",\n data=[\n {\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"},\n {\"vector\": [5.9, 26.5], \"text\": \"There are several kittens playing\"},\n ],\n mode=\"overwrite\",\n)\n\ntable.create_fts_index(\"text\")\ntable.search(\"puppy\").limit(10).select([\"text\"]).to_list()\n# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]\n# ...\n"; - -export const PyBasicFtsAsync = "uri = \"data/sample-lancedb\"\nasync_db = await lancedb.connect_async(uri)\n\nasync_tbl = await async_db.create_table(\n \"my_table_fts_async\",\n data=[\n {\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"},\n {\"vector\": [5.9, 26.5], \"text\": \"There are several kittens playing\"},\n ],\n mode=\"overwrite\",\n)\n\n# async API uses our native FTS algorithm\nawait async_tbl.create_index(\"text\", config=FTS())\nawait (await async_tbl.search(\"puppy\")).select([\"text\"]).limit(10).to_list()\n# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]\n# ...\n"; - -export const PyBasicHybridSearch = "data = [\n {\"text\": \"rebel spaceships striking from a hidden base\"},\n {\"text\": \"have won their first victory against the evil Galactic Empire\"},\n {\"text\": \"during the battle rebel spies managed to steal secret plans\"},\n {\"text\": \"to the Empire's ultimate weapon the Death Star\"},\n]\nuri = \"data/sample-lancedb\"\ndb = lancedb.connect(uri)\ntable = db.create_table(\"documents\", schema=Documents)\n# ingest docs with auto-vectorization\ntable.add(data)\n# Create a fts index before the hybrid search\ntable.create_fts_index(\"text\")\n# hybrid search with default re-ranker\ntable.search(\"flower moon\", query_type=\"hybrid\").to_pandas()\n"; - -export const PyBasicHybridSearchAsync = "uri = \"data/sample-lancedb\"\nasync_db = await lancedb.connect_async(uri)\ndata = [\n {\"text\": \"rebel spaceships striking from a hidden base\"},\n {\"text\": \"have won their first victory against the evil Galactic Empire\"},\n {\"text\": \"during the battle rebel spies managed to steal secret plans\"},\n {\"text\": \"to the Empire's ultimate weapon the Death Star\"},\n]\nasync_tbl = await async_db.create_table(\"documents_async\", schema=Documents)\n# ingest docs with auto-vectorization\nawait async_tbl.add(data)\n# Create a fts index before the hybrid search\nawait async_tbl.create_index(\"text\", config=FTS())\ntext_query = \"flower moon\"\n# hybrid search with default re-ranker\nawait (await async_tbl.search(\"flower moon\", query_type=\"hybrid\")).to_pandas()\n"; - -export const PyBatchSearch = "# Load a batch of query embeddings\nquery_dataset = load_dataset(\n \"sunhaozhepy/ag_news_sbert_keywords_embeddings\", split=\"test[5000:5005]\"\n)\nquery_embeds = query_dataset[\"keywords_embeddings\"]\nbatch_results = table.search(query_embeds).limit(5).to_pandas()\nprint(batch_results)\n"; - -export const PyBruteForceSearch = "tbl.search(np.random.random((1536))).limit(3).to_list()\n"; - -export const PyBypassVectorIndex = "table.search(embedding).bypass_vector_index().limit(5).to_pandas()\n"; - -export const PyClassDefinition = "class Metadata(BaseModel):\n source: str\n timestamp: datetime\n\n\nclass Document(BaseModel):\n content: str\n meta: Metadata\n\n\nclass LanceSchema(LanceModel):\n id: str\n vector: Vector(1536)\n payload: Document\n"; - -export const PyClassDocuments = "class Documents(LanceModel):\n vector: Vector(embeddings.ndims()) = embeddings.VectorField()\n text: str = embeddings.SourceField()\n"; - -export const PyConfigureDistanceMetric = "tbl.search(np.random.random((1536))).distance_type(\"cosine\").limit(10).to_list()\n"; - -export const PyCreateTableAsyncWithNestedSchema = "# Let's add 100 sample rows to our dataset\ndata = [\n LanceSchema(\n id=f\"id{i}\",\n vector=np.random.randn(1536),\n payload=Document(\n content=f\"document{i}\",\n meta=Metadata(source=f\"source{i % 10}\", timestamp=datetime.now()),\n ),\n )\n for i in range(100)\n]\n\nasync_tbl = await async_db.create_table(\n \"documents_async\", data=data, mode=\"overwrite\"\n)\n"; - -export const PyCreateTableWithNestedSchema = "# Let's add 100 sample rows to our dataset\ndata = [\n LanceSchema(\n id=f\"id{i}\",\n vector=np.random.randn(1536),\n payload=Document(\n content=f\"document{i}\",\n meta=Metadata(source=f\"source{i % 10}\", timestamp=datetime.now()),\n ),\n )\n for i in range(100)\n]\n\n# Synchronous client\ntbl = db.create_table(\"documents\", data=data, mode=\"overwrite\")\n"; - -export const PyExactVsApproximateDistances = "# Indexed ANN search without refinement (fast, approximate `_distance`)\nfast_results = (\n table.search(embedding)\n .limit(10)\n .to_pandas()\n)\n\n# Recompute distances on full vectors for reranked candidates\nexact_distance_results = (\n table.search(embedding)\n .limit(10)\n .refine_factor(1)\n .to_pandas()\n)\n\n# Rerank a larger candidate set for better recall (higher latency)\nhigher_recall_results = (\n table.search(embedding)\n .limit(10)\n .refine_factor(20)\n .to_pandas()\n)\n"; - -export const PyExhaustiveSearch = "uri = \"data/sample-lancedb\"\ndb = lancedb.connect(uri)\ndata = [\n {\"vector\": row, \"item\": f\"item {i}\"}\n for i, row in enumerate(np.random.random((10_000, 1536)).astype(\"float32\"))\n]\ntbl = db.create_table(\"vector_search\", data=data, mode=\"overwrite\")\ntbl.search(np.random.random((1536))).limit(10).to_list()\n"; - -export const PyExhaustiveSearchAsync = "uri = \"data/sample-lancedb\"\nasync_db = await lancedb.connect_async(uri)\ndata = [\n {\"vector\": row, \"item\": f\"item {i}\"}\n for i, row in enumerate(np.random.random((10_000, 1536)).astype(\"float32\"))\n]\nasync_tbl = await async_db.create_table(\n \"vector_search_async\", data=data, mode=\"overwrite\"\n)\n(await (await async_tbl.search(np.random.random((1536)))).limit(10).to_list())\n"; - -export const PyExhaustiveSearchAsyncCosine = "(\n await (await async_tbl.search(np.random.random((1536))))\n .distance_type(\"cosine\")\n .limit(10)\n .to_list()\n)\n"; - -export const PyExhaustiveSearchCosine = "tbl.search(np.random.random((1536))).distance_type(\"cosine\").limit(10).to_list()\n"; - -export const PyFastSearch = "table.search(embedding, fast_search=True).limit(5).to_pandas()\n"; - -export const PyFtsConfigFolding = "table.create_fts_index(\n \"text\",\n language=\"French\",\n stem=True,\n ascii_folding=True,\n replace=True,\n)\n"; - -export const PyFtsConfigFoldingAsync = "await async_tbl.create_index(\n \"text\", config=FTS(language=\"French\", stem=True, ascii_folding=True)\n)\n"; - -export const PyFtsConfigStem = "table.create_fts_index(\"text\", tokenizer_name=\"en_stem\", replace=True)\n"; - -export const PyFtsConfigStemAsync = "await async_tbl.create_index(\n \"text\", config=FTS(language=\"English\", stem=True, remove_stop_words=True)\n)\n"; - -export const PyFtsIncrementalIndex = "table.add([{\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"}])\ntable.optimize()\n"; - -export const PyFtsIncrementalIndexAsync = "await async_tbl.add([{\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"}])\nawait async_tbl.optimize()\n"; - -export const PyFtsPostfiltering = "table.search(\"puppy\").limit(10).where(\"text='foo'\", prefilter=False).to_list()\n"; - -export const PyFtsPostfilteringAsync = "await (\n (await async_tbl.search(\"puppy\"))\n .limit(10)\n .where(\"text='foo'\")\n .postfilter()\n .to_list()\n)\n"; - -export const PyFtsPrefiltering = "table.search(\"puppy\").limit(10).where(\"text='foo'\", prefilter=True).to_list()\n"; - -export const PyFtsPrefilteringAsync = "await (await async_tbl.search(\"puppy\")).limit(10).where(\"text='foo'\").to_list()\n"; - -export const PyFtsWithPosition = "table.create_fts_index(\"text\", with_position=True, replace=True)\n"; - -export const PyFtsWithPositionAsync = "await async_tbl.create_index(\"text\", config=FTS(with_position=True))\n"; - -export const PyHybridSearchPassVectorText = "vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]\ntext_query = \"flower moon\"\n(\n table.search(query_type=\"hybrid\")\n .vector(vector_query)\n .text(text_query)\n .limit(5)\n .to_pandas()\n)\n"; - -export const PyHybridSearchPassVectorTextAsync = "vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]\ntext_query = \"flower moon\"\nawait (\n async_tbl.query()\n .nearest_to(vector_query)\n .nearest_to_text(text_query)\n .limit(5)\n .to_pandas()\n)\n"; - -export const PyImportDatetime = "from datetime import datetime\n"; - -export const PyImportEmbeddings = "from lancedb.embeddings import get_registry\n"; - -export const PyImportLancedb = "import lancedb\n"; - -export const PyImportLancedbFts = "from lancedb.index import FTS\n"; - -export const PyImportLancedbPydantic = "from lancedb.pydantic import Vector, LanceModel\n"; - -export const PyImportNumpy = "from lancedb.query import BoostQuery, MatchQuery\nimport numpy as np\nimport pyarrow as pa\n"; - -export const PyImportOpenai = "import openai\n"; - -export const PyImportOs = "import os\n"; - -export const PyImportPydanticBaseModel = "from pydantic import BaseModel\n"; - -export const PyIndexNestedColumn = "table.create_index(vector_column_name=\"image.embedding\")\n"; - -export const PyMultivectorSearch = "query_multi = np.random.random(size=(2, 256))\nresults_multi = tbl.search(query_multi).limit(5).to_pandas()\n"; - -export const PyOpenaiEmbeddings = "# Ingest embedding function in LanceDB table\n# Configuring the environment variable OPENAI_API_KEY\nif \"OPENAI_API_KEY\" not in os.environ:\n # OR set the key here as a variable\n openai.api_key = \"sk-...\"\nembeddings = get_registry().get(\"openai\").create()\n"; - -export const PySearchBinaryVectors = "import numpy as np\nimport pyarrow as pa\n\nschema = pa.schema(\n [\n pa.field(\"id\", pa.int64()),\n # for dim=256, lance stores every 8 bits in a byte\n # so the vector field should be a list of 256 / 8 = 32 bytes\n pa.field(\"vector\", pa.list_(pa.uint8(), 32)),\n ]\n)\ntbl = db.create_table(\"my_binary_vectors\", schema=schema)\n\ndata = []\nfor i in range(1024):\n vector = np.random.randint(0, 2, size=256)\n # pack the binary vector into bytes to save space\n packed_vector = np.packbits(vector)\n data.append(\n {\n \"id\": i,\n \"vector\": packed_vector,\n }\n )\ntbl.add(data)\n\nquery = np.random.randint(0, 2, size=256)\npacked_query = np.packbits(query)\ntbl.search(packed_query).distance_type(\"hamming\").to_arrow()\n"; - -export const PySearchDistanceRange = "query = np.random.random(256)\n\n# Search for the vectors within the range of [0.1, 0.5)\ntbl.search(query).distance_range(0.1, 0.5).to_arrow()\n\n# Search for the vectors with the distance less than 0.5\ntbl.search(query).distance_range(upper_bound=0.5).to_arrow()\n\n# Search for the vectors with the distance greater or equal to 0.1\ntbl.search(query).distance_range(lower_bound=0.1).to_arrow()\n"; - -export const PySearchResultAsList = "tbl.search(np.random.randn(1536)).to_list()\n"; - -export const PySearchResultAsPandas = "tbl.search(np.random.randn(1536)).to_pandas()\n"; - -export const PySearchResultAsPandasFlatten1 = "tbl.search(np.random.randn(1536)).to_pandas(flatten=1)\n"; - -export const PySearchResultAsPandasFlattenTrue = "tbl.search(np.random.randn(1536)).to_pandas(flatten=True)\n"; - -export const PySearchResultAsPyarrow = "tbl.search(np.random.randn(1536)).to_arrow()\n"; - -export const PySearchResultAsPydantic = "tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)\n"; - -export const PySearchResultAsyncAsList = "await (await async_tbl.search(np.random.randn(1536))).to_list()\n"; - -export const PySearchResultAsyncAsPandas = "await (await async_tbl.search(np.random.randn(1536))).to_pandas()\n"; - -export const PySearchResultAsyncAsPyarrow = "await (await async_tbl.search(np.random.randn(1536))).to_arrow()\n"; - -export const PySelectVectorColumn = "import pyarrow as pa\n\nschema = pa.schema([\n pa.field(\"id\", pa.int32()),\n pa.field(\n \"image\",\n pa.struct([pa.field(\"embedding\", pa.list_(pa.float32(), 2))]),\n ),\n])\ntable = db.create_table(\n \"nested\",\n data=[{\"id\": 0, \"image\": {\"embedding\": [0.0, 1.0]}}],\n schema=schema,\n)\n\n# Inferred: the only vector leaf is `image.embedding`.\ntable.search([0.0, 1.0]).limit(1).to_list()\n\n# Explicit: required when more than one vector column matches.\ntable.search([0.0, 1.0], vector_column_name=\"image.embedding\").limit(1).to_list()\n"; - -export const PyVectorSearchPostfilter = "results_post_filtered = (\n table.search(query_embed)\n .where(\"label > 1\", prefilter=False)\n .select([\"text\", \"keywords\", \"label\"])\n .limit(5)\n .to_pandas()\n)\n\nprint(\"Vector search results with post-filter:\")\nprint(results_post_filtered)\n"; - -export const PyVectorSearchPrefilter = "from datasets import load_dataset\n\n# Load query vector from dataset\nquery_dataset = load_dataset(\"sunhaozhepy/ag_news_sbert_keywords_embeddings\", split=\"test[5000:5001]\")\nprint(f\"Query keywords: {query_dataset[0]['keywords']}\")\nquery_embed = query_dataset[\"keywords_embeddings\"][0]\n\n# Open table and perform search\ntable_name = \"lancedb-enterprise-quickstart\"\ntable = db.open_table(table_name)\n\n# Vector search with filters (pre-filtering is the default)\nsearch_results = (\n table.search(query_embed)\n .where(\"label > 2\")\n .select([\"text\", \"keywords\", \"label\"])\n .limit(5)\n .to_pandas()\n)\n\nprint(\"Search results (with pre-filtering):\")\nprint(search_results)\n"; - -export const TsBatchSearch = "// Batch query\nconsole.log(\"Performing batch vector search...\");\nconst batchSize = 5;\nconst queryVectors = Array.from({ length: batchSize }, () =>\n Array.from({ length: dimensions }, () => Math.random() * 2 - 1),\n);\nlet batchQuery = table.search(queryVectors[0]) as lancedb.VectorQuery;\nfor (let i = 1; i < batchSize; i++) {\n batchQuery = batchQuery.addQueryVector(queryVectors[i]);\n}\nconst batchResults = await batchQuery\n .select([\"text\", \"keywords\", \"label\"])\n .limit(5)\n .toArray();\nconsole.log(\"Batch vector search results:\");\nconsole.log(batchResults);\n"; - -export const TsBinarySearch = "const tbl = await db.createTable(\"binary_vectors\", data, {\n mode: \"overwrite\",\n});\nawait tbl.createIndex(\"vector\", {\n config: lancedb.Index.ivfFlat({\n numPartitions: 10,\n distanceType: \"hamming\",\n }),\n});\n\nconst query = Array(32)\n .fill(1)\n .map(() => Math.floor(Math.random() * 255));\nconst results = await tbl.query().nearestTo(query).limit(10).toArray();\n"; - -export const TsBruteForceSearch = "const tbl = await db.openTable(\"my_vectors\");\n\nconst results1 = await tbl.search(Array(128).fill(1.2)).limit(3).toArray();\n"; - -export const TsBypassVectorIndex = "await table\n .query()\n .nearestTo(embedding)\n .bypassVectorIndex()\n .limit(5)\n .toArray();\n"; - -export const TsDistanceRange = "const results3 = await (\n tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery\n)\n .distanceType(\"cosine\")\n .distanceRange(0.1, 0.2)\n .limit(10)\n .toArray();\n"; - -export const TsExactVsApproximate = "// Indexed ANN search without refinement (fast, approximate `_distance`)\nconst fastResults = await (table.search(embedding) as lancedb.VectorQuery)\n .limit(10)\n .toArray();\n\n// Recompute distances on full vectors for reranked candidates\nconst exactDistanceResults = await (\n table.search(embedding) as lancedb.VectorQuery\n)\n .limit(10)\n .refineFactor(1)\n .toArray();\n\n// Rerank a larger candidate set for better recall (higher latency)\nconst higherRecallResults = await (\n table.search(embedding) as lancedb.VectorQuery\n)\n .limit(10)\n .refineFactor(20)\n .toArray();\n"; - -export const TsFastSearch = "await table\n .query()\n .nearestTo(embedding)\n .fastSearch()\n .limit(5)\n .toArray();\n"; - -export const TsImport = "import * as lancedb from \"@lancedb/lancedb\";\n"; - -export const TsImportBinUtil = "import { Field, FixedSizeList, Int32, Schema, Uint8 } from \"apache-arrow\";\n"; - -export const TsIndexNestedColumn = "await table.createIndex(\"image.embedding\");\n"; - -export const TsIngestBinaryData = "const schema = new Schema([\n new Field(\"id\", new Int32(), true),\n new Field(\"vec\", new FixedSizeList(32, new Field(\"item\", new Uint8()))),\n]);\nconst data = lancedb.makeArrowTable(\n Array(1_000)\n .fill(0)\n .map((_, i) => ({\n // the 256 bits would be store in 32 bytes,\n // if your data is already in this format, you can skip the packBits step\n id: i,\n vec: lancedb.packBits(Array(256).fill(i % 2)),\n })),\n { schema: schema },\n);\n\nconst tbl = await db.createTable(\"binary_table\", data);\nawait tbl.createIndex(\"vec\", {\n config: lancedb.Index.ivfFlat({\n numPartitions: 10,\n distanceType: \"hamming\",\n }),\n});\n"; - -export const TsSearch1 = "const db = await lancedb.connect(databaseDir);\nconst tbl = await db.openTable(\"my_vectors\");\n\nconst results1 = await tbl.search(Array(128).fill(1.2)).limit(10).toArray();\n"; - -export const TsSearch2 = "const results2 = await (\n tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery\n)\n .distanceType(\"cosine\")\n .limit(10)\n .toArray();\n"; - -export const TsSearchBinaryData = "const query = Array(32)\n .fill(1)\n .map(() => Math.floor(Math.random() * 255));\nconst results = await tbl.query().nearestTo(query).limit(10).toArrow();\n"; - -export const TsSelectVectorColumn = "const table = await db.openTable(\"nested\");\n\n// Inferred: LanceDB finds the single nested vector leaf automatically.\nawait table.query().nearestTo([0.0, 1.0]).limit(1).toArray();\n\n// Explicit: required when more than one vector column matches.\nawait table\n .query()\n .nearestTo([0.0, 1.0])\n .column(\"image.embedding\")\n .limit(1)\n .toArray();\n"; - -export const TsVectorSearchPostfilter = "const vectorResultsWithPostFilter = await (\n table.search(queryEmbed) as lancedb.VectorQuery\n)\n .where(\"label > 2\")\n .postfilter()\n .select([\"text\", \"keywords\", \"label\"])\n .limit(5)\n .toArray();\n\nconsole.log(\"Vector search results with post-filter:\");\nconsole.log(vectorResultsWithPostFilter);\n"; - -export const TsVectorSearchPrefilter = "// Generate a sample 768-dimension embedding vector (typical for BERT-based models)\n// In real applications, you would get this from an embedding model\nconst dimensions = 768;\nconst queryEmbed = Array.from(\n { length: dimensions },\n () => Math.random() * 2 - 1,\n);\n\n// Open table and perform search\nconst tableName = \"lancedb-enterprise-quickstart\";\nconst table = await db.openTable(tableName);\n\n// Vector search with filters (pre-filtering is the default)\nconst vectorResults = await table\n .search(queryEmbed)\n .where(\"label > 2\")\n .select([\"text\", \"keywords\", \"label\"])\n .limit(5)\n .toArray();\n\nconsole.log(\"Search results (with pre-filtering):\");\nconsole.log(vectorResults);\n"; - -export const RsBatchSearch = "// Search multiple query vectors in one call. Each result row carries a\n// `query_index` mapping it back to the query it matched.\nlet mut results = table\n .vector_search(&query_1)?\n .add_query_vector(&query_2)?\n .limit(5)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsBinarySearch = "// Binary vectors use `hamming` distance over the packed uint8 bytes.\nlet query: Arc = Arc::new(UInt8Array::from(vec![1u8; NUM_BYTES as usize]));\nlet mut results = tbl\n .vector_search(query)?\n .distance_type(DistanceType::Hamming)\n .limit(10)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsBruteForceSearch = "// A plain vector search returns the top-k closest rows.\nlet mut results = table.vector_search(&query_vector)?.limit(3).execute().await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsBypassVectorIndex = "// Force an exhaustive (flat) scan for exact, ground-truth results.\nlet mut results = table\n .vector_search(&query_vector)?\n .bypass_vector_index()\n .limit(5)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsConfigureDistanceMetric = "// Use the same distance metric the index was trained with.\nlet mut results = table\n .vector_search(&query_vector)?\n .distance_type(DistanceType::Cosine)\n .limit(10)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsExactVsApproximate = "// Approximate ANN search (fast, distances may come from the index representation)\nlet mut fast_results = table.vector_search(&query_vector)?.limit(10).execute().await?;\nwhile let Some(batch) = fast_results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n\n// Rerank a larger candidate set on full vectors for better recall\nlet mut refined_results = table\n .vector_search(&query_vector)?\n .limit(10)\n .refine_factor(20)\n .execute()\n .await?;\nwhile let Some(batch) = refined_results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsFastSearch = "// Skip unindexed data for lower latency.\nlet mut results = table\n .vector_search(&query_vector)?\n .fast_search()\n .limit(5)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsSearchDistanceRange = "// Only return rows whose distance falls within [0.1, 0.5).\nlet mut results = table\n .vector_search(&query_vector)?\n .distance_range(Some(0.1), Some(0.5))\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsVectorSearchPostfilter = "// Apply the filter after vector search by calling postfilter().\nlet mut results = table\n .vector_search(&query_vector)?\n .only_if(\"id > 100\")\n .postfilter()\n .select(Select::columns(&[\"id\"]))\n .limit(5)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - -export const RsVectorSearchPrefilter = "// Prefiltering is the default: the filter is applied before vector search.\nlet mut results = table\n .vector_search(&query_vector)?\n .only_if(\"id > 100\")\n .select(Select::columns(&[\"id\"]))\n .limit(5)\n .execute()\n .await?;\nwhile let Some(batch) = results.try_next().await? {\n println!(\"{:?}\", batch);\n}\n"; - diff --git a/docs/snippets/sentence-transformers.mdx b/docs/snippets/sentence-transformers.mdx deleted file mode 100644 index bf4aec1..0000000 --- a/docs/snippets/sentence-transformers.mdx +++ /dev/null @@ -1,14 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const TsQuickstartConnect = "const db = await lancedb.connect(\"data/sample-lancedb\");\n"; - -export const TsQuickstartCreateTable = "const table = await db.createEmptyTable(\"words\", wordsSchema, {\n mode: \"overwrite\",\n});\nawait table.add([{ text: \"hello world\" }, { text: \"goodbye world\" }]);\n"; - -export const TsQuickstartImports = "import * as lancedb from \"@lancedb/lancedb\";\nimport \"@lancedb/lancedb/embedding/transformers\";\nimport { Utf8 } from \"apache-arrow\";\n"; - -export const TsQuickstartInitModel = "const model = (await lancedb.embedding\n .getRegistry()\n .get(\"huggingface\")\n ?.create()) as lancedb.embedding.EmbeddingFunction;\n"; - -export const TsQuickstartQuery = "const query = \"greetings\";\nconst actual = (await table.search(query).limit(1).toArray())[0];\nconsole.log(actual.text);\n"; - -export const TsQuickstartSchema = "const wordsSchema = lancedb.embedding.LanceSchema({\n text: model.sourceField(new Utf8()),\n vector: model.vectorField(),\n});\n"; - diff --git a/docs/snippets/simple.mdx b/docs/snippets/simple.mdx deleted file mode 100644 index 92710bf..0000000 --- a/docs/snippets/simple.mdx +++ /dev/null @@ -1,24 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const RsAdd = "let new_data = create_some_records()?;\ntbl.add(new_data).execute().await.unwrap();\n"; - -export const RsConnect = "#[tokio::main]\nasync fn main() -> Result<()> {\n if std::path::Path::new(\"data\").exists() {\n std::fs::remove_dir_all(\"data\").unwrap();\n }\n // --8<-- [start:connect_uri]\n let uri = \"data/sample-lancedb\";\n let db = connect(uri).execute().await?;\n // --8<-- [end:connect_uri]\n\n // --8<-- [start:list_names]\n println!(\"{:?}\", db.table_names().execute().await?);\n // --8<-- [end:list_names]\n let tbl = create_table(&db).await?;\n create_index(&tbl).await?;\n let batches = search(&tbl).await?;\n println!(\"{:?}\", batches);\n\n create_empty_table(&db).await.unwrap();\n\n // --8<-- [start:delete]\n tbl.delete(\"id > 24\").await.unwrap();\n // --8<-- [end:delete]\n\n // --8<-- [start:drop_table]\n db.drop_table(\"my_table\").await.unwrap();\n // --8<-- [end:drop_table]\n Ok(())\n}\n"; - -export const RsConnectUri = "let uri = \"data/sample-lancedb\";\nlet db = connect(uri).execute().await?;\n"; - -export const RsCreateEmptyTable = "let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int32, false),\n Field::new(\"item\", DataType::Utf8, true),\n]));\ndb.create_empty_table(\"empty_table\", schema).execute().await\n"; - -export const RsCreateIndex = "table.create_index(&[\"vector\"], Index::Auto).execute().await\n"; - -export const RsCreateTable = "let initial_data = create_some_records()?;\nlet tbl = db\n .create_table(\"my_table\", initial_data)\n .execute()\n .await\n .unwrap();\n"; - -export const RsDelete = "tbl.delete(\"id > 24\").await.unwrap();\n"; - -export const RsDropTable = "db.drop_table(\"my_table\").await.unwrap();\n"; - -export const RsListNames = "println!(\"{:?}\", db.table_names().execute().await?);\n"; - -export const RsOpenExistingTbl = "let table = db.open_table(\"my_table\").execute().await.unwrap();\n"; - -export const RsSearch = "table\n .query()\n .limit(2)\n .nearest_to(&[1.0; 128])?\n .execute()\n .await?\n .try_collect::>()\n .await\n"; - diff --git a/docs/snippets/storage.mdx b/docs/snippets/storage.mdx deleted file mode 100644 index 942fd8a..0000000 --- a/docs/snippets/storage.mdx +++ /dev/null @@ -1,64 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyStorageAzureAccount = "db = lancedb.connect(\n \"az://my-container/my-database\",\n storage_options={\n \"account_name\": \"some-account\",\n \"account_key\": \"some-key\",\n },\n)\n"; - -export const PyStorageAzureSas = "db = lancedb.connect(\n \"az://my-container/my-database\",\n storage_options={\n \"azure_storage_account_name\": \"some-account\",\n \"azure_storage_sas_token\": \"\",\n },\n)\n"; - -export const PyStorageConnectAzure = "db = lancedb.connect(\"az://bucket/path\")\n"; - -export const PyStorageConnectGcs = "db = lancedb.connect(\"gs://bucket/path\")\n"; - -export const PyStorageConnectS3 = "db = lancedb.connect(\"s3://bucket/path\")\n"; - -export const PyStorageConnectTimeout = "db = lancedb.connect(\n \"s3://bucket/path\",\n storage_options={\"timeout\": \"60s\"},\n)\n"; - -export const PyStorageGcsServiceAccount = "db = lancedb.connect(\n \"gs://my-bucket/my-database\",\n storage_options={\n \"service_account\": \"path/to/service-account.json\",\n },\n)\n"; - -export const PyStorageS3Ddb = "db = lancedb.connect(\n \"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table\",\n)\n"; - -export const PyStorageS3DdbLocal = "db = lancedb.connect(\n \"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table\",\n storage_options={\n \"endpoint\": \"http://localhost:4566\",\n \"dynamodb_endpoint\": \"http://localhost:4566\",\n \"allow_http\": \"true\",\n },\n)\n"; - -export const PyStorageS3Express = "db = lancedb.connect(\n \"s3://my-bucket--use1-az4--x-s3/path\",\n storage_options={\n \"region\": \"us-east-1\",\n \"s3_express\": \"true\",\n },\n)\n"; - -export const PyStorageS3Minio = "db = lancedb.connect(\n \"s3://bucket/path\",\n storage_options={\n \"region\": \"us-east-1\",\n \"endpoint\": \"http://minio:9000\",\n },\n)\n"; - -export const PyStorageS3SseKms = "db = lancedb.connect(\n \"s3://bucket/path\",\n storage_options={\n \"aws_server_side_encryption\": \"aws:kms\",\n \"aws_sse_kms_key_id\": \"\",\n },\n)\n"; - -export const PyStorageTableTimeout = "table = db.create_table(\n \"table\",\n [{\"a\": 1, \"b\": 2}],\n storage_options={\"timeout\": \"60s\"},\n)\n"; - -export const PyStorageTigrisConnect = "db = lancedb.connect(\n \"s3://your-bucket/path\",\n storage_options={\n \"endpoint\": \"https://t3.storage.dev\",\n \"region\": \"auto\",\n },\n)\n"; - -export const PyStorageCosConnect = "db = lancedb.connect(\n \"cos://my-bucket/my-database\",\n storage_options={\n \"secret_id\": \"\",\n \"secret_key\": \"\",\n \"region\": \"ap-guangzhou\",\n },\n)\n"; - -export const PyStorageGoosefsConnect = "db = lancedb.connect(\"goosefs://my-namespace/my-database\")\n"; - -export const TsStorageAzureAccount = "async function storageAzureAccount() {\n const db = await lancedb.connect(\n \"az://my-container/my-database\",\n {\n storageOptions: {\n accountName: \"some-account\",\n accountKey: \"some-key\",\n },\n },\n );\n return db;\n}\n"; - -export const TsStorageAzureSas = "async function storageAzureSas() {\n const db = await lancedb.connect(\n \"az://my-container/my-database\",\n {\n storageOptions: {\n azureStorageAccountName: \"some-account\",\n azureStorageSasToken: \"\",\n },\n },\n );\n return db;\n}\n"; - -export const TsStorageConnectAzure = "async function storageConnectAzure() {\n const db = await lancedb.connect(\"az://bucket/path\");\n return db;\n}\n"; - -export const TsStorageConnectGcs = "async function storageConnectGcs() {\n const db = await lancedb.connect(\"gs://bucket/path\");\n return db;\n}\n"; - -export const TsStorageConnectS3 = "async function storageConnectS3() {\n const db = await lancedb.connect(\"s3://bucket/path\");\n return db;\n}\n"; - -export const TsStorageConnectTimeout = "async function storageConnectTimeout() {\n const db = await lancedb.connect(\"s3://bucket/path\", {\n storageOptions: { timeout: \"60s\" },\n });\n return db;\n}\n"; - -export const TsStorageGcsServiceAccount = "async function storageGcsServiceAccount() {\n const db = await lancedb.connect(\n \"gs://my-bucket/my-database\",\n {\n storageOptions: {\n serviceAccount: \"path/to/service-account.json\",\n },\n },\n );\n return db;\n}\n"; - -export const TsStorageS3Ddb = "async function storageS3Ddb() {\n const db = await lancedb.connect(\n \"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table\",\n );\n return db;\n}\n"; - -export const TsStorageS3DdbLocal = "async function storageS3DdbLocal() {\n const db = await lancedb.connect(\n \"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table\",\n {\n storageOptions: {\n endpoint: \"http://localhost:4566\",\n dynamodbEndpoint: \"http://localhost:4566\",\n allowHttp: \"true\",\n },\n },\n );\n return db;\n}\n"; - -export const TsStorageS3Express = "async function storageS3Express() {\n const db = await lancedb.connect(\n \"s3://my-bucket--use1-az4--x-s3/path\",\n {\n storageOptions: {\n region: \"us-east-1\",\n s3Express: \"true\",\n },\n },\n );\n return db;\n}\n"; - -export const TsStorageS3Minio = "async function storageS3Minio() {\n const db = await lancedb.connect(\"s3://bucket/path\", {\n storageOptions: {\n region: \"us-east-1\",\n endpoint: \"http://minio:9000\",\n },\n });\n return db;\n}\n"; - -export const TsStorageS3SseKms = "async function storageS3SseKms() {\n const db = await lancedb.connect(\"s3://bucket/path\", {\n storageOptions: {\n awsServerSideEncryption: \"aws:kms\",\n awsSseKmsKeyId: \"\",\n },\n });\n return db;\n}\n"; - -export const TsStorageTableTimeout = "async function storageTableTimeout() {\n const db = await lancedb.connect(\"s3://bucket/path\");\n const table = await db.createTable(\n \"table\",\n [{ a: 1, b: 2 }],\n { storageOptions: { timeout: \"60s\" } },\n );\n return table;\n}\n"; - -export const TsStorageTigrisConnect = "async function storageTigrisConnect() {\n const db = await lancedb.connect(\n \"s3://your-bucket/path\",\n {\n storageOptions: {\n endpoint: \"https://t3.storage.dev\",\n region: \"auto\",\n },\n },\n );\n return db;\n}\n"; - -export const TsStorageGoosefsConnect = "async function storageGoosefsConnect() {\n const db = await lancedb.connect(\"goosefs://my-namespace/my-database\");\n return db;\n}\n"; - diff --git a/docs/snippets/tables.mdx b/docs/snippets/tables.mdx deleted file mode 100644 index 0d8f213..0000000 --- a/docs/snippets/tables.mdx +++ /dev/null @@ -1,370 +0,0 @@ -{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} - -export const PyAddColumnsCalculated = "# Add a discounted price column (10% discount)\ntable.add_columns({\"discounted_price\": \"cast((price * 0.9) as float)\"})\n"; - -export const PyAddColumnsDefaultValues = "# Add a stock status column with default value\ntable.add_columns({\"in_stock\": \"cast(true as boolean)\"})\n"; - -export const PyAddColumnsNullable = "# Add a nullable timestamp column\ntable.add_columns({\"last_ordered\": \"cast(NULL as timestamp)\"})\n"; - -export const PyAddDataNestedModel = "from lancedb.pydantic import LanceModel, Vector\nfrom pydantic import BaseModel\n\nclass Document(BaseModel):\n content: str\n source: str\n\nclass NestedSchema(LanceModel):\n id: str\n vector: Vector(128)\n document: Document\n\n# Create table with nested schema\ntable_name = \"nested_model_example\"\ntable = db.create_table(table_name, schema=NestedSchema, mode=\"overwrite\")\n"; - -export const PyAddDataPydanticModel = "from lancedb.pydantic import LanceModel, Vector\n\n# Define a Pydantic model\nclass Content(LanceModel):\n movie_id: int\n vector: Vector(128)\n genres: str\n title: str\n imdb_id: int\n\n @property\n def imdb_url(self) -> str:\n return f\"https://www.imdb.com/title/tt{self.imdb_id}\"\n\n# Create table with Pydantic model schema\ntable_name = \"pydantic_example\"\ntable = db.create_table(table_name, schema=Content, mode=\"overwrite\")\n"; - -export const PyAddDataToTable = "import pyarrow as pa\n\n# create an empty table with schema\ndata = [\n {\"vector\": [3.1, 4.1], \"item\": \"foo\", \"price\": 10.0},\n {\"vector\": [5.9, 26.5], \"item\": \"bar\", \"price\": 20.0},\n {\"vector\": [10.2, 100.8], \"item\": \"baz\", \"price\": 30.0},\n {\"vector\": [1.4, 9.5], \"item\": \"fred\", \"price\": 40.0},\n]\n\nschema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n pa.field(\"item\", pa.utf8()),\n pa.field(\"price\", pa.float32()),\n ]\n)\n\ntable_name = \"basic_ingestion_example\"\ntable = db.create_table(table_name, schema=schema, mode=\"overwrite\")\n# Add data\ntable.add(data)\n"; - -export const PyAddFeatureColumnsSql = "table.add_columns(\n {\n \"price_per_id\": \"cast(price / id as float)\",\n \"price_log\": \"ln(price)\",\n \"price_score\": \"cast(price / (price + 100.0) as float)\",\n }\n)\n"; - -export const PyAddFromDataset = "import pyarrow.dataset as ds\n\ndataset = ds.dataset(data_path, format=\"parquet\")\ndb = tmp_db\ntable = db.create_table(\"my_table\", schema=dataset.schema, mode=\"overwrite\")\ntable.add(dataset)\n"; - -export const PyAlterColumnsDataType = "# Change price from int32 to int64 for larger numbers\ntable.alter_columns({\"path\": \"price\", \"data_type\": pa.int64()})\n"; - -export const PyAlterColumnsMultiple = "# Rename, change type, and make nullable in one operation\ntable.alter_columns(\n {\n \"path\": \"sale_price\",\n \"rename\": \"final_price\",\n \"data_type\": pa.float64(),\n \"nullable\": True,\n }\n)\n"; - -export const PyAlterColumnsNullable = "# Make the name column nullable\ntable.alter_columns({\"path\": \"name\", \"nullable\": True})\n"; - -export const PyAlterColumnsRename = "# Rename discount_price to sale_price\ntable.alter_columns({\"path\": \"discount_price\", \"rename\": \"sale_price\"})\n"; - -export const PyAlterColumnsWithExpression = "# For custom transforms, create a new column from a SQL expression.\nexpression_table = tmp_db.create_table(\n \"schema_evolution_expression_example\",\n [{\"id\": 1, \"price_text\": \"$100\"}],\n mode=\"overwrite\",\n)\n\nexpression_table.add_columns(\n {\"price_numeric\": \"cast(replace(price_text, '$', '') as int)\"}\n)\nexpression_table.drop_columns([\"price_text\"])\nexpression_table.alter_columns({\"path\": \"price_numeric\", \"rename\": \"price\"})\n"; - -export const PyAlterVectorColumn = "vector_dim = 768 # Your embedding dimension\ntable_name = \"vector_alter_example\"\ndb = tmp_db\ndata = [\n {\n \"id\": 1,\n \"embedding\": np.random.random(vector_dim).tolist(),\n },\n]\ntable = db.create_table(table_name, data, mode=\"overwrite\")\n\ntable.alter_columns(\n dict(path=\"embedding\", data_type=pa.list_(pa.float32(), vector_dim))\n)\n"; - -export const PyBatchDataInsertion = "import pyarrow as pa\n\ndef make_batches():\n for i in range(5): # Create 5 batches\n yield pa.RecordBatch.from_arrays(\n [\n pa.array([[3.1, 4.1], [5.9, 26.5]], pa.list_(pa.float32(), 2)),\n pa.array([f\"item{i * 2 + 1}\", f\"item{i * 2 + 2}\"]),\n pa.array([float((i * 2 + 1) * 10), float((i * 2 + 2) * 10)]),\n ],\n [\"vector\", \"item\", \"price\"],\n )\n\nschema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n pa.field(\"item\", pa.utf8()),\n pa.field(\"price\", pa.float32()),\n ]\n)\n# Create table with batches\ntable_name = \"batch_ingestion_example\"\ntable = db.create_table(table_name, make_batches(), schema=schema, mode=\"overwrite\")\n"; - -export const PyBranchCreate = "# Fork an isolated, writable branch from main's latest version.\n# `create` returns a table handle scoped to the new branch.\nbranch = table.branches.create(\"exp\")\n"; - -export const PyBranchDelete = "# Delete the branch and its branch-local history. Data on main is safe.\ntable.branches.delete(\"exp\")\n"; - -export const PyBranchIndex = "# Build and validate indexes on a branch before using the configuration on\n# main.\ndev = products.branches.create(\"index-dev\")\n\n# A vector (ANN) index and a full-text search index, both branch-scoped.\ndev.create_index(\n \"vector\",\n config=IvfPq(distance_type=\"cosine\", num_partitions=1, num_sub_vectors=2),\n)\ndev.create_index(\"text\", config=FTS())\n\n# Both indexes live only on the branch; main still has none.\nprint([ix.name for ix in dev.list_indices()]) # branch: two indexes\nprint([ix.name for ix in products.list_indices()]) # main: [] (untouched)\n"; - -export const PyBranchReopen = "# Reopen an existing branch by name from the table handle...\nchecked_out = table.branches.checkout(\"exp\")\n# ...or open it directly from the database connection.\nbranch_handle = db.open_table(\"quotes_branches_example\", branch=\"exp\")\nprint(checked_out.count_rows(), branch_handle.count_rows()) # both 4\n"; - -export const PyBranchUpsertToMain = "# This is a row-level upsert, not a merge of branch histories.\n# `merge_insert` updates matching rows and inserts new rows using a stable\n# unique key. Filter the branch read if you only want to apply some results.\nrows_to_apply = candidate.to_arrow()\n(\n table.merge_insert(\"id\")\n .when_matched_update_all() # update rows that already exist on main\n .when_not_matched_insert_all() # insert rows that are new on the branch\n .execute(rows_to_apply)\n)\n"; - -export const PyBranchWrite = "# Writes land on the branch handle only; main is left untouched.\nbranch.add([{\"id\": 4, \"author\": \"Lancelot\", \"quote\": \"For the realm!\"}])\nprint(branch.count_rows()) # 4 rows on the branch\nprint(table.count_rows()) # 3 rows; main is unaffected\n\n# List every branch, each mapped to its metadata (including its fork point).\nprint(table.branches.list())\n"; - -export const PyConsistencyCheckoutLatest = "uri = str(tmp_db.uri)\nwriter_db = lancedb.connect(uri)\nreader_db = lancedb.connect(uri)\nwriter_table = writer_db.create_table(\n \"consistency_checkout_latest_table\", [{\"id\": 1}], mode=\"overwrite\"\n)\nreader_table = reader_db.open_table(\"consistency_checkout_latest_table\")\n\nwriter_table.add([{\"id\": 2}])\nrows_before_refresh = reader_table.count_rows()\nprint(f\"Rows before checkout_latest: {rows_before_refresh}\")\n\nreader_table.checkout_latest()\nrows_after_refresh = reader_table.count_rows()\nprint(f\"Rows after checkout_latest: {rows_after_refresh}\")\n"; - -export const PyConsistencyEventual = "from datetime import timedelta\n\nuri = str(tmp_db.uri)\nwriter_db = lancedb.connect(uri)\nreader_db = lancedb.connect(uri, read_consistency_interval=timedelta(seconds=3600))\nwriter_table = writer_db.create_table(\n \"consistency_eventual_table\", [{\"id\": 1}], mode=\"overwrite\"\n)\nreader_table = reader_db.open_table(\"consistency_eventual_table\")\nwriter_table.add([{\"id\": 2}])\nrows_after_write = reader_table.count_rows()\nprint(f\"Rows visible before eventual refresh interval: {rows_after_write}\")\n"; - -export const PyConsistencyStrong = "from datetime import timedelta\n\nuri = str(tmp_db.uri)\nwriter_db = lancedb.connect(uri)\nreader_db = lancedb.connect(uri, read_consistency_interval=timedelta(0))\nwriter_table = writer_db.create_table(\n \"consistency_strong_table\", [{\"id\": 1}], mode=\"overwrite\"\n)\nreader_table = reader_db.open_table(\"consistency_strong_table\")\nwriter_table.add([{\"id\": 2}])\nrows_after_write = reader_table.count_rows()\nprint(f\"Rows visible with strong consistency: {rows_after_write}\")\n"; - -export const PyCreateEmptyTable = "import pyarrow as pa\n\nschema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n pa.field(\"item\", pa.string()),\n pa.field(\"price\", pa.float32()),\n ]\n)\ndb = tmp_db\ntbl = db.create_table(\"test_empty_table\", schema=schema, mode=\"overwrite\")\n"; - -export const PyCreateEmptyTablePydantic = "from lancedb.pydantic import LanceModel, Vector\n\nclass Item(LanceModel):\n vector: Vector(2)\n item: str\n price: float\n\ndb = tmp_db\ntbl = db.create_table(\n \"test_empty_table_new\", schema=Item.to_arrow_schema(), mode=\"overwrite\"\n)\n"; - -export const PyCreateTableConflictHandling = "# Idempotent open: reuse the existing table if it exists.\n# The provided data is ignored; the schema is validated against the\n# existing table and a mismatch raises an error.\ntbl = db.create_table(\"conflict_table\", data, exist_ok=True)\n\n# Overwrite: drop the existing table and create a new one with the\n# provided data. This permanently discards the old table's data.\ntbl = db.create_table(\"conflict_table\", data, mode=\"overwrite\")\n"; - -export const PyCreateTableCustomSchema = "import pyarrow as pa\n\ncustom_schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\"lat\", pa.float32()),\n pa.field(\"long\", pa.float32()),\n ]\n)\n\ndata = [\n {\"vector\": [1.1, 1.2, 1.3, 1.4], \"lat\": 45.5, \"long\": -122.7},\n {\"vector\": [0.2, 1.8, 0.4, 3.6], \"lat\": 40.1, \"long\": -74.1},\n]\ndb = tmp_db\ntbl = db.create_table(\n \"my_table_custom_schema\", data, schema=custom_schema, mode=\"overwrite\"\n)\n"; - -export const PyCreateTableFromArrow = "import numpy as np\nimport pyarrow as pa\n\ndim = 16\ntotal = 2\nschema = pa.schema(\n [pa.field(\"vector\", pa.list_(pa.float16(), dim)), pa.field(\"text\", pa.string())]\n)\ndata = pa.Table.from_arrays(\n [\n pa.array(\n [np.random.randn(dim).astype(np.float16) for _ in range(total)],\n pa.list_(pa.float16(), dim),\n ),\n pa.array([\"foo\", \"bar\"]),\n ],\n [\"vector\", \"text\"],\n)\ndb = tmp_db\ntbl = db.create_table(\"f16_tbl\", data, schema=schema, mode=\"overwrite\")\n"; - -export const PyCreateTableFromDicts = "data = [\n {\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1},\n]\ndb = tmp_db\ndb.create_table(\"test_table\", data, mode=\"overwrite\")\ntbl = db[\"test_table\"]\ntbl.head()\n"; - -export const PyCreateTableFromIterator = "import pyarrow as pa\n\nschema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\"item\", pa.utf8()),\n pa.field(\"price\", pa.float32()),\n ]\n)\n\ndef make_batches():\n for i in range(5):\n yield pa.RecordBatch.from_arrays(\n [\n pa.array(\n [[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],\n pa.list_(pa.float32(), 4),\n ),\n pa.array([\"foo\", \"bar\"]),\n pa.array([10.0, 20.0]),\n ],\n [\"vector\", \"item\", \"price\"],\n )\n\ndb = tmp_db\ndb.create_table(\"batched_table\", make_batches(), schema=schema, mode=\"overwrite\")\n"; - -export const PyCreateTableFromPandas = "import pandas as pd\n\ndata = pd.DataFrame(\n {\n \"vector\": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],\n \"lat\": [45.5, 40.1],\n \"long\": [-122.7, -74.1],\n }\n)\ndb = tmp_db\ndb.create_table(\"my_table_pandas\", data, mode=\"overwrite\")\ndb[\"my_table_pandas\"].head()\n"; - -export const PyCreateTableFromPolars = "import polars as pl\n\ndata = pl.DataFrame(\n {\n \"vector\": [[3.1, 4.1], [5.9, 26.5]],\n \"item\": [\"foo\", \"bar\"],\n \"price\": [10.0, 20.0],\n }\n)\ndb = tmp_db\ntbl = db.create_table(\"my_table_pl\", data, mode=\"overwrite\")\n"; - -export const PyCreateTableFromPydantic = "from lancedb.pydantic import LanceModel, Vector\n\nclass Content(LanceModel):\n movie_id: int\n vector: Vector(128)\n genres: str\n title: str\n imdb_id: int\n\n @property\n def imdb_url(self) -> str:\n return f\"https://www.imdb.com/title/tt{self.imdb_id}\"\n\ndb = tmp_db\ntbl = db.create_table(\"movielens_small\", schema=Content, mode=\"overwrite\")\n"; - -export const PyCreateTableNestedSchema = "from lancedb.pydantic import LanceModel, Vector\n\n# --8<-- [start:tables_document_model]\nfrom pydantic import BaseModel\n\nclass Document(BaseModel):\n content: str\n source: str\n\n# --8<-- [end:tables_document_model]\n\nclass NestedSchema(LanceModel):\n id: str\n vector: Vector(1536)\n document: Document\n\ndb = tmp_db\ntbl = db.create_table(\"nested_table\", schema=NestedSchema, mode=\"overwrite\")\n"; - -export const PyDeleteOperation = "# delete data\npredicate = \"id = 3\"\ntable.delete(predicate)\n"; - -export const PyDropColumnsMultiple = "# Remove the second temporary column\ntable.drop_columns([\"temp_col2\"])\n"; - -export const PyDropColumnsSingle = "# Remove the first temporary column\ntable.drop_columns([\"temp_col1\"])\n"; - -export const PyDropTable = "db = tmp_db\n# Create a table first\ndata = [{\"vector\": [1.1, 1.2], \"lat\": 45.5}]\ndb.create_table(\"my_table\", data, mode=\"overwrite\")\n\n# Drop the table\ndb.drop_table(\"my_table\")\n"; - -export const PyInsertIfNotExists = "import pyarrow as pa\n\ntable = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n)\n\nincoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n)\n\n(table.merge_insert(\"id\").when_not_matched_insert_all().execute(incoming_users))\n"; - -export const PyMergeDeleteMissingBySource = "import pyarrow as pa\n\ntable = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2, 3],\n \"name\": [\"Alice\", \"Bob\", \"Charlie\"],\n \"login_count\": [10, 20, 5],\n }\n ),\n mode=\"overwrite\",\n)\n\nincoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n)\n\n(\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .when_not_matched_by_source_delete()\n .execute(incoming_users)\n)\n"; - -export const PyMergeMatchedUpdateOnly = "import pyarrow as pa\n\ntable = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n)\n\nincoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n)\n\n(table.merge_insert(\"id\").when_matched_update_all().execute(incoming_users))\n"; - -export const PyMergePartialColumns = "import pyarrow as pa\n\ntable = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n)\n\nincoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n }\n)\n\n(\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(incoming_users)\n)\n"; - -export const PyMergeUpdateInsert = "import pyarrow as pa\n\ntable = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n)\n\nincoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n)\n\n(\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(incoming_users)\n)\n"; - -export const PyOpenExistingTable = "db = tmp_db\n# Create a table first\ndata = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\ndb.create_table(\"test_table\", data, mode=\"overwrite\")\n\n# List table names\nprint(db.list_tables().tables)\n\n# Open existing table\ntbl = db.open_table(\"test_table\")\n"; - -export const PySchemaAddSetup = "table_name = \"schema_evolution_add_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\ntable = tmp_db.create_table(table_name, data, mode=\"overwrite\")\n"; - -export const PySchemaAlterSetup = "table_name = \"schema_evolution_alter_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200,\n \"discount_price\": 1080.0,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800,\n \"discount_price\": 720.0,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\nschema = pa.schema(\n {\n \"id\": pa.int64(),\n \"name\": pa.string(),\n \"price\": pa.int32(),\n \"discount_price\": pa.float64(),\n \"vector\": pa.list_(pa.float32(), 128),\n }\n)\ntable = tmp_db.create_table(table_name, data, schema=schema, mode=\"overwrite\")\n"; - -export const PySchemaDropSetup = "if data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"temp_col1\": \"X\",\n \"temp_col2\": 100,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"temp_col1\": \"Y\",\n \"temp_col2\": 200,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"temp_col1\": \"Z\",\n \"temp_col2\": 300,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\ntable = tmp_db.create_table(\"schema_evolution_drop_example\", data, mode=\"overwrite\")\n"; - -export const PySchemaFieldMetadataMerge = "# Set two metadata keys on the `category` field.\nres = table.update_field_metadata(\n {\"path\": \"category\", \"metadata\": {\"unit\": \"label\", \"pii\": \"false\"}}\n)\nprint(res.version)\n\n# Merge: add a new key, delete one with None, keep the rest.\ntable.update_field_metadata(\n {\"path\": \"category\", \"metadata\": {\"source\": \"import\", \"pii\": None}}\n)\n\n# Arrow stores field metadata as bytes.\nassert table.schema.field(\"category\").metadata == {\n b\"unit\": b\"label\",\n b\"source\": b\"import\",\n}\n"; - -export const PySchemaFieldMetadataReplace = "table.update_field_metadata(\n {\n \"path\": \"category\",\n \"metadata\": {\"owner\": \"search-team\"},\n \"replace\": True,\n }\n)\n"; - -export const PyTablesBasicConnect = "import lancedb\n\nuri = \"data/sample-lancedb\"\ndb = lancedb.connect(uri)\n"; - -export const PyTablesDocumentModel = "from pydantic import BaseModel\n\nclass Document(BaseModel):\n content: str\n source: str\n"; - -export const PyTablesImports = "import lancedb\nimport numpy as np\nimport pandas as pd\nimport pyarrow as pa\nimport pytest\nfrom numpy.random import randint, random\n"; - -export const PyTablesTzValidator = "from datetime import datetime\nfrom zoneinfo import ZoneInfo\n\nfrom lancedb.pydantic import LanceModel\nfrom pydantic import Field, ValidationError, ValidationInfo, field_validator\n\ntzname = \"America/New_York\"\ntz = ZoneInfo(tzname)\n\nclass TestModel(LanceModel):\n dt_with_tz: datetime = Field(json_schema_extra={\"tz\": tzname})\n\n @field_validator(\"dt_with_tz\")\n @classmethod\n def tz_must_match(cls, dt: datetime) -> datetime:\n assert dt.tzinfo == tz\n return dt\n\nok = TestModel(dt_with_tz=datetime.now(tz))\n\ntry:\n TestModel(dt_with_tz=datetime.now(ZoneInfo(\"Asia/Shanghai\")))\n assert 0 == 1, \"this should raise ValidationError\"\nexcept ValidationError:\n print(\"A ValidationError was raised.\")\n pass\n"; - -export const PyUpdateConnectEnterprise = "import lancedb\n\ndb = lancedb.connect(\n uri=\"db://your-project-slug\",\n api_key=\"your-api-key\",\n region=\"us-east-1\",\n)\n"; - -export const PyUpdateConnectLocal = "import lancedb\n\ndb = lancedb.connect(\"./data\")\n"; - -export const PyUpdateExampleTableSetup = "import pyarrow as pa\n\ntable = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n)\n"; - -export const PyUpdateOperation = "import pyarrow as pa\n\ntable = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n)\ntable.update(where=\"id = 2\", values={\"name\": \"Bobby\"})\n"; - -export const PyUpdateOptimizeCleanup = "from datetime import timedelta\n\ntable.optimize(cleanup_older_than=timedelta(days=1))\n"; - -export const PyUpdateUsingSql = "import pyarrow as pa\n\ntable = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n)\ntable.update(where=\"id = 2\", values_sql={\"login_count\": \"login_count + 1\"})\n"; - -export const PyVersioningAddData = "more_data = [\n {\n \"id\": 4,\n \"author\": \"Richard Daniel Sanchez\",\n \"quote\": \"That's the way the news goes!\",\n },\n {\"id\": 5, \"author\": \"Morty\", \"quote\": \"Aww geez, Rick!\"},\n]\ntable.add(more_data)\n"; - -export const PyVersioningBasicSetup = "import pyarrow as pa\n\ndb = tmp_db\n\ntable_name = \"quotes_versioning_example\"\ndata = [\n {\"id\": 1, \"author\": \"Richard\", \"quote\": \"Wubba Lubba Dub Dub!\"},\n {\"id\": 2, \"author\": \"Morty\", \"quote\": \"Rick, what's going on?\"},\n {\n \"id\": 3,\n \"author\": \"Richard\",\n \"quote\": \"I turned myself into a pickle, Morty!\",\n },\n]\n\n# Define schema\nschema = pa.schema(\n [\n pa.field(\"id\", pa.int64()),\n pa.field(\"author\", pa.string()),\n pa.field(\"quote\", pa.string()),\n ]\n)\n\ntable = db.create_table(table_name, data, schema=schema, mode=\"overwrite\")\n"; - -export const PyVersioningCheckInitialVersion = "versions = table.list_versions()\ncurrent_version = table.version\nprint(f\"Number of versions after creation: {len(versions)}\")\nprint(f\"Current version: {current_version}\")\n"; - -export const PyVersioningCheckVersionsAfterMod = "versions = table.list_versions()\nversion_count_after_mod = len(versions)\nversion_after_mod = table.version\nprint(f\"Number of versions after modifications: {version_count_after_mod}\")\nprint(f\"Current version: {version_after_mod}\")\n"; - -export const PyVersioningCheckoutLatest = "table.checkout_latest()\n"; - -export const PyVersioningDeleteData = "table.delete(\"author = 'Morty'\")\nrows_after_deletion = table.count_rows()\nprint(f\"Number of rows after deletion: {rows_after_deletion}\")\n"; - -export const PyVersioningListAllVersions = "versions = table.list_versions()\nfor v in versions:\n print(f\"Version {v['version']}, created at {v['timestamp']}\")\n"; - -export const PyVersioningRollback = "table.restore(version_after_mod)\nversions = table.list_versions()\nversion_count_after_rollback = len(versions)\nprint(f\"Total number of versions after rollback: {version_count_after_rollback}\")\n"; - -export const PyVersioningTags = "# Create a tag pointing at a specific version\ntable.tags.create(\"baseline\", 1)\ntable.tags.create(\"with-edits\", table.version)\n\n# List all tags on this table\nprint(table.tags.list())\n\n# Look up the version a tag points at\nprint(table.tags.get_version(\"baseline\"))\n\n# Move an existing tag to a different version\ntable.tags.update(\"baseline\", 2)\n\n# Check out a version by tag name\ntable.checkout(\"baseline\")\nprint(table.version)\n\n# Delete a tag (does not delete the underlying version)\ntable.tags.delete(\"with-edits\")\n\n# Return to the latest version\ntable.checkout_latest()\n"; - -export const PyVersioningUpdateData = "table.update(where=\"author='Richard'\", values={\"author\": \"Richard Daniel Sanchez\"})\nrows_after_update = table.count_rows(\"author = 'Richard Daniel Sanchez'\")\nprint(f\"Rows updated to Richard Daniel Sanchez: {rows_after_update}\")\n"; - -export const TsAddColumnsCalculated = "// Add a discounted price column (10% discount)\nawait schemaAddTable.addColumns([\n {\n name: \"discounted_price\",\n valueSql: \"cast((price * 0.9) as float)\",\n },\n]);\n"; - -export const TsAddColumnsDefaultValues = "// Add a stock status column with default value\nawait schemaAddTable.addColumns([\n {\n name: \"in_stock\",\n valueSql: \"cast(true as boolean)\",\n },\n]);\n"; - -export const TsAddColumnsNullable = "// Add a nullable timestamp column\nawait schemaAddTable.addColumns([\n {\n name: \"last_ordered\",\n valueSql: \"cast(NULL as timestamp)\",\n },\n]);\n"; - -export const TsAddFeatureColumnsSql = "await schemaAddTable.addColumns([\n {\n name: \"price_per_id\",\n valueSql: \"cast(price / id as float)\",\n },\n {\n name: \"price_log\",\n valueSql: \"ln(price)\",\n },\n {\n name: \"price_score\",\n valueSql: \"cast(price / (price + 100.0) as float)\",\n },\n]);\n"; - -export const TsAlterColumnsDataType = "// Change price from int32 to int64 for larger numbers\nawait schemaAlterTable.alterColumns([\n { path: \"price\", dataType: new arrow.Int64() },\n]);\n"; - -export const TsAlterColumnsMultiple = "// Rename, change type, and make nullable in one operation\nawait schemaAlterTable.alterColumns([\n {\n path: \"sale_price\",\n rename: \"final_price\",\n dataType: new arrow.Float64(),\n nullable: true,\n },\n]);\n"; - -export const TsAlterColumnsNullable = "// Make the name column nullable\nawait schemaAlterTable.alterColumns([{ path: \"name\", nullable: true }]);\n"; - -export const TsAlterColumnsRename = "// Rename discount_price to sale_price\nawait schemaAlterTable.alterColumns([\n { path: \"discount_price\", rename: \"sale_price\" },\n]);\n"; - -export const TsAlterColumnsWithExpression = "// For custom transforms, create a new column from a SQL expression.\nconst expressionTable = await db.createTable(\n \"schema_evolution_expression_example\",\n [{ id: 1, price_text: \"$100\" }],\n { mode: \"overwrite\" },\n);\n\nawait expressionTable.addColumns([\n {\n name: \"price_numeric\",\n valueSql: \"cast(replace(price_text, '$', '') as int)\",\n },\n]);\nawait expressionTable.dropColumns([\"price_text\"]);\nawait expressionTable.alterColumns([\n { path: \"price_numeric\", rename: \"price\" },\n]);\n"; - -export const TsAlterVectorColumn = "const oldDim = 384;\nconst newDim = 1024;\nconst vectorSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\n \"embedding\",\n new arrow.FixedSizeList(\n oldDim,\n new arrow.Field(\"item\", new arrow.Float16(), true),\n ),\n true,\n ),\n]);\nconst vectorData = lancedb.makeArrowTable(\n [{ id: 1, embedding: Array.from({ length: oldDim }, () => Math.random()) }],\n { schema: vectorSchema },\n);\nconst vectorTable = await db.createTable(\"vector_alter_example\", vectorData, {\n mode: \"overwrite\",\n});\n\n// Changing FixedSizeList dimensions (384 -> 1024) is not supported via alterColumns.\n// Use addColumns + dropColumns + alterColumns(rename) to replace the column.\nawait vectorTable.addColumns([\n {\n name: \"embedding_v2\",\n valueSql: `arrow_cast(NULL, 'FixedSizeList(${newDim}, Float16)')`,\n },\n]);\nawait vectorTable.dropColumns([\"embedding\"]);\nawait vectorTable.alterColumns([{ path: \"embedding_v2\", rename: \"embedding\" }]);\n"; - -export const TsBranchCreate = "// Fork an isolated, writable branch from main's latest version.\n// `create` returns a table handle scoped to the new branch.\nconst branch = await branches.create(\"exp\");\n"; - -export const TsBranchDelete = "// Delete the branch and its branch-local history. Data on main is safe.\nawait branches.delete(\"exp\");\n"; - -export const TsBranchIndex = "// Build and validate indexes on a branch before using the configuration on\n// main.\nconst dev = await productBranches.create(\"index-dev\");\n\n// A vector (ANN) index and a full-text search index, both branch-scoped.\nawait dev.createIndex(\"vector\", {\n config: lancedb.Index.ivfPq({\n distanceType: \"cosine\",\n numPartitions: 1,\n numSubVectors: 2,\n }),\n});\nawait dev.createIndex(\"text\", { config: lancedb.Index.fts() });\n\n// Both indexes live only on the branch; main still has none.\nconsole.log((await dev.listIndices()).map((ix) => ix.name)); // branch: two indexes\nconsole.log((await products.listIndices()).map((ix) => ix.name)); // main: [] (untouched)\n"; - -export const TsBranchReopen = "// Reopen an existing branch by name from the table handle...\nconst checkedOut = await branches.checkout(\"exp\");\n// ...or open it directly from the database connection.\nconst branchHandle = await db.openTable(\n \"quotes_branches_example\",\n undefined,\n { branch: \"exp\" },\n);\nconsole.log(await checkedOut.countRows(), await branchHandle.countRows()); // both 4\n"; - -export const TsBranchUpsertToMain = "// This is a row-level upsert, not a merge of branch histories.\n// `mergeInsert` updates matching rows and inserts new rows using a stable\n// unique key. Filter the branch read if you only want to apply some results.\nconst rowsToApply = await candidate.toArrow();\nawait table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll() // update rows that already exist on main\n .whenNotMatchedInsertAll() // insert rows that are new on the branch\n .execute(rowsToApply);\n"; - -export const TsBranchWrite = "// Writes land on the branch handle only; main is left untouched.\nawait branch.add([{ id: 4, author: \"Lancelot\", quote: \"For the realm!\" }]);\nconsole.log(await branch.countRows()); // 4 rows on the branch\nconsole.log(await table.countRows()); // 3 rows; main is unaffected\n\n// List every branch, each mapped to its metadata (including its fork point).\nconsole.log(await branches.list());\n"; - -export const TsConsistencyCheckoutLatest = "const checkoutWriterDb = await lancedb.connect(databaseDir);\nconst checkoutReaderDb = await lancedb.connect(databaseDir);\nconst checkoutWriterTable = await checkoutWriterDb.createTable(\n \"consistency_checkout_latest_table\",\n [{ id: 1 }],\n { mode: \"overwrite\" },\n);\nconst checkoutReaderTable = await checkoutReaderDb.openTable(\n \"consistency_checkout_latest_table\",\n);\nawait checkoutWriterTable.add([{ id: 2 }]);\nconst rowsBeforeRefresh = await checkoutReaderTable.countRows();\nconsole.log(`Rows before checkoutLatest: ${rowsBeforeRefresh}`);\nawait checkoutReaderTable.checkoutLatest();\nconst rowsAfterRefresh = await checkoutReaderTable.countRows();\nconsole.log(`Rows after checkoutLatest: ${rowsAfterRefresh}`);\n"; - -export const TsConsistencyEventual = "const eventualWriterDb = await lancedb.connect(databaseDir);\nconst eventualReaderDb = await lancedb.connect(databaseDir, {\n readConsistencyInterval: 3600,\n});\nconst eventualWriterTable = await eventualWriterDb.createTable(\n \"consistency_eventual_table\",\n [{ id: 1 }],\n { mode: \"overwrite\" },\n);\nconst eventualReaderTable = await eventualReaderDb.openTable(\n \"consistency_eventual_table\",\n);\nawait eventualWriterTable.add([{ id: 2 }]);\nconst eventualRowsAfterWrite = await eventualReaderTable.countRows();\nconsole.log(\n `Rows visible before eventual refresh interval: ${eventualRowsAfterWrite}`,\n);\n"; - -export const TsConsistencyStrong = "const strongWriterDb = await lancedb.connect(databaseDir);\nconst strongReaderDb = await lancedb.connect(databaseDir, {\n readConsistencyInterval: 0,\n});\nconst strongWriterTable = await strongWriterDb.createTable(\n \"consistency_strong_table\",\n [{ id: 1 }],\n { mode: \"overwrite\" },\n);\nconst strongReaderTable = await strongReaderDb.openTable(\n \"consistency_strong_table\",\n);\nawait strongWriterTable.add([{ id: 2 }]);\nconst strongRowsAfterWrite = await strongReaderTable.countRows();\nconsole.log(`Rows visible with strong consistency: ${strongRowsAfterWrite}`);\n"; - -export const TsCreateEmptyTable = "const emptySchema = new arrow.Schema([\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 2,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n new arrow.Field(\"item\", new arrow.Utf8()),\n new arrow.Field(\"price\", new arrow.Float32()),\n]);\nconst emptyTable = await db.createEmptyTable(\n \"test_empty_table\",\n emptySchema,\n {\n mode: \"overwrite\",\n },\n);\n"; - -export const TsCreateTableConflictHandling = "// Idempotent open: reuse the existing table if it exists.\n// The provided data is ignored; the schema is validated against the\n// existing table and a mismatch raises an error.\nlet conflictTable = await db.createTable(\"conflict_table\", data, {\n existOk: true,\n});\n\n// Overwrite: drop the existing table and create a new one with the\n// provided data. This permanently discards the old table's data.\nconflictTable = await db.createTable(\"conflict_table\", data, {\n mode: \"overwrite\",\n});\n"; - -export const TsCreateTableCustomSchema = "const customSchema = new arrow.Schema([\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n new arrow.Field(\"lat\", new arrow.Float32()),\n new arrow.Field(\"long\", new arrow.Float32()),\n]);\n\nconst customSchemaData = lancedb.makeArrowTable(\n [\n { vector: [1.1, 1.2, 1.3, 1.4], lat: 45.5, long: -122.7 },\n { vector: [0.2, 1.8, 0.4, 3.6], lat: 40.1, long: -74.1 },\n ],\n { schema: customSchema },\n);\nconst customSchemaTable = await db.createTable(\n \"my_table_custom_schema\",\n customSchemaData,\n { mode: \"overwrite\" },\n);\n"; - -export const TsCreateTableFromArrow = "const arrowSchema = new arrow.Schema([\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 16,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n new arrow.Field(\"text\", new arrow.Utf8()),\n]);\nconst arrowData = lancedb.makeArrowTable(\n [\n { vector: Array(16).fill(0.1), text: \"foo\" },\n { vector: Array(16).fill(0.2), text: \"bar\" },\n ],\n { schema: arrowSchema },\n);\nconst arrowTable = await db.createTable(\"f32_tbl\", arrowData, {\n mode: \"overwrite\",\n});\n"; - -export const TsCreateTableFromDicts = "type Location = {\n vector: number[];\n lat: number;\n long: number;\n};\n\nconst data: Location[] = [\n { vector: [1.1, 1.2], lat: 45.5, long: -122.7 },\n { vector: [0.2, 1.8], lat: 40.1, long: -74.1 },\n];\nconst table = await db.createTable(\"test_table\", data, {\n mode: \"overwrite\",\n});\n"; - -export const TsCreateTableFromIterator = "const batchSchema = new arrow.Schema([\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n new arrow.Field(\"item\", new arrow.Utf8()),\n new arrow.Field(\"price\", new arrow.Float32()),\n]);\n\nconst tableForBatches = await db.createEmptyTable(\n \"batched_table\",\n batchSchema,\n {\n mode: \"overwrite\",\n },\n);\n\nconst rows = Array.from({ length: 10 }, (_, i) => ({\n vector: [i + 0.1, i + 0.2, i + 0.3, i + 0.4],\n item: `item-${i + 1}`,\n price: (i + 1) * 10,\n}));\n\nconst chunkSize = 2;\nfor (let i = 0; i < rows.length; i += chunkSize) {\n const batch = lancedb.makeArrowTable(rows.slice(i, i + chunkSize), {\n schema: batchSchema,\n });\n await tableForBatches.add(batch);\n}\n"; - -export const TsDeleteOperation = "// delete data\nconst predicate = \"id = 3\";\nawait table.delete(predicate);\n"; - -export const TsDropColumnsMultiple = "// Remove the second temporary column\nawait schemaDropTable.dropColumns([\"temp_col2\"]);\n"; - -export const TsDropColumnsSingle = "// Remove the first temporary column\nawait schemaDropTable.dropColumns([\"temp_col1\"]);\n"; - -export const TsDropTable = "await db.createTable(\"my_table\", [{ vector: [1.1, 1.2], lat: 45.5 }], {\n mode: \"overwrite\",\n});\n\nawait db.dropTable(\"my_table\");\n"; - -export const TsInsertIfNotExists = "const table = await db.createTable(\n \"users_example\",\n [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ],\n { mode: \"overwrite\" },\n);\n\nconst incomingUsers = [\n { id: 2, name: \"Bobby\", login_count: 21 },\n { id: 3, name: \"Charlie\", login_count: 5 },\n];\n\nawait table\n .mergeInsert(\"id\")\n .whenNotMatchedInsertAll()\n .execute(incomingUsers);\n"; - -export const TsMergeDeleteMissingBySource = "const table = await db.createTable(\n \"users_example\",\n [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n { id: 3, name: \"Charlie\", login_count: 5 },\n ],\n { mode: \"overwrite\" },\n);\n\nconst incomingUsers = [\n { id: 2, name: \"Bobby\", login_count: 21 },\n { id: 3, name: \"Charlie\", login_count: 5 },\n];\n\nawait table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll()\n .whenNotMatchedInsertAll()\n .whenNotMatchedBySourceDelete()\n .execute(incomingUsers);\n"; - -export const TsMergeMatchedUpdateOnly = "const table = await db.createTable(\n \"users_example\",\n [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ],\n { mode: \"overwrite\" },\n);\n\nconst incomingUsers = [\n { id: 2, name: \"Bobby\", login_count: 21 },\n { id: 3, name: \"Charlie\", login_count: 5 },\n];\n\nawait table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll()\n .execute(incomingUsers);\n"; - -export const TsMergePartialColumns = "const table = await db.createTable(\n \"users_example\",\n [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ],\n { mode: \"overwrite\" },\n);\n\nconst incomingUsers = [\n { id: 2, name: \"Bobby\" },\n { id: 3, name: \"Charlie\" },\n];\n\nawait table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll()\n .whenNotMatchedInsertAll()\n .execute(incomingUsers);\n"; - -export const TsMergeUpdateInsert = "const table = await db.createTable(\n \"users_example\",\n [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ],\n { mode: \"overwrite\" },\n);\n\nconst incomingUsers = [\n { id: 2, name: \"Bobby\", login_count: 21 },\n { id: 3, name: \"Charlie\", login_count: 5 },\n];\n\nawait table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll()\n .whenNotMatchedInsertAll()\n .execute(incomingUsers);\n"; - -export const TsOpenExistingTable = "const openTableData = [{ vector: [1.1, 1.2], lat: 45.5, long: -122.7 }];\nawait db.createTable(\"test_table_open\", openTableData, {\n mode: \"overwrite\",\n});\n\nconsole.log(await db.tableNames());\n\nconst openedTable = await db.openTable(\"test_table_open\");\n"; - -export const TsSchemaAddSetup = "const schemaAddData = [\n {\n id: 1,\n name: \"Laptop\",\n price: 1200.0,\n vector: Array.from({ length: 128 }, () => Math.random()),\n },\n {\n id: 2,\n name: \"Smartphone\",\n price: 800.0,\n vector: Array.from({ length: 128 }, () => Math.random()),\n },\n {\n id: 3,\n name: \"Headphones\",\n price: 150.0,\n vector: Array.from({ length: 128 }, () => Math.random()),\n },\n];\nconst schemaAddTable = await db.createTable(\n \"schema_evolution_add_example\",\n schemaAddData,\n { mode: \"overwrite\" },\n);\n"; - -export const TsSchemaAlterSetup = "const schemaAlter = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\"name\", new arrow.Utf8()),\n new arrow.Field(\"price\", new arrow.Int32()),\n new arrow.Field(\"discount_price\", new arrow.Float64()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 128,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n]);\nconst schemaAlterData = lancedb.makeArrowTable(\n [\n {\n id: 1,\n name: \"Laptop\",\n price: 1200,\n discount_price: 1080.0,\n vector: Array.from({ length: 128 }, () => Math.random()),\n },\n {\n id: 2,\n name: \"Smartphone\",\n price: 800,\n discount_price: 720.0,\n vector: Array.from({ length: 128 }, () => Math.random()),\n },\n ],\n { schema: schemaAlter },\n);\nconst schemaAlterTable = await db.createTable(\n \"schema_evolution_alter_example\",\n schemaAlterData,\n { mode: \"overwrite\" },\n);\n"; - -export const TsSchemaDropSetup = "const schemaDropData = [\n {\n id: 1,\n name: \"Laptop\",\n price: 1200.0,\n temp_col1: \"X\",\n temp_col2: 100,\n vector: Array.from({ length: 128 }, () => Math.random()),\n },\n {\n id: 2,\n name: \"Smartphone\",\n price: 800.0,\n temp_col1: \"Y\",\n temp_col2: 200,\n vector: Array.from({ length: 128 }, () => Math.random()),\n },\n {\n id: 3,\n name: \"Headphones\",\n price: 150.0,\n temp_col1: \"Z\",\n temp_col2: 300,\n vector: Array.from({ length: 128 }, () => Math.random()),\n },\n];\nconst schemaDropTable = await db.createTable(\n \"schema_evolution_drop_example\",\n schemaDropData,\n { mode: \"overwrite\" },\n);\n"; - -export const TsSchemaFieldMetadataMerge = "// Set two metadata keys on the `category` field.\nconst res = await fieldMetadataTable.updateFieldMetadata([\n { path: \"category\", metadata: { unit: \"label\", pii: \"false\" } },\n]);\nconsole.log(res.version);\n\n// Merge: add a new key, delete one via null, keep the rest.\nawait fieldMetadataTable.updateFieldMetadata([\n { path: \"category\", metadata: { source: \"import\", pii: null } },\n]);\n"; - -export const TsSchemaFieldMetadataReplace = "await fieldMetadataTable.updateFieldMetadata([\n {\n path: \"category\",\n metadata: { owner: \"search-team\" },\n replace: true,\n },\n]);\n"; - -export const TsUpdateConnectEnterprise = "const db = await lancedb.connect(\"db://your-project-slug\", {\n apiKey: \"your-api-key\",\n region: \"us-east-1\",\n});\n"; - -export const TsUpdateConnectLocal = "const db = await lancedb.connect(\"./data\");\n"; - -export const TsUpdateExampleTableSetup = "const table = await db.createTable(\n \"users_example\",\n [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ],\n { mode: \"overwrite\" },\n);\n"; - -export const TsUpdateOperation = "const table = await db.createTable(\n \"users_example\",\n [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ],\n { mode: \"overwrite\" },\n);\nawait table.update({ where: \"id = 2\", values: { name: \"Bobby\" } });\n"; - -export const TsUpdateOptimizeCleanup = "const olderThan = new Date();\nolderThan.setDate(olderThan.getDate() - 1);\nawait table.optimize({ cleanupOlderThan: olderThan });\n"; - -export const TsUpdateUsingSql = "const table = await db.createTable(\n \"users_example\",\n [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ],\n { mode: \"overwrite\" },\n);\nawait table.update({\n where: \"id = 2\",\n valuesSql: { login_count: \"login_count + 1\" },\n});\n"; - -export const TsVersioningAddData = "const moreData = [\n {\n id: 4,\n author: \"Richard Daniel Sanchez\",\n quote: \"That's the way the news goes!\",\n },\n { id: 5, author: \"Morty\", quote: \"Aww geez, Rick!\" },\n];\nawait table.add(moreData);\n"; - -export const TsVersioningBasicSetup = "const tableName = \"quotes_versioning_example\";\nconst data = [\n { id: 1, author: \"Richard\", quote: \"Wubba Lubba Dub Dub!\" },\n { id: 2, author: \"Morty\", quote: \"Rick, what's going on?\" },\n {\n id: 3,\n author: \"Richard\",\n quote: \"I turned myself into a pickle, Morty!\",\n },\n];\nconst table = await db.createTable(tableName, data, { mode: \"overwrite\" });\n"; - -export const TsVersioningCheckInitialVersion = "const versions = await table.listVersions();\nconst currentVersion = await table.version();\nconsole.log(`Number of versions after creation: ${versions.length}`);\nconsole.log(`Current version: ${currentVersion}`);\n"; - -export const TsVersioningCheckVersionsAfterMod = "const versionsAfterMod = await table.listVersions();\nconst versionCountAfterMod = versionsAfterMod.length;\nconst versionAfterMod = await table.version();\nconsole.log(\n `Number of versions after modifications: ${versionCountAfterMod}`,\n);\nconsole.log(`Current version: ${versionAfterMod}`);\n"; - -export const TsVersioningCheckoutLatest = "await table.checkoutLatest();\n"; - -export const TsVersioningDeleteData = "await table.delete(\"author = 'Morty'\");\nconst rowsAfterDeletion = await table.countRows();\nconsole.log(`Number of rows after deletion: ${rowsAfterDeletion}`);\n"; - -export const TsVersioningListAllVersions = "const allVersions = await table.listVersions();\nfor (const v of allVersions) {\n console.log(`Version ${v.version}, created at ${v.timestamp}`);\n}\n"; - -export const TsVersioningRollback = "await table.checkout(versionAfterMod);\nawait table.restore();\nconst versionsAfterRollback = await table.listVersions();\nconst versionCountAfterRollback = versionsAfterRollback.length;\nconsole.log(\n `Total number of versions after rollback: ${versionCountAfterRollback}`,\n);\n"; - -export const TsVersioningTags = "const tags = await tagsTable.tags();\n\n// Create a tag pointing at a specific version\nawait tags.create(\"baseline\", 1);\nawait tags.create(\"with-edits\", await tagsTable.version());\n\n// List all tags on this table\nconsole.log(await tags.list());\n\n// Look up the version a tag points at\nconsole.log(await tags.getVersion(\"baseline\"));\n\n// Move an existing tag to a different version\nawait tags.update(\"baseline\", 2);\n\n// Check out a version by tag name\nawait tagsTable.checkout(\"baseline\");\nconsole.log(await tagsTable.version());\n\n// Delete a tag (does not delete the underlying version)\nawait tags.delete(\"with-edits\");\n\n// Return to the latest version\nawait tagsTable.checkoutLatest();\n"; - -export const TsVersioningUpdateData = "await table.update({\n where: \"author = 'Richard'\",\n values: { author: \"Richard Daniel Sanchez\" },\n});\nconst rowsAfterUpdate = await table.countRows(\n \"author = 'Richard Daniel Sanchez'\",\n);\nconsole.log(`Rows updated to Richard Daniel Sanchez: ${rowsAfterUpdate}`);\n"; - -export const RsAddColumnsCalculated = "// Add a discounted price column (10% discount)\nschema_add_table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"discounted_price\".to_string(),\n \"cast((price * 0.9) as float)\".to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\n"; - -export const RsAddColumnsDefaultValues = "// Add a stock status column with default value\nschema_add_table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"in_stock\".to_string(),\n \"cast(true as boolean)\".to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\n"; - -export const RsAddColumnsNullable = "// Add a nullable timestamp column\nschema_add_table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"last_ordered\".to_string(),\n \"cast(NULL as timestamp)\".to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\n"; - -export const RsAddFeatureColumnsSql = "schema_add_table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![\n (\n \"price_per_id\".to_string(),\n \"cast(price / id as float)\".to_string(),\n ),\n (\"price_log\".to_string(), \"ln(price)\".to_string()),\n (\n \"price_score\".to_string(),\n \"cast(price / (price + 100.0) as float)\".to_string(),\n ),\n ]),\n None,\n )\n .await\n .unwrap();\n"; - -export const RsAlterColumnsDataType = "// Change price from int32 to int64 for larger numbers\nschema_alter_table\n .alter_columns(&[ColumnAlteration::new(\"price\".to_string()).cast_to(DataType::Int64)])\n .await\n .unwrap();\n"; - -export const RsAlterColumnsMultiple = "// Rename, change type, and make nullable in one operation\nschema_alter_table\n .alter_columns(&[ColumnAlteration::new(\"sale_price\".to_string())\n .rename(\"final_price\".to_string())\n .cast_to(DataType::Float64)\n .set_nullable(true)])\n .await\n .unwrap();\n"; - -export const RsAlterColumnsNullable = "// Make the name column nullable\nschema_alter_table\n .alter_columns(&[ColumnAlteration::new(\"name\".to_string()).set_nullable(true)])\n .await\n .unwrap();\n"; - -export const RsAlterColumnsRename = "// Rename discount_price to sale_price\nschema_alter_table\n .alter_columns(&[ColumnAlteration::new(\"discount_price\".to_string())\n .rename(\"sale_price\".to_string())])\n .await\n .unwrap();\n"; - -export const RsAlterColumnsWithExpression = "// For custom transforms, create a new column from a SQL expression.\nlet expression_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"price_text\", DataType::Utf8, false),\n]));\nlet expression_batch = RecordBatch::try_new(\n expression_schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1])),\n Arc::new(StringArray::from(vec![\"$100\"])),\n ],\n)\n.unwrap();\nlet expression_reader: Box = Box::new(RecordBatchIterator::new(\n vec![Ok(expression_batch)].into_iter(),\n expression_schema.clone(),\n));\nlet expression_table = db\n .create_table(\"schema_evolution_expression_example\", expression_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nexpression_table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"price_numeric\".to_string(),\n \"cast(replace(price_text, '$', '') as int)\".to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\nexpression_table.drop_columns(&[\"price_text\"]).await.unwrap();\nexpression_table\n .alter_columns(&[ColumnAlteration::new(\"price_numeric\".to_string())\n .rename(\"price\".to_string())])\n .await\n .unwrap();\n"; - -export const RsAlterVectorColumn = "let old_dim = 384;\nlet new_dim = 1024;\nlet vector_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\n \"embedding\",\n DataType::FixedSizeList(\n Arc::new(Field::new(\"item\", DataType::Float32, true)),\n old_dim,\n ),\n true,\n ),\n]));\nlet vector_batch = RecordBatch::try_new(\n vector_schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![Some(vec![Some(0.1_f32); old_dim as usize])],\n old_dim,\n ),\n ),\n ],\n)\n.unwrap();\nlet vector_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(vector_batch)].into_iter(), vector_schema.clone()));\nlet vector_table = db\n .create_table(\"vector_alter_example\", vector_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\n// Changing FixedSizeList dimensions (384 -> 1024) is not supported via alter_columns.\n// Use add_columns + drop_columns + alter_columns(rename) to replace the column.\nvector_table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"embedding_v2\".to_string(),\n format!(\"arrow_cast(NULL, 'FixedSizeList({}, Float32)')\", new_dim),\n )]),\n None,\n )\n .await\n .unwrap();\nvector_table.drop_columns(&[\"embedding\"]).await.unwrap();\nvector_table\n .alter_columns(&[ColumnAlteration::new(\"embedding_v2\".to_string())\n .rename(\"embedding\".to_string())])\n .await\n .unwrap();\n"; - -export const RsBranchCreate = "// Fork an isolated, writable branch from main's latest version.\n// `create_branch` returns a table handle scoped to the new branch.\nlet branch = branches_table\n .create_branch(\"exp\", Ref::Version(None, None))\n .await\n .unwrap();\n"; - -export const RsBranchDelete = "// Delete the branch and its branch-local history. Data on main is safe.\nbranches_table.delete_branch(\"exp\").await.unwrap();\n"; - -export const RsBranchIndex = "use lancedb::index::scalar::FtsIndexBuilder;\nuse lancedb::index::Index;\n\n// Build and validate indexes on a branch before using the configuration on\n// main.\nlet dev = products\n .create_branch(\"index-dev\", Ref::Version(None, None))\n .await\n .unwrap();\n\n// A vector (ANN) index and a full-text search index, both branch-scoped.\ndev.create_index(&[\"vector\"], Index::Auto)\n .execute()\n .await\n .unwrap();\ndev.create_index(&[\"text\"], Index::FTS(FtsIndexBuilder::default()))\n .execute()\n .await\n .unwrap();\n\n// Both indexes live only on the branch; main still has none.\nprintln!(\"Branch indexes: {}\", dev.list_indices().await.unwrap().len()); // 2\nprintln!(\"Main indexes: {}\", products.list_indices().await.unwrap().len()); // 0\n"; - -export const RsBranchReopen = "// Reopen an existing branch by name from the table handle...\nlet checked_out = branches_table.checkout_branch(\"exp\", None).await.unwrap();\n// ...or open it directly via the connection's builder.\nlet opened = db\n .open_table(\"quotes_branches_example\")\n .branch(\"exp\")\n .execute()\n .await\n .unwrap();\nprintln!(\n \"Reopened rows: {}, {}\",\n checked_out.count_rows(None).await.unwrap(),\n opened.count_rows(None).await.unwrap()\n); // both 4\n"; - -export const RsBranchUpsertToMain = "// This is a row-level upsert, not a merge of branch histories.\n// `merge_insert` updates matching rows and inserts new rows using a stable\n// unique key. Filter the branch read if you only want to apply some results.\nlet schema = candidate.schema().await.unwrap();\nlet batches = candidate\n .query()\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\nlet rows_to_apply = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);\n\nlet mut merge = branches_table.merge_insert(&[\"id\"]);\nmerge\n .when_matched_update_all(None) // update rows that already exist on main\n .when_not_matched_insert_all(); // insert rows that are new on the branch\nmerge.execute(Box::new(rows_to_apply)).await.unwrap();\n"; - -export const RsBranchWrite = "// Writes land on the branch handle only; main is left untouched.\nbranch\n .add(make_quotes_reader(vec![(4, \"Lancelot\", \"For the realm!\")]))\n .execute()\n .await\n .unwrap();\nprintln!(\"Branch rows: {}\", branch.count_rows(None).await.unwrap()); // 4\nprintln!(\"Main rows: {}\", branches_table.count_rows(None).await.unwrap()); // 3\n\n// List every branch, each mapped to its metadata (including its fork point).\nprintln!(\"Branches: {:?}\", branches_table.list_branches().await.unwrap());\n"; - -export const RsConsistencyCheckoutLatest = "let checkout_writer_db = connect(&db_uri).execute().await.unwrap();\nlet checkout_reader_db = connect(&db_uri).execute().await.unwrap();\nlet checkout_writer_table = checkout_writer_db\n .create_table(\n \"consistency_checkout_latest_table\",\n make_users_reader(vec![1], vec![\"Alice\"], None),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\nlet checkout_reader_table = checkout_reader_db\n .open_table(\"consistency_checkout_latest_table\")\n .execute()\n .await\n .unwrap();\ncheckout_writer_table\n .add(make_users_reader(vec![2], vec![\"Bob\"], None))\n .execute()\n .await\n .unwrap();\nlet rows_before_refresh = checkout_reader_table.count_rows(None).await.unwrap();\nprintln!(\"Rows before checkout_latest: {}\", rows_before_refresh);\ncheckout_reader_table.checkout_latest().await.unwrap();\nlet rows_after_refresh = checkout_reader_table.count_rows(None).await.unwrap();\nprintln!(\"Rows after checkout_latest: {}\", rows_after_refresh);\n"; - -export const RsConsistencyEventual = "let eventual_writer_db = connect(&db_uri).execute().await.unwrap();\nlet eventual_reader_db = connect(&db_uri)\n .read_consistency_interval(StdDuration::from_secs(3600))\n .execute()\n .await\n .unwrap();\nlet eventual_writer_table = eventual_writer_db\n .create_table(\n \"consistency_eventual_table\",\n make_users_reader(vec![1], vec![\"Alice\"], None),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\nlet eventual_reader_table = eventual_reader_db\n .open_table(\"consistency_eventual_table\")\n .execute()\n .await\n .unwrap();\neventual_writer_table\n .add(make_users_reader(vec![2], vec![\"Bob\"], None))\n .execute()\n .await\n .unwrap();\nlet eventual_rows_after_write = eventual_reader_table.count_rows(None).await.unwrap();\nprintln!(\n \"Rows visible before eventual refresh interval: {}\",\n eventual_rows_after_write\n);\n"; - -export const RsConsistencyStrong = "let strong_writer_db = connect(&db_uri).execute().await.unwrap();\nlet strong_reader_db = connect(&db_uri)\n .read_consistency_interval(StdDuration::from_secs(0))\n .execute()\n .await\n .unwrap();\nlet strong_writer_table = strong_writer_db\n .create_table(\n \"consistency_strong_table\",\n make_users_reader(vec![1], vec![\"Alice\"], None),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\nlet strong_reader_table = strong_reader_db\n .open_table(\"consistency_strong_table\")\n .execute()\n .await\n .unwrap();\nstrong_writer_table\n .add(make_users_reader(vec![2], vec![\"Bob\"], None))\n .execute()\n .await\n .unwrap();\nlet strong_rows_after_write = strong_reader_table.count_rows(None).await.unwrap();\nprintln!(\n \"Rows visible with strong consistency: {}\",\n strong_rows_after_write\n);\n"; - -export const RsCreateEmptyTable = "let empty_schema = Arc::new(Schema::new(vec![\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n false,\n ),\n Field::new(\"item\", DataType::Utf8, false),\n Field::new(\"price\", DataType::Float32, false),\n]));\nlet empty_table = db\n .create_empty_table(\"test_empty_table\", empty_schema)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsCreateTableConflictHandling = "// Idempotent open: reuse the existing table if it exists.\n// The provided data is ignored; the schema is validated against the\n// existing table and a mismatch raises an error.\nlet _conflict_table = db\n .create_table(\"conflict_table\", exist_ok_reader)\n .mode(CreateTableMode::exist_ok(|req| req))\n .execute()\n .await\n .unwrap();\n\n// Overwrite: drop the existing table and create a new one with the\n// provided data. This permanently discards the old table's data.\nlet conflict_table = db\n .create_table(\"conflict_table\", overwrite_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsCreateTableCustomSchema = "let custom_schema = Arc::new(Schema::new(vec![\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n Field::new(\"lat\", DataType::Float32, false),\n Field::new(\"long\", DataType::Float32, false),\n]));\n\nlet custom_batch = RecordBatch::try_new(\n custom_schema.clone(),\n vec![\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![\n Some(vec![Some(1.1), Some(1.2), Some(1.3), Some(1.4)]),\n Some(vec![Some(0.2), Some(1.8), Some(0.4), Some(3.6)]),\n ],\n 4,\n ),\n ),\n Arc::new(Float32Array::from(vec![45.5, 40.1])),\n Arc::new(Float32Array::from(vec![-122.7, -74.1])),\n ],\n)\n.unwrap();\nlet custom_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(custom_batch)].into_iter(), custom_schema.clone()));\nlet custom_table = db\n .create_table(\"my_table_custom_schema\", custom_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsCreateTableFromArrow = "let arrow_schema = Arc::new(Schema::new(vec![\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 16),\n false,\n ),\n Field::new(\"text\", DataType::Utf8, false),\n]));\n\nlet arrow_batch = RecordBatch::try_new(\n arrow_schema.clone(),\n vec![\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![Some(vec![Some(0.1); 16]), Some(vec![Some(0.2); 16])],\n 16,\n ),\n ),\n Arc::new(StringArray::from(vec![\"foo\", \"bar\"])),\n ],\n)\n.unwrap();\nlet arrow_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(arrow_batch)].into_iter(), arrow_schema.clone()));\nlet arrow_table = db\n .create_table(\"arrow_table_example\", arrow_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsCreateTableFromDicts = "struct Location {\n vector: [f32; 2],\n lat: f32,\n long: f32,\n}\n\nlet data = vec![\n Location {\n vector: [1.1, 1.2],\n lat: 45.5,\n long: -122.7,\n },\n Location {\n vector: [0.2, 1.8],\n lat: 40.1,\n long: -74.1,\n },\n];\n\nlet schema = Arc::new(Schema::new(vec![\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n false,\n ),\n Field::new(\"lat\", DataType::Float32, false),\n Field::new(\"long\", DataType::Float32, false),\n]));\n\nlet batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n data.iter()\n .map(|row| Some(row.vector.iter().copied().map(Some).collect::>())),\n 2,\n ),\n ),\n Arc::new(Float32Array::from_iter_values(\n data.iter().map(|row| row.lat),\n )),\n Arc::new(Float32Array::from_iter_values(\n data.iter().map(|row| row.long),\n )),\n ],\n)\n.unwrap();\nlet reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone()));\nlet table = db\n .create_table(\"test_table\", reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsCreateTableFromIterator = "let batch_schema = Arc::new(Schema::new(vec![\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n Field::new(\"item\", DataType::Utf8, false),\n Field::new(\"price\", DataType::Float32, false),\n]));\n\nlet batches = (0..5)\n .map(|i| {\n RecordBatch::try_new(\n batch_schema.clone(),\n vec![\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![\n Some(vec![Some(3.1 + i as f32), Some(4.1), Some(5.1), Some(6.1)]),\n Some(vec![\n Some(5.9),\n Some(26.5 + i as f32),\n Some(4.7),\n Some(32.8),\n ]),\n ],\n 4,\n ),\n ),\n Arc::new(StringArray::from(vec![\n format!(\"item{}\", i * 2 + 1),\n format!(\"item{}\", i * 2 + 2),\n ])),\n Arc::new(Float32Array::from(vec![\n ((i * 2 + 1) * 10) as f32,\n ((i * 2 + 2) * 10) as f32,\n ])),\n ],\n )\n .unwrap()\n })\n .collect::>();\n\nlet batch_reader: Box =\n Box::new(RecordBatchIterator::new(batches.into_iter().map(Ok), batch_schema.clone()));\nlet batch_table = db\n .create_table(\"batched_table\", batch_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsDeleteOperation = "// delete data\nlet predicate = \"id = 3\";\ntable.delete(predicate).await.unwrap();\n"; - -export const RsDropColumnsMultiple = "// Remove the second temporary column\nschema_drop_table.drop_columns(&[\"temp_col2\"]).await.unwrap();\n"; - -export const RsDropColumnsSingle = "// Remove the first temporary column\nschema_drop_table.drop_columns(&[\"temp_col1\"]).await.unwrap();\n"; - -export const RsDropTable = "let drop_schema = Arc::new(Schema::new(vec![\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n false,\n ),\n Field::new(\"lat\", DataType::Float32, false),\n]));\nlet drop_batch = RecordBatch::try_new(\n drop_schema.clone(),\n vec![\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![Some(vec![Some(1.1), Some(1.2)])],\n 2,\n ),\n ),\n Arc::new(Float32Array::from(vec![45.5])),\n ],\n)\n.unwrap();\nlet drop_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(drop_batch)].into_iter(), drop_schema.clone()));\ndb.create_table(\"my_table\", drop_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\ndb.drop_table(\"my_table\", &[]).await.unwrap();\n"; - -export const RsInsertIfNotExists = "let table = db\n .create_table(\n \"users_example\",\n make_users_reader(vec![1, 2], vec![\"Alice\", \"Bob\"], Some(vec![10, 20])),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nlet mut merge_insert = table.merge_insert(&[\"id\"]);\nmerge_insert.when_not_matched_insert_all();\nmerge_insert\n .execute(make_users_reader(\n vec![2, 3],\n vec![\"Bobby\", \"Charlie\"],\n Some(vec![21, 5]),\n ))\n .await\n .unwrap();\n"; - -export const RsMergeDeleteMissingBySource = "let table = db\n .create_table(\n \"users_example\",\n make_users_reader(\n vec![1, 2, 3],\n vec![\"Alice\", \"Bob\", \"Charlie\"],\n Some(vec![10, 20, 5]),\n ),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nlet mut merge_insert = table.merge_insert(&[\"id\"]);\nmerge_insert\n .when_matched_update_all(None)\n .when_not_matched_insert_all()\n .when_not_matched_by_source_delete(None);\nmerge_insert\n .execute(make_users_reader(\n vec![2, 3],\n vec![\"Bobby\", \"Charlie\"],\n Some(vec![21, 5]),\n ))\n .await\n .unwrap();\n"; - -export const RsMergeMatchedUpdateOnly = "let table = db\n .create_table(\n \"users_example\",\n make_users_reader(vec![1, 2], vec![\"Alice\", \"Bob\"], Some(vec![10, 20])),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nlet mut merge_insert = table.merge_insert(&[\"id\"]);\nmerge_insert.when_matched_update_all(None);\nmerge_insert\n .execute(make_users_reader(\n vec![2, 3],\n vec![\"Bobby\", \"Charlie\"],\n Some(vec![21, 5]),\n ))\n .await\n .unwrap();\n"; - -export const RsMergePartialColumns = "let table = db\n .create_table(\n \"users_example\",\n make_users_reader(vec![1, 2], vec![\"Alice\", \"Bob\"], Some(vec![10, 20])),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nlet mut merge_insert = table.merge_insert(&[\"id\"]);\nmerge_insert\n .when_matched_update_all(None)\n .when_not_matched_insert_all();\nmerge_insert\n .execute(make_users_reader(vec![2, 3], vec![\"Bobby\", \"Charlie\"], None))\n .await\n .unwrap();\n"; - -export const RsMergeUpdateInsert = "let table = db\n .create_table(\n \"users_example\",\n make_users_reader(vec![1, 2], vec![\"Alice\", \"Bob\"], Some(vec![10, 20])),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nlet mut merge_insert = table.merge_insert(&[\"id\"]);\nmerge_insert\n .when_matched_update_all(None)\n .when_not_matched_insert_all();\nmerge_insert\n .execute(make_users_reader(\n vec![2, 3],\n vec![\"Bobby\", \"Charlie\"],\n Some(vec![21, 5]),\n ))\n .await\n .unwrap();\n"; - -export const RsOpenExistingTable = "let open_schema = Arc::new(Schema::new(vec![\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n false,\n ),\n Field::new(\"lat\", DataType::Float32, false),\n Field::new(\"long\", DataType::Float32, false),\n]));\nlet open_batch = RecordBatch::try_new(\n open_schema.clone(),\n vec![\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![Some(vec![Some(1.1), Some(1.2)])],\n 2,\n ),\n ),\n Arc::new(Float32Array::from(vec![45.5])),\n Arc::new(Float32Array::from(vec![-122.7])),\n ],\n)\n.unwrap();\nlet open_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(open_batch)].into_iter(), open_schema.clone()));\ndb.create_table(\"test_table\", open_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nprintln!(\"{:?}\", db.table_names().execute().await.unwrap());\n\nlet opened_table = db.open_table(\"test_table\").execute().await.unwrap();\n"; - -export const RsSchemaAddSetup = "let schema_add_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n Field::new(\"price\", DataType::Float64, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n false,\n ),\n]));\nlet schema_add_batch = RecordBatch::try_new(\n schema_add_schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\"Laptop\", \"Smartphone\", \"Headphones\"])),\n Arc::new(Float64Array::from(vec![1200.0, 800.0, 150.0])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![\n Some(vec![Some(0.1_f32); 128]),\n Some(vec![Some(0.2_f32); 128]),\n Some(vec![Some(0.3_f32); 128]),\n ],\n 128,\n ),\n ),\n ],\n)\n.unwrap();\nlet schema_add_reader: Box = Box::new(RecordBatchIterator::new(\n vec![Ok(schema_add_batch)].into_iter(),\n schema_add_schema.clone(),\n));\nlet schema_add_table = db\n .create_table(\"schema_evolution_add_example\", schema_add_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsSchemaAlterSetup = "let schema_alter_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n Field::new(\"price\", DataType::Int32, false),\n Field::new(\"discount_price\", DataType::Float64, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n false,\n ),\n]));\nlet schema_alter_batch = RecordBatch::try_new(\n schema_alter_schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2])),\n Arc::new(StringArray::from(vec![\"Laptop\", \"Smartphone\"])),\n Arc::new(Int32Array::from(vec![1200, 800])),\n Arc::new(Float64Array::from(vec![1080.0, 720.0])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![Some(vec![Some(0.1_f32); 128]), Some(vec![Some(0.2_f32); 128])],\n 128,\n ),\n ),\n ],\n)\n.unwrap();\nlet schema_alter_reader: Box = Box::new(RecordBatchIterator::new(\n vec![Ok(schema_alter_batch)].into_iter(),\n schema_alter_schema.clone(),\n));\nlet schema_alter_table = db\n .create_table(\"schema_evolution_alter_example\", schema_alter_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsSchemaDropSetup = "let schema_drop_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n Field::new(\"price\", DataType::Float64, false),\n Field::new(\"temp_col1\", DataType::Utf8, false),\n Field::new(\"temp_col2\", DataType::Int32, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n false,\n ),\n]));\nlet schema_drop_batch = RecordBatch::try_new(\n schema_drop_schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\"Laptop\", \"Smartphone\", \"Headphones\"])),\n Arc::new(Float64Array::from(vec![1200.0, 800.0, 150.0])),\n Arc::new(StringArray::from(vec![\"X\", \"Y\", \"Z\"])),\n Arc::new(Int32Array::from(vec![100, 200, 300])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![\n Some(vec![Some(0.1_f32); 128]),\n Some(vec![Some(0.2_f32); 128]),\n Some(vec![Some(0.3_f32); 128]),\n ],\n 128,\n ),\n ),\n ],\n)\n.unwrap();\nlet schema_drop_reader: Box = Box::new(RecordBatchIterator::new(\n vec![Ok(schema_drop_batch)].into_iter(),\n schema_drop_schema.clone(),\n));\nlet schema_drop_table = db\n .create_table(\"schema_evolution_drop_example\", schema_drop_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsSchemaFieldMetadataMerge = "// Set two metadata keys on the `category` field.\nlet res = field_metadata_table\n .update_field_metadata(&[FieldMetadataUpdate::new(\"category\")\n .set(\"unit\", \"label\")\n .set(\"pii\", \"false\")])\n .await\n .unwrap();\nprintln!(\"version: {}\", res.version);\n\n// Merge: add a new key, delete one with `.remove`, keep the rest.\nfield_metadata_table\n .update_field_metadata(&[FieldMetadataUpdate::new(\"category\")\n .set(\"source\", \"import\")\n .remove(\"pii\")])\n .await\n .unwrap();\n"; - -export const RsSchemaFieldMetadataReplace = "field_metadata_table\n .update_field_metadata(&[FieldMetadataUpdate::new(\"category\")\n .set(\"owner\", \"search-team\")\n .replace()])\n .await\n .unwrap();\n"; - -export const RsUpdateConnectEnterprise = "let uri = \"db://your-project-slug\";\nlet api_key = \"your-api-key\";\nlet region = \"us-east-1\";\n"; - -export const RsUpdateConnectLocal = "let db = connect(\"./data\").execute().await.unwrap();\n"; - -export const RsUpdateExampleTableSetup = "let table = db\n .create_table(\n \"users_example\",\n make_users_reader(vec![1, 2], vec![\"Alice\", \"Bob\"], Some(vec![10, 20])),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsUpdateMakeUsersReader = "fn make_users_reader(\n ids: Vec,\n names: Vec<&str>,\n login_counts: Option>,\n) -> Box {\n let mut fields = vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n ];\n let mut columns: Vec> =\n vec![Arc::new(Int64Array::from(ids)), Arc::new(StringArray::from(names))];\n\n if let Some(login_counts) = login_counts {\n fields.push(Field::new(\"login_count\", DataType::Int64, true));\n columns.push(Arc::new(Int64Array::from(login_counts)));\n }\n\n let schema = Arc::new(Schema::new(fields));\n let batch = RecordBatch::try_new(schema.clone(), columns).unwrap();\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n Box::new(reader)\n}\n"; - -export const RsUpdateOperation = "let table = db\n .create_table(\n \"users_example\",\n make_users_reader(vec![1, 2], vec![\"Alice\", \"Bob\"], Some(vec![10, 20])),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\ntable\n .update()\n .only_if(\"id = 2\")\n .column(\"name\", \"'Bobby'\")\n .execute()\n .await\n .unwrap();\n"; - -export const RsUpdateOptimizeCleanup = "table\n .optimize(OptimizeAction::Prune {\n older_than: Some(Duration::days(1)),\n delete_unverified: None,\n error_if_tagged_old_versions: None,\n })\n .await\n .unwrap();\n"; - -export const RsUpdateUsingSql = "let table = db\n .create_table(\n \"users_example\",\n make_users_reader(vec![1, 2], vec![\"Alice\", \"Bob\"], Some(vec![10, 20])),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\ntable\n .update()\n .only_if(\"id = 2\")\n .column(\"login_count\", \"login_count + 1\")\n .execute()\n .await\n .unwrap();\n"; - -export const RsVersioningAddData = "let more_data = vec![\n (4, \"Richard Daniel Sanchez\", \"That's the way the news goes!\"),\n (5, \"Morty\", \"Aww geez, Rick!\"),\n];\ntable\n .add(make_quotes_reader(more_data))\n .execute()\n .await\n .unwrap();\n"; - -export const RsVersioningBasicSetup = "let table_name = \"quotes_versioning_example\";\nlet data = vec![\n (1, \"Richard\", \"Wubba Lubba Dub Dub!\"),\n (2, \"Morty\", \"Rick, what's going on?\"),\n (3, \"Richard\", \"I turned myself into a pickle, Morty!\"),\n];\n\nlet table = db\n .create_table(table_name, make_quotes_reader(data))\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; - -export const RsVersioningCheckInitialVersion = "let versions = table.list_versions().await.unwrap();\nlet current_version = table.version().await.unwrap();\nprintln!(\"Number of versions after creation: {}\", versions.len());\nprintln!(\"Current version: {}\", current_version);\n"; - -export const RsVersioningCheckVersionsAfterMod = "let versions_after_mod = table.list_versions().await.unwrap();\nlet version_count_after_mod = versions_after_mod.len();\nlet version_after_mod = table.version().await.unwrap();\nprintln!(\n \"Number of versions after modifications: {}\",\n version_count_after_mod\n);\nprintln!(\"Current version: {}\", version_after_mod);\n"; - -export const RsVersioningCheckoutLatest = "table.checkout_latest().await.unwrap();\n"; - -export const RsVersioningDeleteData = "table.delete(\"author = 'Morty'\").await.unwrap();\nlet rows_after_deletion = table.count_rows(None).await.unwrap();\nprintln!(\"Number of rows after deletion: {}\", rows_after_deletion);\n"; - -export const RsVersioningListAllVersions = "let all_versions = table.list_versions().await.unwrap();\nfor v in &all_versions {\n println!(\"Version {}, created at {}\", v.version, v.timestamp);\n}\n"; - -export const RsVersioningMakeQuotesReader = "fn make_quotes_reader(rows: Vec<(i64, &str, &str)>) -> Box {\n let ids: Vec = rows.iter().map(|(id, _, _)| *id).collect();\n let authors: Vec<&str> = rows.iter().map(|(_, author, _)| *author).collect();\n let quotes: Vec<&str> = rows.iter().map(|(_, _, quote)| *quote).collect();\n\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"author\", DataType::Utf8, false),\n Field::new(\"quote\", DataType::Utf8, false),\n ]));\n\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(ids)),\n Arc::new(StringArray::from(authors)),\n Arc::new(StringArray::from(quotes)),\n ],\n )\n .unwrap();\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n Box::new(reader)\n}\n"; - -export const RsVersioningRollback = "table.checkout(version_after_mod).await.unwrap();\ntable.restore().await.unwrap();\nlet versions_after_rollback = table.list_versions().await.unwrap();\nlet version_count_after_rollback = versions_after_rollback.len();\nprintln!(\n \"Total number of versions after rollback: {}\",\n version_count_after_rollback\n);\n"; - -export const RsVersioningTags = "let mut tags = tags_table.tags().await.unwrap();\n\n// Create a tag pointing at a specific version\ntags.create(\"baseline\", 1).await.unwrap();\nlet current_version = tags_table.version().await.unwrap();\ntags.create(\"with-edits\", current_version).await.unwrap();\n\n// List all tags on this table\nlet all_tags = tags.list().await.unwrap();\nprintln!(\"Tags: {:?}\", all_tags);\n\n// Look up the version a tag points at\nlet baseline_version = tags.get_version(\"baseline\").await.unwrap();\nprintln!(\"baseline -> v{}\", baseline_version);\n\n// Move an existing tag to a different version\ntags.update(\"baseline\", 2).await.unwrap();\n\n// Check out a version by tag name (separate method in Rust)\ntags_table.checkout_tag(\"baseline\").await.unwrap();\nprintln!(\"Current version: {}\", tags_table.version().await.unwrap());\n\n// Delete a tag (does not delete the underlying version)\ntags.delete(\"with-edits\").await.unwrap();\n\n// Return to the latest version\ntags_table.checkout_latest().await.unwrap();\n"; - -export const RsVersioningUpdateData = "table\n .update()\n .only_if(\"author = 'Richard'\")\n .column(\"author\", \"'Richard Daniel Sanchez'\")\n .execute()\n .await\n .unwrap();\nlet rows_after_update = table\n .count_rows(Some(\"author = 'Richard Daniel Sanchez'\".to_string()))\n .await\n .unwrap();\nprintln!(\n \"Rows updated to Richard Daniel Sanchez: {}\",\n rows_after_update\n);\n"; - diff --git a/docs/static/assets/images/demos/sda_hero.jpg b/docs/static/assets/images/demos/sda_hero.jpg deleted file mode 100644 index afe4322..0000000 Binary files a/docs/static/assets/images/demos/sda_hero.jpg and /dev/null differ diff --git a/docs/static/assets/images/demos/video_demo_hero.png b/docs/static/assets/images/demos/video_demo_hero.png deleted file mode 100644 index 89722df..0000000 Binary files a/docs/static/assets/images/demos/video_demo_hero.png and /dev/null differ diff --git a/docs/static/assets/images/demos/wiki_hero.png b/docs/static/assets/images/demos/wiki_hero.png deleted file mode 100644 index a78015b..0000000 Binary files a/docs/static/assets/images/demos/wiki_hero.png and /dev/null differ diff --git a/docs/static/assets/images/enterprise/architecture.png b/docs/static/assets/images/enterprise/architecture.png deleted file mode 100644 index aa00854..0000000 Binary files a/docs/static/assets/images/enterprise/architecture.png and /dev/null differ diff --git a/docs/static/assets/images/faq/recall-vs-latency.webp b/docs/static/assets/images/faq/recall-vs-latency.webp deleted file mode 100644 index 5aafcf4..0000000 Binary files a/docs/static/assets/images/faq/recall-vs-latency.webp and /dev/null differ diff --git a/docs/static/assets/images/geneva/console_screenshot.png b/docs/static/assets/images/geneva/console_screenshot.png deleted file mode 100644 index 16d595d..0000000 Binary files a/docs/static/assets/images/geneva/console_screenshot.png and /dev/null differ diff --git a/docs/static/assets/images/geneva/eks-auth.png b/docs/static/assets/images/geneva/eks-auth.png deleted file mode 100644 index 998b6a8..0000000 Binary files a/docs/static/assets/images/geneva/eks-auth.png and /dev/null differ diff --git a/docs/static/assets/images/geneva/geneva-security-reqs.png b/docs/static/assets/images/geneva/geneva-security-reqs.png deleted file mode 100644 index 579d779..0000000 Binary files a/docs/static/assets/images/geneva/geneva-security-reqs.png and /dev/null differ diff --git a/docs/static/assets/images/geneva/preview-image.png b/docs/static/assets/images/geneva/preview-image.png deleted file mode 100644 index 94f0d20..0000000 Binary files a/docs/static/assets/images/geneva/preview-image.png and /dev/null differ diff --git a/docs/static/assets/images/get-started/main-cloud-cta.png b/docs/static/assets/images/get-started/main-cloud-cta.png deleted file mode 100644 index 98bdd66..0000000 Binary files a/docs/static/assets/images/get-started/main-cloud-cta.png and /dev/null differ diff --git a/docs/static/assets/images/indexing/ivfpq_ivf_desc.webp b/docs/static/assets/images/indexing/ivfpq_ivf_desc.webp deleted file mode 100644 index 82e60d7..0000000 Binary files a/docs/static/assets/images/indexing/ivfpq_ivf_desc.webp and /dev/null differ diff --git a/docs/static/assets/images/indexing/ivfpq_pq_desc.png b/docs/static/assets/images/indexing/ivfpq_pq_desc.png deleted file mode 100644 index 8cc4913..0000000 Binary files a/docs/static/assets/images/indexing/ivfpq_pq_desc.png and /dev/null differ diff --git a/docs/static/assets/images/indexing/ivfpq_query_vector.webp b/docs/static/assets/images/indexing/ivfpq_query_vector.webp deleted file mode 100644 index 72f07f8..0000000 Binary files a/docs/static/assets/images/indexing/ivfpq_query_vector.webp and /dev/null differ diff --git a/docs/static/assets/images/integrations/voxel.gif b/docs/static/assets/images/integrations/voxel.gif deleted file mode 100644 index b74d112..0000000 Binary files a/docs/static/assets/images/integrations/voxel.gif and /dev/null differ diff --git a/docs/static/assets/images/namespaces/lance-namespace.png b/docs/static/assets/images/namespaces/lance-namespace.png deleted file mode 100644 index 01573ab..0000000 Binary files a/docs/static/assets/images/namespaces/lance-namespace.png and /dev/null differ diff --git a/docs/static/assets/images/overview/lancedb-suite.png b/docs/static/assets/images/overview/lancedb-suite.png deleted file mode 100644 index a7455f4..0000000 Binary files a/docs/static/assets/images/overview/lancedb-suite.png and /dev/null differ diff --git a/docs/static/assets/images/overview/lancedb-suite.svg b/docs/static/assets/images/overview/lancedb-suite.svg deleted file mode 100644 index 92a9335..0000000 --- a/docs/static/assets/images/overview/lancedb-suite.svg +++ /dev/null @@ -1,95 +0,0 @@ - - LanceDB suite - LanceDB OSS supports search. LanceDB Enterprise is a multimodal lakehouse for curation, feature engineering, search and retrieval, and training. Both are built on the Lance open lakehouse format for multimodal AI. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - LanceDB OSS - Search - - - - - - - - - - - - - - - LanceDB Enterprise - Multimodal Lakehouse - - Curation - Feature - Engineering - Search & - Retrieval - Training - - - - - - - - - - Lance Open Lakehouse Format for Multimodal AI - - diff --git a/docs/static/assets/images/overview/multimodal.png b/docs/static/assets/images/overview/multimodal.png deleted file mode 100644 index 4c40bb9..0000000 Binary files a/docs/static/assets/images/overview/multimodal.png and /dev/null differ diff --git a/docs/static/assets/images/overview/training-data-lifecycle.svg b/docs/static/assets/images/overview/training-data-lifecycle.svg deleted file mode 100644 index a533c70..0000000 --- a/docs/static/assets/images/overview/training-data-lifecycle.svg +++ /dev/null @@ -1,52 +0,0 @@ - - Training data lifecycle - Four pillars of the LanceDB training data lifecycle: Curation, Feature Engineering, Search and Retrieval, and Training. - - - - - - - - - - - - - - - - - - - - - - - - - - - - Curation - - - - - - Feature - Engineering - - - - - - Search & - Retrieval - - - - - - Training - diff --git a/docs/static/assets/images/overview/understanding-tables.png b/docs/static/assets/images/overview/understanding-tables.png deleted file mode 100644 index f61d904..0000000 Binary files a/docs/static/assets/images/overview/understanding-tables.png and /dev/null differ diff --git a/docs/static/assets/images/quickstart/sir-lancelot.jpg b/docs/static/assets/images/quickstart/sir-lancelot.jpg deleted file mode 100644 index e987d60..0000000 Binary files a/docs/static/assets/images/quickstart/sir-lancelot.jpg and /dev/null differ diff --git a/docs/static/assets/images/search/knn_search.png b/docs/static/assets/images/search/knn_search.png deleted file mode 100644 index 05803f2..0000000 Binary files a/docs/static/assets/images/search/knn_search.png and /dev/null differ diff --git a/docs/static/assets/images/search/multivector/multivector-1.png b/docs/static/assets/images/search/multivector/multivector-1.png deleted file mode 100644 index f34f11c..0000000 Binary files a/docs/static/assets/images/search/multivector/multivector-1.png and /dev/null differ diff --git a/docs/static/assets/images/search/multivector/multivector-2.png b/docs/static/assets/images/search/multivector/multivector-2.png deleted file mode 100644 index 1a99c7b..0000000 Binary files a/docs/static/assets/images/search/multivector/multivector-2.png and /dev/null differ diff --git a/docs/static/assets/images/search/multivector/multivector-3.png b/docs/static/assets/images/search/multivector/multivector-3.png deleted file mode 100644 index 2f2f23b..0000000 Binary files a/docs/static/assets/images/search/multivector/multivector-3.png and /dev/null differ diff --git a/docs/static/assets/images/search/multivector/multivector-4.png b/docs/static/assets/images/search/multivector/multivector-4.png deleted file mode 100644 index 3333610..0000000 Binary files a/docs/static/assets/images/search/multivector/multivector-4.png and /dev/null differ diff --git a/docs/static/assets/images/search/multivector/multivector-5.png b/docs/static/assets/images/search/multivector/multivector-5.png deleted file mode 100644 index 4aa7715..0000000 Binary files a/docs/static/assets/images/search/multivector/multivector-5.png and /dev/null differ diff --git a/docs/static/assets/images/search/multivector/multivector-6.png b/docs/static/assets/images/search/multivector/multivector-6.png deleted file mode 100644 index b33a3cf..0000000 Binary files a/docs/static/assets/images/search/multivector/multivector-6.png and /dev/null differ diff --git a/docs/static/assets/images/search/vector-db-basics.png b/docs/static/assets/images/search/vector-db-basics.png deleted file mode 100644 index 7174f36..0000000 Binary files a/docs/static/assets/images/search/vector-db-basics.png and /dev/null differ diff --git a/docs/static/assets/images/storage/aws.jpg b/docs/static/assets/images/storage/aws.jpg deleted file mode 100644 index c727cb4..0000000 Binary files a/docs/static/assets/images/storage/aws.jpg and /dev/null differ diff --git a/docs/static/assets/images/storage/azure.jpg b/docs/static/assets/images/storage/azure.jpg deleted file mode 100644 index 07247f9..0000000 Binary files a/docs/static/assets/images/storage/azure.jpg and /dev/null differ diff --git a/docs/static/assets/images/storage/gcp.jpg b/docs/static/assets/images/storage/gcp.jpg deleted file mode 100644 index c5a8db7..0000000 Binary files a/docs/static/assets/images/storage/gcp.jpg and /dev/null differ diff --git a/docs/static/assets/images/storage/lancedb_storage_tradeoffs.png b/docs/static/assets/images/storage/lancedb_storage_tradeoffs.png deleted file mode 100644 index 0fccf43..0000000 Binary files a/docs/static/assets/images/storage/lancedb_storage_tradeoffs.png and /dev/null differ diff --git a/docs/static/assets/images/storage/tigris.jpg b/docs/static/assets/images/storage/tigris.jpg deleted file mode 100644 index 7dc6309..0000000 Binary files a/docs/static/assets/images/storage/tigris.jpg and /dev/null differ diff --git a/docs/static/assets/images/training/distant_person_00.jpg b/docs/static/assets/images/training/distant_person_00.jpg deleted file mode 100644 index be50634..0000000 Binary files a/docs/static/assets/images/training/distant_person_00.jpg and /dev/null differ diff --git a/docs/static/assets/images/training/nighttime_person_01.jpg b/docs/static/assets/images/training/nighttime_person_01.jpg deleted file mode 100644 index d5f6b81..0000000 Binary files a/docs/static/assets/images/training/nighttime_person_01.jpg and /dev/null differ diff --git a/docs/static/assets/images/training/rider_04.jpg b/docs/static/assets/images/training/rider_04.jpg deleted file mode 100644 index 93f8eac..0000000 Binary files a/docs/static/assets/images/training/rider_04.jpg and /dev/null differ diff --git a/docs/static/assets/images/training/vlm-finetuning/textvqa-domino-sugar.jpg b/docs/static/assets/images/training/vlm-finetuning/textvqa-domino-sugar.jpg deleted file mode 100644 index 3711c75..0000000 Binary files a/docs/static/assets/images/training/vlm-finetuning/textvqa-domino-sugar.jpg and /dev/null differ diff --git a/docs/static/assets/images/training/vlm-finetuning/textvqa-lego-box.jpg b/docs/static/assets/images/training/vlm-finetuning/textvqa-lego-box.jpg deleted file mode 100644 index 6e81b09..0000000 Binary files a/docs/static/assets/images/training/vlm-finetuning/textvqa-lego-box.jpg and /dev/null differ diff --git a/docs/static/assets/images/training/vlm-finetuning/textvqa-phone-time.jpg b/docs/static/assets/images/training/vlm-finetuning/textvqa-phone-time.jpg deleted file mode 100644 index 1d4caf7..0000000 Binary files a/docs/static/assets/images/training/vlm-finetuning/textvqa-phone-time.jpg and /dev/null differ diff --git a/docs/static/assets/images/training/vlm-finetuning/textvqa-warning-sign.jpg b/docs/static/assets/images/training/vlm-finetuning/textvqa-warning-sign.jpg deleted file mode 100644 index e6a8eb3..0000000 Binary files a/docs/static/assets/images/training/vlm-finetuning/textvqa-warning-sign.jpg and /dev/null differ diff --git a/docs/static/assets/logo/dark-lancedb-logo.svg b/docs/static/assets/logo/dark-lancedb-logo.svg deleted file mode 100644 index 0227590..0000000 --- a/docs/static/assets/logo/dark-lancedb-logo.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/docs/static/assets/logo/huggingface-logo.svg b/docs/static/assets/logo/huggingface-logo.svg deleted file mode 100644 index eceb4e3..0000000 --- a/docs/static/assets/logo/huggingface-logo.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/docs/static/assets/logo/lance-logo-gray.svg b/docs/static/assets/logo/lance-logo-gray.svg deleted file mode 100644 index 764b68d..0000000 --- a/docs/static/assets/logo/lance-logo-gray.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - diff --git a/docs/static/assets/logo/lancedb-icon-gray.svg b/docs/static/assets/logo/lancedb-icon-gray.svg deleted file mode 100644 index e74f013..0000000 --- a/docs/static/assets/logo/lancedb-icon-gray.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/docs/static/assets/logo/light-lancedb-logo.svg b/docs/static/assets/logo/light-lancedb-logo.svg deleted file mode 100644 index b074674..0000000 --- a/docs/static/assets/logo/light-lancedb-logo.svg +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json deleted file mode 100644 index 5851d2b..0000000 --- a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json +++ /dev/null @@ -1,106 +0,0 @@ -[ - { - "id": 1, - "name": "King Arthur", - "role": "King of Camelot", - "description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.", - "stats": { - "strength": 2, - "courage": 5, - "magic": 1, - "wisdom": 4 - }, - "img": "data/img/arthur.jpg" - }, - { - "id": 2, - "name": "Merlin", - "role": "Wizard and Advisor", - "description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.", - "stats": { - "strength": 2, - "courage": 4, - "magic": 5, - "wisdom": 5 - }, - "img": "data/img/merlin.jpg" - }, - { - "id": 3, - "name": "Queen Guinevere", - "role": "Queen of Camelot", - "description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.", - "stats": { - "strength": 1, - "courage": 3, - "magic": 1, - "wisdom": 4 - }, - "img": "data/img/guinevere.jpg" - }, - { - "id": 4, - "name": "Sir Lancelot", - "role": "Knight of the Round Table", - "description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.", - "stats": { - "strength": 5, - "courage": 5, - "magic": 1, - "wisdom": 3 - }, - "img": "data/img/sir_lancelot.jpg" - }, - { - "id": 5, - "name": "Sir Gawain", - "role": "Knight of the Round Table", - "description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.", - "stats": { - "strength": 4, - "courage": 5, - "magic": 1, - "wisdom": 4 - }, - "img": "data/img/sir_gawain.jpg" - }, - { - "id": 6, - "name": "Sir Galahad", - "role": "Knight of the Round Table", - "description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.", - "stats": { - "strength": 4, - "courage": 5, - "magic": 2, - "wisdom": 5 - }, - "img": "data/img/sir_galahad.jpg" - }, - { - "id": 7, - "name": "Sir Percival", - "role": "Knight of the Round Table", - "description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.", - "stats": { - "strength": 4, - "courage": 4, - "magic": 1, - "wisdom": 3 - }, - "img": "data/img/sir_percival.jpg" - }, - { - "id": 8, - "name": "Mordred", - "role": "Traitor Knight", - "description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.", - "stats": { - "strength": 4, - "courage": 2, - "magic": 1, - "wisdom": 2 - }, - "img": "data/img/mordred.jpg" - } -] diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/arthur.jpg b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/arthur.jpg deleted file mode 100644 index 6c60e70..0000000 Binary files a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/arthur.jpg and /dev/null differ diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/guinevere.jpg b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/guinevere.jpg deleted file mode 100644 index ce76f0a..0000000 Binary files a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/guinevere.jpg and /dev/null differ diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/merlin.jpg b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/merlin.jpg deleted file mode 100644 index df0d5fc..0000000 Binary files a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/merlin.jpg and /dev/null differ diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/mordred.jpg b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/mordred.jpg deleted file mode 100644 index 6728462..0000000 Binary files a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/mordred.jpg and /dev/null differ diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_galahad.jpg b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_galahad.jpg deleted file mode 100644 index de9e0c0..0000000 Binary files a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_galahad.jpg and /dev/null differ diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_gawain.jpg b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_gawain.jpg deleted file mode 100644 index 949360e..0000000 Binary files a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_gawain.jpg and /dev/null differ diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_lancelot.jpg b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_lancelot.jpg deleted file mode 100644 index 186ad23..0000000 Binary files a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_lancelot.jpg and /dev/null differ diff --git a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_percival.jpg b/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_percival.jpg deleted file mode 100644 index c784fc9..0000000 Binary files a/docs/static/assets/tutorials/build-with-ai-agents/camelot/data/img/sir_percival.jpg and /dev/null differ diff --git a/docs/static/favicon.ico b/docs/static/favicon.ico deleted file mode 100644 index 911581b..0000000 Binary files a/docs/static/favicon.ico and /dev/null differ diff --git a/docs/static/styles/style.css b/docs/static/styles/style.css deleted file mode 100644 index db71d8b..0000000 --- a/docs/static/styles/style.css +++ /dev/null @@ -1,25 +0,0 @@ -:root { - --banner-left: #e4d8f8; - --banner-right: #e55a2b; -} - -/* Mintlify banner gradient */ -#banner, -:where(.banner, [data-banner], [class*="Banner_banner"]) { - background: linear-gradient(90deg, var(--banner-left) 0%, var(--banner-right) 100%) !important; - color: #1b0f09 !important; - border: none !important; -} - -#banner a, -:where(.banner, [data-banner], [class*="Banner_banner"]) a { - color: #1b0f09 !important; - font-size: 0.9rem; - font-weight: 500; - text-decoration-color: rgba(27, 15, 9, 0.35); -} - -#banner button, -:where(.banner, [data-banner], [class*="Banner_banner"]) button { - color: #1b0f09 !important; -} diff --git a/docs/storage/configuration.mdx b/docs/storage/configuration.mdx deleted file mode 100644 index 45d8bec..0000000 --- a/docs/storage/configuration.mdx +++ /dev/null @@ -1,341 +0,0 @@ ---- -title: "Configuring Cloud Storage in LanceDB" -sidebarTitle: "Configuring storage" -description: "Configure LanceDB to use S3, GCS, Azure Blob, and S3-compatible object stores with environment variables or storage options." -icon: "wrench" ---- -import { - PyStorageAzureAccount, - PyStorageAzureSas, - PyStorageConnectAzure, - PyStorageConnectGcs, - PyStorageConnectS3, - PyStorageConnectTimeout, - PyStorageGcsServiceAccount, - PyStorageS3Express, - PyStorageS3Minio, - PyStorageS3SseKms, - PyStorageTableTimeout, - PyStorageTigrisConnect, - PyStorageCosConnect, - PyStorageGoosefsConnect, - TsStorageAzureAccount, - TsStorageAzureSas, - TsStorageConnectAzure, - TsStorageConnectGcs, - TsStorageConnectS3, - TsStorageConnectTimeout, - TsStorageGcsServiceAccount, - TsStorageS3Express, - TsStorageS3Minio, - TsStorageS3SseKms, - TsStorageTableTimeout, - TsStorageTigrisConnect, - TsStorageGoosefsConnect, -} from '/snippets/storage.mdx'; - -When using LanceDB OSS, you can choose where to store your data. The tradeoffs between storage options are covered in the [storage architecture guide](/storage). This page shows how to configure each backend. - - -**LanceDB Enterprise storage configuration** - -In LanceDB Enterprise, you connect with `db://...` and the cluster owns the storage credentials, so `storage_options` are not passed at runtime. Cloud auth is set at deployment time. For federated databases, the namespace service vends per-request credentials automatically. See the [quickstart](/quickstart), [Enterprise overview](/enterprise/), and [Azure deployment guide](/enterprise/deployment/azure) for the Enterprise flow. - - -## Object stores {#object-stores} - -LanceDB supports AWS S3 (and compatible stores), Azure Blob Storage, and Google Cloud Storage. The URI scheme in your `connect` call selects the backend. - - - - {PyStorageConnectS3} - - - {TsStorageConnectS3} - - - - - - {PyStorageConnectGcs} - - - {TsStorageConnectGcs} - - - - - - {PyStorageConnectAzure} - - - {TsStorageConnectAzure} - - - -### Configuration options {#configuration-options} - -When running inside the target cloud with correct IAM bindings, LanceDB often needs no extra configuration. When running elsewhere, provide credentials via environment variables or `storage_options`. - - - - {PyStorageConnectTimeout} - - - {TsStorageConnectTimeout} - - - - -**Storage option casing** - -Keys are case-insensitive. Use lowercase in `storage_options` and uppercase in environment variables. - - -Table-level `storage_options` inherit every key from the connection and override on a per-key basis. Pass them to `create_table` or `open_table` for options that should apply to a single table: - - - - {PyStorageTableTimeout} - - - {TsStorageTableTimeout} - - - - -**Inspect the effective options** - -On `AsyncTable`, `await table.initial_storage_options()` returns the options the table was opened with, and `await table.latest_storage_options()` returns the current options after any provider-driven refresh. The deprecated `table.storage_options()` method will be removed in a future release. - - -#### General object store options {#general-object-store-options} - -| Key | Description | -| :-- | :-- | -| `allow_http` | Allow non-TLS connections. | -| `allow_invalid_certificates` | Skip certificate validation for TLS connections. | -| `connect_timeout` | Timeout for the connect phase. | -| `timeout` | Timeout for the full request. | -| `user_agent` | User agent string sent with requests. | -| `proxy_url` | Proxy URL to route requests through. | -| `proxy_ca_certificate` | PEM-formatted CA certificate for proxy connections. | -| `proxy_excludes` | Comma-separated hosts that bypass the proxy (domains or CIDR). | -| `download_retry_count` | Number of retries when downloading objects. | -| `client_max_retries` | Maximum retries for object-store client requests. | -| `client_retry_timeout` | Total retry timeout (seconds) for object-store client requests. | - - -**Option support varies by backend** - -These are commonly used options. Cloud-specific keys (for example `region`, `endpoint`, `service_account`, and Azure credential keys) are backend-dependent and can be provided in `storage_options` as needed. - - -#### New table configuration {#new-table-configuration} - -These options control the Lance file format and features used when creating new tables. Pass them via `storage_options` at connection or table level. They are evaluated only at table creation; setting them on an existing connection does not rewrite or alter tables that already exist. - -| Key | Values | Default | Description | -| :-- | :-- | :-- | :-- | -| `new_table_data_storage_version` | `legacy`, `stable` | `stable` | Lance file format version for new tables. Use `legacy` for backward compatibility with older clients, or `stable` for the current format with better performance. | -| `new_table_enable_v2_manifest_paths` | `true`, `false` | `false` | Use v2 manifest path naming. Requires LanceDB >= 0.10.0 to read. | -| `new_table_enable_stable_row_ids` | `true`, `false` | `false` | Keep row IDs stable across compaction, delete, and merge operations. | - -```mermaid -flowchart TD - A[Creating a new table] --> B{Need backward compatibility\nwith older LanceDB clients?} - B -->|Yes| C[new_table_data_storage_version: legacy] - B -->|No| D[new_table_data_storage_version: stable\nDefault — recommended] - D --> E{Need stable row IDs\nacross compaction and deletes?} - E -->|Yes| F[new_table_enable_stable_row_ids: true] - E -->|No| G[Default: false] - D --> H{All clients on LanceDB >= 0.10.0?} - H -->|Yes| I[new_table_enable_v2_manifest_paths: true] - H -->|No| J[Default: false] -``` - - -```python Python icon="python" -import lancedb - -# Set the Lance file format version at connection level -db = lancedb.connect( - "s3://bucket/path", - storage_options={ - "new_table_data_storage_version": "stable", - }, -) -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; - -// Set the Lance file format version at connection level -const db = await lancedb.connect("s3://bucket/path", { - storageOptions: { - newTableDataStorageVersion: "stable", - }, -}); -``` - - - -**Deprecated parameter** - -The `data_storage_version` parameter on `create_table()` is deprecated. Use `new_table_data_storage_version` in `storage_options` instead. - - -## AWS S3 {#aws-s3} - -![](/static/assets/images/storage/aws.jpg) - -Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optionally `AWS_SESSION_TOKEN` as environment variables or pass them in `storage_options`. Region is optional for AWS but required for most S3-compatible stores. - -Minimum permissions usually include `s3:PutObject`, `s3:GetObject`, `s3:DeleteObject`, `s3:ListBucket`, and `s3:GetBucketLocation` scoped to the relevant bucket/prefix. - -### S3-compatible stores {#s3-compatible-stores} - - - - {PyStorageS3Minio} - - - {TsStorageS3Minio} - - - -If the endpoint is `http://` (common in local development), also set `ALLOW_HTTP=true` or pass `allow_http=True` in `storage_options`. - -### S3 Express {#s3-express} - - - - {PyStorageS3Express} - - - {TsStorageS3Express} - - - -Consult AWS networking requirements for S3 Express before enabling. - - -**Clean up failed multipart uploads** - -LanceDB aborts multipart uploads on graceful shutdown, but crashes can leave incomplete uploads. Add an S3 lifecycle rule to delete in-progress uploads after a few days. - - -### Server-side encryption with KMS {#server-side-encryption-with-kms} - -To encrypt at rest with an AWS KMS key, set `aws_server_side_encryption` to `aws:kms` and `aws_sse_kms_key_id` to the key ID or ARN. The same options apply at connection or table level and combine with bucket-level default encryption. - - - - {PyStorageS3SseKms} - - - {TsStorageS3SseKms} - - - -The IAM principal needs `kms:Encrypt`, `kms:Decrypt`, and `kms:GenerateDataKey` on the configured KMS key. - -## Google Cloud Storage {#google-cloud-storage} - -![](/static/assets/images/storage/gcp.jpg) - -Provide credentials via `GOOGLE_SERVICE_ACCOUNT` (path to JSON) or include the path in `storage_options`. GCS defaults to HTTP/1; set `HTTP1_ONLY=false` if you need HTTP/2. - - - - {PyStorageGcsServiceAccount} - - - {TsStorageGcsServiceAccount} - - - -## Azure Blob Storage {#azure-blob-storage} - -![](/static/assets/images/storage/azure.jpg) - -Set `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_ACCOUNT_KEY` as environment variables, or pass them via `storage_options`. - - - - {PyStorageAzureAccount} - - - {TsStorageAzureAccount} - - - -For SAS-token auth, set `azure_storage_account_name` and `azure_storage_sas_token`: - - - - {PyStorageAzureSas} - - - {TsStorageAzureSas} - - - -Other supported keys include service principal credentials (`azure_client_id`, `azure_client_secret`, `azure_tenant_id`), managed identities, and custom endpoints. - -## Tigris Object Storage {#tigris-object-storage} - -![](/static/assets/images/storage/tigris.jpg) - -Tigris exposes an S3-compatible API. Configure the endpoint and region: - - - - {PyStorageTigrisConnect} - - - {TsStorageTigrisConnect} - - - -Environment variables `AWS_ENDPOINT=https://t3.storage.dev` and `AWS_DEFAULT_REGION=auto` achieve the same configuration. - -## Tencent COS {#tencent-cos} - -[Tencent Cloud Object Storage (COS)](https://www.tencentcloud.com/products/cos) is the primary object store for workloads running in the China region. Use the `cos://` URI scheme to connect directly to a COS bucket. - - - - {PyStorageCosConnect} - - - -Supported keys include `secret_id`, `secret_key`, `region`, and `endpoint`. You can also authenticate via the `TENCENTCLOUD_SECRET_ID` and `TENCENTCLOUD_SECRET_KEY` environment variables. - - -**Availability** - -COS is bundled in the Python wheel by default. To use it from Rust or the Node binding, build with the `cos` Cargo feature enabled. - - -## GooseFS {#goosefs} - -[GooseFS](https://www.tencentcloud.com/document/product/1424) is Tencent Cloud's distributed cache acceleration layer for COS and S3. It is a common choice when the same hot dataset is read repeatedly, such as vector search and AI training workloads. Connect using the `goosefs://` URI scheme. - - - - {PyStorageGoosefsConnect} - - - {TsStorageGoosefsConnect} - - - -GooseFS reads credentials and endpoint configuration from the GooseFS client environment. See the [GooseFS documentation](https://www.tencentcloud.com/document/product/1424) for cluster setup. - - -**Availability** - -GooseFS is bundled by default in the Python wheel and the Node binding. To use it from Rust, build with the `goosefs` Cargo feature enabled. - - diff --git a/docs/storage/index.mdx b/docs/storage/index.mdx deleted file mode 100644 index 1d57dce..0000000 --- a/docs/storage/index.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: "Storage Architecture in LanceDB" -sidebarTitle: "Storage options" -description: "Understand LanceDB storage backends, tradeoffs, and how to pick the right option for your latency, scale, and cost goals." -icon: "database" ---- - -LanceDB's storage layer is built on modular, disk-first components. That design makes it flexible enough to run across local NVMe, EBS, EFS, and any object store that exposes an S3-compatible API. It also supports region-specific backends such as [Tencent COS](/storage/configuration#tencent-cos) and cache-acceleration layers such as [GooseFS](/storage/configuration#goosefs). - -Choosing a backend is a balance between latency, scalability, cost, and operational complexity. Use this guide to pick the right fit for your workload. - -## Storage backend selection guide {#storage-backend-selection-guide} - -![](/static/assets/images/storage/lancedb_storage_tradeoffs.png) - -When architecting your system, ask yourself: - -- **Latency**: How fast do I need results? What do the p50 and p95 look like? -- **Scalability**: Can I scale data volume and QPS easily? -- **Cost**: What is the all-in cost of storage plus serving? -- **Reliability/Availability**: How will replication and disaster recovery work? - -## Storage backend comparison {#storage-backend-comparison} - -Below is a high-level comparison ordered from lowest cost to lowest latency. - -### 1\. Object storage (S3, GCS, Azure Blob) {#1-object-storage-s3-gcs-azure-blob} -- **Latency**: Highest; expect hundreds of milliseconds and higher p95. -- **Scalability**: Effectively unlimited storage; QPS bound by concurrency limits. -- **Cost**: Lowest overall. -- **Reliability/Availability**: Highly available, backed by cloud SLAs. - -LanceDB separates storage and compute and writes immutable fragments, making it a strong fit for stateless, horizontally scalable deployments. - - -**Concurrent writers on S3** - -S3 and S3 Express now support atomic writes natively, so LanceDB handles concurrent writers against the same table out-of-the-box — no external commit coordinator is required. Bucket-level [server-side encryption with KMS](/storage/configuration#server-side-encryption-with-kms) and [S3 Express One Zone](/storage/configuration#s3-express) are also supported on this tier. - - -### 2\. File storage (EFS, GCS Filestore, Azure File) {#2-file-storage-efs-gcs-filestore-azure-file} -- **Latency**: Better than object storage; p95 under ~<100ms is typical. -- **Scalability**: High, but limited by provisioned IOPS per volume. -- **Cost**: More than object storage but cheaper than in-memory options; cold data can tier down automatically. -- **Reliability/Availability**: Highly available; replication/backup must be managed separately. - -Keep a copy of data in object storage for disaster recovery. If zero downtime is required, provision a second network file system with replicated data. - -### 3\. Third-party storage (e.g., MinIO, WekaFS) {#3-third-party-storage-e-g--minio-wekafs} - -- **Latency**: Similar to EFS; typically under <100ms. -- **Scalability**: Determined by the chosen vendor’s cluster sizing. -- **Cost**: Higher than S3; may edge above EFS at larger scales. -- **Reliability/Availability**: Shareable across many nodes; replication depends on vendor capabilities. - -### 4\. Block storage (EBS, GCP Persistent Disk, Azure Managed Disk) {#4-block-storage-ebs-gcp-persistent-disk-azure-managed-disk} -- **Latency**: Near-local performance; often <30ms. -- **Scalability**: Not shareable across instances; shard or copy data when scaling. -- **Cost**: Higher than networked file systems, plus potential I/O charges. -- **Reliability/Availability**: Persists through instance restarts; backups and sharding must be managed. - -### 5\. Local storage (SSD, NVMe) {#5-local-storage-ssd-nvme} -- **Latency**: Fastest; p95 often under <10ms. -- **Scalability**: Hard to scale in cloud environments; requires sharding or additional copies for higher QPS. -- **Cost**: Highest; tightly coupling compute and storage makes horizontal scaling difficult. -- **Reliability/Availability**: Data is tied to the instance; backups must be rigorous. - -Use local disk only when you need extremely low latency and are comfortable owning the operational overhead. - -## File-format choices that interact with the backend {#file-format-choices-that-interact-with-the-backend} - -A few `storage_options` keys shape new tables in ways that depend on the backend you picked above. They are documented in full on the [configuration page](/storage/configuration#new-table-configuration); the architecture-level summary is: - -- `new_table_enable_v2_manifest_paths` matters most on object stores, where opening a table with many versions is dominated by listing cost. Leave it off for backward compatibility with clients older than LanceDB 0.10.0. -- `new_table_enable_stable_row_ids` keeps row IDs stable across compaction, delete, and merge. The choice is independent of the backend but affects any system that joins on row ID. -- `new_table_data_storage_version` selects the on-disk format. The default `stable` is recommended for all new tables; pick `legacy` only when older readers must keep working. - diff --git a/docs/storage/monitoring.mdx b/docs/storage/monitoring.mdx deleted file mode 100644 index e62828c..0000000 --- a/docs/storage/monitoring.mdx +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: "Monitor LanceDB with OpenTelemetry" -sidebarTitle: "Monitoring" -description: "Export LanceDB object store request counts, bytes, latency, errors, and throttles to any OpenTelemetry backend." -icon: "chart-line" -keywords: ["monitoring", "observability", "opentelemetry", "otel", "metrics", "prometheus", "object store"] ---- - -LanceDB emits internal metrics (currently object store request counts, bytes transferred, request latency, retryable errors, and throttles) and can bridge them into any [OpenTelemetry](https://opentelemetry.io/) backend. Use this to watch how your application interacts with S3, GCS, Azure Blob, or the local filesystem in production: spot latency regressions, catch retry storms, and size your storage tier from real workload data. - -The bridge is available in the Python and TypeScript SDKs. It is a thin wrapper over LanceDB's `metrics` recorder; your application supplies and configures the OpenTelemetry SDK. - - -This page covers LanceDB OSS. LanceDB Enterprise clusters emit their own Prometheus/OpenTelemetry metrics from the server side — see the [Enterprise overview](/enterprise/) for that flow. - - -## What you get {#what-you-get} - -Once instrumented, LanceDB registers one observable instrument per metric on your `MeterProvider`. The current catalog covers the object store layer: - -| Metric | Kind | Description | -|--------|------|-------------| -| `lance_object_store_requests_total` | Counter | Total object store requests, labelled by `operation` and `base` (store scheme). | -| `lance_object_store_request_duration_seconds` | Histogram | Request latency in seconds. | -| `lance_object_store_bytes_transferred_total` | Counter | Bytes read from or written to the store. | -| `lance_object_store_retryable_responses_total` | Counter | Requests that returned a retryable error (throttles, transient failures). | -| `lance_object_store_in_flight_requests` | Gauge | Currently outstanding object store requests. | - -The recorder is process-global and pull-based: your configured `MetricReader` collects on its own schedule, so there is no hot-path overhead beyond the atomic aggregation that LanceDB does anyway. - - -**Histograms are exported Prometheus-style.** OpenTelemetry has no asynchronous histogram instrument, so each histogram surfaces as three observable counters: `_bucket` (with an `le` attribute per bucket boundary, including `+Inf`), `_count`, and `_sum`. Only `_sum` carries the histogram's unit; `_bucket` and `_count` are cumulative sample counts. - - -## Python {#python} - -Install LanceDB with the `otel` extra to pull in the OpenTelemetry API, plus an OpenTelemetry SDK of your choice. The SDK is intentionally not bundled, so you configure it and its readers and exporters however your platform expects. - -```bash -pip install "lancedb[otel]" opentelemetry-sdk -``` - -Call `instrument_lancedb_metrics()` once at startup, before opening any tables. It returns `True` when the recorder is installed and instruments are registered. - -```python Python icon="python" -import lancedb -from lancedb.otel import instrument_lancedb_metrics -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import ( - PeriodicExportingMetricReader, -) -from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, -) - -reader = PeriodicExportingMetricReader(OTLPMetricExporter()) -provider = MeterProvider(metric_readers=[reader]) - -instrument_lancedb_metrics(provider) - -# Any object store activity from this point on is now recorded. -db = lancedb.connect("s3://my-bucket/lancedb") -``` - -If you omit `meter_provider`, LanceDB uses the global provider returned by `opentelemetry.metrics.get_meter_provider()`. - - -`instrument_lancedb_metrics()` returns `False` and emits a warning if another `metrics`-crate recorder is already installed in the process. Only one global recorder is permitted, so instrument LanceDB before any other library that installs its own recorder. - - -## TypeScript {#typescript} - -The Node SDK depends on `@opentelemetry/api` directly, so no extra install step is needed to expose the entry point. You still need an OpenTelemetry SDK to actually export. - -```bash -npm install @opentelemetry/sdk-metrics @opentelemetry/exporter-metrics-otlp-grpc -``` - -```typescript TypeScript icon="square-js" -import { connect, instrumentLanceDbMetrics } from "@lancedb/lancedb"; -import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; -import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-grpc"; - -const reader = new PeriodicExportingMetricReader({ - exporter: new OTLPMetricExporter(), -}); -const provider = new MeterProvider({ readers: [reader] }); - -instrumentLanceDbMetrics(provider); - -const db = await connect("s3://my-bucket/lancedb"); -``` - -`instrumentLanceDbMetrics()` also accepts no arguments, in which case it uses the global provider from `@opentelemetry/api`. Calling it more than once is safe: instruments are created only on the first successful call. - -## What to watch {#what-to-watch} - -A few starting points for dashboards and alerts: - -- **Request rate by operation:** `rate(lance_object_store_requests_total[1m])` broken down by `operation` shows read vs. write pressure and helps size ingestion and serving traffic separately. -- **Tail latency:** histogram quantiles over `lance_object_store_request_duration_seconds_bucket` catch object store slowdowns before they surface as query timeouts. -- **Retryable responses:** a rising `lance_object_store_retryable_responses_total` typically means you are being throttled and should back off or shard writes. -- **In-flight requests:** a growing `lance_object_store_in_flight_requests` gauge without a matching rise in throughput indicates queueing. - -## Where to go next {#where-to-go-next} - - - - Tune ingestion, indexing, and query patterns once metrics highlight a hot spot. - - - Configure the object store backends whose requests these metrics measure. - - diff --git a/docs/tables-and-namespaces.mdx b/docs/tables-and-namespaces.mdx deleted file mode 100644 index 6899d42..0000000 --- a/docs/tables-and-namespaces.mdx +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: "Tables and Namespaces" -sidebarTitle: "Tables and Namespaces" -description: "Learn more about the table abstraction and namespaces in LanceDB." -icon: "table" ---- - -Despite its name, LanceDB is not a "database" in the traditional sense. It is a **Multimodal Lakehouse** built on Lance tables plus a catalog abstraction. -As you dive deeper into LanceDB, it helps to separate two ideas: -- A **table** is where your data lives and is queried. -- A **namespace** is how groups of tables are organized and resolved at the catalog level. - -## Understanding tables {#understanding-tables} - -A table is the core data abstraction in LanceDB: a structured dataset with schema, indexes, and versioned updates. -What changes between deployments is how that table is addressed and accessed. - -The mental model below clarifies table types by connection mode: - -- **`LanceTable`**: direct table access (local path, `file://`, `s3://`, and similar object-store paths). This is the common mode in LanceDB OSS. -- **`RemoteTable`**: catalog-backed table access through a server/cluster (`db://...`). This is the mode you will use in LanceDB Enterprise. - -![](/static/assets/images/overview/understanding-tables.png) - -From an application perspective, both expose a familiar table API: create/open tables, mutate rows, and query data. -The main difference is where resolution and execution happen (directly against storage vs through a remote catalog service). - -## Semantic difference between tables and namespaces {#semantic-difference-between-tables-and-namespaces} - -The easiest way to think about this is: -- A **table** answers: "What data do I store and query?" -- A **namespace** answers: "Where does this table name live in my catalog hierarchy?" - -In other words, tables are data objects; namespaces are catalog objects. - -| Concept | Scope | Owns | Typical operations | -| --- | --- | --- | --- | -| Table | Data layer | Schema, rows, indexes, versions | `create_table`, `open_table`, inserts/updates/deletes, search/query | -| Namespace | Catalog layer | Hierarchy of names, table grouping, table name resolution | `create_namespace`, `list_namespaces`, `drop_namespace`, table ops with `namespace` | - -For simple use cases where you have a relatively flat set of tables, you can ignore namespaces and just use table paths directly. -As your application needs evolve and your tables grow in number and complexity, you may move from table-centric thinking -to catalog-centric thinking. Check out the [Namespaces and the Catalog Model](/namespaces) guide to learn more. diff --git a/docs/tables/branching.mdx b/docs/tables/branching.mdx deleted file mode 100644 index 2f4f0c3..0000000 --- a/docs/tables/branching.mdx +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: "Branches" -sidebarTitle: "Branches" -description: "Fork isolated, writable lines of table history in LanceDB. Run experiments, backfills, and index rebuilds without disturbing production reads on main." -icon: "code-branch" ---- -import { - PyConnect, - TsConnect, - RsConnect, - PyConnectEnterpriseQuickstart, - TsConnectEnterpriseQuickstart, - RsConnectEnterpriseQuickstart, -} from '/snippets/connection.mdx'; -import { - PyBranchCreate as BranchCreate, - TsBranchCreate, - RsBranchCreate, - PyBranchWrite as BranchWrite, - TsBranchWrite, - RsBranchWrite, - PyBranchReopen as BranchReopen, - TsBranchReopen, - RsBranchReopen, - PyBranchDelete as BranchDelete, - TsBranchDelete, - RsBranchDelete, - PyBranchUpsertToMain as BranchUpsertToMain, - TsBranchUpsertToMain, - RsBranchUpsertToMain, - PyBranchIndex as BranchIndex, - TsBranchIndex, - RsBranchIndex, -} from '/snippets/tables.mdx'; - -A branch is an isolated, writable line of history forked from `main` (or from any -other branch). Anything you do on a branch (e.g., adding rows, changing the schema, -building an index) stays on that branch, so `main` keeps serving production -reads exactly as before. Branches are a natural fit when you want to: - -- Experiment with a new index, schema change, or reprocessing step without - affecting live queries on `main`. -- Run a backfill or migration you'd like to validate before applying it to - `main`. -- Hand a collaborator a frozen point-in-time fork while you keep writing to `main`. - -## How branches relate to versions and tags {#how-branches-relate-to-versions-and-tags} - -Every LanceDB table already tracks a linear history of [versions](/tables/versioning), -and you can [tag](/tables/versioning#tag-based-versioning) a version or `checkout` -one to read it. Branches add the missing piece: a *separate, writable* line of -history. Where a tag is a read-only label and `checkout` is a read-only view, a -branch forks from a point in history and then evolves on its own. Creating or -checking out a branch hands you a new table handle whose reads and writes are -scoped to that branch. The [comparison table](#branches-vs-tags-vs-checkout) at -the end of this page lays out when to reach for each. - - -Branches are supported on local and namespace-backed tables in LanceDB OSS, as -well as on LanceDB Enterprise (remote) tables. - - -## Connect to a table {#connect-to-a-table} - -The branch API is identical no matter how you connect — only the connection -itself differs between OSS and Enterprise. Establish a connection (`db`) and open -a `table` (see [Create a table](/tables/create)), then use the same branch calls -in every example that follows. - -### LanceDB OSS {#lancedb-oss} - -Point LanceDB at a local directory (or an object-storage URI) to use it as an -embedded library. - - - - {PyConnect} - - - - {TsConnect} - - - - { "use lancedb::connect;\n\n" } - {RsConnect} - - - -### LanceDB Enterprise {#lancedb-enterprise} - -Enterprise - -Branching on LanceDB Enterprise works the same way as on OSS, but you connect to your -Enterprise deployment with a `db://` URI, an API key, and your -region. Once you have a connection, open a table and use the same branch calls in -every example that follows. - - - - {PyConnectEnterpriseQuickstart} - - - - {TsConnectEnterpriseQuickstart} - - - - {RsConnectEnterpriseQuickstart} - - - -## Work with branches {#work-with-branches} - -The lifecycle of a branch is short and predictable: fork it, write to it, reopen -it whenever you need it, and delete it once you're done. The examples below use a -small `quotes` table with three rows on `main`. - -### Create a branch {#create-a-branch} - -Forking from `main` returns a table handle scoped to the new branch. `main` is -the reserved default source, so `create` needs only a name; to fork from -somewhere else, pass a branch name, a specific version, or both. - - - - {BranchCreate} - - - - {TsBranchCreate} - - - - {RsBranchCreate} - - - -### Write to a branch {#write-to-a-branch} - -Writes go through the branch handle and stay there — the `main` handle keeps -reporting its original row count. Listing branches returns a mapping of each -branch name to its metadata, including the version it was forked from. - - - - {BranchWrite} - - - - {TsBranchWrite} - - - - {RsBranchWrite} - - - -### Reopen a branch {#reopen-a-branch} - -A branch outlives the handle that created it. Reopen it later by name — either -from an existing table handle or straight from the connection when you open the -table. Both routes give you a writable handle tracking the branch's latest state. - - - - {BranchReopen} - - - - {TsBranchReopen} - - - - {RsBranchReopen} - - - -### Delete a branch {#delete-a-branch} - -Deleting a branch removes it and its branch-local history; `main` is untouched. -Before deleting a branch, make sure you've retained any results you need — see -[Apply branch-tested changes to `main`](#apply-branch-tested-changes-to-main) -below. - - - - {BranchDelete} - - - - {TsBranchDelete} - - - - {RsBranchDelete} - - - -## Apply branch-tested changes to `main` {#apply-branch-tested-changes-to-main} - - -A branch has its own writable history. Outside of the [diff and merge -APIs](#compare-and-merge-a-branch-into-main) — which are available on -LanceDB Enterprise and promote added columns only — LanceDB does not reconcile -one branch's history with another or detect conflicts between them. To carry -other accepted work forward, rerun the validated operation against `main` or -explicitly write selected results to it. - - -How you apply a validated change depends on the type of work: - -- **Added columns (Enterprise):** review the branch with `diff`, then promote - the new columns onto `main` with `merge`. See - [Compare and merge a branch into `main`](#compare-and-merge-a-branch-into-main). -- **Backfill or transformation:** rerun the validated job against `main`. -- **Schema change:** apply the same reviewed schema operation to `main`. -- **Index change:** build the index on `main` using the configuration validated - on the branch. -- **Selected row results:** upsert those rows into `main` using a stable unique - key. - -### Upsert selected branch rows into `main` {#upsert-selected-branch-rows-into-main} - -If the result you want to retain is a set of inserted or updated rows, use -[`merge_insert`](/tables/update#merge-incoming-rows-by-key) to write them to -`main`. Despite its name, `merge_insert` is a row-ingestion operation: it -matches incoming rows by key and does not merge branch histories. - -Read the rows you want from the reviewed branch, then upsert them into `main` on -your key column — `id` in this example. Here, `candidate` is the branch handle: - - - - {BranchUpsertToMain} - - - - {TsBranchUpsertToMain} - - - - {RsBranchUpsertToMain} - - - -This operation transfers only the inserted or updated rows in its input. It does -not transfer branch history, schema changes, indexes, or branch-local deletions, -and it does not determine which rows changed after the branch was created. - -Reapplying an identical payload is idempotent on the key, but the operation is -not conflict-aware. If `main` has diverged, incoming branch rows can overwrite -newer values with the same key. Read and upsert the whole branch only when that -overwrite is intentional; otherwise, filter the branch read to the rows you -intend to apply. - -## Compare and merge a branch into `main` {#compare-and-merge-a-branch-into-main} - -Enterprise - -On LanceDB Enterprise, branches expose two review-and-land calls that let you -inspect what a branch has changed relative to `main` and then promote its new -columns onto `main` in place — without reissuing the branch's writes. - - -`diff` and `merge` are available on Enterprise (remote) tables only. On local -tables both calls raise `NotSupported`. `merge` currently promotes added -columns; use the [upsert](#upsert-selected-branch-rows-into-main) or rerun -patterns above for row and index changes. - - -### Diff a branch {#diff-a-branch} - -`diff` reads the branch and `main`, and returns a summary of what has changed: -which columns were added, removed, or altered; which indexes were added or -removed; row-count deltas; and — most importantly — a list of merge blockers -explaining why the branch cannot currently be merged, if any. - - -```python Python icon="python" -diff = table.branches.diff("exp") - -print(diff["addedColumns"]) # columns the branch introduced -print(diff["mergeable"]) # True when there are no blockers -print(diff["mergeBlockers"]) # list of {"code", "message"} entries -``` - -```typescript TypeScript icon="square-js" -const diff = await table.branches.diff("exp"); - -console.log(diff.addedColumns); // columns the branch introduced -console.log(diff.mergeable); // true when there are no blockers -console.log(diff.mergeBlockers); // array of { code, message } entries -``` - - -Common merge blocker codes include `BaseMoved` (the branch's parent no longer -matches `main`'s latest), `RowsChanged`, `ColumnRemoved`, `ColumnChanged`, -`NoMergeableChanges`, `NoColumnChanges`, `InputColumnDependency`, and -`ParentNotMain`. Newer server codes surface as `Unknown` so older clients -keep working. - -### Merge a branch {#merge-a-branch} - -`merge` promotes a branch's added columns onto `main`. It is a review-and-land -operation: the server re-evaluates the diff at request time, and either lands -the promotion or rejects it with the same blockers `diff` would report. A -rejected merge is not an exception — it resolves with `status="rejected"` so -you can inspect the blockers and decide what to do next. - -Set `dry_run=True` (Python) or `dryRun: true` (TypeScript) to preview the -merge without landing it. The result includes a `preview.promoted_columns` -list showing which columns the merge would (or did) promote. - - -```python Python icon="python" -# Preview first — this does not modify main. -preview = table.branches.merge("exp", dry_run=True) -print(preview["status"]) # "ready" | "rejected" | ... -print(preview["preview"]["promotedColumns"]) - -# Land the merge. -result = table.branches.merge("exp") -if result["status"] == "merged": - print("landed at main version", result["mainVersionAfter"]) -elif result["status"] == "rejected": - for blocker in result["diff"]["mergeBlockers"]: - print(blocker["code"], blocker["message"]) -``` - -```typescript TypeScript icon="square-js" -// Preview first — this does not modify main. -const preview = await table.branches.merge("exp", true); -console.log(preview.status); // "ready" | "rejected" | ... -console.log(preview.preview.promotedColumns); - -// Land the merge. -const result = await table.branches.merge("exp"); -if (result.status === "merged") { - console.log("landed at main version", result.mainVersionAfter); -} else if (result.status === "rejected") { - for (const blocker of result.diff.mergeBlockers) { - console.log(blocker.code, blocker.message); - } -} -``` - - -Possible `status` values are `ready` (returned by `dry_run` when the merge -would land), `merged` (a real merge that landed), `rejected` (server declined; -see `diff.mergeBlockers`), `notImplemented`, and `unknown` for forward -compatibility. Merge requests are not retried on rejection — the response -carries everything you need to decide next steps. - -## Build indexes on a branch {#build-indexes-on-a-branch} - -One of the most useful things a branch buys you is a safe place to build and -validate an index without affecting what's in production on the `main` branch. -Fork a branch, create your vector (ANN) and full-text search (FTS) -indexes on it, and check recall and latency. Once you have selected a -configuration, build the corresponding index on `main` through your normal -deployment workflow. Because the indexes live on the branch, queries against -`main` never see a half-built index and are never slowed down by the branch's -build. - -This pattern is especially valuable on Enterprise -deployments, where `main` is typically serving production traffic while you tune -an index configuration on the side. - - - - {BranchIndex} - - - - {TsBranchIndex} - - - - {RsBranchIndex} - - - -Schema changes such as adding, altering, or dropping columns are branch-scoped in -the same way, so you can stage and review a larger reshaping of a table before -applying the same schema operations to `main`. - -## Branches vs. tags vs. versions {#branches-vs-tags-vs-versions} - -Now that you're familiar with branches, you can see how they complement the other ways LanceDB -give you to work with table history. Choose these approaches based on whether you need to -*label*, *read*, or *write* data at a point in history: - -| Feature | Writable? | Purpose | -| --- | --- | --- | -| **[Branch](#work-with-branches)** | ✅ Yes | Write on top of a point in history without touching `main`. | -| **[Tag](/tables/versioning#tag-based-versioning)** | ❌ No | Attach a human-readable label to an existing version; protects it from cleanup. | -| **[`checkout(version)`](/tables/versioning#rollback-to-previous-versions)** | ❌ No | Read a historical version of `main` without forking. Read-only until you `restore`. | - - -For linear version history — creating versions, listing them, rolling back, and -tagging — see the [Versioning and Reproducibility](/tables/versioning) guide. - diff --git a/docs/tables/consistency.mdx b/docs/tables/consistency.mdx deleted file mode 100644 index a4152f8..0000000 --- a/docs/tables/consistency.mdx +++ /dev/null @@ -1,154 +0,0 @@ ---- -title: Consistency -sidebarTitle: "Consistency" -description: Learn about consistency settings and versioning in LanceDB tables. -icon: "water" ---- -import { - PyConsistencyStrong as ConsistencyStrong, - TsConsistencyStrong as TsConsistencyStrong, - RsConsistencyStrong as RsConsistencyStrong, - PyConsistencyEventual as ConsistencyEventual, - TsConsistencyEventual as TsConsistencyEventual, - RsConsistencyEventual as RsConsistencyEventual, - PyConsistencyCheckoutLatest as ConsistencyCheckoutLatest, - TsConsistencyCheckoutLatest as TsConsistencyCheckoutLatest, - RsConsistencyCheckoutLatest as RsConsistencyCheckoutLatest, - RsUpdateMakeUsersReader as RsConsistencyMakeUsersReader, -} from '/snippets/tables.mdx'; - -You can set `read_consistency_interval` on the connection to control how often reads check for updates from other writers. - -There are three possible settings for `read_consistency_interval`: - -1. **Unset (default)**: no automatic cross-process refresh checks. -2. **Zero seconds**: check for updates on every read (strongest freshness). -3. **Non-zero interval**: check for updates after the interval elapses (eventual refresh). - -The value you set depends on your application's consistency needs and performance requirements. -For example, a real-time dashboard might require strong consistency, while a batch analytics job might be -fine with eventual consistency. Stronger consistency is not free — the smaller the interval, the more -often each read pays the cost of refreshing against object storage, which raises per-read latency and cost. - -This setting works for both local ([LanceTable](/tables-and-namespaces#understanding-tables)) and remote -tables. It only affects read operations — -write operations are always consistent. - - -**Consistency in Remote Tables** - -For remote tables (`db://` connections), `read_consistency_interval` is also -respected by the client. The interval is sent to the server as a freshness bound on each read: - -- **Unset (default)**: no freshness header is sent; reads use the server's cached view of the table. -- **Zero seconds**: every read asks the server for the latest committed version. -- **Non-zero interval**: reads accept data at least as fresh as `now - interval`. - -In addition, after any write or after a `checkout_latest` / `restore` on a table handle, subsequent -reads on that same handle carry a freshness floor so you read your own writes without extra -configuration. The floor is the later of the configured interval and the moment of the last -write or refresh, and it is shared across handles to the same table on the same connection. - -Each remote table handle also tracks the highest dataset version it has observed in a read -response and sends it back with every subsequent read, so successive reads on the same handle -never observe an older version even when a load balancer routes them to query nodes with -differently-cached views. `checkout_latest` resets this watermark. - -Stronger consistency is not free — the smaller the interval, the more often each read pays the cost -of refreshing against storage, which raises per-read latency and cost. - -In Enterprise deployments, the server-side default freshness is still -controlled by the cluster-level `weak_read_consistency_interval_seconds` parameter; the client setting -tightens that bound on a per-connection basis. - - -## Configure Consistency Parameters {#configure-consistency-parameters} - -To set strong consistency, set the interval to 0: - - - - {ConsistencyStrong} - - - - {TsConsistencyStrong} - - - - {RsConsistencyStrong} - - - - - - - {RsConsistencyMakeUsersReader} - - - - -For eventual consistency, use a non-zero interval: - - - - {ConsistencyEventual} - - - - {TsConsistencyEventual} - - - - {RsConsistencyEventual} - - - - -With the default unset interval, tables do not auto-refresh from other writers. -To manually check for updates, use `checkout_latest` / `checkoutLatest`: - - - - {ConsistencyCheckoutLatest} - - - - {TsConsistencyCheckoutLatest} - - - - {RsConsistencyCheckoutLatest} - - - -For reproducible reads, you can also pin a table to a specific snapshot with `checkout(...)` or -a tag, restore a table to a prior version, then return to the live table with -`checkout_latest` / `checkoutLatest`. See -[Versioning](/tables/versioning/) for the full version and tag workflow. - -## Handle bad vectors {#handle-bad-vectors} - - -This section is currently specific to the Python SDK. - - -In LanceDB Python, you can use the `on_bad_vectors` parameter to choose how -invalid vector values are handled. Invalid vectors are vectors that are not valid -because: - -1. They are the wrong dimension -2. They contain NaN values -3. They are null but are on a non-nullable field - -By default, LanceDB will raise an error if it encounters a bad vector. You can -also choose one of the following options: - -* `drop`: Ignore rows with bad vectors -* `fill`: Replace bad values (NaNs) or missing values (too few dimensions) with - the fill value specified in the `fill_value` parameter. An input like - `[1.0, NaN, 3.0]` will be replaced with `[1.0, 0.0, 3.0]` if `fill_value=0.0`. -* `null`: Replace bad vectors with null (only works if the column is nullable). - A bad vector `[1.0, NaN, 3.0]` will be replaced with `null` if the column is - nullable. If the vector column is non-nullable, then bad vectors will cause an - error diff --git a/docs/tables/create.mdx b/docs/tables/create.mdx deleted file mode 100644 index 167a0d4..0000000 --- a/docs/tables/create.mdx +++ /dev/null @@ -1,442 +0,0 @@ ---- -title: Ingesting Data -sidebarTitle: "Ingesting data" -description: Learn about different methods to ingest data into tables in LanceDB, including from various data sources and empty tables. -icon: "cookie" ---- -import { TsConnect, RsConnect } from '/snippets/connection.mdx'; -import { - PyCreateTableFromDicts as CreateTableFromDicts, - TsCreateTableFromDicts as TsCreateTableFromDicts, - RsCreateTableFromDicts as RsCreateTableFromDicts, - PyCreateTableConflictHandling as CreateTableConflictHandling, - TsCreateTableConflictHandling as TsCreateTableConflictHandling, - RsCreateTableConflictHandling as RsCreateTableConflictHandling, - PyCreateTableFromPandas as CreateTableFromPandas, - PyCreateTableCustomSchema as CreateTableCustomSchema, - TsCreateTableCustomSchema as TsCreateTableCustomSchema, - RsCreateTableCustomSchema as RsCreateTableCustomSchema, - PyCreateTableFromPolars as CreateTableFromPolars, - PyCreateTableFromArrow as CreateTableFromArrow, - TsCreateTableFromArrow as TsCreateTableFromArrow, - RsCreateTableFromArrow as RsCreateTableFromArrow, - PyCreateTableFromPydantic as CreateTableFromPydantic, - PyCreateTableNestedSchema as CreateTableNestedSchema, - PyAddFromDataset as AddFromDataset, - PyCreateTableFromIterator as CreateTableFromIterator, - TsCreateTableFromIterator as TsCreateTableFromIterator, - RsCreateTableFromIterator as RsCreateTableFromIterator, - TsAddProgress, - PyOpenExistingTable as OpenExistingTable, - TsOpenExistingTable as TsOpenExistingTable, - RsOpenExistingTable as RsOpenExistingTable, - PyCreateEmptyTable as CreateEmptyTable, - TsCreateEmptyTable as TsCreateEmptyTable, - RsCreateEmptyTable as RsCreateEmptyTable, - PyCreateEmptyTablePydantic as CreateEmptyTablePydantic, - PyDropTable as DropTable, - TsDropTable as TsDropTable, - RsDropTable as RsDropTable, - PyTablesBasicConnect as TablesBasicConnect, - PyTablesDocumentModel as TablesDocumentModel, - PyTablesTzValidator as TablesTzValidator, -} from '/snippets/tables.mdx'; - -In LanceDB, tables store records with a defined schema that specifies column names and types. Across the SDKs, you can create tables from row-oriented data and Apache Arrow data structures. The Python SDK additionally supports: - -- PyArrow schemas for explicit schema control -- `LanceModel` for Pydantic-based validation - -## Create a table with data {#create-a-table-with-data} - -Initialize a LanceDB connection and create a table - - - - {TablesBasicConnect} - - - - {TsConnect} - - - - {RsConnect} - - - -Depending on the SDK, LanceDB can ingest arrays of records, Arrow tables or record batches, and Arrow batch iterators or readers. Let's take a look at some of the common patterns. - -### From list of objects {#from-list-of-objects} - -You can provide a list of objects to create a table. The Python and TypeScript SDKs -support lists/arrays of dictionaries, while the Rust SDK supports lists of structs. -In Python, pass a list or other batch-like object; a single bare `dict` or single -`LanceModel` is rejected. - - - - {CreateTableFromDicts} - - - - {TsCreateTableFromDicts} - - - - {RsCreateTableFromDicts} - - - -### Handle existing tables {#handle-existing-tables} - -By default, `create_table` raises an error if a table with the same name already exists. -You can change this behavior with two parameters that resolve the conflict in different ways: - -- **Idempotent open**: return the existing table without modifying it. Use when your - code may run more than once (notebooks, retries, init scripts) and you want to reuse - the table on subsequent runs. The provided data is ignored, but the schema is - validated against the existing table and a mismatch raises an error. -- **Overwrite**: drop the existing table and create a new one with the provided data. - Use this for test fixtures or when you intentionally want to replace prior contents. - This permanently discards the old table's data. - - - - {CreateTableConflictHandling} - - - - {TsCreateTableConflictHandling} - - - - {RsCreateTableConflictHandling} - - - - -`exist_ok` / `existOk` does not append the provided data to an existing table. Use -[`table.add()`](/tables/update) for that. If you need to ensure a table exists *and* -contains specific rows, prefer the [empty-table-then-add pattern](#create-empty-table). - - -### From a custom schema {#from-a-custom-schema} - -You can define a custom Arrow schema for the table. This is useful when you want to have more control over the column types and metadata. - - - - {CreateTableCustomSchema} - - - - {TsCreateTableCustomSchema} - - - - {RsCreateTableCustomSchema} - - - -An explicit schema is also where you control nullability. If later writes omit a -non-nullable column, or provide actual nulls for it, ingestion fails; nullable columns can be -omitted or written with null values. Without an explicit schema, Python infers list-like vector -values as fixed-size `float32` vector fields from the observed dimension. - -For Python ingest, malformed vector values fail by default. If you expect occasional wrong-length, -null, or NaN vectors, choose an `on_bad_vectors` policy: `"drop"` removes those rows, `"fill"` writes -`fill_value`, and `"null"` writes nulls. - -### From an Arrow Table {#from-an-arrow-table} -You can also create LanceDB tables directly from Arrow tables. -Rust uses an Arrow `RecordBatchReader` for the same Arrow-native ingest flow. - - - - {CreateTableFromArrow} - - - - {TsCreateTableFromArrow} - - - - {RsCreateTableFromArrow} - - - - -### From a Pandas DataFrame {#from-a-pandas-dataframe} -Python Only - - - - {CreateTableFromPandas} - - - - -Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly. - - - -The **`vector`** column needs to be a [Vector](/integrations/data/pydantic#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type. - - -### From a Polars DataFrame {#from-a-polars-dataframe} -Python Only - -LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library -written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow -under the hood. A deeper integration between LanceDB Tables and Polars DataFrames -is on the way. - - - - {CreateTableFromPolars} - - - -### From Pydantic Models {#from-pydantic-models} -Python Only - -When you create an empty table without data, you must specify the table schema. -LanceDB supports creating tables by specifying a PyArrow schema or a specialized -Pydantic model called `LanceModel`. - -For example, the following Content model specifies a table with 5 columns: -`movie_id`, `vector`, `genres`, `title`, and `imdb_id`. When you create a table, you can -pass the class as the value of the `schema` parameter to `create_table`. -The `vector` column is a `Vector` type, which is a specialized Pydantic type that -can be configured with the vector dimensions. It is also important to note that -LanceDB only understands subclasses of `lancedb.pydantic.LanceModel` -(which itself derives from `pydantic.BaseModel`). - - - - {CreateTableFromPydantic} - - - -#### Nested schemas {#nested-schemas} - -Sometimes your data model may contain nested objects. For example, you may want to store the document string and the document source name as a nested Document object: - - - - {TablesDocumentModel} - - - -This can be used as the type of a LanceDB table column: - - - - {CreateTableNestedSchema} - - - -This creates a struct column called "document" that has two subfields -called "content" and "source": - -```bash -In [28]: tbl.schema -Out[28]: -id: string not null -vector: fixed_size_list[1536] not null - child 0, item: float -document: struct not null - child 0, content: string not null - child 1, source: string not null -``` - -#### Validators {#validators} - -Because `LanceModel` inherits from Pydantic's `BaseModel`, you can combine them with Pydantic's -[field validators](https://docs.pydantic.dev/latest/concepts/validators). The example -below shows how to add a validator to ensure that only valid timezone-aware datetime objects are used -for a `created_at` field. - - - - {TablesTzValidator} - - - -When you run this code it, should raise the `ValidationError`. - -### Loading Large Datasets {#loading-large-datasets} - -When ingesting large datasets, use `table.add()` on an existing table rather than -passing all data to `create_table()`. The `add()` method auto-parallelizes large -writes, while `create_table(name, data)` does not. - - -For best performance with large datasets, create an empty table first and then call -`table.add()`. This enables automatic write parallelism for materialized data sources. - - -#### From files (Parquet, CSV, etc.) {#from-files-parquet-csv-etc} -Python Only - -For file-based data, pass a `pyarrow.dataset.Dataset` to `table.add()`. This streams -data from disk without loading the entire dataset into memory. - - - - {AddFromDataset} - - - - -`pa.dataset()` input is currently Python-only. TypeScript and Rust support for -file-based dataset ingestion is tracked in -[lancedb#3173](https://github.com/lancedb/lancedb/issues/3173). - - -#### From iterators (custom batch generation) {#from-iterators-custom-batch-generation} - -When you need custom batch logic — generating embeddings on the fly, transforming -rows from an external source, etc. — use an iterator of `RecordBatch` objects. - - - - {CreateTableFromIterator} - - - - {TsCreateTableFromIterator} - - - - {RsCreateTableFromIterator} - - - -Use this pattern when: - -- Your source data already arrives in Arrow batches, readers, datasets, or streams. -- Materializing the entire ingest as one giant in-memory list or array would be too expensive. -- You want to control chunk size explicitly during ingestion. - -Python can also consume iterators of other supported types like Pandas DataFrames or Python lists. - -#### Write parallelism {#write-parallelism} - - -For materialized data (`pa.Table`, `pd.DataFrame`, `pa.dataset()`), LanceDB -automatically parallelizes large writes — no configuration needed. Auto-parallelism -targets approximately 1M rows or 2GB per write partition. - -For streaming sources (iterators, `RecordBatchReader`), LanceDB cannot determine -total size upfront. A `parallelism` parameter to control this manually is planned -but not yet exposed in Python or TypeScript -([tracking issue](https://github.com/lancedb/lancedb/issues/3173)). - - -#### Tracking ingestion progress {#tracking-ingestion-progress} -TypeScript Only - -For long-running writes, pass a `progress` callback to `table.add()` to surface -per-batch progress in your UI, logs, or metrics pipeline. The callback fires -once per batch written and once more with `done: true` when the write completes. - -Each invocation receives a `WriteProgress` object: - -| Field | Description | -|:------|:------------| -| `outputRows` | Rows written so far. | -| `outputBytes` | Bytes written so far. | -| `totalRows` | Expected total rows when the input source reports one. Always set on the final callback. | -| `elapsedSeconds` | Wall-clock seconds since the write started. | -| `activeTasks` | Parallel write tasks currently in flight. | -| `totalTasks` | Total parallel write tasks (the write parallelism). | -| `done` | `true` only for the final callback. | - - - - {TsAddProgress} - - - -A few things to know before you wire this up: - -- Back-pressures the writer: callback invocations are serialized and run inline with each batch, so a slow callback will slow the write rather than drop updates. Every batch update is delivered, and the final `done: true` callback always fires (even on error or cancellation). Keep the callback cheap — offload heavy work to a queue you drain elsewhere. -- Errors swallowed: anything your callback throws is logged with `console.warn` and won't abort the write, so keep the callback side-effect-only and don't rely on it for control flow. -- Row totals: `totalRows` is only populated when the input source can report it up front (for example, a materialized `arrow.Table`). For streaming sources it stays `undefined` until the final callback, where it falls back to the actual rows written. - -## Create empty table {#create-empty-table} -You can create an empty table for scenarios where you want to add data to the table later. -An example would be when you want to collect data from a stream/external file and then add it to a table in -batches. - -An empty table can be initialized via an Arrow schema. - - - - {CreateEmptyTable} - - - - {TsCreateEmptyTable} - - - - {RsCreateEmptyTable} - - - -Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not -directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel` -that has been extended to support LanceDB specific types like `Vector`. - - - - {CreateEmptyTablePydantic} - - - -Once the empty table has been created, you can append to it or modify its contents, -as explained in the [updating and modifying tables](/tables/update) section. - -## Open an existing table {#open-an-existing-table} - -You can open an existing table by specifying the name of the table to the `open_table` / `openTable` method. -If you forget the name of your table, you can always get a listing of all table names. - - - - {OpenExistingTable} - - - - {TsOpenExistingTable} - - - - {RsOpenExistingTable} - - - -## Drop a table {#drop-a-table} - -Use the `drop_table()` method on the database to remove a table. - - - - {DropTable} - - - - {TsDropTable} - - - - {RsDropTable} - - - -This permanently removes the table and is not recoverable, unlike deleting rows. -By default, if the table does not exist an exception is raised. To suppress this, -you can pass in `ignore_missing=True`. diff --git a/docs/tables/index.mdx b/docs/tables/index.mdx deleted file mode 100644 index de34d4e..0000000 --- a/docs/tables/index.mdx +++ /dev/null @@ -1,672 +0,0 @@ ---- -title: "Basic Table Operations" -sidebarTitle: "Basic usage" -description: "Create tables, search vectors, and append data in LanceDB." -icon: "table" -keywords: ["create table", "polars", "pandas", "pyarrow", "dataframe", "nested data"] ---- - -import { PyConnect, PyConnectEnterpriseQuickstart, TsConnect, TsConnectEnterpriseQuickstart, RsConnect, RsConnectEnterpriseQuickstart } from '/snippets/connection.mdx'; -import { - PyBasicImports, - PyDataLoad, - PyBasicCreateTable, - PyBasicOpenTable, - PyBasicCreateEmptyTable, - PyBasicCreateTablePandas, - PyBasicCreateTablePolars, - PyBasicAsyncApi, - PyBasicVectorSearch, - PyBasicVectorSearchQ1, - PyBasicVectorSearchQ2, - PyBasicVectorSearchQ3, - PyBasicVectorSearchQ4, - PyBasicSortPolars, - PyBasicDeleteRows, - PyBasicAddData, - PyBasicAddColumns, - PyBasicDropColumns, - PyBasicDropTable, - TsBasicImports, - TsDataLoad, - TsBasicCreateTable, - TsBasicOpenTable, - TsBasicCreateEmptyTable, - TsBasicVectorSearchQ1, - TsBasicVectorSearchQ2, - TsBasicVectorSearchQ3, - TsBasicVectorSearchQ4, - TsBasicDeleteRows, - TsBasicAddData, - TsBasicAddColumns, - TsBasicDropColumns, - TsBasicDropTable, - RsBasicImports, - RsDataLoad, - RsBasicCreateTable, - RsBasicOpenTable, - RsBasicCreateEmptyTable, - RsBasicAddData, - RsBasicVectorSearchQ1, - RsBasicVectorSearchQ2, - RsBasicVectorSearchQ3, - RsBasicVectorSearchQ4, - RsBasicDeleteRows, - RsBasicAddColumns, - RsBasicDropColumns, - RsBasicDropTable, -} from '/snippets/basic_usage.mdx'; - -Now that you've completed the [LanceDB quickstart](/quickstart), you're ready to -explore some more table operations you'll typically need when working with LanceDB. - -- **Ingest data into tables** from JSON data (and in Python, Pandas or Polars DataFrames) -- **Create empty tables** by defining explicit Arrow schemas -- **Vector similarity search** with filtering and projections -- **Filtered queries** that can operate on nested structs -- **Query Lance tables in DuckDB** via the Lance extension for SQL analytics (including joins) - - -This page uses **synchronous** Python snippets. If your Python app uses `asyncio`, -the same flow works with `connect_async(...)` and `await`-based table/query calls. -Use the example below as a template, and see [Quickstart](/quickstart#python-sync-and-async-apis) -for example snippets on both sync and async Python usage. - - -## Dataset {#dataset} - -We'll work with this small dataset based on characters from the legends of Camelot. Note that -the `vector` column holds 4-dimensional embeddings, and the `stats` column is a nested struct -with several integer fields, indicating each character's attributes. - -```json camelot.json icon="brackets-curly" expandable=true -[ - { - "id": 1, - "name": "King Arthur", - "role": "King of Camelot", - "description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.", - "vector": [0.72, -0.28, 0.60, 0.86], - "stats": { "strength": 2, "courage": 5, "magic": 1, "wisdom": 4 } - }, - { - "id": 2, - "name": "Merlin", - "role": "Wizard and Advisor", - "description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.", - "vector": [0.05, 0.88, 0.62, 0.85], - "stats": { "strength": 2, "courage": 4, "magic": 5, "wisdom": 5 } - }, - { - "id": 3, - "name": "Queen Guinevere", - "role": "Queen of Camelot", - "description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.", - "vector": [0.22, -0.22, 0.42, 0.82], - "stats": { "strength": 1, "courage": 3, "magic": 1, "wisdom": 4 } - }, - { - "id": 4, - "name": "Sir Lancelot", - "role": "Knight of the Round Table", - "description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.", - "vector": [0.86, -0.35, 0.38, 0.55], - "stats": { "strength": 5, "courage": 5, "magic": 1, "wisdom": 3 } - }, - { - "id": 5, - "name": "Sir Gawain", - "role": "Knight of the Round Table", - "description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.", - "vector": [0.82, -0.32, 0.52, 0.60], - "stats": { "strength": 4, "courage": 5, "magic": 1, "wisdom": 4 } - }, - { - "id": 6, - "name": "Sir Galahad", - "role": "Knight of the Round Table", - "description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.", - "vector": [0.80, -0.20, 0.70, 0.78], - "stats": { "strength": 4, "courage": 5, "magic": 2, "wisdom": 5 } - }, - { - "id": 7, - "name": "Sir Percival", - "role": "Knight of the Round Table", - "description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.", - "vector": [0.78, -0.36, 0.48, 0.52], - "stats": { "strength": 4, "courage": 4, "magic": 1, "wisdom": 3 } - }, - { - "id": 8, - "name": "Mordred", - "role": "Traitor Knight", - "description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.", - "vector": [0.68, -0.30, -0.65, 0.20], - "stats": { "strength": 4, "courage": 2, "magic": 1, "wisdom": 2 } - } -] -``` - - -The `vector` arrays here are synthetic and for demonstration purposes only. In your real-world -applications, you'd generate these vectors from the raw text fields using a suitable embedding model. - - -## Connect to a database {#connect-to-a-database} - -### Option 1: Direct table access {#option-1-direct-table-access} - -We start by connecting to a LanceDB database path. The example below uses a local path in LanceDB OSS. - - - - {PyConnect} - - - - {TsConnect} - - - - {RsConnect} - - - -You can also connect LanceDB OSS directly to object storage. For credentials, endpoints, and provider-specific options, see -[Configuring storage](/storage/configuration). - -### Option 2: Remote tables {#option-2-remote-tables} - -If you're using LanceDB [Enterprise](/enterprise), you can connect using a `db://` URI, -along with any necessary credentials. Simply replace the local path with a remote `uri` -that points to where your data is stored, and you're ready to go. - - - - { "import lancedb\n\n" } - {PyConnectEnterpriseQuickstart} - - - - { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } - {TsConnectEnterpriseQuickstart} - - - - { "use lancedb::connect;\n\n" } - {RsConnectEnterpriseQuickstart} - - - - -- When you connect to a remote URI (Enterprise), `open_table(...)` returns a *remote* table. -Remote tables support core operations (ingest, search, update, delete), but some convenience -methods for bulk data export are not available. -- In the Python SDK, `table.to_arrow()` and `table.to_pandas()` are not implemented for remote tables. -To retrieve data, use search queries instead: `table.search(query).limit(n).to_arrow()`. - - -## Create a table and ingest data {#create-a-table-and-ingest-data} - -### From JSON {#from-json} -LanceDB stores records in Lance tables. Each row is a record and each column -holds a field or related metadata. The simplest way to start is to obtain the source -data as a list of JSON records that includes a vector column and any metadata -fields you care about. - - - - {PyBasicImports} - - - - {TsBasicImports} - - - - {RsBasicImports} - - - -Load the data from the JSON file: - - - {PyDataLoad} - - - - {TsDataLoad} - - - - {RsDataLoad} - - - -You can now create a LanceDB table from the loaded data. By default, creating a table with a name -that already exists raises an error. Use `mode="overwrite"` only when you intentionally want to -replace the existing table and its data, or use `exist_ok` / `existOk` when repeatable setup should -reuse the existing table instead of writing the supplied rows again. - - - - {PyBasicCreateTable} - - - - {TsBasicCreateTable} - - - - {RsBasicCreateTable} - - - - -If you want to avoid overwriting an existing table, omit the overwrite mode. For append-only -ingestion into a table that already exists, open the table and call `add(...)` instead of -`create_table(...)`. For repeatable setup with `exist_ok` / `existOk`, see -[Handle existing tables](/tables/create#handle-existing-tables). - - -For more ingestion patterns, including PyArrow tables, Python `pyarrow.dataset.Dataset` inputs, -empty tables, and Python `LanceModel` schemas with nested fields, see -[Ingesting data](/tables/create/). - -### From Pandas DataFrames {#from-pandas-dataframes} -Python Only - -You can create LanceDB tables directly from [Pandas](https://pandas.pydata.org/) DataFrames. Simply -obtain the source data as a Pandas DataFrame, then create the table -and directly ingest to it. - - - - {PyBasicCreateTablePandas} - - - -### From Polars DataFrames {#from-polars-dataframes} -Python Only - -You can also create LanceDB tables directly from [Polars](https://www.pola.rs/) DataFrames. Simply -obtain the source data as a Polars DataFrame, then create the table -and directly ingest to it. - - - - {PyBasicCreateTablePolars} - - - -### From an Arrow schema {#from-an-arrow-schema} - -If you want to create an _empty_ table without any data -- say you want to -define the schema first and then incrementally add data later -- you can -do so by defining an Arrow schema explicitly. - - - - {PyBasicCreateEmptyTable} - - - - {TsBasicCreateEmptyTable} - - - - {RsBasicCreateEmptyTable} - - - -Once the empty table is defined, LanceDB is ready to accept new data via -the `add` method, as shown in the next section. - - -LanceDB tables are type-aware, leveraging Apache Arrow under the hood. -You can display a given table's schema using the `schema` property or -method. For example, in Python, running `print(table.schema)` would show -something like the following: - -```txt expandable=true -id: int64 -name: string -role: string -description: string -vector: fixed_size_list[4] - child 0, item: float -stats: struct - child 0, courage: int64 - child 1, magic: int64 - child 2, strength: int64 - child 3, wisdom: int64 -``` - - -## Append data to a table {#append-data-to-a-table} - -LanceDB tables are mutable, and you can append new records to existing tables. -If you're starting with a fresh session, connect to the database and open the -existing table named `camelot`. - - - - {PyBasicOpenTable} - - - - {TsBasicOpenTable} - - - - {RsBasicOpenTable} - - - -Prepare the new records to add. Here, we add two new magical characters -via the `add` method. For the Rust snippet, you can find the helper functions in the -[code](https://github.com/lancedb/docs/blob/main/tests/rs/basic_usage.rs). - - - - {PyBasicAddData} - - - - {TsBasicAddData} - - - - {RsBasicAddData} - - - -We now have two new records in the table. Let's begin to query our data! - -## Vector search {#vector-search} - -It's straightforward to run vector similarity search in LanceDB. Let's answer -some questions about the data using vector search with projections (returning only -the desired columns). - -> Q1: _Who are the characters similar to "wizard"?_ - - - - {PyBasicVectorSearchQ1} - - - - {TsBasicVectorSearchQ1} - - - - {RsBasicVectorSearchQ1} - - - -| name | role | description | -| --- | --- | --- | -| Merlin | Wizard and Advisor | A powerful wizard and prophet | -| The Lady of the Lake | Mystical Guardian | A mysterious supernatural figu… | -| Morgan le Fay | Sorceress | A powerful enchantress, Arthur… | -| Queen Guinevere | Queen of Camelot | Arthur's queen, admired for he… | -| Sir Galahad | Knight of the Round Table | The purest and most virtuous k… | - -We have Merlin, The Lady of the Lake, and Morgan le Fay in the top results, who -all have magical abilities. - -Next, let's try to answer a different question that involves vector search while -filtering on a nested struct field. Filtering is done using the `where` method, -into which you can pass SQL-like expressions. - -> Q2: _Who are the characters similar to "wizard" with high magic stats?_ - - - - {PyBasicVectorSearchQ2} - - - - {TsBasicVectorSearchQ2} - - - - {RsBasicVectorSearchQ2} - - - -| name | role | description | -| --- | --- | --- | -| Merlin | Wizard and Advisor | A powerful wizard and prophet | -| The Lady of the Lake | Mystical Guardian | A mysterious supernatural figu… | -| Morgan le Fay | Sorceress | A powerful enchantress, Arthur… | - -Only three characters have magical abilities greater than 3. Merlin is -clearly the most magical of them all! - -## Filtered search {#filtered-search} - -You can also run traditional analytics-style search queries that do not -involve vectors. For example, let's find the strongest characters in -the dataset. In the query below, we leave the `search` method empty to indicate -that we don't want to use any vector for similarity search (in TypeScript/Rust, -use `query()` instead), and use the `where` method to filter on the `strength` field. - -> Q3: _Who are the strongest characters?_ - - - - {PyBasicVectorSearchQ3} - - - - {TsBasicVectorSearchQ3} - - - - {RsBasicVectorSearchQ3} - - - -| name | role | description | -| --- | --- | --- | -| Sir Galahad | Knight of the Round Table | The purest and most virtuous k… | -| Sir Gawain | Knight of the Round Table | A noble and honorable knight k… | -| Sir Percival | Knight of the Round Table | A loyal and innocent knight wh… | -| Sir Lancelot | Knight of the Round Table | Arthur's most skilled knight, … | -| Mordred | Traitor Knight | Arthur's treacherous son or ne… | - -Clearly, the strongest characters are all Knights of the Round Table! - - -Need SQL analytics like filters, aggregations, or joins on Lance tables? Use the DuckDB -Lance extension to query Lance tables directly with SQL. See the -[DuckDB integration guide](/integrations/data/duckdb). - - -## Add column {#add-column} - -We can also add new columns to an existing LanceDB table using the `add_columns` method. -For this example, let's add a new float column named `power` that shows the average -of each character's strength, courage, magic, and wisdom stats. - - - - {PyBasicAddColumns} - - - - {TsBasicAddColumns} - - - - {RsBasicAddColumns} - - - -The example above sums up the individual stats and divides by 4 to compute the average. -The resulting average total stats is cast to an Arrow float type under the hood for the -Lance table. - -We can display the results of this column in descending order of power. - -> Q4: _Who are the most powerful characters?_ - - - - {PyBasicVectorSearchQ4} - - - - {TsBasicVectorSearchQ4} - - - - {RsBasicVectorSearchQ4} - - - -Note that LanceDB's `where` only filters rows, but doesn't sort them by applying an `ORDER BY` -clause that you may be used to when working with SQL databases. - -You can also sort the results after converting them to a Polars DataFrame. -In TypeScript/Rust, you can sort the -returned array in application code. - - -{PyBasicSortPolars} - - -| name | role | description | power | -| --- | --- | --- | --- | -| Merlin | Wizard and Advisor | A powerful wizard and prophet … | 4.0 | -| Sir Galahad | Knight of the Round Table | The purest and most virtuous k… | 4.0 | -| The Lady of the Lake | Mystical Guardian | A mysterious supernatural figu… | 3.75 | -| Sir Lancelot | Knight of the Round Table | Arthur's most skilled knight, … | 3.5 | -| Sir Gawain | Knight of the Round Table | A noble and honorable knight k… | 3.5 | - -Merlin and Sir Galahad are the most powerful characters when considering the average of -all their abilities! Sir Lancelot and the Lady of the Lake follow closely behind. - -For column renames, type changes, nullability changes, and grouping multiple schema changes into -one operation, see [Schema and data evolution](/tables/schema/). - -## Delete data {#delete-data} - -You can delete rows from a LanceDB table using the `delete` method with -a filtering expression. - -Say we want to remove Mordred, the traitor knight, from our table. - - - - {PyBasicDeleteRows} - - - - {TsBasicDeleteRows} - - - - {RsBasicDeleteRows} - - - -This will delete the row(s) where the `role` value matches "Traitor Knight". -You can verify that the row has been deleted by running a search query again, -and confirming that Mordred no longer appears in the results. - -## Drop column {#drop-column} - -If you want to remove or delete a column from an existing LanceDB table, you can use -the `drop_columns` method. - - - - {PyBasicDropColumns} - - - - {TsBasicDropColumns} - - - - {RsBasicDropColumns} - - - -This will remove the `power` column we added earlier from the table schema. - -## Drop table {#drop-table} - -If you want to delete an entire table from the database, you can use the -`drop_table` method. - - - - {PyBasicDropTable} - - - - {TsBasicDropTable} - - - - {RsBasicDropTable} - - - -This will delete the `camelot` table from the connected LanceDB database. - - -See the full code for these examples (including helper functions) in the -`basic_usage` file for the appropriate client language in the -[docs repo](https://github.com/lancedb/docs/tree/main/tests). - - -## What about vector indexes? {#what-about-vector-indexes} - -LanceDB supports vector indexes to speed up similarity search on large datasets. -For datasets up to a few hundred thousand vectors, LanceDB's highly efficient kNN -(brute-force) retrieval of nearest neighbors is often sufficient. As your dataset -grows larger, you can create vector indexes on your vector columns to accelerate -search. See the [indexing](/indexing/) documentation for details on how to create and use -vector indexes in LanceDB. - -## Next steps {#next-steps} -Now that you've learned the basics of creating tables, adding data, running -vector search, and modifying table schemas, you're ready to explore more -advanced features of LanceDB. Below are some suggested next pages. - - - - Learn the different approaches to creating and ingesting data into LanceDB tables from various sources. - - - Learn how to update and modify existing LanceDB tables and their data. - - - Understand how to evolve your table schemas and data over time with LanceDB. - - - Explore LanceDB's built-in table versioning and time travel capabilities. - - diff --git a/docs/tables/multimodal.mdx b/docs/tables/multimodal.mdx deleted file mode 100644 index 6da5000..0000000 --- a/docs/tables/multimodal.mdx +++ /dev/null @@ -1,237 +0,0 @@ ---- -title: Multimodal Data (Blobs) -sidebarTitle: "Working with multimodal data" -description: Learn how to store and query multimodal data (images, audio, video) directly in LanceDB using binary columns. -icon: "images" -keywords: ["blob", "large binary", "blobs", "multimodal"] ---- - -import { - PyMultimodalImports as MultimodalImports, - TsMultimodalImports as TsMultimodalImports, - RsMultimodalImports as RsMultimodalImports, - PyCreateDummyData as CreateDummyData, - TsCreateDummyData as TsCreateDummyData, - RsCreateDummyData as RsCreateDummyData, - PyDefineSchema as DefineSchema, - TsDefineSchema as TsDefineSchema, - RsDefineSchema as RsDefineSchema, - PyIngestData as IngestData, - TsIngestData as TsIngestData, - RsIngestData as RsIngestData, - PySearchData as SearchData, - TsSearchData as TsSearchData, - RsSearchData as RsSearchData, - PyProcessResults as ProcessResults, - TsProcessResults as TsProcessResults, - RsProcessResults as RsProcessResults, - PyBlobApiSchema as BlobApiSchema, - TsBlobApiSchema as TsBlobApiSchema, - RsBlobApiSchema as RsBlobApiSchema, - PyBlobApiIngest as BlobApiIngest, - TsBlobApiIngest as TsBlobApiIngest, - RsBlobApiIngest as RsBlobApiIngest, - PyBlobApiToPandas as BlobApiToPandas, - PyQueryToPandasKwargs as QueryToPandasKwargs, -} from '/snippets/multimodal.mdx'; - -LanceDB handles multimodal data—images, audio, video, and PDF files—natively by storing the raw bytes in a binary column alongside your vectors and metadata. This approach simplifies your data infrastructure by keeping the raw assets and their embeddings in the same database, eliminating the need for separate object storage for many use cases. - -This guide demonstrates how to ingest, store, and retrieve image data using standard binary columns, and also introduces the **Lance Blob API** for optimized handling of larger multimodal files. - -## Store binary data {#store-binary-data} - -To store binary data, define a binary Arrow field in your schema (`pa.binary()` in Python, `Binary` in TypeScript, and `DataType::Binary` in Rust). - -### 1\. Setup and imports {#1-setup-and-imports} - -First, import the necessary libraries for LanceDB and Arrow in your SDK. - - - - {MultimodalImports} - - - - {TsMultimodalImports} - - - - {RsMultimodalImports} - - - -### 2\. Prepare data {#2-prepare-data} - -For this example, we'll create some dummy in-memory images. In a real application, you would read these from files or an API. The key is to convert your data (image, audio, etc.) into a raw `bytes` object. - - - - {CreateDummyData} - - - - {TsCreateDummyData} - - - - {RsCreateDummyData} - - - -### 3\. Define the schema {#3-define-the-schema} - -When creating the table, it is **highly recommended** to define the schema explicitly. This ensures that your binary data is correctly interpreted as a `binary` type by Arrow/LanceDB and not as a generic string or list. - - - - {DefineSchema} - - - - {TsDefineSchema} - - - - {RsDefineSchema} - - - -### 4\. Ingest data {#4-ingest-data} - -Now, create the table using the data and the defined schema. - - - - {IngestData} - - - - {TsIngestData} - - - - {RsIngestData} - - - -## Retrieve and use blobs {#retrieve-and-use-blobs} - -When you search your LanceDB table, you can retrieve the binary column just like any other metadata. - - - - {SearchData} - - - - {TsSearchData} - - - - {RsSearchData} - - - -### Convert bytes back to objects {#convert-bytes-back-to-objects} - -Once you have the bytes back from the search result, you can decode them into the original format (for example, an image object or audio buffer). - - - - {ProcessResults} - - - - {TsProcessResults} - - - - {RsProcessResults} - - - -## Large Blobs (Blob API) {#large-blobs-blob-api} - -For larger files like high-resolution images or videos, Lance provides a specialized **Blob API**. By using a large-binary Arrow type (`pa.large_binary()` in Python, `LargeBinary` in TypeScript, and `DataType::LargeBinary` in Rust) and specific metadata, you enable **lazy loading** and optimized encoding. This allows you to work with massive datasets without loading all binary data into memory upfront. - -### 1\. Define a blob schema {#1-define-a-blob-schema} - -To use the Blob API, you must mark the column with `{"lance-encoding:blob": "true"}` metadata. - - - - {BlobApiSchema} - - - - {TsBlobApiSchema} - - - - {RsBlobApiSchema} - - - -### 2\. Ingest large blobs {#2-ingest-large-blobs} - -You can then ingest data normally, and Lance will handle the optimized storage. - - - - {BlobApiIngest} - - - - {TsBlobApiIngest} - - - - {RsBlobApiIngest} - - - - -For more advanced usage, including random access and file-like reading of blobs, see the -Lance format's [blob API documentation](https://lance.org/guide/blob/). - - -### 3\. Convert blob tables to pandas {#3-convert-blob-tables-to-pandas} - -When you call `to_pandas()` on a local LanceDB table that contains Blob API columns, the `blob_mode` argument controls how those columns materialize. This is available in the Python SDK on local tables; remote tables raise `NotImplementedError`. - -`blob_mode` accepts: - -- `"lazy"` (default): returns blob columns as lazy `BlobFile` objects without eagerly materializing their payloads. Use this when you want to stream blob bytes on demand or only inspect a subset of rows. Namespace-backed local tables also use the Lance native blob-aware pandas conversion for lazy blobs; in-memory datasets fall back to the standard PyArrow `to_pandas()` path. -- `"bytes"`: eagerly materializes each blob as `bytes`. Use this when you need the raw payload in the DataFrame, for example to decode an image or audio clip in-process. -- `"descriptions"`: returns blob descriptors (offsets, sizes, and positions) instead of the data itself. Use this when you want to plan I/O without paying the cost of loading every blob. - -`"bytes"` and `"descriptions"` require a filesystem-backed Lance dataset and are not supported on in-memory tables. - -Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas conversion, so you can also pass options like `split_blocks` or `self_destruct`: - - - - {BlobApiToPandas} - - - -Query builders also accept `blob_mode` on their `to_pandas()` method: - -- Plain scans support `"lazy"`, `"bytes"`, and `"descriptions"` with filters, projections, aliases, `limit`, and `offset`. -- Vector, FTS, hybrid, and ordered queries can't materialize blob columns through `to_pandas()`; omit blob columns from the projection for those query shapes. -- This works on both sync and async query builders. Extra PyArrow kwargs like `split_blocks` and `self_destruct` are still forwarded. - - - - {QueryToPandasKwargs} - - - -## Other modalities {#other-modalities} - -The `pa.binary()` and `pa.large_binary()` types are universal. You can use this same pattern for other types of multimodal data: - -- **Audio:** Read `.wav` or `.mp3` files as bytes. -- **Video:** Store video transitions or full clips using the Blob API. -- **PDFs/Documents:** Store the raw file content for document search. diff --git a/docs/tables/schema.mdx b/docs/tables/schema.mdx deleted file mode 100644 index ef8db19..0000000 --- a/docs/tables/schema.mdx +++ /dev/null @@ -1,486 +0,0 @@ ---- -title: "Schema and Data Evolution" -sidebarTitle: "Schema & data evolution" -description: Learn how to manage table schemas in LanceDB, including adding, altering, and dropping columns. -icon: "boxes-stacked" ---- -import { - PySchemaAddSetup as SchemaAddSetup, - TsSchemaAddSetup as TsSchemaAddSetup, - RsSchemaAddSetup as RsSchemaAddSetup, - PyAddColumnsCalculated as AddColumnsCalculated, - TsAddColumnsCalculated as TsAddColumnsCalculated, - RsAddColumnsCalculated as RsAddColumnsCalculated, - PyAddColumnsDefaultValues as AddColumnsDefaultValues, - TsAddColumnsDefaultValues as TsAddColumnsDefaultValues, - RsAddColumnsDefaultValues as RsAddColumnsDefaultValues, - PyAddColumnsNullable as AddColumnsNullable, - TsAddColumnsNullable as TsAddColumnsNullable, - RsAddColumnsNullable as RsAddColumnsNullable, - PyAddFeatureColumnsSql as AddFeatureColumnsSql, - TsAddFeatureColumnsSql as TsAddFeatureColumnsSql, - RsAddFeatureColumnsSql as RsAddFeatureColumnsSql, - PySchemaAlterSetup as SchemaAlterSetup, - TsSchemaAlterSetup as TsSchemaAlterSetup, - RsSchemaAlterSetup as RsSchemaAlterSetup, - PyAlterColumnsRename as AlterColumnsRename, - TsAlterColumnsRename as TsAlterColumnsRename, - RsAlterColumnsRename as RsAlterColumnsRename, - PyAlterColumnsDataType as AlterColumnsDataType, - TsAlterColumnsDataType as TsAlterColumnsDataType, - RsAlterColumnsDataType as RsAlterColumnsDataType, - PyAlterColumnsNullable as AlterColumnsNullable, - TsAlterColumnsNullable as TsAlterColumnsNullable, - RsAlterColumnsNullable as RsAlterColumnsNullable, - PyAlterColumnsMultiple as AlterColumnsMultiple, - TsAlterColumnsMultiple as TsAlterColumnsMultiple, - PyAlterColumnsWithExpression as AlterColumnsWithExpression, - TsAlterColumnsWithExpression as TsAlterColumnsWithExpression, - RsAlterColumnsMultiple as RsAlterColumnsMultiple, - RsAlterColumnsWithExpression as RsAlterColumnsWithExpression, - PySchemaDropSetup as SchemaDropSetup, - TsSchemaDropSetup as TsSchemaDropSetup, - RsSchemaDropSetup as RsSchemaDropSetup, - PyDropColumnsSingle as DropColumnsSingle, - TsDropColumnsSingle as TsDropColumnsSingle, - RsDropColumnsSingle as RsDropColumnsSingle, - PyDropColumnsMultiple as DropColumnsMultiple, - TsDropColumnsMultiple as TsDropColumnsMultiple, - RsDropColumnsMultiple as RsDropColumnsMultiple, - PyAlterVectorColumn as AlterVectorColumn, - TsAlterVectorColumn as TsAlterVectorColumn, - RsAlterVectorColumn as RsAlterVectorColumn, - PySchemaFieldMetadataMerge as SchemaFieldMetadataMerge, - TsSchemaFieldMetadataMerge as TsSchemaFieldMetadataMerge, - RsSchemaFieldMetadataMerge as RsSchemaFieldMetadataMerge, - PySchemaFieldMetadataReplace as SchemaFieldMetadataReplace, - TsSchemaFieldMetadataReplace as TsSchemaFieldMetadataReplace, - RsSchemaFieldMetadataReplace as RsSchemaFieldMetadataReplace, -} from '/snippets/tables.mdx'; - -Schema evolution enables non-breaking modifications to a database table's structure — such as adding columns, altering data types, or dropping fields — to adapt to evolving data requirements without service interruptions. -LanceDB supports ACID-compliant schema evolution through granular operations (add/alter/drop columns), allowing you to: - -* Iterate Safely: Modify schemas in production with versioned datasets and backward compatibility -* Scale Seamlessly: Handle ML model iterations, regulatory changes, or feature additions -* Optimize Continuously: Remove unused fields or enforce new constraints without downtime - -## Schema evolution operations {#schema-evolution-operations} - -LanceDB supports four primary schema evolution operations: - -1. **Adding new columns**: Extend your table with additional attributes -2. **Altering existing columns**: Change column names, data types, or nullability -3. **Updating field metadata**: Attach or change per-column Arrow metadata -4. **Dropping columns**: Remove unnecessary columns from your schema - - - -Schema evolution operations are applied immediately but do not typically require rewriting all data. However, data type changes may involve more substantial operations. - - -Each schema evolution operation commits a new table version and returns status metadata such as -the committed `version`. Run these operations from a mutable table handle; if you checked out an -older version for reads, call `checkout_latest` / `checkoutLatest` before modifying the schema. - -## Add new columns {#add-new-columns} - -You can add new columns to a table with the [`add_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.add_columns) -method in Python, [`addColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#addcolumns) in TypeScript/JavaScript, or `add_columns` in Rust. -New columns are populated based on SQL expressions you provide. - - -### Set up the example table {#set-up-the-example-table} - -First, let's create a sample table with product data to demonstrate schema evolution: - - - - {SchemaAddSetup} - - - - {TsSchemaAddSetup} - - - - {RsSchemaAddSetup} - - - -### Add derived columns {#add-derived-columns} - -You can add new columns that are derived from existing data using SQL expressions. -For feature engineering on large existing tables, group related derived features into -one `add_columns` operation instead of running many separate writes. This creates one -new table version for the schema change and computes the new columns from the existing -rows, which avoids growing the table's version history with many small updates. - - - - {AddColumnsCalculated} - - - - {TsAddColumnsCalculated} - - - - {RsAddColumnsCalculated} - - - -The same call can add multiple derived columns at once. For example, if you are -building several lightweight features from existing product fields, pass all of the -new column expressions together: - - - - {AddFeatureColumnsSql} - - - - {TsAddFeatureColumnsSql} - - - - {RsAddFeatureColumnsSql} - - - - -LanceDB `add_columns` does not currently accept Python callables, batch UDFs, or -PyArrow `RecordBatch` iterators for populating new columns. New column values must be -defined with SQL expressions, or added as NULL columns from an Arrow field or schema. -If your transformation cannot be expressed in SQL, compute the values outside -`add_columns` before writing them back through another workflow. - - -### Add columns with default values {#add-columns-with-default-values} - -Add boolean columns with default values for status tracking: - - - - {AddColumnsDefaultValues} - - - - {TsAddColumnsDefaultValues} - - - - {RsAddColumnsDefaultValues} - - - -### Add nullable columns {#add-nullable-columns} - -Add timestamp columns that can contain NULL values: - - - - {AddColumnsNullable} - - - - {TsAddColumnsNullable} - - - - {RsAddColumnsNullable} - - - - -When adding columns that should contain NULL values, be sure to cast the NULL to the appropriate type, e.g., `cast(NULL as timestamp)`. - - -## Alter existing columns {#alter-existing-columns} - -You can alter columns using the [`alter_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.alter_columns) -method in Python, [`alterColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#altercolumns) in TypeScript/JavaScript, or `alter_columns` in Rust. This allows you to: - -- Rename a column -- Change a column's data type -- Modify nullability (whether a column can contain NULL values) - - -### Set up the example table {#set-up-the-example-table-2} - -Create a table with a custom schema to demonstrate column alterations: - - - - {SchemaAlterSetup} - - - - {TsSchemaAlterSetup} - - - - {RsSchemaAlterSetup} - - - -### Rename columns {#rename-columns} - -Change column names to better reflect their purpose: - - - - {AlterColumnsRename} - - - - {TsAlterColumnsRename} - - - - {RsAlterColumnsRename} - - - -### Change data types {#change-data-types} - -Convert column data types for better performance or compatibility: - - - - {AlterColumnsDataType} - - - - {TsAlterColumnsDataType} - - - - {RsAlterColumnsDataType} - - - -### Make columns nullable {#make-columns-nullable} - -You can alter columns to contain NULL values: - - - - {AlterColumnsNullable} - - - - {TsAlterColumnsNullable} - - - - {RsAlterColumnsNullable} - - - -Changing a column to nullable affects future writes and merges too: missing values are accepted -only when the target column is nullable. - -### Multiple changes at once {#multiple-changes-at-once} - -Apply several alterations in a single operation: - - - - {AlterColumnsMultiple} - - - - {TsAlterColumnsMultiple} - - - - {RsAlterColumnsMultiple} - - - -### Expression-based type changes {#expression-based-type-changes} - -For transformations that are not simple casts (for example, converting `"$100"` to an integer), use a SQL-expression column add, then drop and rename: - - - - {AlterColumnsWithExpression} - - - - {TsAlterColumnsWithExpression} - - - - {RsAlterColumnsWithExpression} - - - -### Alter embedding types and dimensions {#alter-embedding-types-and-dimensions} - -It's quite common to need to change an embedding column's schema, in case a new model becomes available with a different embedding dimension. -- In Python, the example shows an in-place type update when the cast is compatible. -- In TypeScript and Rust, the example shows a dimension change (`384 -> 1024`), which cannot be cast in-place. - -For dimension changes, use this 3-step pattern: add a new column with the target type, drop the old column, then rename the new column to the original name. - - - - {AlterVectorColumn} - - - - {TsAlterVectorColumn} - - - - {RsAlterVectorColumn} - - - - -**`FixedSizeList` Dimension Changes in TypeScript and Rust** - -`alterColumns` / `alter_columns` can cast between compatible types, but changing `FixedSizeList` dimensions (for example `384 -> 1024`) is not a compatible cast. -For such cases, use `addColumns` / `add_columns` (with `arrow_cast`), then `dropColumns` / `drop_columns`, then rename the replacement column. - - - -Changing data types requires rewriting the column data and may be resource-intensive for large tables. Renaming columns or changing nullability is more efficient as it only updates metadata. - - -## Update field metadata {#update-field-metadata} - -Each column in a LanceDB table can carry a small key/value map of Arrow field metadata — useful -for annotating columns with units, provenance, PII flags, embedding model versions, or any other -schema-level context your application needs. - -Use [`update_field_metadata`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.update_field_metadata) -in Python, [`updateFieldMetadata`](https://lancedb.github.io/lancedb/js/classes/Table/#updatefieldmetadata) -in TypeScript/JavaScript, or `update_field_metadata` in Rust to add, change, or remove these -key/value pairs without rewriting the column data. Each call commits a new table version and returns -the new `version`. - -Each update targets one field by **dot-path**: top-level columns are addressed by name (for -example `"embedding"`), and nested fields by their full path (for example `"address.zip"`). By -default, the keys you pass are **merged** into the field's existing metadata — keys you do not -mention are preserved, and passing `None` (Python) or `null` (TypeScript) deletes a key. Set -`replace: true` to swap the field's entire metadata map instead of merging. - - - - {SchemaFieldMetadataMerge} - - - - {TsSchemaFieldMetadataMerge} - - - - {RsSchemaFieldMetadataMerge} - - - -To overwrite a field's metadata entirely instead of merging, set `replace` to `true`: - - - - {SchemaFieldMetadataReplace} - - - - {TsSchemaFieldMetadataReplace} - - - - {RsSchemaFieldMetadataReplace} - - - - -You can pass multiple updates in a single call to change metadata on several fields at once — -each call commits a single new table version. - - -### Canonical metadata keys {#canonical-metadata-keys} - -You can use any string as a metadata key. By convention, LanceDB treats keys prefixed with -`lancedb:` as canonical. LanceDB Enterprise displays these keys in the UI, and agents can rely -on finding this information in a consistent place. Use them when they fit your use case: - -| Key | Purpose | Example | -| --- | --- | --- | -| `lancedb:description` | Human-readable description of the field. | `"CLIP embedding of the product image"` | -| `lancedb:tag:` | User-defined key/value tag, where the suffix names the tag category. | `"lancedb:tag:model": "clip"` | -| `lancedb:logical-column` | Groups related columns into one logical column. For example, `feature_v1` and `feature_v2` might belong to the same logical column. | `"lancedb:logical-column": "feature"` | -| `lancedb:status` | Life cycle state of the column: `production`, `candidate`, `deprecated`, or `archived`. | `"lancedb:status": "production"` | - - -These keys are conventions, not enforced constraints. LanceDB does not validate their values, -but following the conventions keeps your tables consistent with Enterprise tooling. - - -## Drop columns {#drop-columns} - -You can remove columns using the [`drop_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.drop_columns) - method in Python, [`dropColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#dropcolumns) in TypeScript/JavaScript, or `drop_columns` in Rust. - - -### Set Up the example table {#set-up-the-example-table-3} - -Create a table with temporary columns that we'll remove: - - - - {SchemaDropSetup} - - - - {TsSchemaDropSetup} - - - - {RsSchemaDropSetup} - - - -### Drop single columns {#drop-single-columns} - -Remove individual columns that are no longer needed: - - - - {DropColumnsSingle} - - - - {TsDropColumnsSingle} - - - - {RsDropColumnsSingle} - - - -### Drop multiple columns {#drop-multiple-columns} - -Remove several columns at once for efficiency: - - - - {DropColumnsMultiple} - - - - {TsDropColumnsMultiple} - - - - {RsDropColumnsMultiple} - - - - -Dropping columns cannot be undone. Make sure you have backups or are certain before removing columns. - diff --git a/docs/tables/update.mdx b/docs/tables/update.mdx deleted file mode 100644 index 64e169e..0000000 --- a/docs/tables/update.mdx +++ /dev/null @@ -1,440 +0,0 @@ ---- -title: "Updating and Modifying Table Data" -sidebarTitle: "Update/modify data" -description: "Learn how to update, merge, and delete rows in a LanceDB table." -icon: "clone" ---- -import { - PyUpdateConnectEnterprise as UpdateConnectEnterprise, - PyUpdateConnectLocal as UpdateConnectLocal, - PyUpdateExampleTableSetup as UpdateExampleTableSetup, - PyUpdateOperation as UpdateOperation, - PyUpdateUsingSql as UpdateUsingSql, - PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly, - PyInsertIfNotExists as InsertIfNotExists, - PyMergeUpdateInsert as MergeUpdateInsert, - PyMergeDeleteMissingBySource as MergeDeleteMissingBySource, - PyMergePartialColumns as MergePartialColumns, - PyDeleteOperation as DeleteOperation, - PyUpdateOptimizeCleanup as UpdateOptimizeCleanup, - TsUpdateConnectEnterprise, - TsUpdateConnectLocal, - TsUpdateExampleTableSetup, - TsUpdateOperation, - TsUpdateUsingSql, - TsMergeMatchedUpdateOnly, - TsInsertIfNotExists, - TsMergeUpdateInsert, - TsMergeDeleteMissingBySource, - TsMergePartialColumns, - TsDeleteOperation, - TsUpdateOptimizeCleanup, - RsUpdateConnectEnterprise, - RsUpdateConnectLocal, - RsUpdateExampleTableSetup, - RsUpdateMakeUsersReader, - RsUpdateOperation, - RsUpdateUsingSql, - RsMergeMatchedUpdateOnly, - RsInsertIfNotExists, - RsMergeUpdateInsert, - RsMergeDeleteMissingBySource, - RsMergePartialColumns, - RsDeleteOperation, - RsUpdateOptimizeCleanup, -} from '/snippets/tables.mdx'; - -Updating or modifying data involves changing rows in an existing table. -LanceDB provides two families of write operations that can modify data in a table: - -- `update(...)`: mutate existing rows that match a SQL filter. -- `merge_insert(...)`: compare incoming rows to existing rows by key, then choose what to do for each case. - -The `update` method is simpler to use when you already know which rows you want to modify and you do not need to compare against an incoming dataset. The `merge_insert` method is more powerful when you have a new dataset that you want to merge into an existing table, and you want LanceDB to handle the logic of comparing against existing rows by key. - -Let's look at an example that demonstrates these operations in practice. - -## Connect to LanceDB {#connect-to-lancedb} - -Connect to your local LanceDB instance: - - - - {UpdateConnectLocal} - - - - {TsUpdateConnectLocal} - - - - {RsUpdateConnectLocal} - - - -Or, connect to LanceDB Enterprise: - - - - {UpdateConnectEnterprise} - - - - {TsUpdateConnectEnterprise} - - - - {RsUpdateConnectEnterprise} - - - -In the Rust snippets, a `make_users_reader` helper is used to build Arrow input data. - - - - {RsUpdateMakeUsersReader} - - - -## Create the example table {#create-the-example-table} - -We'll start by creating a simple table with `id`, `name`, and `login_count` columns. All examples below use the same table. - - - {UpdateExampleTableSetup} - - - - {TsUpdateExampleTableSetup} - - - - {RsUpdateExampleTableSetup} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | - -The example above shows a PyArrow schema. You can just as well create the table using other -table creation patterns (Pandas, Polars, Pydantic, iterators, etc.) -- see the [ingestion](/tables/create/) guide for more details. - -## Choose a write method {#choose-a-write-method} - -| Family | Method | Use this when... | -| --------------- | --------------- | ---------------- | -| `update` | `update(where=..., values=...)` | You want to edit rows that already exist, using a SQL filter. | -| `merge_insert` | `.when_matched_update_all()` | You have incoming rows and want to update keys that already exist in the table. | -| `merge_insert` | `.when_not_matched_insert_all()` | You have incoming rows and want to insert keys that do not exist yet. | -| `merge_insert` | `.when_matched_update_all()` + `.when_not_matched_insert_all()` | You want both behaviors together (often called **upsert**: update existing keys **and** insert missing keys in the same operation). | -| `merge_insert` | `.when_not_matched_by_source_delete(...)` | You want to remove target rows that are missing from the incoming source set. | - -Write operations return status metadata. For example, `update` reports updated row count and -committed `version`, `delete` reports deleted row count and `version`, and `merge_insert` reports -inserted, updated, deleted, retry-attempt, and `version` fields. Writes require a mutable table -handle; after checking out an older version for reads, call `checkout_latest` / `checkoutLatest` -before modifying data. -The committed `version` advances even for writes that affect zero rows, such as a delete predicate -that matches nothing. - -## Update rows {#update-rows} - -Use `update` when you already know which target rows to modify and you do not need to compare against an incoming dataset. - - - - {UpdateOperation} - - - - {TsUpdateOperation} - - - - {RsUpdateOperation} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bobby | 20 | - - -Updating nested columns is not yet supported. - - -## Update rows with SQL expressions {#update-rows-with-sql-expressions} - -Use `values_sql` when you want to use SQL-like expressions to update rows. This is useful for operations like incrementing a counter, or setting a column value based on another column. - - - - {UpdateUsingSql} - - - - {TsUpdateUsingSql} - - - - {RsUpdateUsingSql} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 21 | - - -See the [SQL queries](/search/sql/) page for more information on the supported SQL syntax. - - -When rows are updated, they are moved out of any existing index. The row will still show up in search queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider triggering an index rebuild afterwards. - -## Merge incoming rows by key {#merge-incoming-rows-by-key} - -Merging is different from updating because it involves comparing incoming rows to existing rows by key, and then choosing what to do based on whether the key exists in the target table or not. -The `merge_insert(""..."")` method lets you do this. - -In merge operations, rows are split into three groups: - -- **Matched**: key exists in both source and target. -- **Not matched**: key exists only in source. -- **Not matched by source**: key exists only in target. - -Conditional merge clauses can compare old and new values. Use the `target.` prefix for the -existing table row and `source.` for the incoming row, for example -`target.last_update < source.last_update`. - - -**Use scalar indexes to speed up merge insert** - -The merge insert command performs a join between the input data and the target table `on` the key you provide. This requires scanning that entire column, which can be expensive for large tables. To speed up this operation, create a scalar index on the join column, which will allow LanceDB to find matches without scanning the whole table. - -Read more about scalar indices in the [Scalar Index](/indexing/scalar-index/) guide. - - - -If you see this HTTP 400 error from `merge_insert`: `Bad request: Merge insert cannot be performed because the number of unindexed rows exceeds the maximum of 10000`. Verify that the scalar index on the join column is up to date before retrying. - - - -**Rust: build merge predicates with DataFusion expressions** - -In the Rust SDK, you can pass a `datafusion_expr::Expr` directly instead of a SQL string by using -`when_matched_update_all_expr` and `when_not_matched_by_source_delete_expr` on `MergeInsertBuilder`. -This is useful when you are constructing predicates programmatically and want to avoid building a SQL string. - -These methods are only supported on local tables. Calling them against a remote table returns a -`NotSupported` error — use the SQL string variants (`when_matched_update_all` / -`when_not_matched_by_source_delete`) for remote tables. - - -Like the create table and add APIs, the merge insert API will automatically compute embeddings based on the [embedding registry](/embedding/index#embedding-registry) if the table has an embedding definition in its schema. - -During `merge_insert`, if the input data doesn't contain the source column (i.e., the original field used to generate embeddings, such as text for a text embedding model or `image_uri` for an image model), or if a vector value is already provided, LanceDB skips embedding generation for that row. Embeddings are only auto-generated when that source field is present in the incoming data, **and** the vector field is empty. - -Primary keys in LanceDB are metadata used by operations such as `merge_insert`; they are not -enforced as uniqueness constraints on ordinary writes. Keep using `merge_insert` or your own -deduplication logic when you need key-based upsert semantics. - -### Update matched rows only {#update-matched-rows-only} - -This updates keys that already exist in the target table. Source rows with new keys are ignored. - - - - {MergeMatchedUpdateOnly} - - - - {TsMergeMatchedUpdateOnly} - - - - {RsMergeMatchedUpdateOnly} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bobby | 21 | - -### Insert unmatched rows only {#insert-unmatched-rows-only} - -This inserts only brand-new keys from the source. Existing keys are left unchanged. - - - - {InsertIfNotExists} - - - - {TsInsertIfNotExists} - - - - {RsInsertIfNotExists} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | -| 3 | Charlie | 5 | - -### Update matched rows and insert unmatched rows {#update-matched-rows-and-insert-unmatched-rows} - -Use both `when_matched_update_all()` and `when_not_matched_insert_all()` when you want to update existing keys and insert missing keys in one operation. - - -This is a conventional **upsert**. - - - - - {MergeUpdateInsert} - - - - {TsMergeUpdateInsert} - - - - {RsMergeUpdateInsert} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bobby | 21 | -| 3 | Charlie | 5 | - -### Delete target rows that are missing from source {#delete-target-rows-that-are-missing-from-source} - -Use `when_not_matched_by_source_delete()` when you want to remove any target row that does not appear in the incoming source data. - - - - {MergeDeleteMissingBySource} - - - - {TsMergeDeleteMissingBySource} - - - - {RsMergeDeleteMissingBySource} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 2 | Bobby | 21 | -| 3 | Charlie | 5 | - -In the example above, LanceDB matches rows by `id`. Rows with `id=2` and `id=3` exist in both the table and incoming data, so they are updated. Row `id=1` exists only in the target, so it is deleted. - -### Use partial columns in merge updates {#use-partial-columns-in-merge-updates} - -Merge updates do not require you to provide values for all columns. You can provide only a subset of columns in source rows. For matched rows, only the provided columns are updated. - - - - {MergePartialColumns} - - - - {TsMergePartialColumns} - - - - {RsMergePartialColumns} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bobby | 20 | -| 3 | Charlie | null | - -Note that in the example above, when `merge_insert` creates a new row, any missing columns are written as `null`. If a missing column is non-nullable in your schema, the insert will fail. - -## Delete rows {#delete-rows} - -Delete operations **soft delete** rows that match a given condition. -The underlying data is not immediately removed, but is marked -for deletion (in the [deletion files](https://lance.org/format/table/#deletion-files) at the Lance format level) and excluded from query results. - - - - {DeleteOperation} - - - - {TsDeleteOperation} - - - - {RsDeleteOperation} - - - -Expected table contents: - -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | - - - -**Deleting rows removes them from the index** - -When rows are deleted, those rows are also excluded from the index segments, so indexed queries will not return them either. If ALL the rows are deleted (i.e., the table is emptied), ensure that you recreate the index after ingesting new data. - - -To permanently remove deleted rows, you can optimize the table, which will run compaction and cleans up the soft-deleted rows, which frees up storage space. - -- In LanceDB OSS, compaction and cleanup are manual. Run `table.optimize()` regularly to free up disk space. -- In LanceDB Enterprise, files aren't cleaned up by default. You can configure automatic compaction and cleanup behavior at cluster setup time to suit your organization's retention policy. - -By default, table cleanup removes data up to 7 days ago. If you need to reclaim space from deleted rows more aggressively, manually call `table.optimize()` use a shorter retention window as follows: - - - - {UpdateOptimizeCleanup} - - - - {TsUpdateOptimizeCleanup} - - - - {RsUpdateOptimizeCleanup} - - diff --git a/docs/tables/versioning.mdx b/docs/tables/versioning.mdx deleted file mode 100644 index ca2f507..0000000 --- a/docs/tables/versioning.mdx +++ /dev/null @@ -1,299 +0,0 @@ ---- -title: "Versioning and Reproducibility" -sidebarTitle: "Versioning" -description: "Learn how to implement versioning and ensure reproducibility in LanceDB. Includes version control, data snapshots, and audit trails." -icon: "clock" ---- -import { - PyVersioningBasicSetup as VersioningBasicSetup, - TsVersioningBasicSetup as TsVersioningBasicSetup, - RsVersioningBasicSetup as RsVersioningBasicSetup, - PyVersioningCheckInitialVersion as VersioningCheckInitialVersion, - TsVersioningCheckInitialVersion as TsVersioningCheckInitialVersion, - RsVersioningCheckInitialVersion as RsVersioningCheckInitialVersion, - PyVersioningUpdateData as VersioningUpdateData, - TsVersioningUpdateData as TsVersioningUpdateData, - RsVersioningUpdateData as RsVersioningUpdateData, - PyVersioningAddData as VersioningAddData, - TsVersioningAddData as TsVersioningAddData, - RsVersioningAddData as RsVersioningAddData, - PyVersioningCheckVersionsAfterMod as VersioningCheckVersionsAfterMod, - TsVersioningCheckVersionsAfterMod as TsVersioningCheckVersionsAfterMod, - RsVersioningCheckVersionsAfterMod as RsVersioningCheckVersionsAfterMod, - PyVersioningListAllVersions as VersioningListAllVersions, - TsVersioningListAllVersions as TsVersioningListAllVersions, - RsVersioningListAllVersions as RsVersioningListAllVersions, - PyVersioningRollback as VersioningRollback, - TsVersioningRollback as TsVersioningRollback, - RsVersioningRollback as RsVersioningRollback, - PyVersioningCheckoutLatest as VersioningCheckoutLatest, - TsVersioningCheckoutLatest as TsVersioningCheckoutLatest, - RsVersioningCheckoutLatest as RsVersioningCheckoutLatest, - PyVersioningDeleteData as VersioningDeleteData, - TsVersioningDeleteData as TsVersioningDeleteData, - RsVersioningDeleteData as RsVersioningDeleteData, - PyVersioningTags as VersioningTags, - TsVersioningTags as TsVersioningTags, - RsVersioningTags as RsVersioningTags, - RsVersioningMakeQuotesReader as RsVersioningMakeQuotesReader, -} from '/snippets/tables.mdx'; - -This page shows the core table-versioning APIs used in the code snippets for Python, TypeScript, and Rust. -Each operation below maps directly to methods shown in the examples. - -## Basic Versioning Example {#basic-versioning-example} - -Let's create a table with sample data to demonstrate LanceDB's versioning capabilities: - -### Set Up the Table {#set-up-the-table} - -First, let's create a table with some sample data: - - - - {VersioningBasicSetup} - - - - {TsVersioningBasicSetup} - - - - {RsVersioningBasicSetup} - - - - - - - {RsVersioningMakeQuotesReader} - - - - -### Check Initial Version {#check-initial-version} - -After creating the table, let's check the initial version information: - - - - {VersioningCheckInitialVersion} - - - - {TsVersioningCheckInitialVersion} - - - - {RsVersioningCheckInitialVersion} - - - -## Modify Data {#modify-data} - -When you modify data through operations like update or delete, LanceDB automatically creates new versions. - -### Update Existing Data {#update-existing-data} - -Let's update some existing records to see versioning in action: - - - - {VersioningUpdateData} - - - - {TsVersioningUpdateData} - - - - {RsVersioningUpdateData} - - - -### Add New Data {#add-new-data} - -Now let's add more records to the table: - - - - {VersioningAddData} - - - - {TsVersioningAddData} - - - - {RsVersioningAddData} - - - -### Check Version Changes {#check-version-changes} - -Let's see how the versions have changed after our modifications: - - - - {VersioningCheckVersionsAfterMod} - - - - {TsVersioningCheckVersionsAfterMod} - - - - {RsVersioningCheckVersionsAfterMod} - - - -## Rollback to Previous Versions {#rollback-to-previous-versions} - -LanceDB supports fast rollbacks to any previous version without data duplication. - -### View All Versions {#view-all-versions} - -First, let's see all the versions we've created: - - - - {VersioningListAllVersions} - - - - {TsVersioningListAllVersions} - - - - {RsVersioningListAllVersions} - - - -### Restore a Version Snapshot {#restore-a-version-snapshot} - -Now let's restore a captured version snapshot: - - - - {VersioningRollback} - - - - {TsVersioningRollback} - - - - {RsVersioningRollback} - - - -## Tag-Based Versioning {#tag-based-versioning} - -Numeric table versions like `v3` or `v17` are precise but hard to remember. Tags -let you attach human-readable labels (e.g., `"prod"`, `"baseline"`, -`"q3-evaluation"`) to specific versions and check those out by name. They are -conceptually similar to git tags, and unlike numeric versions, **tagged versions -are preserved when old versions are pruned** (see the cleanup note at the bottom -of this page). - -The tags API supports the standard CRUD operations — create, list, update, delete — -plus checking out by tag name. - - - - {VersioningTags} - - - - {TsVersioningTags} - - - - {RsVersioningTags} - - - - -Deleting a tag only removes the label, not the version it points to. After -deletion, the underlying table version becomes eligible for cleanup again. - - -## Branches {#branches} - -Beyond linear history, LanceDB also supports **branches** — isolated, writable -lines of history forked from `main` (or a specific version). Whereas tags and -`checkout` give you read-only views of existing versions, a branch has its own -writable history, making it ideal for experiments, backfills, and migrations -that you want to keep separate from production reads on `main`. - -Branches are covered in their own guide: see [Branches](/tables/branching). - -## Delete Data From the Table {#delete-data-from-the-table} - -Let's demonstrate how deletions also create new versions: - -### Go Back to Latest Version {#go-back-to-latest-version} - -First, let's return to the latest version: - - - - {VersioningCheckoutLatest} - - - - {TsVersioningCheckoutLatest} - - - - {RsVersioningCheckoutLatest} - - - -### Delete Data {#delete-data} - -Now let's delete some data to see how it affects versioning: - - - - {VersioningDeleteData} - - - - {TsVersioningDeleteData} - - - - {RsVersioningDeleteData} - - - -### Version History and Operations {#version-history-and-operations} - -On a fresh table, the snippets in this guide produce this version sequence: - -1. `v1`: create table (`create_table` / `createTable` / `create_table`) -2. `v2`: update rows (`update`) -3. `v3`: add rows (`add`) -4. `v4`: restore snapshot (`restore`) from `version_after_mod`/`versionAfterMod` -5. `v5`: delete rows (`delete`) - -Read-only and checkout operations shown here (`list_versions`/`listVersions`, `version`, `checkout`, `checkout_latest`/`checkoutLatest`) do not create new versions. - -The version metadata fields can differ by backend. Direct table-backed version listing exposes a -timestamp, while namespace-backed listing may expose fields such as `manifest_path`, -`manifest_size`, `e_tag`, and `timestamp_millis`. In deployments that use managed versioning, -prefer the table version APIs exposed by LanceDB Enterprise or the namespace service instead of -mixing in lower-level Lance file operations. - - -**System Operations** - -System operations like `optimize()`, index updates, and table compaction also increment table version numbers. -In LanceDB OSS and Enterprise, `optimize()` can prune older versions based on its retention setting (`cleanup_older_than`, 7 days by default), -which is when old-version files are removed and disk space is reclaimed. - -**Tagged versions are exempt from cleanup.** A version with a tag pointing at it is -retained regardless of age, and its files are not removed by `optimize()`. To make -a tagged version eligible for pruning, [delete the tag](#tag-based-versioning) first. - diff --git a/docs/training/index.mdx b/docs/training/index.mdx deleted file mode 100644 index a1510a2..0000000 --- a/docs/training/index.mdx +++ /dev/null @@ -1,512 +0,0 @@ ---- -title: "Loading Data for Model Training" -sidebarTitle: "Data loading and shuffles" -description: "Stream, shuffle, transform, and resume model training data from LanceDB." -icon: boxes-stacked ---- - -LanceDB makes an excellent data backend for training machine learning models. A `Table` can be used directly as input -to a data loader, but this is typically limited. A `Permutation` gives you control over which rows are accessed and in -what order. For a more complete solution, LanceDB also provides a streaming data loader through `StreamingDataset`. -This PyTorch `IterableDataset` adapts the lower-level `Permutation` API and adds prefetching, elastic determinism, -resumability, and multithreaded transformations. - -## Basic data loading {#basic-data-loading} - -Most model training frameworks iterate through data in batches and feed this data into the model. This process is -often referred to as **data loading**. The simplest way to load data into a model is to iterate a LanceDB table in -a loop and feed the data into the model. - - -```py Python icon=Python -import lancedb - -db = lancedb.connect("file://some/db/path") -table = db.open_table("some_table") - -for batch in table: - print(batch.to_pydict()) -``` - - -In practice, this is too simplistic for effective training. We may not want to load all the data, or we may want -to load the data in a different order, or we may need to apply some sort of processing to the data before training. -To achieve this, we can use the `StreamingDataset`. - - -```py Python icon=Python -import lancedb -import torch -from lancedb.streaming import StreamingDataset - -db = lancedb.connect("file://some/db/path") -table = db.open_table("some_table") - -dataset = StreamingDataset(table, shuffle_seed=42) -dataloader = torch.utils.data.DataLoader( - dataset, - batch_size=128, - num_workers=0, -) - -for batch in dataloader: - train_step(batch) -``` - - -`StreamingDataset` yields plain Python dictionaries by default. PyTorch's default collation function combines those -samples into batches. You can also iterate the dataset directly when you do not need collation. - - -`StreamingDataset` is built on the permutation API and works with both local LanceDB tables when using OSS, and -remote tables accessed through LanceDB Enterprise. The underlying table data can live on local disk or object storage. - - -Use the streaming data loader when the training data does not fit in memory, when you need deterministic global batches -across cluster sizes, or when you want filtering and prefetching to happen before PyTorch requests individual samples. -Use `Permutation` directly when you need map-style random access instead. Only one iterator can be active on a -`StreamingDataset` instance at a time; create a separate instance for each concurrent consumer. - -## Advanced data loading {#advanced-data-loading} - -The `StreamingDataset` wraps a LanceDB `Table` and, by default, adds prefetching and conversion from Arrow to Python. -It can also handle more advanced scenarios. To explain these, consider a model trained with stochastic gradient -descent (SGD) and distributed data parallelism (DDP). In this example, we need to load batches onto multiple GPUs -across multiple servers. After each batch is processed, the GPUs exchange weights and the next batch is loaded. We -will use the following PyTorch terms: - -- **World size** - The number of GPUs that we are loading. For example, if we have 2 servers and each server has 4 - GPUs, the world size is 8. -- **Rank** - The identifier of the process loading data for a GPU. It is an integer in the range `[0, world_size)`. - Each rank gets its own portion of the data. -- **Global batch size** - The number of rows processed across all GPUs in each step of the SGD algorithm. For - example, if we have 8 GPUs and a global batch size of 1024, we load 128 rows onto each GPU for each step. -- **Batch size** - The number of rows processed by one GPU in each step. In the preceding example, the batch size - is 128. - -Other concepts, such as read batch size and `num_workers`, are introduced in the relevant sections below. - -### Prefetching {#prefetching} - -PyTorch datasets were originally built around in-memory structures like a Pandas DataFrame. When they are iterated, -they yield a single sample at a time. This makes sense for a simple in-memory structure, but accessing data on object -storage one row at a time introduces too much per-call overhead. To avoid this, `StreamingDataset` fetches and -transforms data in batches. The `read_batch_size` parameter controls how many rows are fetched from each split per -storage request and defaults to `64`. - -In addition to batching requests, the prefetching mechanism reads ahead in the background. While one batch is being -transformed and processed by the GPU, `StreamingDataset` reads subsequent batches. The `prefetch_batches` parameter -controls how many batches are kept in flight per split and defaults to `4`. A larger value can provide more buffering -against jittery workloads, but it also increases memory use and I/O concurrency. - - -```py Python icon=Python -ds = StreamingDataset( - table, - shuffle_seed=42, - read_batch_size=256, # rows fetched per LanceDB call per split - prefetch_batches=8, # batches to keep in flight per split -) -``` - - -### Transformation {#transformation} - -Many model training workloads require a transformation step between loading the data and training the model. For -example, we may need to decode images, tokenize text, or normalize data. A transformation function can be provided -using the `transform` parameter. Transformations can be expensive, so `StreamingDataset` applies them with a -`ThreadPoolExecutor` whose worker count equals the number of available CPUs. - -Transformations are applied to batches, not individual samples, to amortize per-batch overhead. A transformation -function receives a PyArrow `RecordBatch` and must return an iterable with exactly one output sample for every input -row. The sample format should match what your data loader expects. For example, PyTorch's default collation function -accepts several sample types, with a Python dictionary being one of the most common. When no transform is provided, -the default transform converts the Arrow record batch into Python dictionaries without further processing. - - -```py Python icon=Python -import pyarrow as pa - -def normalize(batch: pa.RecordBatch) -> list[dict]: - # This pure-Python loop holds the GIL and is shown for illustration only. - # In practice, prefer a library like torchvision or numpy that releases the - # GIL so the ThreadPoolExecutor can run transforms in parallel. - rows = batch.to_pylist() - for row in rows: - row["image"] = [v / 255.0 for v in row["image"]] - return rows - -ds = StreamingDataset(table, shuffle_seed=42, transform=normalize) -``` - - -#### DataLoader workers {#dataloader-workers} - -The thread-based transformation model that `StreamingDataset` uses by default is only effective when the transform -function releases the GIL. This is true for many Python scientific libraries, including NumPy, PyArrow, and -TorchVision, but not for pure-Python transforms. PyTorch can launch multiple DataLoader worker processes per rank, -and `StreamingDataset` uses `get_worker_info()` to give each worker a non-overlapping group of splits. - -Multiprocessing adds pickling and transfer overhead and increases memory use. Start with `num_workers=0`, which keeps -loading in the rank's main process, and add workers only after confirming an unavoidable GIL bottleneck. If you use -workers, `num_splits` must be divisible by `world_size * num_workers`. Use the `forkserver` or `spawn` multiprocessing -context because LanceDB uses internal threads. - -```py Python icon=Python -dataloader = torch.utils.data.DataLoader( - dataset, - batch_size=batch_size, - num_workers=2, - multiprocessing_context="forkserver", - persistent_workers=True, -) -``` - -By default, `StreamingDataset` serializes enough table state to reopen the table in each worker. For custom connection -setup, pass a picklable `connection_factory` callable that accepts a table name and returns an open table. This avoids -serializing connection credentials into worker state. - -### Observability and performance {#observability-and-performance} - -Optimizing data loader performance is tricky because it can be difficult to locate the bottleneck. What is often -blamed on I/O can be a CPU bottleneck in the transform stage, or vice versa. To help distinguish them, -`StreamingDataset` exposes pipeline counters. `raw_queue_depth` is the number of loaded rows waiting to be transformed, -while `prefetch_queue_depth` is the number of transformed rows ready to be consumed. `unscanned_rows` and -`consumed_rows` show how far the current iterator has progressed. - -If the `prefetch_queue_depth` is consistently zero but the `raw_queue_depth` is not then you have a CPU -transformation bottleneck. You should investigate GIL bottlenecks or look for ways to optimize your transformation. -This can often be done by batching the compute work. If both `prefetch_queue_depth` and `raw_queue_depth` are -consistently zero while the consumer is waiting, I/O is the likely bottleneck. A larger read batch size or clumped -shuffling could help. - - -```py Python icon=Python -import threading, time - -ds = StreamingDataset(table, shuffle_seed=42) - -def log_pipeline_health(): - while True: - print( - f"unscanned={ds.unscanned_rows} " - f"raw={ds.raw_queue_depth} " - f"cooked={ds.prefetch_queue_depth} " - f"consumed={ds.consumed_rows}" - ) - time.sleep(1.0) - -monitor = threading.Thread(target=log_pipeline_health, daemon=True) -monitor.start() - -for sample in ds: - train_step(sample) - -print( - f"bytes loaded: {ds.bytes_loaded} " - f"fetch time: {ds.fetch_time:.2f}s " - f"transform time: {ds.transform_time:.2f}s" -) -``` - - -`bytes_loaded` measures raw Arrow buffer bytes before transformation. `bytes_loaded`, `fetch_time`, and `transform_time` -are cumulative across iterations of the same dataset instance; because work runs concurrently, the summed stage times -can exceed wall-clock time. The queue depths report rows currently waiting in the pipeline, and the progress counters -reflect the latest iterator snapshot. - -### Filtering data {#filtering-data} - -By default, the streaming data loader includes all rows and columns. LanceDB is a columnar database that also supports -efficient random access. Reducing the number of columns you load has a direct impact on I/O performance. Reducing the -number of rows can also help, especially with a selective filter, large values, local data, or the LanceDB Enterprise -cache. - -Use the `columns` parameter to specify which columns to load and the `filter` parameter to specify which rows to load. -The filter is evaluated once when the permutation is constructed, before the rows are divided into splits. Filtered-out -rows are not loaded from storage during iteration, and split sizes reflect the filtered row count. - - -```py Python icon=Python -ds = StreamingDataset( - table, - shuffle_seed=42, - columns=["image", "label"], # skip all other columns - filter="category = 'train'", # only training rows -) -``` - - -### Shuffling rows {#shuffling-rows} - -By default, `StreamingDataset` sets `shuffle=True` and randomly assigns rows to splits. This helps prevent the model -from learning artifacts from storage order. Set `shuffle=False` to divide rows into splits sequentially, which is -useful for deterministic evaluation and debugging. - -The effective shuffle seed combines `shuffle_seed` with `epoch`, so each epoch has a different ordering while runs -with the same inputs remain reproducible. Keep `epoch=0` if you want the same ordering across iterations. The default -`shuffle_seed` is `0`; set it to `None` to generate a random seed when the dataset is constructed. - - -```py Python icon=Python -# Training loop: each epoch gets a different shuffled ordering -for epoch in range(num_epochs): - ds = StreamingDataset( - table, - shuffle_seed=42, - epoch=epoch, # changes the permutation each epoch - ) - for sample in ds: - train_step(sample) - -# Evaluation: deterministic sequential order, no shuffle -eval_ds = StreamingDataset(table, shuffle=False) -for sample in eval_ds: - eval_step(sample) -``` - - - -Shuffling can have significant impacts on I/O performance, especially if you are loading data from cloud storage. -In many cases the GPU pipeline is slow enough that this penalty will not be noticeable. However, -you can use the `shuffle_clump_size` parameter to shuffle the data in clumps (small contiguous batches that get -shuffled together). This will give some penalty to the randomness of the shuffle, but will significantly improve -I/O performance. - - - -```py Python icon=Python -# Clumped shuffle: groups of 16 contiguous rows are shuffled together, -# preserving read locality while still randomising the global ordering. -ds = StreamingDataset( - table, - shuffle_seed=42, - shuffle_clump_size=16, -) -``` - - -### Data splits and elasticity {#data-splits-and-elasticity} - -`StreamingDataset` partitions the permutation into a fixed number of equal-sized groups called splits. Each rank gets -a contiguous group of splits, and each DataLoader worker gets a contiguous subgroup of its rank's splits. Samples are -then yielded by round-robining over the assigned splits. - -If `num_splits` is omitted, it defaults to `world_size`. This works when `num_workers=0`, but it ties the split layout -to the current number of ranks. A run resumed with a different world size would then construct a different layout and -would not see the same global batches. - -To get **elastic determinism**, choose a fixed `num_splits` that is compatible with every topology you plan to use. -For example, a run with `num_splits=8` and 4 GPUs assigns 2 splits to each rank. The ranks pull from their splits in -round-robin order, producing the same global batches as a run with 8 GPUs and the same `num_splits`, `shuffle_seed`, -and `epoch`. - -The following constraints apply: - -- `num_splits` must be divisible by `world_size`. -- With DataLoader worker processes, `num_splits` must be divisible by `world_size * num_workers`. -- For the same samples to form each global training step across topologies, `global_batch_size` must be a multiple of - `num_splits`. -- The filtered row count must be at least `num_splits`. If it is not evenly divisible by `num_splits`, up to - `num_splits - 1` surplus rows are dropped so that every split has the same length. - -For example, to switch between 8 and 6 GPUs, use a `num_splits` value divisible by both, such as 24. Highly composite -values can support several layouts: 48 supports 1, 2, 3, 4, 6, 8, 12, 16, 24, and 48 GPUs, while 60 supports 1, 2, 3, -4, 5, 6, 10, 12, 15, 20, 30, and 60 GPUs. Remember to include `num_workers` when checking divisibility. - - -```py Python icon=Python -import torch.distributed as dist - -dist.init_process_group("nccl") -rank = dist.get_rank() -world_size = dist.get_world_size() - -# num_splits=48 is divisible by 1, 2, 3, 4, 6, 8, 12, 16, 24, 48 -# so this dataset works unchanged as you scale up or down GPUs. -ds = StreamingDataset( - table, - num_splits=48, - shuffle_seed=42, - epoch=current_epoch, - rank=rank, - world_size=world_size, -) - -for sample in ds: - train_step(sample) -``` - - -### Checkpointing and resumability {#checkpointing-and-resumability} - -Model training is expensive, and failures can occur partway through a run. A model checkpoint is not enough for an -exact resume: the streaming data loader must also continue from the same position. `StreamingDataset.state_dict()` -captures the number of samples consumed from every split in a plain Python dictionary, and `load_state_dict()` restores -that position. - -Save this state at a global-step boundary where every split has contributed equally. The easiest way to guarantee this -is to make `global_batch_size` a multiple of `num_splits`, as in the example below. If you checkpoint partway through a -round-robin cycle, the state reflects the preceding complete cycle and those partial-cycle samples can be replayed. - - -```py Python icon=Python -import torch - -global_batch_size = 48 # one sample per split in each global step -batch_size = global_batch_size // world_size - -dataset = StreamingDataset( - table, - num_splits=48, - shuffle_seed=42, - epoch=current_epoch, - rank=rank, - world_size=world_size, -) -dataloader = torch.utils.data.DataLoader( - dataset, - batch_size=batch_size, - num_workers=0, -) - -for step, batch in enumerate(dataloader): - train_step(batch) - - if (step + 1) % checkpoint_interval == 0: - torch.save( - {"model": model.state_dict(), "dataset": dataset.state_dict()}, - f"checkpoint_{step + 1}.pt", - ) - -# --- resuming after a crash --- -checkpoint = torch.load("checkpoint_100.pt") -model.load_state_dict(checkpoint["model"]) - -dataset_state = checkpoint["dataset"] -dataset = StreamingDataset( - table, - num_splits=dataset_state["num_splits"], - shuffle_seed=dataset_state["shuffle_seed"], - epoch=dataset_state["epoch"], - rank=rank, - world_size=world_size, # may differ from the run that saved the checkpoint -) -dataset.load_state_dict(dataset_state) -dataloader = torch.utils.data.DataLoader( - dataset, - batch_size=global_batch_size // world_size, - num_workers=0, -) - -for batch in dataloader: - train_step(batch) -``` - - -`load_state_dict()` rejects a checkpoint whose `num_splits` or `shuffle_seed` differs from the new dataset. For an -exact resume, also use the same table snapshot, `epoch`, `shuffle`, `filter`, and other data-selection settings. The -`world_size` and number of DataLoader workers may change as long as the split and batch-size divisibility constraints -still hold. - -#### Checkpointing with multiple DataLoader workers {#checkpointing-with-multiple-dataloader-workers} - -The plain `torch.utils.data.DataLoader` only produces a safe checkpoint when `num_workers=0`. With `num_workers > 0`, -PyTorch runs `StreamingDataset.__iter__` in separate worker processes and prefetches batches ahead of the trainer, so -the counters on the parent dataset do not reflect what the trainer has actually consumed. Calling `state_dict()` on the -parent in that setup raises `RuntimeError` and points you at `StreamingDataLoader`. - -Use `StreamingDataLoader` when you want multi-worker prefetch and exact resumability. It carries a state snapshot -alongside every internal batch and commits it to the parent `StreamingDataset` only when that batch is returned to the -trainer. The trainer receives the same collated batch it would receive from a standard `DataLoader`. - - -```py Python icon=Python -from lancedb.streaming import StreamingDataset, StreamingDataLoader - -dataset = StreamingDataset( - table, - num_splits=48, - shuffle_seed=42, - epoch=current_epoch, - rank=rank, - world_size=world_size, -) -dataloader = StreamingDataLoader( - dataset, - batch_size=batch_size, - num_workers=4, -) - -for step, batch in enumerate(dataloader): - train_step(batch) - - if (step + 1) % checkpoint_interval == 0: - torch.save( - {"model": model.state_dict(), "dataset": dataset.state_dict()}, - f"checkpoint_{step + 1}.pt", - ) -``` - - -`StreamingDataLoader` accepts the same arguments as `torch.utils.data.DataLoader` with a few restrictions: - -- `dataset` must be a `StreamingDataset` (subclasses that override `__iter__` are not supported). -- `in_order=True` is required so that consumer-committed checkpoints stay deterministic. -- `persistent_workers=True` is not supported, because prefetched worker copies cannot be restored from parent-committed state. -- `drop_last=True` is not supported, because incomplete tails discarded by worker replicas cannot be checkpointed topology-independently. - -With more than one worker, call `state_dict()` at a complete logical step boundary where every split assigned to the -rank has the same consumed-sample count. Calling it mid-step raises `RuntimeError` asking you to consume more batches -first. - -#### Resuming across different topologies {#resuming-across-different-topologies} - -When training across ranks, each rank owns its own subset of splits and only its own splits have exact progress. To -resume on a different `world_size`, collect the `state_dict()` from every rank of the previous run and merge them with -`StreamingDataset.merge_state_dicts` before calling `load_state_dict()` on the new run. The merge is topology-agnostic: -pass the full list of per-rank states in, and hand the identical merged dict to every rank of the resumed job, -regardless of whether the topology grew, shrank, or stayed the same. - - -```py Python icon=Python -# On each rank of the previous run: -state = dataset.state_dict() -# ...gather `state` from every rank into a single list `states` (for example -# via torch.distributed all-gather or by saving one file per rank). - -merged = StreamingDataset.merge_state_dicts(states) - -# On each rank of the resumed run (world_size may differ): -dataset = StreamingDataset( - table, - num_splits=merged["num_splits"], - shuffle_seed=merged["shuffle_seed"], - epoch=merged["epoch"], - rank=rank, - world_size=new_world_size, -) -dataset.load_state_dict(merged) -``` - - -## Permutations {#permutations} - -In more complicated scenarios, you may want the flexibility to shuffle, split, and select data without using the full -iterable streaming data loader. In these cases, use `Permutation`, the lower-level class on which `StreamingDataset` -is built. A `Permutation` defines a custom ordering of the data and supports map-style access through `__getitem__()` -and batched access through `__getitems__()`. - -### Base table version pinning {#base-table-version-pinning} - -A `Permutation` is pinned to the version of the base table at the time it was built, and every read (including reads -from a `StreamingDataset` that wraps it, and every DataLoader worker after a `fork`) resolves against that pinned -version. This makes iteration deterministic across compactions and worker forks: the permutation only addresses rows -that existed when it was built, and rows appended to the base table afterwards are not visible through the existing -permutation. - - -To include newly appended rows, build a new `Permutation` (and rebuild any `StreamingDataset` that depends on it) -against the updated table. Permutations written before this behavior shipped carry no pinned version and continue to -read as they did before. - diff --git a/docs/training/object-detection.mdx b/docs/training/object-detection.mdx deleted file mode 100644 index 2b516d6..0000000 --- a/docs/training/object-detection.mdx +++ /dev/null @@ -1,368 +0,0 @@ ---- -title: "Object Detection for AV Perception" -sidebarTitle: "Example: Object Detection" -description: End-to-end fine-tuning of a Faster R-CNN object detector on curated dashcam slices, using LanceDB as the data backbone from raw frames to checkpoints. -icon: car ---- - -This example walks through fine-tuning an autonomous vehicle (AV) perception model on targeted failure-mode slices of [BDD100K](https://www.bdd100k.com/) — riders, nighttime pedestrians, and distant pedestrians — using LanceDB as a single multimodal table from raw JPEG bytes through to the PyTorch training loop. - -The full pipeline lives in the [lancedb/training](https://github.com/lancedb/training/tree/main/object-detection) repository. This page focuses on the parts most relevant to training: defining curated splits as materialized views, loading them through the [`Permutation`](/training/) API, and pinning checkpoints to an exact data version. - -## What you get {#what-you-get} - -Fine-tuning Faster R-CNN ResNet50 FPN v2 for 10 epochs on each curated slice (batch size 64, AMP, A100), starting from the same COCO-pretrained checkpoint and evaluating on the matching validation view: - -| Failure mode | Metric | Baseline (COCO) | Fine-tuned | Δ% | -|---|---|---|---|---| -| **Nighttime pedestrian** | mAP@0.5 | 0.4025 | **0.5192** | **+29.0%** | -| | Recall | 0.5923 | **0.7570** | **+27.8%** | -| **Rider** | mAP@0.5 | 0.5563 | **0.6676** | **+20.0%** | -| | Recall | 0.6788 | **0.7847** | **+15.6%** | -| **Distant pedestrian** | mAP@0.5 | 0.4746 | **0.5788** | **+22.0%** | -| | Recall | 0.6794 | **0.8024** | **+18.1%** | - -No external data added — only training-distribution correction via SQL filters over a single Lance table. Each panel below shows the same frame with three overlaid predictions: **green** = ground truth · **red** = pretrained COCO baseline · **blue** = fine-tuned model. - -![Rider detection — ground truth vs baseline vs fine-tuned](/static/assets/images/training/rider_04.jpg) - -![Nighttime pedestrian detection — ground truth vs baseline vs fine-tuned](/static/assets/images/training/nighttime_person_01.jpg) - -![Distant pedestrian detection — ground truth vs baseline vs fine-tuned](/static/assets/images/training/distant_person_00.jpg) - -The rest of the page walks through the pipeline that produced these checkpoints. - -## The failure modes {#the-failure-modes} - -A perception model fine-tuned on a generic dataset typically misses the long-tail scenarios that matter most in deployment. Three common failure modes drive this example: - -| Failure mode | Curation signal | -|---|---| -| **Riders** (person on bike/motorcycle) | `has_rider = true` | -| **Nighttime pedestrians** | `timeofday = 'night' AND has_person = true` | -| **Distant pedestrians** | `has_person = true AND person_bbox_area_pct < 30.0` | - -Each curated slice becomes a [materialized view](/geneva/jobs/materialized-views) — a named, refreshable SQL filter over the source table — and the training script loads it by name. New footage flows in through `add()` → `backfill()` → `refresh()`; no manifests, no exports, no reshuffling on disk. - -## 1\. Schema {#1-schema} - -The source table holds raw image bytes alongside structured annotations. Bounding boxes are stored as a parallel list (one element per box) rather than a nested struct so they remain directly queryable with SQL. - -```py Python icon=Python -import pyarrow as pa - -BDD_SCHEMA = pa.schema([ - pa.field("image_id", pa.string()), - pa.field("split", pa.string()), # "train" | "val" - pa.field("image_bytes", pa.large_binary()), # raw JPEG - pa.field("width", pa.int32()), - pa.field("height", pa.int32()), - - # scene metadata - pa.field("weather", pa.string()), - pa.field("scene", pa.string()), - pa.field("timeofday", pa.string()), - - # annotations — parallel lists, one element per box - pa.field("ann_categories", pa.list_(pa.string())), - pa.field("ann_bboxes", pa.list_(pa.list_(pa.float32()))), - pa.field("ann_occluded", pa.list_(pa.bool_())), -]) -``` - -Ingestion streams `pa.RecordBatch`es of raw frames + annotations directly into a Lance table — no intermediate preprocessing job. The table can live on local disk, S3, GCS, or Azure; everything downstream (backfills, views, the training loader) opens it in place via `lancedb.connect("s3://...")` with no local copy step. - -## 2\. Backfill curation features with Geneva {#2-backfill-curation-features-with-geneva} - -Curation signals are added as columns on the same table using [Geneva UDFs](/geneva/). Backfills are incremental and checkpointed: re-running the command after new footage arrives only computes the new rows. - -```py Python icon=Python -import pyarrow as pa -from geneva.transformer import udf - -# Tier 1 — CPU, derived from annotations alone -@udf(data_type=pa.bool_(), input_columns=["ann_categories"]) -def has_rider(ann_categories: list[str]) -> bool: - return "rider" in (ann_categories or []) - -# Tier 2 — GPU, runs a Faster R-CNN to find the largest detected person -# as a percentage of frame area. <30% = a distant or small pedestrian, -# the hard case we want to upweight in training. -@udf(data_type=pa.float32(), - input_columns=["image_bytes", "width", "height"], - cuda=True, num_gpus=1) -class PersonBboxAreaPct: - def __init__(self): - self._model = None - - def __call__(self, image_bytes, width, height): - # lazy model load — runs once per Ray worker, then reused - ... -``` - -Run the backfill against the live table: - -```py Python icon=Python -import geneva - -gconn = geneva.connect("data/bdd100k/lancedb") -tbl = gconn.open_table("bdd100k") - -tbl.add_columns({"has_rider": has_rider}) -tbl.add_columns({"person_bbox_area_pct": PersonBboxAreaPct()}) - -with gconn.local_ray_context(): - tbl.backfill("has_rider") - tbl.backfill("person_bbox_area_pct") -``` - - -Because the curation features are flat scalar columns on the same table, all four retrieval modes — SQL, full-text search, vector search, and SQL-filtered vector search — work directly without joins or exports. See the [Geneva end-to-end example](/geneva/end-to-end) for more on the backfill pattern. - - -## 3\. Define training splits as materialized views {#3-define-training-splits-as-materialized-views} - -A training split is a named SQL filter, not a CSV manifest. Each view stays in sync with the source table and bumps its `version` on every refresh — the link between a checkpoint and the exact data that produced it. - -```py Python icon=Python -import geneva - -gconn = geneva.connect("data/bdd100k/lancedb") -gtbl = gconn.open_table("bdd100k") - -VIEWS = { - "bdd100k_rider_train": - "has_rider = true AND split = 'train'", - "bdd100k_rider_val": - "has_rider = true AND split = 'val'", - "bdd100k_nighttime_person_train": - "timeofday = 'night' AND has_person = true AND split = 'train'", - "bdd100k_nighttime_person_val": - "timeofday = 'night' AND has_person = true AND split = 'val'", - "bdd100k_distant_person_train": - "has_person = true AND person_bbox_area_pct < 30.0 AND split = 'train'", - "bdd100k_distant_person_val": - "has_person = true AND person_bbox_area_pct < 30.0 AND split = 'val'", -} - -with gconn.local_ray_context(): - for name, sql_filter in VIEWS.items(): - query = gtbl.search().where(sql_filter) - mv = gconn.create_materialized_view(name, query) - mv.refresh() - print(f"[{name}] {mv.count_rows()} rows (version {mv.version})") -``` - -## 4\. PyTorch DataLoader via the Permutation API {#4-pytorch-dataloader-via-the-permutation-api} - -The training script doesn't know about the filter — it opens a view by name and reads through the [`Permutation`](/training/) API. Each DataLoader worker reopens its own connection lazily, reads Arrow batches directly from Lance (zero-copy, no intermediate file format), and the collate function decodes the whole batch in one pass. `Permutation` provides random-access indexing over the table, so shuffling is a cheap pointer rewrite rather than a full-dataset shuffle on disk. - -```py Python icon=Python -import lancedb -import torch -import torchvision.io as tio -from lancedb.permutation import Permutation - -DETECTION_COLS = ["image_bytes", "ann_categories", "ann_bboxes"] - -class LanceDetectionDataset(torch.utils.data.Dataset): - def __init__(self, uri: str, table_name: str): - self.uri, self.table_name = uri, table_name - self._perm = None - self.length = len(lancedb.connect(uri).open_table(table_name)) - - def __len__(self): - return self.length - - def __getstate__(self): - # Permutation holds Rust async state — zero it so each worker reopens - state = self.__dict__.copy() - state["_perm"] = None - return state - - def _ensure_open(self): - if self._perm is None: - tbl = lancedb.connect(self.uri).open_table(self.table_name) - self._perm = ( - Permutation.identity(tbl) - .select_columns(DETECTION_COLS) - .with_format("arrow") # zero-copy - ) - - def __getitems__(self, indices: list[int]): - self._ensure_open() - return self._perm.__getitems__(indices) -``` - -The collate function decodes JPEG bytes and converts BDD category strings into COCO class IDs (so the comparison against the pretrained checkpoint is valid): - -```py Python icon=Python -BDD_LABEL_MAP = { - "person": 1, "rider": 1, "bicycle": 2, "car": 3, "motorcycle": 4, - "bus": 6, "train": 7, "truck": 8, "traffic light": 10, -} - - -def detection_collate(batch): - images, targets = [], [] - for raw, cats, bboxes in zip( - batch.column("image_bytes").to_pylist(), - batch.column("ann_categories").to_pylist(), - batch.column("ann_bboxes").to_pylist(), - ): - buf = torch.frombuffer(bytearray(raw), dtype=torch.uint8) - images.append(tio.decode_image(buf, tio.ImageReadMode.RGB).float() / 255.0) - - valid_boxes, valid_labels = [], [] - for cat, box in zip(cats or [], bboxes or []): - lid = BDD_LABEL_MAP.get(cat) - if lid is None or box[2] <= box[0] or box[3] <= box[1]: - continue - valid_boxes.append(box) - valid_labels.append(lid) - targets.append({ - "boxes": torch.tensor(valid_boxes or [], dtype=torch.float32).reshape(-1, 4), - "labels": torch.tensor(valid_labels or [], dtype=torch.int64), - }) - return images, targets -``` - -Wire it into a standard `torch.utils.data.DataLoader`: - -```py Python icon=Python -def make_loader(uri, table_name, batch_size=64, num_workers=8, shuffle=False): - dataset = LanceDetectionDataset(uri, table_name) - sampler = torch.utils.data.RandomSampler(dataset) if shuffle else None - return torch.utils.data.DataLoader( - dataset, - batch_size=batch_size, - sampler=sampler, - num_workers=num_workers, - collate_fn=detection_collate, - pin_memory=torch.cuda.is_available(), - persistent_workers=(num_workers > 0), - multiprocessing_context="spawn" if num_workers > 0 else None, - ) -``` - - -`with_format("arrow")` keeps batches as zero-copy `pa.RecordBatch`es — no per-row Python boxing, no pickling between worker and main. Each DataLoader worker reopens its own `Permutation` after fork (the Rust async handle is cleared in `__getstate__`), so reads scale with `num_workers` and stream straight from the underlying object store. JPEG decode overlaps with GPU compute via `pin_memory` + `prefetch_factor`, which is what keeps the loader from becoming the bottleneck on a fast GPU. - - -## 5\. Fine-tune Faster R-CNN {#5-fine-tune-faster-r-cnn} - -The training loop is plain PyTorch — the Lance integration ends at the loader. Mixed precision is enabled on CUDA for ~2× speedup on Ampere GPUs. - -```py Python icon=Python -import time -import torch -from torchvision.models.detection import ( - fasterrcnn_resnet50_fpn_v2, FasterRCNN_ResNet50_FPN_V2_Weights, -) - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -use_amp = device.type == "cuda" - -# COCO pretrained weights — head left intact since BDD uses a subset of COCO IDs -model = fasterrcnn_resnet50_fpn_v2( - weights=FasterRCNN_ResNet50_FPN_V2_Weights.COCO_V1 -).to(device) - -train_loader = make_loader("data/bdd100k/lancedb", - "bdd100k_rider_train", - batch_size=64, num_workers=14, shuffle=True) -val_loader = make_loader("data/bdd100k/lancedb", - "bdd100k_rider_val", - batch_size=64, num_workers=14) - -optimizer = torch.optim.SGD( - [p for p in model.parameters() if p.requires_grad], - lr=0.04, momentum=0.9, weight_decay=1e-4, -) -scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1) -scaler = torch.cuda.amp.GradScaler() if use_amp else None - -for epoch in range(1, 11): - model.train() - t0 = time.time() - for images, targets in train_loader: - images = [img.to(device) for img in images] - targets = [{k: v.to(device) for k, v in t.items()} for t in targets] - if all(t["labels"].numel() == 0 for t in targets): - continue - - with torch.cuda.amp.autocast(enabled=use_amp): - losses = sum(model(images, targets).values()) - - optimizer.zero_grad() - if use_amp: - scaler.scale(losses).backward() - scaler.unscale_(optimizer) - torch.nn.utils.clip_grad_norm_(model.parameters(), 10.0) - scaler.step(optimizer) - scaler.update() - else: - losses.backward() - torch.nn.utils.clip_grad_norm_(model.parameters(), 10.0) - optimizer.step() - - scheduler.step() - print(f"epoch {epoch} ({time.time() - t0:.1f}s)") -``` - -## 6\. Pin the checkpoint to a data version {#6-pin-the-checkpoint-to-a-data-version} - -Every Lance table — including a materialized view — exposes a monotonically increasing `version`. Logging it next to the weights gives a permanent, deterministic link between a checkpoint and the exact data snapshot that produced it. - -```py Python icon=Python -import json -from pathlib import Path - -train_tbl = lancedb.connect("data/bdd100k/lancedb").open_table("bdd100k_rider_train") - -out = Path("checkpoints/rider") -out.mkdir(parents=True, exist_ok=True) -torch.save(model.state_dict(), out / "fasterrcnn_bdd_finetuned.pt") -with open(out / "metadata.json", "w") as f: - json.dump({ - "train_table": train_tbl.name, - "table_version": train_tbl.version, - "row_count": len(train_tbl), - }, f, indent=2) -``` - -To reproduce a run, [time-travel](/tables/versioning) the view to the recorded version before opening the loader: - -```py Python icon=Python -tbl = lancedb.connect("data/bdd100k/lancedb").open_table("bdd100k_rider_train") -tbl.checkout(version=7) # exact snapshot the checkpoint was trained on -``` - -## 7\. Continuous updates {#7-continuous-updates} - -When new footage arrives, the same three calls update every downstream view — no view definitions change, no training-script edits required: - -```py Python icon=Python -# 1. ingest the new footage into the source table -table.add(new_record_batches) - -# 2. backfill computes only the new rows (incremental, checkpointed) -with gconn.local_ray_context(): - tbl.backfill("has_rider") - tbl.backfill("person_bbox_area_pct") - -# 3. refresh appends qualifying new rows to every materialized view -for view_name in gconn.table_names(): - if view_name == "bdd100k": - continue - mv = gconn.open_table(view_name) - before = mv.count_rows() - mv.refresh() - print(f"[{view_name}] {before} → {mv.count_rows()} rows (version {mv.version})") -``` - -The next training run picks up the new data automatically — and pins itself to the new `version`. - -## Full source {#full-source} - -The complete code, including a synthetic-data mode for pipeline verification (`--synthetic 500`), GPU UDFs for CLIP embeddings and dHash deduplication, and the EDA notebook, is in this [GitHub repository](https://github.com/lancedb/training/tree/main/object-detection). diff --git a/docs/training/torch.mdx b/docs/training/torch.mdx deleted file mode 100644 index 39c778a..0000000 --- a/docs/training/torch.mdx +++ /dev/null @@ -1,190 +0,0 @@ ---- -title: "PyTorch Integration" -sidebarTitle: "PyTorch integration" -description: Learn how to use LanceDB with PyTorch for training and inference. -icon: fire ---- - -LanceDB provides a seamless integration with PyTorch for training and inference. This allows you to use LanceDB as a backend for your PyTorch models, and to use PyTorch for training and inference. You can use LanceDB to store your data, and PyTorch to train your models. - -## Quickstart {#quickstart} - -The `Table` class in LanceDB implements a contract for a PyTorch -[Dataset](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.Dataset). - This means you can simply use a LanceDB table in a PyTorch dataloader directly. - -```py Python icon=Python -import lancedb -import torch -import pyarrow as pa -from lancedb.util import tbl_to_tensor - -mem_db = lancedb.connect("memory://") -table = mem_db.create_table("test_table", pa.table({"a": range(1000)})) - -# Any LanceDB table can be used as a PyTorch Dataset -dataloader = torch.utils.data.DataLoader( - table, batch_size=1024, shuffle=True, collate_fn=tbl_to_tensor -) - -for batch in dataloader: - print(batch) -``` - -Although the `Table` class in LanceDB implements the `torch.utils.data.Dataset` interface, you may find that using -a table [Permutation](/training/) is more flexible. - -```py Python icon=Python -from lancedb.permutation import Permutation - -permutation = Permutation.identity(table) -dataloader = torch.utils.data.DataLoader(permutation) -``` - -## Output Formats {#output-formats} - -By default, a `Table` data loader will emit Arrow data. `collate_fn` is PyTorch's batching hook: PyTorch calls it to -turn the fetched items into one batch. PyTorch's default collate function only knows how to combine tensors, NumPy -arrays, numbers, dicts, and lists, so it does not accept Arrow data directly. When using a `Table` directly, pass -LanceDB's `lancedb.util.tbl_to_tensor` helper as PyTorch's `collate_fn`; it converts numeric Arrow columns into a -column-major `torch.Tensor` with shape `(columns, rows)`. - -`Permutation` works differently: its default output is a list of Python dicts, which PyTorch's default collate function -can batch into a dict of tensors. This is usually more convenient when you are getting started. However, there is a -significant performance penalty converting from Arrow, Lance's internal representation, to this default format. If you -want the default PyTorch dict-of-tensors behavior, use a `Permutation` as-is; if you want direct Arrow-to-tensor -conversion, either pass `lancedb.util.tbl_to_tensor` as `collate_fn` with a direct `Table` or configure a `Permutation` -with one of the transform formats described below. - -To address this, the `Permutation` class provides a set of builtin transform functions that can be applied to map -the Arrow data in different ways. The `arrow` and `polars` formats will always avoid data copies. However, `numpy`, -`pandas`, and `torch_col` formats will also avoid data copies in most cases. The `python`, `python_col`, and -`torch` formats will all require at least one full copy of the data and are the slowest options. - -### Using the torch column format with a torch data loader {#using-the-torch-column-format-with-a-torch-data-loader} - -The `torch_col` format is the most efficient way to convert from Arrow to a `torch.Tensor`. It will convert the -entire Arrow batch to a _column-major_ `torch.Tensor`. In other words, given C columns and R rows, the resulting -Tensor will have shape `(C, R)`. However, this format generates an error if you are using a -`torch.utils.data.DataLoader` with the default collation function: - -```py Python icon=Python -TypeError: stack(): argument 'tensors' (position 1) must be tuple of Tensors, not Tensor -``` - -This error occurs because the default collation function does not currently expect a single two-dimensional tensor. -It expects a list of tensors which it will then stack. This is what is output by the `torch` format but that format -requires a data copy. To avoid this error, and avoid data copies, you will need to provide a custom collation function -in addition to specifying the `torch_col` format. - -```py Python icon=Python -from lancedb.permutation import Permutation - -permutation = Permutation.identity(table).with_format("torch_col") -dataloader = torch.utils.data.DataLoader(permutation, collate_fn=lambda x: x) -``` - -This will now output a single two-dimensional tensor for each batch. - -## Selecting columns {#selecting-columns} - -By default, the `Table` class will return all columns in the table when used as input to PyTorch. If you only need -a subset of columns, you can significantly reduce your I/O requirements by selecting only the columns you need. The -`Permutation` class provides a `select_columns` method that provides this functionality. - -```py Python icon=Python -from lancedb.permutation import Permutation - -permutation = Permutation.identity(table).select_columns(["id", "prompt"]) -dataloader = torch.utils.data.DataLoader( - permutation, batch_size=1024, shuffle=True -) - -for batch in dataloader: - print(batch.schema) -``` - -## Using multiple DataLoader workers {#using-multiple-dataloader-workers} - -Set `num_workers > 0` to read from LanceDB in multiple PyTorch worker processes. LanceDB tables and `Permutation` objects are picklable, so each worker reopens the table after it starts. - -Prefer the `forkserver` start method when using multiple workers. LanceDB uses internal threads, so the default `fork` method is unsafe; `forkserver` avoids that while being cheaper to start than `spawn`, and it is set to become the Python default. See [the performance guide](/performance) for more multiprocessing guidance. - - -`forkserver` is only available on POSIX systems (Linux and macOS). On Windows, use `spawn` instead — it is the only start method available there. - - -```py Python icon=Python -import torch -from lancedb.permutation import Permutation - -permutation = Permutation.identity(table) -dataloader = torch.utils.data.DataLoader( - permutation, - batch_size=1024, - shuffle=True, - num_workers=4, - multiprocessing_context="forkserver", - persistent_workers=True, -) -``` - -### Remote tables in DataLoader workers {#remote-tables-in-dataloader-workers} - -Remote LanceDB Enterprise tables (`db://...`) work the same way: workers reopen the table from the pickled connection state. - -```py Python icon=Python -import lancedb -import torch -from lancedb.util import tbl_to_tensor - -db = lancedb.connect( - "db://my-database", - api_key="sk-...", - region="us-east-1", -) -table = db.open_table("my_table") - -dataloader = torch.utils.data.DataLoader( - table, - batch_size=512, - num_workers=4, - multiprocessing_context="forkserver", - collate_fn=tbl_to_tensor, -) -``` - - -This sends the connection state, including the API key, to each worker. Use a connection factory if credentials should be loaded inside the worker or your `client_config` contains a non-serializable `header_provider`. - - -### Providing a custom connection factory {#providing-a-custom-connection-factory} - -`Permutation.with_connection_factory` lets each worker reopen the base table with custom logic. The factory takes the table name, returns a LanceDB table, and must be picklable. - -```py Python icon=Python -import os -import lancedb -import torch -from lancedb.permutation import Permutation - -def open_table(name: str): - db = lancedb.connect( - "db://my-database", - api_key=os.environ["LANCEDB_API_KEY"], - region="us-east-1", - ) - return db.open_table(name) - -table = open_table("my_table") -permutation = ( - Permutation.identity(table) - .with_connection_factory(open_table) -) -dataloader = torch.utils.data.DataLoader( - permutation, - batch_size=512, - num_workers=4, - multiprocessing_context="forkserver", -) -``` diff --git a/docs/training/vlm-finetuning.mdx b/docs/training/vlm-finetuning.mdx deleted file mode 100644 index 39abf24..0000000 --- a/docs/training/vlm-finetuning.mdx +++ /dev/null @@ -1,522 +0,0 @@ ---- -title: "Fine-tuning a VLM on TextVQA" -sidebarTitle: "Example: VLM finetuning" -description: End-to-end fine-tuning of Qwen2.5-VL on a curated TextVQA slice, using LanceDB and Geneva to materialize expensive vision-language features once and train from cached columns. -icon: image ---- - -This example walks through a vision-language model (VLM) fine-tuning pipeline for [TextVQA](https://textvqa.org/), where the task is to answer questions that require reasoning over text _inside an image_. The base model is `Qwen2.5-VL-3B-Instruct`, fine-tuned with the [QLoRA](https://arxiv.org/abs/2305.14314) method. The data backbone is one Lance table that evolves from raw multimodal rows into training-ready features. - -The key idea is simple: in this QLoRA fine-tuning setup, we freeze the VLM's image encoder and train only a small adapter on the language-model side. We call that encoder the **vision tower** in this example: it is the part of the model that turns image pixels into visual hidden states before the language model reads them alongside the text prompt. - -Because the vision tower's weights do not change during fine-tuning, its output for a given image also does not change. That means the pipeline can compute those visual hidden states once, store them as a fixed-size Lance column, and reuse them in every epoch instead of recomputing them in every training step. This also helps the run fit comfortably on a small GPU, because the training job does not need to keep the vision encoder active or pay for its forward pass on every batch. - - - - Run the Colab-sized workflow on a free T4: download the pre-baked Lance subset, explore it, benchmark Lance vs Parquet, fine-tune with QLoRA, and evaluate base vs tuned answers. - - - Full demo repository with the notebook, Geneva UDFs, direct backfill fallback, dataloader, training loop, and evaluation scripts. - - - -The Colab notebook uses a pre-baked subset of the TextVQA dataset: it downloads a curated Lance subset whose expensive feature columns have already been computed. This page explains the complete end-to-end pipeline that produced that subset, then shows how the notebook applies it to produce a fine-tuned model that improves performance on the TextVQA task. - -## What you get {#what-you-get} - -On the curated `text_dense` TextVQA slice, the demo fine-tunes `Qwen2.5-VL-3B-Instruct` with QLoRA and evaluates on held-out images: - -| Setup | TextVQA accuracy | -|---|---| -| Base model | 0.799 | -| LoRA-tuned model | **0.820** | -| Lift | **+2.1 percentage points** | - -The larger point is not the absolute score, because you could just as well fine-tune a better base model on more data. The main takeaways are the workflow and quality-of-life improvements that you get when you combine LanceDB and Geneva: - -1. **Add expensive features** as new columns without rewriting the raw dataset. -2. **Read fixed-size model features efficiently** for shuffled PyTorch batches. -3. **Iterate quickly** from feature idea to scalable CPU/GPU backfill, using Geneva UDFs. - -## Why LanceDB fits this workflow {#why-lancedb-fits-this-workflow} - -VLM fine-tuning pipelines spend a lot of time between "I have an experiment idea" and "I trained the model." LanceDB shortens that loop in three places. - - - - Lance can append derived columns such as `ocr_token_count`, `dhash`, `vision_tower_hiddens`, and tokenized SFT prompts without rewriting the existing image/question/answer columns or managing sidecar files. - - - Lance is optimized for scans and random access over fixed-size lists, which are common in model training: embeddings, hidden states, token IDs, masks, and labels. - - - Geneva lets AI engineers express feature work as UDFs, run those UDFs across CPU or GPU workers, and materialize the results directly into the same Lance table. - - - -In this pipeline, those three properties combine into the core optimization: compute the VLM vision features once, store them cheaply, then train by reading only the cached columns the model needs. - -## Pipeline overview {#pipeline-overview} - -The runnable demo uses the exact Colab subset hosted at [`lance-format/textvqa-lance-colab`](https://huggingface.co/datasets/lance-format/textvqa-lance-colab). It is derived from the Lance-formatted TextVQA corpus and stores inline JPEG bytes, questions, answers, OCR tokens, object classes, CLIP image/question embeddings, and the cached training features used by this example. The full demo pipeline adds three tiers of derived features on top. - - - Cheap CPU columns such as `question_length`, `answer_length`, `question_type`, and `ocr_token_count`. - - - - Image-derived columns such as `dhash`, computed by decoding the JPEG once and storing a perceptual hash. - - - - GPU-heavy columns: `vision_tower_hiddens` plus SFT token fields (`input_ids`, `attention_mask`, `labels`). - - -The Colab notebook's workflow starts after all three tiers have been computed. It downloads a small curated subset and runs the training/evaluation path without needing to run Geneva or the vision-tower backfill on the notebook GPU. - -## 1\. Start with a multimodal LanceDB table {#1-start-with-a-multimodal-lancedb-table} - -The base schema comes from the TextVQA Lance dataset. One row contains the image bytes, natural-language question, reference answers, OCR tokens, scene tags, and retrieval embeddings. - -```py Python icon=Python -import pyarrow as pa - -BASE_SCHEMA = pa.schema([ - pa.field("id", pa.int64()), - pa.field("image", pa.large_binary()), - pa.field("image_id", pa.string()), - pa.field("question_id", pa.string()), - pa.field("question", pa.string()), - pa.field("answers", pa.list_(pa.string())), - pa.field("answer", pa.string()), - pa.field("image_emb", pa.list_(pa.float32(), 512)), - pa.field("question_emb", pa.list_(pa.float32(), 512)), - pa.field("ocr_tokens", pa.list_(pa.string())), - pa.field("image_classes", pa.list_(pa.string())), - pa.field("set_name", pa.string()), -]) -``` - -Because the raw image, text, OCR, and embedding features live together, the same table supports curation, retrieval, feature engineering, and training. For example, the notebook can run a text-to-image retrieval demo by searching `image_emb` with a question embedding that already exists in the row. - -## 2\. Add feature columns with Geneva {#2-add-feature-columns-with-geneva} - -Geneva turns feature engineering into UDF definitions plus backfills. The UDFs can be simple text functions, image-processing functions, or stateful GPU model calls. - -The Tier 1 features are ordinary CPU UDFs: - -```py Python icon=Python -import re -import pyarrow as pa -from geneva.transformer import udf - -_QUESTION_TYPE_PATTERNS = [ - ("how_many", re.compile(r"^\s*how\s+many\b", re.IGNORECASE)), - ("what_brand", re.compile(r"^\s*what\s+(is\s+the\s+)?(brand|company|make)\b", re.IGNORECASE)), - ("what", re.compile(r"^\s*what\b", re.IGNORECASE)), -] - -@udf(data_type=pa.string(), input_columns=["question"]) -def question_type(question: str) -> str: - for label, pattern in _QUESTION_TYPE_PATTERNS: - if pattern.search(question or ""): - return label - return "other" - -@udf(data_type=pa.int32(), input_columns=["ocr_tokens"]) -def ocr_token_count(ocr_tokens: list[str] | None) -> int: - return len(ocr_tokens) if ocr_tokens else 0 -``` - -The Tier 3 feature is heavier: run Qwen2.5-VL's frozen vision tower once, then store the merged visual hidden states as a fixed-size fp16 list. - -```py Python icon=Python -IMAGE_PX = 560 -LLM_TOKENS_PER_IMAGE = 400 -VISION_HIDDEN = 2048 - -@udf( - data_type=pa.list_(pa.float16(), LLM_TOKENS_PER_IMAGE * VISION_HIDDEN), - input_columns=["image"], -) -class VisionTowerEmbedder: - def __init__(self): - self._model = None - self._processor = None - - def _lazy_load(self): - if self._model is not None: - return - import torch - from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration - - self._torch = torch - self._model = Qwen2_5_VLForConditionalGeneration.from_pretrained( - "Qwen/Qwen2.5-VL-3B-Instruct", - torch_dtype=torch.bfloat16, - device_map="cuda:0", - ).model.visual.eval() - self._processor = AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-3B-Instruct") - - def __call__(self, image: bytes) -> list[float]: - self._lazy_load() - # Decode image, resize to IMAGE_PX, run the frozen vision tower, - # and return fp16[400, 2048] flattened as one fixed-size list. - ... -``` - -The fixed shape matters. With `IMAGE_PX = 560`, Qwen2.5-VL produces 400 merged visual tokens, each with hidden size 2048. That becomes one `fp16[400 * 2048]` column per row. Training can scan and randomly access that column without decoding images or running the vision tower in the hot loop, saving GPU compute at training time. - -Run the tiered backfill with Geneva: - -```bash bash icon="terminal" -python -m vlm.backfill_geneva --tier 1 # CPU text columns -python -m vlm.backfill_geneva --tier 2 # image decode + dhash -python -m vlm.backfill_geneva --tier 3 # vision tower + SFT tokens -``` - - -The same Tier 3 work can be done manually by creating PyArrow batches and calling Lance's column-evolution APIs directly. The demo repo includes [`backfill_direct.py`](https://github.com/lancedb/tmls-2026-demo/blob/main/vlm/backfill_direct.py) for that path. Geneva is the preferred abstraction when you want to scale the same feature code across CPU or GPU workers and keep backfills incremental. - - -See the full UDF registry in [`vlm/geneva_udfs.py`](https://github.com/lancedb/tmls-2026-demo/blob/main/vlm/geneva_udfs.py) and the backfill driver in [`vlm/backfill_geneva.py`](https://github.com/lancedb/tmls-2026-demo/blob/main/vlm/backfill_geneva.py). - -## 3\. Curate a training slice {#3-curate-a-training-slice} - -The demo uses a `text_dense` slice: TextVQA examples whose images contain many OCR tokens. The slice was chosen empirically because it gave the clearest LoRA lift over the already-strong base model. - -```py Python icon=Python -TEXT_DENSE_OCR_THRESHOLD = 16 - -def matches_text_dense(row: dict) -> bool: - return len(row.get("ocr_tokens") or []) >= TEXT_DENSE_OCR_THRESHOLD -``` - -The Colab-ready bake ingests a small train split, backfills Tier 3 on that train table, ingests a held-out validation split, and optionally pushes the result to Hugging Face: - -```bash bash icon="terminal" -python -m vlm.colab_prepare \ - --out data/colab \ - --slice text_dense \ - --train-rows 600 \ - --val-rows 400 \ - --hf-repo lance-format/textvqa-lance-colab \ - --push -``` - -The train table contains cached Tier 3 columns because training reads them directly. The validation table keeps raw images because evaluation should run the full VLM on unseen images. - -## 4\. Explore the prepared table {#4-explore-the-prepared-table} - -Before training, it helps to look at the actual task. Each row pairs an image with a question whose answer is often visible as text in the image: a product label, phone screen, sign, book spine, or package. - - - - **A:** TWA - - **OCR:** 7h the Finest... 74 1E TWA 8 SALT REESE PEPPER - - - **A:** 12:39 am - - **OCR:** AT&T 12:39 AM TV CS WATCH P PANDORA YouTube Ustream - - - **A:** lego - - **OCR:** LEGO CITY Ages/edades 5-12 POLICE B-403 4473 112 112 pcs - - - **A:** warning - - **OCR:** WARNING Controlled Area Itis unlawf enter thisre without permission nstallation - - - -The notebook downloads the public pre-baked subset: - -```py Python icon=Python -from huggingface_hub import snapshot_download -import lancedb -import os - -local = snapshot_download( - repo_id="lance-format/textvqa-lance-colab", - repo_type="dataset", - local_dir="data/colab", -) - -def open_tbl(path: str): - name = os.path.basename(path).removesuffix(".lance") - return lancedb.connect(os.path.dirname(path)).open_table(name) - -train_tbl = open_tbl(f"{local}/textvqa_colab_train.lance") -val_tbl = open_tbl(f"{local}/textvqa_colab_val.lance") -``` - -Because the table also ships CLIP embeddings, you can run cross-modal retrieval without loading a model: - -```py Python icon=Python -import numpy as np - -seed = ( - train_tbl.search() - .select(["question", "question_emb"]) - .limit(40) - .to_arrow() - .to_pylist()[11] -) - -hits = ( - train_tbl.search( - np.asarray(seed["question_emb"], dtype=np.float32), - vector_column_name="image_emb", - ) - .select(["image", "question", "answer", "_distance"]) - .limit(5) - .to_arrow() - .to_pylist() -) -``` - -This is the same table that later feeds training. There is no separate feature store, image directory, Parquet export, or manifest to keep synchronized. - -## 5\. Benchmark Lance vs Parquet-style reads {#5-benchmark-lance-vs-parquet-style-reads} - -Many training pipelines start with Parquet. Parquet is excellent for columnar analytics, but training commonly needs shuffled batches and fixed-size tensor columns. The notebook compares Lance and Parquet on two access patterns: - -| Column group | Why it matters | -|---|---| -| `image`, `question`, `answer` | Raw multimodal rows: the baseline "decode and tokenize during training" path. | -| `vision_tower_hiddens` | Cached fixed-size fp16 VLM features: the optimized training path. | - -The notebook mirrors those column groups to uncompressed Parquet, then measures sequential scans and shuffled random batches: - -```py Python icon=Python -RAW = ["image", "question", "answer"] -VEC = ["vision_tower_hiddens"] -BATCH = 8 - -lance_ds = train_tbl.to_lance() -n = train_tbl.count_rows() - -def seq(ds, cols): - t0 = time.time() - for _ in ds.to_batches(columns=cols, batch_size=BATCH): - pass - return n / (time.time() - t0) - -def shuf(ds, cols, num_batches=20): - batches = [ - sorted(rng.choice(n, BATCH, replace=False).tolist()) - for _ in range(num_batches) - ] - t0 = time.time() - for idx in batches: - ds.take(idx, columns=cols) - return (num_batches * BATCH) / (time.time() - t0) -``` - -One Colab run produced the following throughput: - -| Throughput, rows/s | LanceDB | Parquet | -|---|---:|---:| -| `image` + `question` + `answer`, sequential | 2,603 | 8,311 | -| `image` + `question` + `answer`, shuffled | 2,613 | 352 | -| `vision_tower_hiddens` fp16, sequential | 1,452 | 90 | -| `vision_tower_hiddens` fp16, shuffled | 2,149 | -- | - -The takeaways are workload-specific: - -- For a traditional sequential scan over raw image/question/answer columns, Parquet is faster in this run: 8,311 rows/s vs 2,603 rows/s. -- For shuffled raw multimodal batches, Lance is faster because training reads scattered rows repeatedly instead of streaming the file once. -- For cached fp16 fixed-size arrays, Lance is about 16x faster than Parquet on the sequential scan. This is the training-relevant path in this example: the model reads `vision_tower_hiddens`, token IDs, masks, and labels as fixed-size columns. -- The benchmark intentionally skips the Parquet fp16 shuffled case. Parquet would re-decode whole row groups for each random batch, which is slow enough to distract from the real use case. The sequential fp16 row already shows the layout gap, while Lance shuffled reads remain fast. - -The numbers shown above are central to the example. The Tier 3 feature is only useful if the storage format can read it efficiently in the way a trainer actually needs: projected columns, repeated scans, and shuffled batches. **Lance specializes in exactly that access pattern**, including fixed-size list columns stored on disk. - -## 6\. Load cached columns with the Permutation API {#6-load-cached-columns-with-the-permutation-api} - -The training DataLoader projects only the columns needed by the cached training loop: - -```py Python icon=Python -from lancedb.permutation import Permutation - -CACHED_COLS = [ - "vision_tower_hiddens", - "input_ids", - "attention_mask", - "labels", -] - -class LancePermutationDataset(torch.utils.data.Dataset): - def __init__(self, uri: str, table_name: str): - self.uri = uri - self.table_name = table_name - self._perm = None - self.length = len(lancedb.connect(uri).open_table(table_name)) - - def __len__(self): - return self.length - - def __getstate__(self): - state = self.__dict__.copy() - state["_perm"] = None - return state - - def _ensure_open(self): - if self._perm is None: - tbl = lancedb.connect(self.uri).open_table(self.table_name) - self._perm = ( - Permutation.identity(tbl) - .select_columns(CACHED_COLS) - .with_format("arrow") - ) - - def __getitems__(self, indices: list[int]): - self._ensure_open() - return self._perm.__getitems__(indices) -``` - -Each worker opens its own `Permutation`, reads Arrow batches directly from Lance, and avoids per-row Python object conversion until the collate function converts arrays into tensors. - -The training batch contains: - -| Field | Shape | -|---|---| -| `vision_hiddens` | `fp16[B, 400, 2048]` | -| `input_ids` | `int64[B, 512]` | -| `attention_mask` | `int64[B, 512]` | -| `labels` | `int64[B, 512]` | - -## 7\. Fine-tune without loading the vision tower {#7-fine-tune-without-loading-the-vision-tower} - -The training process loads the language-model side of Qwen2.5-VL in 4-bit, deletes the vision tower, and wraps the LLM projections with LoRA adapters. - -During the forward pass, the model embeds the token IDs, finds the `<|image_pad|>` positions, and inserts the cached visual hidden states into those positions: - -```py Python icon=Python -def forward_cached(model, batch, image_pad_id: int): - base = model.get_base_model() if hasattr(model, "get_base_model") else model - inner = base.model - - inputs_embeds = inner.get_input_embeddings()(batch.input_ids) - _, _, hidden_dim = inputs_embeds.shape - - mask = ( - (batch.input_ids == image_pad_id) - .unsqueeze(-1) - .expand_as(inputs_embeds) - ) - - vision_flat = batch.vision_hiddens.to(inputs_embeds.dtype).reshape(-1, hidden_dim) - inputs_embeds = inputs_embeds.masked_scatter(mask, vision_flat) - - return model( - inputs_embeds=inputs_embeds, - attention_mask=batch.attention_mask, - labels=batch.labels, - ).loss -``` - -At this point, the LanceDB integration is done. The rest is plain PyTorch: optimizer, gradient accumulation, checkpointing, and saving the LoRA adapter. - -```py Python icon=Python -loader = make_cached_loader( - "data/colab/textvqa_colab_train.lance", - batch_size=2, - shuffle=True, -) - -for batch in loader: - batch = batch.to(device) - loss = forward_cached(model, batch, image_pad_id) - (loss / grad_accum).backward() - ... -``` - -This produces a training log like the following. The loss falls as the adapter learns from the cached features, and peak VRAM stays at 5.3 GB because QLoRA trains without keeping the vision tower active: - -```text -step 10/300 loss=2.6694 5.9 samples/s -step 20/300 loss=2.3133 6.1 samples/s - . - . - . -step 290/300 loss=0.0359 6.3 samples/s -step 300/300 loss=0.4750 6.3 samples/s -saved adapter to runs/colab_lora/lora | peak VRAM 5.3 GB -``` - -The training loop pays zero per-step cost for image decode, vision-tower forward, or prompt tokenization. Those costs were moved into feature engineering, where LanceDB and Geneva make them durable, incremental, and reusable. - -## 8\. Evaluate on held-out images {#8-evaluate-on-held-out-images} - -Evaluation uses the held-out validation table and loads the full VLM, including the vision tower. That is intentional: inference should see raw unseen images, not the cached train features. - -```py Python icon=Python -rows = ( - val_tbl.search() - .select(["image", "question", "answer", "answers"]) - .limit(256) - .to_arrow() - .to_pylist() -) - -base_model, processor = load_model(adapter_dir=None, load_4bit=True) -tuned_model, processor = load_model(adapter_dir="runs/colab_lora/lora", load_4bit=True) - -base_score = score_textvqa(base_model, processor, rows) -tuned_score = score_textvqa(tuned_model, processor, rows) -``` - -In this end-to-end example, the held-out curated validation split produced: - -| Model | TextVQA accuracy | -|---|---:| -| Base `Qwen2.5-VL-3B-Instruct` | 0.799 | -| QLoRA-tuned adapter | **0.820** | -| Lift | **+2.1 percentage points** | - -The tuned adapter is not meant to be a state-of-the-art TextVQA checkpoint. It is the proof point for the pipeline: the same Lance table supports curation, feature engineering, efficient training reads, and evaluation on held-out raw images. - -The notebook renders side-by-side examples: image, question, base answer, tuned answer, and ground truth. This closes the loop from feature idea to trained model while keeping the source data, derived features, training batches, and evaluation split in Lance. - -## Full source {#full-source} - -The complete demo implementation with helper scripts and usage instructions is in [this repo](https://github.com/lancedb/tmls-2026-demo). - - - - The runnable Colab workflow: download, explore, benchmark, train, and evaluate. - - - Tier 1, Tier 2, and Tier 3 feature definitions. - - - Geneva-powered feature materialization. - - - QLoRA training from cached Lance columns. - - diff --git a/docs/training/why-lancedb.mdx b/docs/training/why-lancedb.mdx deleted file mode 100644 index a74b0ee..0000000 --- a/docs/training/why-lancedb.mdx +++ /dev/null @@ -1,118 +0,0 @@ ---- -title: "Why LanceDB for Training" -sidebarTitle: "Why LanceDB for training" -description: "Use LanceDB as the multimodal data layer for model training, fine-tuning, curation, and feature engineering workflows." -icon: fire ---- - -LanceDB is built for AI teams that need a practical data layer between raw multimodal datasets and model training. -Instead of moving data through separate systems for curation, feature engineering, search, manifests, and training, -you can keep the whole workflow attached to one versioned LanceDB table. - -That table can hold images, video, audio, text, annotations, metadata, embeddings, tokenized fields, model outputs, -quality signals, and training-ready tensors. As the dataset evolves, LanceDB lets you add new columns, filter rows, -pin versions, and read batches without rewriting the original data. - -![Training data lifecycle: Curation, Feature Engineering, Search and Retrieval, Training](/static/assets/images/overview/training-data-lifecycle.svg) - -LanceDB gives these stages one platform, so curation, feature engineering, retrieval, and training stay connected. - -## A connected data lifecycle {#a-connected-data-lifecycle} - -Training pipelines usually need more than a pile of files. They need curation, derived features, reproducible splits, -fast random access, and a clean path into frameworks such as PyTorch. LanceDB keeps these pieces connected through -the same table model, whether you organize a workflow as one table or several related tables. - - - - Use filters, vector search, full-text search, and retrieval workflows to find the examples that matter: hard negatives, - long-tail failure modes, duplicate clusters, low-quality samples, or targeted fine-tuning slices. - - - Add embeddings, detections, OCR output, labels, token IDs, hidden states, deduplication flags, or quality scores as - new columns. Lance's columnar layout and schema evolution avoid rewriting large raw media columns when you add features. - - - Build filtered splits and materialized views from the table instead of exporting CSV manifests. Data versions and tags - make it possible to tie a checkpoint back to the exact rows and features used for training. - - - Use fast random access and column projection to read only the columns a training step needs. LanceDB tables can be read - from local storage or object storage, and integrate with data loading patterns such as PyTorch datasets. - - - -## Lance as the foundation {#lance-as-the-foundation} - -LanceDB is built on [Lance](https://lance.org/), an open-source lakehouse format designed for multimodal AI data. -The table below highlights the Lance features that enable the multimodal lakehouse on top. - -| Capability | Why it matters for training | -|---|---| -| **Multimodal columns** | Store raw bytes, annotations, metadata, embeddings, and features together. | -| **Fast random access** | Support shuffled and sampled reads without reshuffling the dataset on disk. | -| **Column projection** | Read only images, tokens, labels, embeddings, or hidden states needed by a given run. | -| **Schema evolution** | Add new feature columns without rewriting existing media columns. | -| **Versioning** | Reproduce experiments against the same table snapshot, even as the dataset evolves. | -| **Search and filtering** | Find and materialize useful training slices directly from the table. | - -## Search inside training workflows {#search-inside-training-workflows} - -Search is not limited to QA systems, agents, or production retrieval apps. It is also a practical way to inspect, -curate, and improve training data: - -- Find visually similar examples when debugging model failures. -- Retrieve hard negatives or near-duplicates for contrastive training. -- Combine vector search, full-text search, and metadata filters to build targeted fine-tuning slices. -- Reuse the same table for both offline curation and production retrieval. - -In LanceDB, retrieval and training workflows can operate over the same multimodal tables instead of forcing teams to -manage separate data systems for each stage. - -## Projects using LanceDB for training workflows {#projects-using-lancedb-for-training-workflows} - - - - A platform for reproducible world-model research built on a LanceDB data layer, reporting faster data loading on Push-T workloads. - - - A joint-embedding predictive world model from pixels, trained on the stable-worldmodel platform and its LanceDB data layer. - - - A drop-in LanceDB backend for Hugging Face LeRobot datasets with faster loading across robotics datasets. - - - -In the world-model ecosystem, [stable-worldmodel](https://github.com/galilai-group/stable-worldmodel) reports -3-4x faster data loading on Push-T versus HDF5 / MP4 at a fraction of the disk footprint. Across these projects, -LanceDB and Lance provide the multimodal data layer that keeps raw observations, annotations, features, and training -access patterns in one format instead of scattering them across task-specific stores. - -## Next steps {#next-steps} - - - - Learn how to use LanceDB permutations to select rows, project columns, split datasets, and shuffle training reads. - - - Use LanceDB tables and permutations with `torch.utils.data.DataLoader`. - - - Fine-tune an AV perception model on curated failure-mode slices backed by one LanceDB table. - - - Fine-tune a VLM on TextVQA using LanceDB and Geneva to cache expensive training features. - - diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx deleted file mode 100644 index bd46b83..0000000 --- a/docs/troubleshooting.mdx +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: "Troubleshooting" -sidebarTitle: "Troubleshooting" -description: "Tips for troubleshooting basic LanceDB issues." -icon: "tools" ---- - -## Frequently-asked questions {#frequently-asked-questions} - -For commonly asked questions about LanceDB, please refer to our [FAQ section](/faq). - -## Getting technical support {#getting-technical-support} - -If you're using LanceDB OSS, the best place to get help is in our -[Discord community](https://discord.gg/AUEWnJ7Txb), -under the relevant language channel for Python, TypeScript, or Rust. -By asking in the language-specific channel, you can get help from the community -and our engineering team. - -If you are a LanceDB Enterprise user, please contact our support team at [support@lancedb.com](mailto:support@lancedb.com) for dedicated assistance. - -## General issues {#general-issues} - -### Slow or unexpected query results {#slow-or-unexpected-query-results} - -If you have slow queries or unexpected query results, it can be helpful to -print the resolved query plan. - -LanceDB provides two powerful tools for query analysis and optimization: `explain_plan` and `analyze_plan`. - -Read the full guide on [Query Optimization](/search/optimize-queries/). - -### The Python multiprocessing module {#the-python-multiprocessing-module} -Multiprocessing with `fork` is not supported. You should use `spawn` instead. diff --git a/docs/tutorials/agents/index.mdx b/docs/tutorials/agents/index.mdx deleted file mode 100644 index 448e781..0000000 --- a/docs/tutorials/agents/index.mdx +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: "RAG and Agents" -sidebarTitle: "Notebooks" -description: "Explore a variety of RAG (Retrieval-Augmented Generation) and agent applications built with LanceDB." ---- - -The table below lists example notebooks we've prepared for a variety of RAG (Retrieval-Augmented Generation) -and agent applications built with LanceDB. - -| Project | Description | -|:----------|:------------| -| **Contextual RAG**
Open In Colab[View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Contextual-RAG) | Improves retrieval by combatting the "lost in the middle" problem. This technique uses an LLM to generate succinct context for each document chunk, then prepends that context to the chunk before embedding, leading to more accurate retrieval. | -| **NVIDIA RAG Blueprint with LanceDB**
[Read the tutorial](/tutorials/agents/nvidia-rag-blueprint/)
[View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/nvidia-rag-blueprint-lancedb) | Shows how to use LanceDB as the retrieval layer for NVIDIA RAG Blueprint with a Docker-first, retrieval-only integration path that includes hybrid search and pluggable rerankers. | -| **Matryoshka Embeddings**
Open In Colab[View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex) | Demonstrates a RAG pipeline using Matryoshka Embeddings with LanceDB and LlamaIndex. This method allows for efficient storage and retrieval of nested, variable-sized embeddings. | -| **HyDE (Hypothetical Document Embeddings)**
Open In Colab[View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Advance-RAG-with-HyDE) | An advanced RAG technique that uses an LLM to generate a "hypothetical" document in response to a query. This hypothetical document is then used to retrieve actual, similar documents, improving relevance. | -| **Late Chunking**
Open In Colab[View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Advanced_RAG_Late_Chunking) | An advanced RAG method where documents are retrieved first, and then chunking is performed on the retrieved documents just before synthesis. This helps maintain context that might be lost with pre-chunking. | -| **Agentic RAG**
Open In Colab[View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Agentic_RAG) | This tutorial demonstrates how to build a RAG system where multiple AI agents collaborate to retrieve information and generate answers, leading to more robust and intelligent applications. | diff --git a/docs/tutorials/agents/multimodal-agent/index.mdx b/docs/tutorials/agents/multimodal-agent/index.mdx deleted file mode 100644 index 039f01c..0000000 --- a/docs/tutorials/agents/multimodal-agent/index.mdx +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: "Multimodal Agent" -sidebarTitle: "Multimodal agent" -description: "Build an AI agent that understands both text and images to help users find recipes using LanceDB and PydanticAI" ---- - - -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1pxavAGoXa-KSh_4HxNpvP2AjHPcIRpbq?usp=sharing) - -Ever wanted to combine the power of text and images in a single AI agent? In this tutorial, -you'll build an agent that can understand both text and images to help users discover recipes that are relevant to them. The approach shown combines LanceDB's multimodal capabilities with [Pydantic AI](https://ai.pydantic.dev/) for the agentic workflow. - -## Key Technologies {#key-technologies} - -- **LanceDB**: Embedded retrieval library and multimodal lakehouse for efficient storage and retrieval -- **PydanticAI**: Modern AI agent framework with type safety -- **Sentence Transformers**: Text embeddings for semantic search -- **CLIP**: Vision-language model for image understanding -- **Streamlit**: Interactive web application framework - -## Tutorial Overview {#tutorial-overview} - -### Option 1: Notebook {#option-1-notebook} -The notebook shows how to work through the steps and prepare a small sample recipe dataset, generate both text and image -embeddings, store everything efficiently in LanceDB, and then build a PydanticAI agent with custom tools to -query it. You'll finish by testing the agent against a few example questions to see the full multimodal flow -end to end. - - -This simple tutorial provides a step-by-step workflow with a small demo dataset of 4 examples. -No local setup required - just click and start learning about multimodal agents. -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1pxavAGoXa-KSh_4HxNpvP2AjHPcIRpbq?usp=sharing) - - -### Option 2: Demo Application (Local Setup) {#option-2-demo-application-local-setup} -The demo application is the full codebase: you'll download and process a real recipe dataset with thousands -of items, run a Streamlit chat interface that supports image upload, and follow a structure that includes -production-minded touches like error handling, logging, and monitoring. Everything you need to deploy is -included. - - -Download the files for the full demo application here. - - -### Dataset Information {#dataset-information} -- **Source**: [Kaggle Recipe Dataset](https://www.kaggle.com/datasets/pes12017000148/food-ingredients-and-recipe-dataset-with-images) -- **Size**: Thousands of recipes with images -- **Format**: CSV file with recipe data and image references - -### Setup {#setup} - - -```bash bash icon="code" -# 1. Extract the downloaded files to a folder -# 2. Navigate to the folder in terminal -cd multimodal-recipe-agent - -# 3. Install dependencies with uv -uv sync - -# 4. Download the Kaggle dataset -# Visit: https://www.kaggle.com/datasets/pes12017000148/food-ingredients-and-recipe-dataset-with-images -# Extract recipes.csv to the data/ folder - -# 5. Import the dataset -uv run python import.py - -# 6. Run the complete Streamlit chat application -uv run streamlit run app.py -``` - diff --git a/docs/tutorials/agents/nvidia-rag-blueprint/index.mdx b/docs/tutorials/agents/nvidia-rag-blueprint/index.mdx deleted file mode 100644 index 667c876..0000000 --- a/docs/tutorials/agents/nvidia-rag-blueprint/index.mdx +++ /dev/null @@ -1,175 +0,0 @@ ---- -title: "NVIDIA RAG Blueprint with LanceDB" -sidebarTitle: "NVIDIA RAG Blueprint" -description: "Use LanceDB as the retrieval layer for NVIDIA RAG Blueprint with a Docker-first, retrieval-only reference integration." ---- - -## What this tutorial shows {#what-this-tutorial-shows} - -If you are using [NVIDIA RAG Blueprints](https://build.nvidia.com/blueprints) and want to evaluate LanceDB in that stack, this tutorial gives you a concrete starting point. It shows how to use LanceDB as the retrieval layer for a Docker-based NVIDIA RAG deployment with a small, script-driven reference integration where LanceDB OSS is embedded directly in the NVIDIA containers, the collection is prepared ahead of time, and the RAG server retrieves from it for search and generation. The example is intentionally retrieval-only, but it also includes hybrid search and reranker selection so you can see how LanceDB fits into a realistic NVIDIA retrieval workflow. - - -The runnable example for this tutorial lives in the -[VectorDB recipes repository](https://github.com/lancedb/vectordb-recipes/tree/main/examples/nvidia-rag-blueprint-lancedb). - - - -## How NVIDIA organizes vector databases {#how-nvidia-organizes-vector-databases} - -NVIDIA's [RAG Blueprint documentation](https://docs.nvidia.com/rag/latest/readme.html) effectively describes three different patterns for vector database support. -1. There are built-in backends such as Milvus, where NVIDIA already owns both ingestion and -retrieval. -2. There are built-in alternatives such as Elasticsearch, where NVIDIA still owns -the end-to-end flow but switches the backend through configuration. -3. Then, there is the custom vector database path, where you implement a `VDBRag` backend yourself and register it in NVIDIA's -factory. - -The LanceDB example shown below fits into the third category. More specifically, it follows NVIDIA's -**retrieval-only** custom backend path: the data is prepared in LanceDB ahead of time, and NVIDIA -RAG Blueprint is then pointed at that existing collection for search and generation. It does not -yet teach NVIDIA's ingestor how to write new documents into LanceDB automatically. - -## Deployment model {#deployment-model} - -This reference integration uses **LanceDB OSS as an embedded retrieval library**, not as a separate -database service. In practice, `APP_VECTORSTORE_NAME` is set to `lancedb`, `APP_VECTORSTORE_URL` -points to a local filesystem path inside the NVIDIA containers, the LanceDB collection is prepared -ahead of time, and the NVIDIA RAG server loads the LanceDB adapter to retrieve directly from that -local dataset. - -## What the recipe contains {#what-the-recipe-contains} - -The recipe at -[`examples/nvidia-rag-blueprint-lancedb`](https://github.com/lancedb/vectordb-recipes/tree/main/examples/nvidia-rag-blueprint-lancedb) -is organized around a small number of practical pieces. The data-prep script builds a demo -LanceDB collection from scratch, generates embeddings through the LanceDB embedding registry, and -creates a full-text index so hybrid retrieval works immediately. The adapter file shows the -retrieval-only integration point for NVIDIA RAG Blueprint, while the Docker override and NVIDIA -change guide show the minimal configuration and source changes needed to run the example against -NVIDIA's containers. - -## End-to-end flow {#end-to-end-flow} - -### 1\. Prepare the LanceDB collection {#1-prepare-the-lancedb-collection} - -From the [recipe directory](https://github.com/lancedb/vectordb-recipes/tree/main/examples/nvidia-rag-blueprint-lancedb): - -```bash -uv sync -uv run prepare_lancedb.py --embedder demo-keyword --reranker mrr -``` - -That script creates: - -- a local LanceDB dataset under `data/` -- a collection named `nvidia_blueprint_demo` -- automatic embeddings generated at ingest time -- an FTS index for hybrid search - -The default embedder is an offline demo embedder so the example stays easy to run. If you want a -more realistic setup, the same script can switch to a sentence-transformers embedder. - -### 2\. Patch the NVIDIA blueprint {#2-patch-the-nvidia-blueprint} - -Follow the instructions in the recipe's -[`nvidia_blueprint_changes.md`](https://github.com/lancedb/vectordb-recipes/tree/main/examples/nvidia-rag-blueprint-lancedb/nvidia_blueprint_changes.md). -The essential changes are: - -- add LanceDB dependencies to the NVIDIA environment -- copy `lancedb_vdb.py` into the NVIDIA source tree -- register the `lancedb` branch in NVIDIA's VDB factory - -NVIDIA's [RAG blueprint documentation](https://docs.nvidia.com/rag/latest/readme.html) and custom-VDB guide provide useful background if you want more context before applying the LanceDB-specific changes. - -### 3\. Start the Docker deployment {#3-start-the-docker-deployment} - -Set the absolute path to the recipe directory: - -```bash -export LANCEDB_RECIPE_DIR=/absolute/path/to/vectordb-recipes/examples/nvidia-rag-blueprint-lancedb -``` - -Then from the NVIDIA repo root: - -```bash -docker compose \ - -f deploy/compose/docker-compose-rag-server.yaml \ - -f "$LANCEDB_RECIPE_DIR"/docker-compose.override.yml \ - up -d --build - -docker compose \ - -f deploy/compose/docker-compose-ingestor-server.yaml \ - -f "$LANCEDB_RECIPE_DIR"/docker-compose.override.yml \ - up -d --build -``` - -The key environment values are: - -- `APP_VECTORSTORE_NAME=lancedb` -- `APP_VECTORSTORE_URL=/opt/lancedb-recipe/data` -- `COLLECTION_NAME=nvidia_blueprint_demo` -- `APP_VECTORSTORE_SEARCHTYPE=hybrid` -- `LANCEDB_RERANKER=mrr` - -## Verifying the integration {#verifying-the-integration} - -### Search {#search} - -```bash -curl -X POST http://localhost:8081/v1/search \ - -H 'Content-Type: application/json' \ - -d '{ - "query": "How do I replace Milvus in the NVIDIA RAG blueprint with LanceDB?", - "use_knowledge_base": true, - "collection_names": ["nvidia_blueprint_demo"], - "vdb_top_k": 3, - "reranker_top_k": 0 - }' -``` - -### Generate {#generate} - -```bash -curl -N -X POST http://localhost:8081/v1/generate \ - -H 'Content-Type: application/json' \ - -d '{ - "messages": [{"role":"user","content":"Summarize the LanceDB integration approach."}], - "use_knowledge_base": true, - "collection_names": ["nvidia_blueprint_demo"], - "vdb_top_k": 3, - "reranker_top_k": 0 - }' -``` - -## Hybrid retrieval and rerankers {#hybrid-retrieval-and-rerankers} - -This example is meant to prove more than a trivial vector lookup. - -- LanceDB hybrid retrieval combines vector search with full-text search -- the recipe creates the FTS index as part of dataset prep -- the adapter supports `RRFReranker`, `MRRReranker`, and `CrossEncoderReranker` -- the default example uses `MRRReranker`, not a plain weighted linear combination - -That matters for NVIDIA partner workloads because product names, storage platforms, and technical -jargon often need exact lexical matching as well as semantic retrieval. - -## How this can be extended {#how-this-can-be-extended} - -The current example follows NVIDIA's **custom retrieval-only backend** path. In practice, that -means the LanceDB collection is created ahead of time and NVIDIA RAG Blueprint is then pointed at -that existing collection for search and generation. The sample data in `prepare_lancedb.py` exists -only to make that flow runnable end to end: it creates a small local collection, inserts a few -documents, generates embeddings, and builds an FTS index so the NVIDIA side has something real to -query. - -A fuller integration is possible. NVIDIA's custom `VDBRag` interface also supports the pattern used -by built-in backends such as Milvus and Elasticsearch, where NVIDIA owns both ingestion and -retrieval. To make LanceDB work that way, a complete LanceDB backend would need to implement the -ingestion methods NVIDIA documents, especially `create_collection` and `write_to_index`, along with -the retrieval and collection-management methods expected by the rest of the stack. - -The open work is in defining how NVIDIA's ingestor should write -records into LanceDB, how that storage is shared between the ingestor and the RAG server, and how -document and collection metadata should be exposed so the broader NVIDIA APIs behave correctly. -Until those pieces exist, this example should be read as: prepare LanceDB first, then let NVIDIA -retrieve from it. diff --git a/docs/tutorials/agents/time-travel-rag/index.mdx b/docs/tutorials/agents/time-travel-rag/index.mdx deleted file mode 100644 index 3f33533..0000000 --- a/docs/tutorials/agents/time-travel-rag/index.mdx +++ /dev/null @@ -1,179 +0,0 @@ ---- -title: "Time-Travel RAG with versioned data" -sidebarTitle: "Time-travel RAG" -description: "Learn how to build production-ready RAG systems with LanceDB's time-travel capabilities for regulatory compliance and audit trails." ---- - - -All the scripts and code for this tutorial are available in the -[vectorDB recipes](https://github.com/lancedb/vectordb-recipes/tree/main/examples/time-travel-rag) repository. - - -## Use case: Financial services regulatory knowledge base {#use-case-financial-services-regulatory-knowledge-base} - -Imagine you're a major investment bank. Your team is tasked with building a critical Retrieval-Augmented Generation (RAG) system. This system must provide instant, accurate answers to compliance officers about ever-changing financial regulations. A wrong or out-of-date answer isn't just an inconvenience—it could lead to multi-million dollar fines, reputational damage, and regulatory audits. - -Your knowledge base is a living entity, constantly evolving with: - -* Daily regulatory updates from government bodies. -* New internal policy documents and interpretations. -* A/B testing of different embedding models and text chunking strategies to improve accuracy. - -This dynamic environment creates a series of high-stakes challenges that traditional -vector databases are ill-equipped to handle. - -## Pain points solved by LanceDB {#pain-points-solved-by-lancedb} - -1. "Our RAG gave different answers yesterday versus today. Which version was used in the official compliance report?" Without versioning, you can't prove what the AI knew at a specific point in time, making audits impossible. - -2. "The new embedding model we deployed corrupted half the vectors. Can we instantly roll back our 10TB dataset?" With traditional systems, a rollback means a painful, hours-long (or days-long) process of re-indexing from a backup, leading to significant downtime. - -3. "Regulators want to audit an AI-assisted decision from three months ago. How can we prove what data the model had access to at that exact moment?" Reproducibility is key for compliance. You must be able to reconstruct the exact state of the knowledge base for any historical query. - -4. "We need to A/B test a new chunking strategy, but we can't disrupt the production system or duplicate the entire dataset." Experimentation is vital for improvement, but it can't come at the cost of production stability or a massive infrastructure bill. - -LanceDB's [zero-cost data evolution](/tables/schema) and [time-travel capabilities](https://docs.lancedb.com/tables/versioning) directly address these critical enterprise pain points, providing the foundation for a reliable, auditable, and production-ready RAG system. - -## Dataset: The U.S. Federal Register {#dataset-the-us-federal-register} - -To make this use case realistic, we'll use a perfect real-world dataset: The U.S. Federal Register, the official daily journal of the United States Government. - -It contains all new rules, proposed rules, and notices from federal agencies. It is the canonical source for regulatory changes, and it's updated every business day. It even has a public API, allowing us to simulate the real-time ingestion of new documents. - -An example output of the workflow defined in main.py -is shown below. - -```bash main.py expandable ---- Initializing Database Environment --- -Removed old database at ./lancedb -Loading embedding model: all-MiniLM-L6-v2... - ---- STEP 1: Initial Data Ingestion --- - -Fetching 500 documents for publication date: 2024-08-19... -Successfully fetched 86 documents. -Embedding 86 documents... -Batches: 100%|███████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:00<00:00, 7.12it/s] -Successfully embedded 86 documents. - -Creating table 'federal_register'... -✅ Table 'federal_register' created. Version: 1, Rows: 86 - ---- STEP 2: Simulating Sequential Daily Updates --- - -Fetching 500 documents for publication date: 2024-08-20... -Successfully fetched 102 documents. -Embedding 102 documents... -Batches: 100%|███████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 10.66it/s] -Successfully embedded 102 documents. -✅ Data added to 'federal_register'. New Version: 2, Total Rows: 188 - -Fetching 500 documents for publication date: 2024-08-21... -Successfully fetched 114 documents. -Embedding 114 documents... -Batches: 100%|███████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 10.19it/s] -Successfully embedded 114 documents. -✅ Data added to 'federal_register'. New Version: 3, Total Rows: 302 - - -======================================================== -= PART 1: AUDITING KNOWLEDGE BASE ACROSS TIME = -======================================================== - -Running audit for query: 'cybersecurity reporting requirements for public companies' - -Attempting to open 'federal_register' and checkout version 1... -✅ Successfully checked out Version 1 of 'federal_register'. Total rows: 86 -Querying table 'federal_register' (Version 1)... - ---- Top Result for Version: V1 (all-MiniLM-L6-v2) --- -📄 Title: Public Company Accounting Oversight Board; Extension of Approval Periods for Proposed Rules on a Firm's System of Quality Control and Related Amendments to PCAOB Standards, Proposed Rules on Amendments Related to Aspects of Designing and Performing Audit Procedures That Involve Technology-Assisted Analysis of Information in Electronic Form, and Proposed Rules on Amendment to PCAOB Rule 3502 Governing Contributory Liability -🗓️ Date: 2024-08-19 -📏 Distance: 1.1667 -📝 Abstract: -[No abstract available for this document] --------------------------------------- - -Attempting to open 'federal_register' and checkout version 2... -✅ Successfully checked out Version 2 of 'federal_register'. Total rows: 188 -Querying table 'federal_register' (Version 2)... - ---- Top Result for Version: V2 (all-MiniLM-L6-v2) --- -📄 Title: Information Collection Being Reviewed by the Federal Communications Commission Under Delegated Authority -🗓️ Date: 2024-08-20 -📏 Distance: 1.1436 -📝 Abstract: -As part of its continuing effort to reduce paperwork burdens, and as required by the Paperwork -Reduction Act (PRA) of 1995, the Federal Communications Commission (FCC or the Commission) invites -the general public and other Federal agencies to take this opportunity to comment on the following -information collection. Comments are requested concerning: whether the proposed collection of -information is necessary for the proper performance of the functions of the Commission, including -whether the information shall have practical utility; the accuracy of the Commission's burden -estimate; ways to enhance the quality, utility, and clarity of the information collected; ways to -minimize the burden of the collection of information on the respondents, including the use of -automated collection techniques or other forms of information technology; and ways to further reduce -the information collection burden on small business concerns with fewer than 25 employees. The FCC -may not conduct or sponsor a collection of information unless it displays a currently valid control -number. No person shall be subject to any penalty for failing to comply with a collection of -information subject to the PRA that does not display a valid Office of Management and Budget (OMB) -control number. --------------------------------------- - -Attempting to open 'federal_register' and checkout version 3... -✅ Successfully checked out Version 3 of 'federal_register'. Total rows: 302 -Querying table 'federal_register' (Version 3)... - ---- Top Result for Version: V3 (all-MiniLM-L6-v2) --- -📄 Title: Equipment, Systems, and Network Information Security Protection -🗓️ Date: 2024-08-21 -📏 Distance: 1.0942 -📝 Abstract: -This proposed rulemaking would impose new design standards to address cybersecurity threats for -transport category airplanes, engines, and propellers. The intended effect of this proposed action -is to standardize the FAA's criteria for addressing cybersecurity threats, reducing certification -costs and time while maintaining the same level of safety provided by current special conditions. --------------------------------------- - -✅ Date-based audit complete. Results show how knowledge evolves over time. This demonstrates LanceDB's powerful [versioning capabilities](/tutorials/tables/consistency#versioning) for maintaining audit trails. - - -============================================================= -= PART 2: A/B TESTING DIFFERENT EMBEDDING MODELS = -============================================================= -Loading embedding model: all-mpnet-base-v2... -Embedding 302 documents... -Batches: 100%|█████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:03<00:00, 2.55it/s] -Successfully embedded 302 documents. - -Creating table 'federal_register_experimental'... -✅ Table 'federal_register_experimental' created. Version: 1, Rows: 302 - -Comparing search results for the same data with different models: -Querying table 'federal_register' (Version 3)... - ---- Top Result for Version: Latest Prod V3 (all-MiniLM-L6-v2) --- -📄 Title: Equipment, Systems, and Network Information Security Protection -🗓️ Date: 2024-08-21 -📏 Distance: 1.0942 -📝 Abstract: -This proposed rulemaking would impose new design standards to address cybersecurity threats for -transport category airplanes, engines, and propellers. The intended effect of this proposed action -is to standardize the FAA's criteria for addressing cybersecurity threats, reducing certification -costs and time while maintaining the same level of safety provided by current special conditions. --------------------------------------- -Querying table 'federal_register_experimental' (Version 1)... - ---- Top Result for Version: Experimental (all-mpnet-base-v2) --- -📄 Title: Commission Information Collection Activities (FERC-725B); Comment Request; Extension -🗓️ Date: 2024-08-20 -📏 Distance: 1.0827 -📝 Abstract: -In compliance with the requirements of the Paperwork Reduction Act of 1995, the Federal Energy -Regulatory Commission (Commission or FERC) is soliciting public comment on the currently approved -information collection, FERC-725B, Mandatory Reliability Standards, Critical Infrastructure -Protection (CIP) (Update for CIP-012-1 to version CIP-012-02) Cyber Security--Communications between -Control Centers. The 60-day notice comment period ended on July 23, 2024, with no comments received. --------------------------------------- - -✅ A/B test complete. Notice the difference in relevance (distance score) between models. This showcases how LanceDB enables [experimentation with different embedding models](/docs/embeddings/) without disrupting production systems. -``` diff --git a/docs/tutorials/feature-engineering/index.mdx b/docs/tutorials/feature-engineering/index.mdx deleted file mode 100644 index ad18e99..0000000 --- a/docs/tutorials/feature-engineering/index.mdx +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: "Feature Engineering" -sidebarTitle: "Feature Engineering" -description: "Learn how to build features for your data in LanceDB." -layout: wide ---- - -| Example | Description | -|--------|:------------| -| **Feature Engineering 101**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/feature-engineering/feature-engineering-101.ipynb) | This example demonstrates how to use LanceDB's feature engineering platform to add new derived features. -| **Materialized Views**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/feature-engineering/materialized-views.ipynb) | This example shows how to create materialized views: query results persisted as physical tables. - - -## Read the docs {#read-the-docs} - -The relevant section of the documentation are listed below. - -| Feature | Description | -|:--------|:------------| -| [Feature Engineering](/geneva/) | Learn the fundamentals of feature engineering. | -| [Materialized Views](/geneva/jobs/materialized-views) | Learn more about creating materialized views. | -| [Contexts](/geneva/jobs/contexts) | Learn how to run your job on a Ray cluster for production use. | diff --git a/docs/tutorials/index.mdx b/docs/tutorials/index.mdx deleted file mode 100644 index ec25dc0..0000000 --- a/docs/tutorials/index.mdx +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: "Tutorials" -sidebarTitle: "Overview" -description: "Step-by-step tutorials for building applications with LanceDB" ---- - -Explore tutorials organized by use case: - -| Tutorial | Description | -|:---------|:------------| -| [Search & advanced retrieval](/tutorials/search/) | Learn how to perform vector search and use advanced retrieval techniques. | -| [Agents](/tutorials/agents/) | Build Retrieval-Augmented Generation (RAG) applications and agents with LanceDB. | -| [Working with tables in LanceDB](/tables/) | Learn the basics of working with tables in LanceDB: creation, ingestion and schema evolution. | - -## Recipes {#recipes} - -If you're looking for ideas and hands-on code examples, we've worked on a collection of practical -projects in the repository linked below. - - -Check out past code examples and tutorials [here](https://github.com/lancedb/vectordb-recipes) -on GitHub. - diff --git a/docs/tutorials/search/index.mdx b/docs/tutorials/search/index.mdx deleted file mode 100644 index 035ab38..0000000 --- a/docs/tutorials/search/index.mdx +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: "Search Tutorials" -sidebarTitle: "Overview" -description: "Learn how vector, full-text and hybrid search work in LanceDB." -layout: wide ---- - -The table below shows examples of applications built with LanceDB for search use cases. - -| Example | Description | -|--------|:------------| -| **Hybrid search & reranking on BEIR**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Inbuilt-Hybrid-Search) | This example demonstrates how to use LanceDB's built-in hybrid search feature, which combines the strengths of both semantic and full-text search. By using the BEIR dataset, it shows how to achieve more relevant results by searching for both the meaning of a query and the specific keywords it contains. | -| **Semantic search across videos**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/v-jepa-video-search) | Learn how to build a video search application using V-JEPA (Video Joint Embedding Predictive Architecture) and LanceDB. This example shows how to generate vector embeddings for videos and then use LanceDB to perform similarity searches, allowing you to find videos that are visually similar to a given query. | -| **Semantic result merging**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Vector-Arithmetic-with-LanceDB) | Explore the concept of vector arithmetic with LanceDB. This notebook demonstrates how you can manipulate vector embeddings to capture more complex relationships in your data. For instance, you can modify a search query by adding or subtracting vector representations of different concepts, enabling more nuanced and powerful semantic search. | -| **Reddit concept summarizer**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Reddit-summarization-and-search) | This project showcases a complete pipeline for acquiring text data from Reddit, transforming it into meaningful vector representations using embeddings, and then storing and managing those vectors in LanceDB. It demonstrates how to build applications on top of this data, such as summarization and powerful semantic search. | -| **NER-powered vector search**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/NER-powered-Semantic-Search) | This example demonstrates how to use Named Entity Recognition (NER) to power vector search. By extracting entities (like people, places, and organizations) from text and creating vector embeddings of them, you can significantly improve the accuracy of your search results. | -| **Multi-vector search with XTR**
Open In Colab [View on GitHub](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multivector_xtr) | This notebook dives into LanceDB's advanced multivector search capabilities, enhanced by the XTR (ConteXtualized Token Retriever) technique. It shows how to represent complex data with multiple vectors for more nuanced meaning and how XTR speeds up retrieval by prioritizing the most important tokens. | -| **Needle-in-a-haystack multi-vector search**

[Read the tutorial](/tutorials/search/multivector-needle-in-a-haystack/)
| This tutorial complements the XTR multi-vector example by comparing several retrieval strategies on a token-level "needle in a haystack" benchmark. It shows when full multi-vector search, pooling, and reranking help or hurt when the goal is to find an exact page rather than just a relevant document. | - - -## Read the docs {#read-the-docs} - -The relevant section of the documentation are listed below. - -| Feature | Description | -|:--------|:------------| -| [Vector search](/search/vector-search/) | Learn the fundamentals of vector search, including how to perform similarity searches, use different distance metrics, and optimize performance. | -| [Hybrid search](/search/hybrid-search/) | Combine keyword-based search with vector search to improve retrieval accuracy and relevance. | -| [Full-text search](/search/full-text-search/) | Perform full-text search on your text data, and combine it with vector search for a powerful hybrid search experience. | -| [Reranking](/reranking/) | Refine your search results using reranking models to improve the relevance of the top-k results. | -| [Multi-vector search](/search/multivector-search/) | Use multiple vector embeddings per document to perform more nuanced and accurate searches. | diff --git a/docs/tutorials/search/multivector-needle-in-a-haystack.mdx b/docs/tutorials/search/multivector-needle-in-a-haystack.mdx deleted file mode 100644 index 7a030ae..0000000 --- a/docs/tutorials/search/multivector-needle-in-a-haystack.mdx +++ /dev/null @@ -1,330 +0,0 @@ ---- -title: "Finding the Needle in a Haystack: Comparing Multi-vector Search Strategies" -sidebarTitle: "Multi-vector search: Needle in a Haystack" -description: "Tutorial on token-level retrieval with LanceDB multivector search." ---- - -![](/static/assets/images/search/multivector/multivector-1.png) - -In the development of advanced search and retrieval systems, moving from keyword matching to semantic understanding is a critical step. However, a key distinction exists between finding a relevant document and locating a specific piece of information within that document with precision. While there are techniques that perform well for retrieving documents, most of them work by extracting summarized semantic meaning of the document. - -This can be seen as these models trying to understand the "gist" of the documents. Both single-vector search and late-interaction approaches work well for these conditions with various tradeoffs involved. But there's another type of problem where the goal is not just to understand the overall topic of a document in general, but to also specifically account for the requested detail within the document. This "needle in a haystack" problem is a significant challenge, and addressing it is essential for building high-precision retrieval systems. - -This guide provides a technical analysis of multivector search for high-precision information retrieval. We will examine various optimization strategies and analyze their performance. This guide should be seen as complementary to resources like [this blog post by Answer.AI](https://www.answer.ai/posts/colbert-pooling.html) on ColBERT pooling, which explains how pooling strategies can be effective for document-level retrieval. Here, we will demonstrate why those same techniques can be counterproductive when precision at an intra-document, token level is the primary objective. - -To reproduce the work below, see the code [here](https://github.com/lancedb/research/tree/main/multivector-needle-haystack-bench). - -## The Dataset {#the-dataset} - -This task is different from benchmarks like BEIR, which focus on text-based doc retrieval, finding the most relevant documents from a large collection. Here, we want *intra-document localization*, where the goal is to find a precise piece of information within a single, dense document, in a multimodal setting. - -### The Task: The Document Haystack Dataset {#the-task-the-document-haystack-dataset} - -Our benchmark is built on the **[AmazonScience/document-haystack](https://huggingface.co/datasets/AmazonScience/document-haystack)** dataset, which contains 25 visually complex source documents (e.g., financial reports, academic papers). To create a rigorous test, our evaluation follows a per-document methodology: - -* We process each of the 25 source documents independently. -* **Table Creation:** For a single source document (e.g., "AIG"), we ingest all pages from all of its page-length variants (from 5 to 200 pages long). This creates a temporary LanceDB table containing approximately 1,230 pages. -* **The Task:** We then query this table using a set of "needle" questions, where the goal is to retrieve the **exact page number** containing the answer. A successful retrieval means the correct page number is within the top K results. -* **Target metric:** We measure both retrieval accuracy (Hit@K) and the average search latency for each query against this table. - -**Dataset example** -The documents contain "text needles" like these - -![](/static/assets/images/search/multivector/multivector-2.png) -![](/static/assets/images/search/multivector/multivector-3.png) - -During evaluation, the queries processed are somewhat like this: -``` -What is the secret currency in the document? -What is the secret object #3 in the document? -``` -The intention of this task is to find the page which has the text needle that answers this questions - - -## Models and Architectures {#models-and-architectures} - -Our testbed includes a baseline single-vector model and a family of advanced multivector models. - -### Single-Vector (Bi-Encoder) Baseline: CLIP ViT-Base-Patch32 {#single-vector-bi-encoder-baseline-clip-vit-base-patch32} -A bi-encoder maps an entire piece of content (a query, a document page) to a *single* vector. The search process is simple: pre-compute one vector for every page, and at query time, find the page vector closest to the query vector. - -* **Strength:** Speed and simplicity. -* **Weakness:** This creates an **information bottleneck**. All the nuanced details, keywords, and semantic relationships on a page must be compressed into a single, fixed-size vector. For finding a needle, this is like trying to describe a specific person's face using only one word. - -### Multi-vector (Late-Interaction) Models {#multi-vector-late-interaction-models} - -Multi-vector models, pioneered by ColBERT, take a different approach. Instead of one vector per page, they generate a *set of vectors* for each page—one for every token (or image patch). - -* **Mechanism (MaxSim):** The search process is more sophisticated. For each token in the query, the system finds the most similar token on the page. These maximum similarity scores are then summed up to get the final relevance score. This "late-interaction" preserves fine-grained, token-level details. -* **The Models:** We used several vision-language models adapted for this architecture, including `ColPali`, `ColQwen2`, and `ColSmol`. While their underlying transformer backbones differ, they all share the ColBERT philosophy of representing documents as a bag of contextualized token embeddings. - -![](/static/assets/images/search/multivector/multivector-4.png) - -## Different Retrieval Strategies Used {#different-retrieval-strategies-used} - -A full multivector search is powerful but computationally intensive. Here are five strategies for managing it, complete with LanceDB implementation details. - - -### 1\. `base`: The Gold Standard (Full Multi-vector Search) {#1-base-the-gold-standard-full-multi-vector-search} - -This is the pure, baseline late-interaction search. It offers the highest potential for accuracy by considering every token. - -**LanceDB also integrates with ConteXtualized Token Retriever (XTR)** , an advanced retrieval model that prioritizes the most semantically important document tokens during search. This integration enhances the quality of search results by focusing on the most relevant token matches. -**LanceDB Implementation:** - -```python Python icon="python" -import lancedb -import pyarrow as pa - -# Schema for multivector data -# Assumes embeddings are 128-dimensional -schema = pa.schema([ - pa.field("page_num", pa.int32()), - pa.field("vector", pa.list_(pa.list_(pa.float32(), 128))) -]) - -db = lancedb.connect("./lancedb") -tbl = db.create_table("document_pages_base", schema=schema) - -# Ingesting multi-token embeddings for a page -# multi_token_embeddings is a NumPy array of shape (num_tokens, 128) -tbl.add([{"page_num": 1, "vector": multi_token_embeddings.tolist()}]) - -# Searching with a multi-token query -# query_multi_vector is also shape (num_query_tokens, 128) -results = tbl.search(query_multi_vector).limit(5).to_list() -``` - -### 2\. `flatten`: Mean Pooling {#2-flatten-mean-pooling} - -This strategy "flattens" the set of token vectors into a single vector by averaging them. This transforms the search into a standard, fast approximate nearest neighbor (ANN) search. - -**LanceDB Implementation:** - -```python Python icon="python" -# Schema for single-vector data -schema_flat = pa.schema([ - pa.field("page_num", pa.int32()), - pa.field("vector", pa.list_(pa.float32(), 128)) -]) -tbl_flat = db.create_table("document_pages_flat", schema=schema_flat) - -# Ingesting the mean-pooled vector -mean_vector = multi_token_embeddings.mean(axis=0) -tbl_flat.add([{"page_num": 1, "vector": mean_vector.tolist()}]) - -# Searching with a single averaged query vector -query_mean_vector = query_multi_vector.mean(axis=0) -results = tbl_flat.search(query_mean_vector).limit(5).to_list() -``` - -### 3\. Max pooling {#3-max-pooling} - -This is a variation of `flatten`. `max_pooling` takes the element-wise max across all token vectors instead of the mean. The implementation is identical to `flatten`, just with a different aggregation method (`.max(axis=0)`). - -### 4\. `flatten and multivector rerank`: The Hybrid Optimization {#4-flatten-and-multivector-rerank-the-hybrid-optimization} -This two-stage strategy aims for the best of both worlds. First, use a fast, pooled-vector search to find a set of promising candidates. Then, run the full, accurate multivector search on *only* those candidates. - -**LanceDB Implementation:** -This requires a table with two vector columns. - -```python Python icon="python" -# Schema with both flat and multivector columns -schema_rerank = pa.schema([ - pa.field("page_num", pa.int32()), - pa.field("vector_flat", pa.list_(pa.float32(), 128)), - pa.field("vector_multi", pa.list_(pa.list_(pa.float32(), 128))) -]) -tbl_rerank = db.create_table("document_pages_rerank", schema=schema_rerank) - -# Ingest both vectors -tbl_rerank.add([ - { - "page_num": 1, - "vector_flat": multi_token_embeddings.mean(axis=0).tolist(), - "vector_multi": multi_token_embeddings.tolist() - } -]) - -# --- Two-Stage Search --- -# Stage 1: Fast search on the flat vector -query_flat = query_multi_vector.mean(axis=0) -candidates = tbl_rerank.search(query_flat, vector_column_name="vector_flat") \ - .limit(100) \ - .with_row_id(True) \ - .to_pandas() - -# Stage 2: Precise multivector search on candidates -candidate_ids = tuple(candidates["_rowid"].to_list()) -final_results = tbl_rerank.search(query_multi_vector, vector_column_name="vector_multi") \ - .where(f"_rowid IN {candidate_ids}") \ - .limit(5) \ - .to_list() -``` - -### 5\. `hierarchical token pooling`: Compressing the Haystack {#5-hierarchical-token-pooling-compressing-the-haystack} - -This is an indexing-time strategy that aims to reduce the storage footprint and computational cost of multivector search by reducing the number of vectors per document. Instead of using every token vector, it clusters semantically similar tokens together and replaces them with a single, averaged vector. - -* **Mechanism:** For each document, it computes the similarity between all token vectors, performs hierarchical clustering to group them, and then mean-pools the vectors within each cluster. This results in a smaller, more compact set of token vectors representing the document. -* **Goal:** To reduce memory and disk usage while attempting to preserve the most important semantic information, potentially offering a middle ground between the high accuracy of `base` search and the speed of pooled methods. - -**LanceDB Implementation:** -The schema is identical to the `base` multivector search, but the data is pre-processed before ingestion. - -```python Python icon="python" -from utils import pool_embeddings_hierarchical -import numpy as np - -# Schema is the same as the base multivector schema -schema = pa.schema([ - pa.field("page_num", pa.int32()), - pa.field("vector", pa.list_(pa.list_(pa.float32(), 128))) -]) -tbl_hierarchical = db.create_table("document_pages_hierarchical", schema=schema) - -# Pool the embeddings before ingestion -# multi_token_embeddings is a NumPy array of shape (num_tokens, 128) -pooled_embeddings = pool_embeddings_hierarchical( - multi_token_embeddings, - pool_factor=4 # Reduce vector count by a factor of 4 -) - -# Ingest the smaller set of multi-token embeddings -# pooled_embeddings is now shape (approx. num_tokens / 4, 128) -tbl_hierarchical.add([{"page_num": 1, "vector": pooled_embeddings.tolist()}]) - -# Search is identical to the base multivector search -results = tbl_hierarchical.search(query_multi_vector).limit(5).to_list() -``` - -## The Results {#the-results} - -For a "needle in a haystack" task, retrieval accuracy is the primary metric of success. The benchmark results reveal a significant performance gap between the full multivector search strategy and common optimization techniques. - -### Baseline Performance: Single-Vector Bi-Encoder {#baseline-performance-single-vector-bi-encoder} - -First, we establish a baseline using a standard single-vector bi-encoder model, `openai/clip-vit-base-patch32`. This represents a common approach to semantic search but, as the data shows, is ill-suited for this task's precision requirements. - -| Model | Strategy | Hit@1 | Hit@5 | Hit@20 | Avg. Latency (s) | -| :--- | :--- | :--- | :--- | :--- | :--- | -| `openai/clip-vit-base-patch32` | `base` | 1.6% | 4.7% | 11.8% | **0.008 s** | - -With a Hit@20 rate of just under 12%, the baseline model struggles to reliably locate the correct page. This performance level is insufficient for applications requiring high precision. - -### Multi-vector Model Performance {#multi-vector-model-performance} - -We now examine the performance of multivector models using different strategies. The following table compares the `base` (full multivector), `flatten` (mean pooling), and `rerank` (hybrid) strategies across several late-interaction models. - -| Model | Strategy | Hit@1 | Hit@5 | Hit@20 | Avg. Latency (s) | -| :--- | :--- | :--- | :--- | :--- | :--- | -| `vidore/colqwen2-v1.0` | `flatten` | 1.9% | 5.5% | 11.9% | 0.010 s | -| `vidore/colqwen2-v1.0` | `flatten and multivector rerank` | 0.3% | 1.5% | 7.3% | 0.692 s | -| **`vidore/colqwen2-v1.0`** | **`hierarchical token pooling`** | **13.7%** | **60.5%** | **91.6%** | **0.144 s** | -| **`vidore/colqwen2-v1.0`** | **`base`** | **14.0%** | **65.4%** | **95.5%** | 0.668 s | -| `vidore/colpali-v1.3` | `flatten` | 1.7% | 4.5% | 9.3% | 0.008 s | -| `vidore/colpali-v1.3` | `flatten and multivector rerank` | 0.6% | 2.3% | 6.9% | 0.949 s | -| **`vidore/colpali-v1.3`** | **`hierarchical token pooling`** | **10.8%** | **41.7%** | **64.8%** | **0.189 s** | -| **`vidore/colpali-v1.3`** | **`base`** | **11.3%** | **42.3%** | **65.6%** | 0.936 s | -| `vidore/colSmol-256M` | `flatten` | 1.6% | 4.7% | 10.5% | 0.008 s | -| `vidore/colSmol-256M` | `flatten and multivector rerank` | 0.3% | 1.6% | 7.0% | 0.853 s | -| **`vidore/colSmol-256M`** | **`base`** | **14.4%** | **64.0%** | **91.7%** | 0.848 s | - -The data shows a consistent pattern: the `base` strategy outperforms all other techniques. The flattned pooling and reranking strategies perform no better than the single-vector baseline. However, hierarchical token pooling seems like a decent alternative to base considering speed vs accuracy tradeoff. Let's look at the numbers in detail. - -### In-Depth Analysis of Pooling Strategies {#in-depth-analysis-of-pooling-strategies} - -To further understand the failure of optimization techniques, we compared different methods for pooling token vectors into a single vector: `mean` (`flatten`), `max`. - -| Model & Pooling Strategy | Hit@1 | Hit@5 | Hit@20 | Avg. Latency (s) | -| :--- | :--- | :--- | :--- | :--- | -| `vidore/colqwen2-v1.0` (`mean_pooling`) | 1.9% | 5.5% | 11.9% | 0.010 s | -| `vidore/colqwen2-v1.0` (`max_pooling`) | 1.4% | 4.2% | 11.2% | 0.011 s | -| **`vidore/colqwen2-v1.0` (`base`)** | **14.0%** | **65.4%** | **95.5%** | 0.668 s | - -All flattened pooling methods perform poorly, confirming that the aggregation of token vectors into a single representation loses the fine-grained detail required for this task. - -**Finding the Right Trade-Off** - -![](/static/assets/images/search/multivector/multivector-5.png) - -1. **The Failure of Simple Pooling:** The `flatten` (mean pooling) and `max_pooling` strategies fail to improve upon the baseline. This is because their aggressive compression **destroys the essential localization signal**. The resulting single vector represents the *topic* of the page, not the *specific needle* on it. -2. **The Failure of `flatten and multivector rerank`:** This hybrid strategy is the *worst-performing* of all. The reason is a fundamental flaw in its design for this task: the first stage uses a simple pooled vector to retrieve candidates. Since this pooling eliminates the localization signal, the initial candidate set is effectively random. -3. **`hierarchical token pooling`:** By clustering and pooling tokens at indexing time, it reduces the number of vectors per page (in our case, by a factor of 4). This intelligently compresses the data, while preserving enough token-level detail in multivector setting. It achieves a **Hit@20 of 91.6%**, only slightly behind the `base` strategy's 95.5%, but is significantly faster. -4. **`base` multivector Search:** The vanilla, un-optimized `base` multivector search remains the most accurate strategy. Preserving every token vector provides the highest guarantee of finding the needle, but this comes at the highest computational cost. - -### Latency: {#latency} - -![](/static/assets/images/search/multivector/multivector-6.png) - - -The "optimizations" are not all created equal. While simple pooling is fast, its inaccuracy makes it unusable. Hierarchical pooling, however, offers a compelling balance of speed and accuracy. - -| Strategy (on `vidore/colqwen2-v1.0`) | Avg. Search Latency (s) | Hit@20 Accuracy | -| :--- | :--- | :--- | -| `flatten` (Fast but Ineffective) | **0.010 s** | 11.9% | -| `flatten and multivector rerank` (Slower and Ineffective) | 0.692 s | 7.3% | -| **`hierarchical token pooling` (Accurate & Fast)** | **0.144 s** | **91.6%** | -| `base` (Most Accurate) | 0.668 s | **95.5%** | -_Latency reported is as seen on NVIDIA H100 GPUs_ - -## Practical Considerations {#practical-considerations} - -The accuracy of `base` multivector search is impressive, but its computational intensity has historically limited its use. `hierarchical token pooling` as a viable strategy creates a new, practical sweet spot on the accuracy-latency curve, making high-precision search accessible for a wider range of applications. - -### Search Latency and Computational Complexity {#search-latency-and-computational-complexity} - - -As the benchmark data shows, the search latency for `base` multivector search is orders of magnitude higher than for single-vector (or pooled-vector) search. It's important to note that the reported ~670ms latency is an average from per-document evaluations. In this benchmark, each of the 25 documents is processed independently. All pages from a single document's variants (ranging from 5 to 200 pages) are ingested into a temporary table, resulting in a table size of approximately **1,230 rows (pages)** per evaluation. The search is performed on this table, and then the table is discarded. This highlights a significant performance cost even on a relatively small, per-document scale. This stems from a fundamental difference in computational complexity: - -* **Modern ANN Search (for single vectors):** Algorithms like HNSW (Hierarchical Navigable Small World) provide sub-linear search times, often close to `O(log N)`, where `N` is the number of items in the index. This allows them to scale to billions of vectors with millisecond-level latency. -* **Late-Interaction Search (Multi-vector):** The search process is far more intensive. For each query, it must compute similarity scores between query tokens and the tokens of many candidate documents. The complexity is closer to `O(M * Q * D)`, where `M` is the number of candidate documents to score, `Q` is the number of query tokens, and `D` is the average number of tokens per document. `Hierarchical token pooling` directly attacks this problem by reducing `D`, leading to a significant reduction in search latency. - -### When to Use Multi-Vector Search {#when-to-use-multi-vector-search} - -Given these constraints, the choice of strategy depends on the specific requirements of the application. - -* **For Maximum Precision (`base`):** In domains where the cost of missing the needle is extremely high, the full `base` search is the most reliable option. -* **For a Balance of Precision and Performance (`hierarchical token pooling`):** This is the ideal choice for many applications. It makes high-precision search practical for larger datasets and more interactive use cases where the sub-second latency of the `base` search may be too high. It significantly lowers the barrier to entry for adopting multivector search. It should still not be seen as a drop-in replacement for ANN, as it still requires more computational resources than single-vector search. -* **For General-Purpose Document Retrieval (`flatten` / single-vector):** For large-scale retrieval where understanding the "gist" is sufficient or where in cases where large-context text-based models suffice, single-vector search remains the most practical and scalable solution. - -## Appendix: Full Benchmark Results {#appendix-full-benchmark-results} - -The full benchmark results are shown below. - -```text -| name | _runtime | _step | _timestamp | _wandb | avg_inference_latency | avg_search_latency | hit_rates | model_name | strategy | -|:--------------------------------------------|-----------:|--------:|--------------:|:-------------------|------------------------:|---------------------:|:------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------|:---------------------| -| vidore/colqwen2-v0.1_base | 13410 | 0 | 1.75873e+09 | {'runtime': 13410} | 0.0418646 | 0.751151 | {'1': 0.1355151515151515, '10': 0.888, '20': 0.9597575757575758, '3': 0.3936969696969697, '5': 0.6349090909090909} | vidore/colqwen2-v0.1 | base | -| vidore/colqwen2-v1.0_rerank | 13008 | 0 | 1.75873e+09 | {'runtime': 13008} | 0.0426252 | 0.692482 | {'1': 0.003393939393939394, '10': 0.03296969696969697, '20': 0.07296969696969698, '3': 0.010666666666666666, '5': 0.015272727272727271} | vidore/colqwen2-v1.0 | rerank | -| vidore/colpali-v1.3_flatten | 2998 | 0 | 1.75872e+09 | {'runtime': 2998} | 0.0296511 | 0.00833224 | {'1': 0.017454545454545455, '10': 0.06448484848484848, '20': 0.09333333333333334, '3': 0.034666666666666665, '5': 0.04509090909090909} | vidore/colpali-v1.3 | flatten | -| vidore/colqwen2-v0.1_rerank | 13512 | 0 | 1.75873e+09 | {'runtime': 13512} | 0.0418855 | 0.662372 | {'1': 0.002909090909090909, '10': 0.026424242424242423, '20': 0.05987878787878788, '3': 0.0075151515151515155, '5': 0.014545454545454544} | vidore/colqwen2-v0.1 | rerank | -| vidore/colqwen2-v1.0_base | 12933 | 0 | 1.75873e+09 | {'runtime': 12933} | 0.0416839 | 0.667898 | {'1': 0.14012121212121212, '10': 0.8846060606060606, '20': 0.9553939393939394, '3': 0.4111515151515152, '5': 0.6538181818181819} | vidore/colqwen2-v1.0 | base | -| vidore/colpali-v1.3_rerank | 12090 | 0 | 1.75873e+09 | {'runtime': 12090} | 0.0310783 | 0.949047 | {'1': 0.006060606060606061, '10': 0.03878787878787879, '20': 0.06933333333333333, '3': 0.014545454545454544, '5': 0.022787878787878788} | vidore/colpali-v1.3 | rerank | -| vidore/colqwen2-v1.0_flatten | 4846 | 0 | 1.75874e+09 | {'runtime': 4846} | 0.0504031 | 0.010443 | {'1': 0.018666666666666668, '10': 0.07854545454545454, '20': 0.11903030303030304, '3': 0.041212121212121214, '5': 0.055030303030303034} | vidore/colqwen2-v1.0 | flatten | -| vidore/colqwen2-v0.1_base | 10838 | 0 | 1.75874e+09 | {'runtime': 10838} | 0.04627 | 0.692836 | {'1': 0.13575757575757577, '10': 0.8870303030303031, '20': 0.96, '3': 0.3941818181818182, '5': 0.6351515151515151} | vidore/colqwen2-v0.1 | base | -| vidore/colqwen2-v0.1_flatten | 4828 | 0 | 1.75874e+09 | {'runtime': 4828} | 0.0486335 | 0.0103028 | {'1': 0.018424242424242423, '10': 0.07224242424242425, '20': 0.10521212121212122, '3': 0.03903030303030303, '5': 0.05212121212121213} | vidore/colqwen2-v0.1 | flatten | -| vidore/colpali-v1.3_base | 10253 | 0 | 1.75874e+09 | {'runtime': 10253} | 0.0312334 | 0.93611 | {'1': 0.11272727272727272, '10': 0.551030303030303, '20': 0.6555151515151515, '3': 0.29987878787878786, '5': 0.4232727272727273} | vidore/colpali-v1.3 | base | -| vidore/colqwen2-v0.1_flatten | 4745 | 0 | 1.75874e+09 | {'runtime': 4745} | 0.0472825 | 0.00990508 | {'1': 0.018424242424242423, '10': 0.07296969696969698, '20': 0.10496969696969696, '3': 0.03951515151515152, '5': 0.05236363636363636} | vidore/colqwen2-v0.1 | flatten | -| vidore/colqwen2.5-v0.2_base | 17218 | 0 | 1.75875e+09 | {'runtime': 17218} | 0.0540356 | 0.694855 | {'1': 0.11903030303030304, '10': 0.7127272727272728, '20': 0.8366060606060606, '3': 0.336, '5': 0.5258181818181819} | vidore/colqwen2.5-v0.2 | base | -| vidore/colqwen2.5-v0.2_rerank | 16859 | 0 | 1.75875e+09 | {'runtime': 16859} | 0.0518383 | 0.71693 | {'1': 0.0026666666666666666, '10': 0.025212121212121213, '20': 0.060848484848484846, '3': 0.006787878787878788, '5': 0.01187878787878788} | vidore/colqwen2.5-v0.2 | rerank | -| vidore/colqwen2-v0.1_rerank | 9484 | 0 | 1.75875e+09 | {'runtime': 9484} | 0.0445599 | 0.691609 | {'1': 0.005333333333333333, '10': 0.030545454545454542, '20': 0.064, '3': 0.010666666666666666, '5': 0.017696969696969697} | vidore/colqwen2-v0.1 | rerank | -| vidore/colSmol-256M_flatten | 6822 | 0 | 1.75875e+09 | {'runtime': 6822} | 0.0404544 | 0.00822329 | {'1': 0.01575757575757576, '10': 0.06836363636363636, '20': 0.10496969696969696, '3': 0.03442424242424243, '5': 0.04678787878787879} | vidore/colSmol-256M | flatten | -| vidore/colSmol-500M_base | 12681 | 0 | 1.75875e+09 | {'runtime': 12681} | 0.0408026 | 0.850902 | {'1': 0.136, '10': 0.8029090909090909, '20': 0.8993939393939394, '3': 0.3806060606060606, '5': 0.5975757575757575} | vidore/colSmol-500M | base | -| vidore/colSmol-500M_rerank | 13066 | 0 | 1.75876e+09 | {'runtime': 13066} | 0.0440974 | 0.927632 | {'1': 0.003636363636363637, '10': 0.028606060606060607, '20': 0.07054545454545455, '3': 0.00896969696969697, '5': 0.015030303030303033} | vidore/colSmol-500M | rerank | -| vidore/colSmol-256M_rerank | 12646 | 0 | 1.75876e+09 | {'runtime': 12646} | 0.0391553 | 0.853279 | {'1': 0.003393939393939394, '10': 0.02909090909090909, '20': 0.07006060606060606, '3': 0.008727272727272728, '5': 0.015515151515151517} | vidore/colSmol-256M | rerank | -| vidore/colqwen2.5-v0.2_flatten | 6348 | 0 | 1.75876e+09 | {'runtime': 6348} | 0.0509971 | 0.00772653 | {'1': 0.017696969696969697, '10': 0.06545454545454546, '20': 0.09284848484848485, '3': 0.03515151515151515, '5': 0.045575757575757575} | vidore/colqwen2.5-v0.2 | flatten | -| vidore/colSmol-256M_base | 11554 | 0 | 1.75876e+09 | {'runtime': 11554} | 0.0366467 | 0.848463 | {'1': 0.1435151515151515, '10': 0.8426666666666667, '20': 0.9173333333333332, '3': 0.40824242424242424, '5': 0.6404848484848484} | vidore/colSmol-256M | base | -| vidore/colSmol-500M_flatten | 6395 | 0 | 1.75876e+09 | {'runtime': 6395} | 0.0384664 | 0.00716238 | {'1': 0.018424242424242423, '10': 0.07345454545454545, '20': 0.11393939393939394, '3': 0.03903030303030303, '5': 0.05090909090909091} | vidore/colSmol-500M | flatten | -| openai/clip-vit-base-patch32_base | 815 | 0 | 1.75876e+09 | {'runtime': 815} | 0.00533487 | 0.00794629 | {'1': 0.016, '10': 0.07636363636363637, '20': 0.11757575757575756, '3': 0.03296969696969697, '5': 0.04703030303030303} | openai/clip-vit-base-patch32 | base | -| vidore/colqwen2-v0.1_max_pooling | 8758 | 0 | 1.7595e+09 | {'runtime': 8758} | 0.0985753 | 0.0106716 | {'1': 0.015515151515151517, '10': 0.07296969696969698, '20': 0.11248484848484848, '3': 0.032484848484848484, '5': 0.04703030303030303} | vidore/colqwen2-v0.1 | max_pooling | -| vidore/colpali-v1.3_max_pooling | 4728 | 0 | 1.75949e+09 | {'runtime': 4728} | 0.0718573 | 0.0103053 | {'1': 0.011393939393939394, '10': 0.05672727272727273, '20': 0.08872727272727272, '3': 0.02666666666666667, '5': 0.03709090909090909} | vidore/colpali-v1.3 | max_pooling | -| vidore/colqwen2-v1.0_max_pooling | 8760 | 0 | 1.7595e+09 | {'runtime': 8760} | 0.0981102 | 0.0106316 | {'1': 0.013575757575757576, '10': 0.06933333333333333, '20': 0.112, '3': 0.02812121212121212, '5': 0.041939393939393936} | vidore/colqwen2-v1.0 | max_pooling | -| vidore/colqwen2-v0.1_max_pooling | 7696 | 0 | 1.7595e+09 | {'runtime': 7696} | 0.105203 | 0.011907 | {'1': 0.016242424242424242, '10': 0.07321212121212121, '20': 0.1132121212121212, '3': 0.03442424242424243, '5': 0.04945454545454545} | vidore/colqwen2-v0.1 | max_pooling | -| vidore/colSmol-256M_max_pooling | 13982 | 0 | 1.75951e+09 | {'runtime': 13982} | 0.0999708 | 0.0121211 | {'1': 0.00993939393939394, '10': 0.05818181818181818, '20': 0.09187878787878788, '3': 0.025212121212121213, '5': 0.037575757575757575} | vidore/colSmol-256M | max_pooling | -| vidore/colSmol-500M_max_pooling | 14191 | 0 | 1.75951e+09 | {'runtime': 14191} | 0.111858 | 0.0121703 | {'1': 0.012363636363636365, '10': 0.07539393939393939, '20': 0.13187878787878787, '3': 0.02787878787878788, '5': 0.0416969696969697} | vidore/colSmol-500M | max_pooling | -| vidore/colqwen2-v0.1_hierarchical_pooling | 4507 | 0 | 1.7599e+09 | {'runtime': 4507} | 0.0320023 | 0.133653 | {'1': 0.1296969696969697, '10': 0.8504242424242424, '20': 0.9343030303030304, '3': 0.37527272727272726, '5': 0.6041212121212122} | vidore/colqwen2-v0.1 | hierarchical_pooling | -| vidore/colpali-v1.3_hierarchical_pooling | 5816 | 0 | 1.7599e+09 | {'runtime': 5816} | 0.0217625 | 0.188727 | {'1': 0.10763636363636364, '10': 0.5372121212121213, '20': 0.6482424242424243, '3': 0.29333333333333333, '5': 0.4167272727272727} | vidore/colpali-v1.3 | hierarchical_pooling | -| vidore/colqwen2.5-v0.2_hierarchical_pooling | 8430 | 0 | 1.75991e+09 | {'runtime': 8430} | 0.043073 | 0.141276 | {'1': 0.11103030303030303, '10': 0.6892121212121212, '20': 0.8203636363636364, '3': 0.3185454545454545, '5': 0.4989090909090909} | vidore/colqwen2.5-v0.2 | hierarchical_pooling | -| vidore/colqwen2-v1.0_hierarchical_pooling | 5685 | 0 | 1.75991e+09 | {'runtime': 5685} | 0.0343824 | 0.144062 | {'1': 0.13745454545454547, '10': 0.822060606060606, '20': 0.9156363636363636, '3': 0.3856969696969697, '5': 0.6050909090909091} | vidore/colqwen2-v1.0 | hierarchical_pooling | -``` \ No newline at end of file diff --git a/scripts/assemble.py b/scripts/assemble.py index 82b6f34..dd1b592 100644 --- a/scripts/assemble.py +++ b/scripts/assemble.py @@ -32,6 +32,7 @@ import argparse import json import re +import os import shutil import sys import urllib.error @@ -49,6 +50,11 @@ # and reordering, and the key Enterprise overlays will join on from A5. ANCHOR_RE = re.compile(r"^#{1,6}\s+.*\{#([A-Za-z0-9][A-Za-z0-9._-]*)\}\s*$", re.M) PAGE_SUFFIXES = (".mdx", ".md") +# The reference root ships a complete `docs.json` so it can be served on its own. +# Every other root contributes a `docs.nav.json` fragment instead: tabs merged by +# name into the base. Neither file is content, so neither is copied to the output. +NAV_BASE = "docs.json" +NAV_FRAGMENT = "docs.nav.json" class AssembleError(Exception): @@ -75,6 +81,19 @@ class Resolved: files: dict[str, tuple[Root, Path]] = field(default_factory=dict) overlays: dict[str, tuple[Root, Path]] = field(default_factory=dict) + fragments: dict[str, Path] = field(default_factory=dict) + + +ENV_RE = re.compile(r"\$\{([A-Z_][A-Z0-9_]*)(?::-([^}]*))?\}") + + +def expand(value: str) -> str: + """Resolve ${VAR} and ${VAR:-default} in a configured path. + + Root paths point at sibling checkouts, which sit somewhere different in CI + than on a laptop. Everything else in the config stays literal. + """ + return ENV_RE.sub(lambda m: os.environ.get(m.group(1), m.group(2) or ""), value) def load_config(path: Path = CONFIG_PATH) -> Config: @@ -86,14 +105,14 @@ def load_config(path: Path = CONFIG_PATH) -> Config: role = entry.get("role", "reference") if role not in ("reference", "overlay"): raise AssembleError(f"root {entry['name']}: unknown role {role!r}") - root_path = (REPO_ROOT / entry["path"]).resolve() + root_path = (REPO_ROOT / expand(entry["path"])).resolve() if not root_path.is_dir(): raise AssembleError(f"root {entry['name']}: {root_path} is not a directory") roots.append(Root(name=entry["name"], path=root_path, role=role)) if not any(r.role == "reference" for r in roots): raise AssembleError("at least one reference root is required") return Config( - output=(REPO_ROOT / raw["output"]).resolve(), + output=(REPO_ROOT / expand(raw["output"])).resolve(), roots=roots, openapi=raw.get("openapi"), ) @@ -117,6 +136,9 @@ def resolve(config: Config) -> Resolved: if not src.is_file(): continue rel = src.relative_to(root.path).as_posix() + if rel == NAV_FRAGMENT: + resolved.fragments[root.name] = src + continue target = resolved.overlays if root.role == "overlay" else resolved.files if rel in target: other = target[rel][0] @@ -241,6 +263,121 @@ def merge(resolved: Resolved) -> None: # --------------------------------------------------------------------------- # +def merge_nav(base: dict, fragment: dict) -> dict: + """Fold a root's navigation fragment into the base navigation. + + Tabs are matched by name: a fragment tab that already exists contributes its + groups to it, and a new tab is inserted. Both carry an optional `after` + naming the sibling they follow, because appending is not good enough -- + sidebar order is what a reader navigates by, and dropping Geneva below + Support or Datasets past Use Cases silently reorders the whole site. + + This is the same shape sophon's Enterprise fragment will use in A5. + """ + + def descend(node: dict | list, path: list[str]) -> list: + """Follow a list of group names to the container that should hold an entry.""" + items = node["navigation"]["tabs"] if isinstance(node, dict) else node + for name in path: + match = next( + ( + i + for i in items + # Page paths sit alongside groups in the same list. + if isinstance(i, dict) + and (i.get("tab") == name or i.get("group") == name) + ), + None, + ) + if match is None: + raise AssembleError( + f"navigation fragment targets {name!r}, which does not exist" + ) + items = match.setdefault("groups" if "groups" in match else "pages", []) + return items + + def insert_value(items: list, value: str, after: str | None) -> None: + """Insert a bare page path after a named sibling.""" + if after is None: + items.insert(0, value) + return + for index, item in enumerate(items): + name = item.get("tab") or item.get("group") if isinstance(item, dict) else item + if name == after: + items.insert(index + 1, value) + return + items.append(value) + + def insert(items: list, entry: dict, key: str) -> None: + after = entry.pop("after", None) + if after is None: + items.append(entry) + return + for index, item in enumerate(items): + # A container list holds tabs, groups and bare page paths, so match + # on whichever names the entry rather than on one fixed key. + name = ( + (item.get("tab") or item.get("group")) + if isinstance(item, dict) + else item + ) + if name == after: + items.insert(index + 1, entry) + return + raise AssembleError( + f"navigation fragment wants to follow {after!r}, which does not exist" + ) + + # Entries that name a nested container, e.g. the Enterprise group inside + # "Get started". A5's Enterprise fragment uses the same mechanism. + for spec in fragment.get("insert", []): + target = descend(base, spec["into"]) + entry = spec["entry"] + if isinstance(entry, str): + insert_value(target, entry, spec.get("after")) + else: + insert(target, dict(entry, after=spec.get("after")), "group") + + # Keys lifted out of the base because the file they name lives here. + for spec in fragment.get("set", []): + container = descend(base, spec["into"][:-1] or []) + name = spec["into"][-1] + target = next( + ( + i + for i in container + if isinstance(i, dict) + and (i.get("tab") == name or i.get("group") == name) + ), + None, + ) + if target is None: + raise AssembleError(f"navigation fragment targets {name!r}, which does not exist") + target[spec["key"]] = spec["value"] + + tabs = base.setdefault("navigation", {}).setdefault("tabs", []) + by_name = {t.get("tab"): t for t in tabs} + for incoming in fragment.get("tabs", []): + existing = by_name.get(incoming.get("tab")) + if existing is None: + insert(tabs, incoming, "tab") + by_name[incoming.get("tab")] = incoming + continue + for key, value in incoming.items(): + if key in ("tab", "after"): + continue + if key == "groups": + for group in value: + insert(existing.setdefault("groups", []), group, "group") + else: + # `openapi` and friends: the root that owns the generated pages + # owns how they are generated. + existing[key] = value + if fragment.get("redirects"): + base["redirects"] = fragment["redirects"] + return base + + def assemble_nav(resolved: Resolved) -> tuple[dict, bytes | None]: """Produce the published docs.json. @@ -256,11 +393,20 @@ def assemble_nav(resolved: Resolved) -> tuple[dict, bytes | None]: existing URLs survive, and under its version path so it stays addressable. Both genuinely change the navigation, and both will serialize. """ - entry = resolved.files.get("docs.json") + entry = resolved.files.get(NAV_BASE) if entry is None: - raise AssembleError("no docs.json found in any reference root") + raise AssembleError(f"no {NAV_BASE} found in any reference root") raw = entry[1].read_bytes() - return json.loads(raw.decode("utf-8")), raw + base = json.loads(raw.decode("utf-8")) + + fragments = sorted(resolved.fragments.items()) + if not fragments: + # Nothing to merge: emit the source bytes so the tree stays literally + # byte-identical rather than merely equivalent. + return base, raw + for _name, path in fragments: + base = merge_nav(base, json.loads(path.read_text(encoding="utf-8"))) + return base, None # --------------------------------------------------------------------------- # @@ -306,7 +452,7 @@ def emit( written = 0 for rel, (_root, src) in sorted(resolved.files.items()): - if rel == "docs.json": + if rel == NAV_BASE: continue dest = (output / rel).resolve() # A root containing `../` in a name would otherwise write outside the @@ -318,11 +464,14 @@ def emit( written += 1 if nav_raw is not None: - (output / "docs.json").write_bytes(nav_raw) + (output / NAV_BASE).write_bytes(nav_raw) else: - (output / "docs.json").write_text( - json.dumps(docs_json, indent=2, ensure_ascii=False) + "\n", - encoding="utf-8", + # `ensure_ascii=True` matches how the navigation was written before the + # assembler existed. It is not cosmetic: the banner text carries an + # em-dash, and escaping it differently changes the bytes Mintlify hashes + # its CSS and JS bundles from, which renames those assets on every page. + (output / NAV_BASE).write_text( + json.dumps(docs_json, indent=2) + "\n", encoding="utf-8" ) return written + 1 diff --git a/scripts/mdx_snippets_gen.py b/scripts/mdx_snippets_gen.py deleted file mode 100644 index 1c2930f..0000000 --- a/scripts/mdx_snippets_gen.py +++ /dev/null @@ -1,362 +0,0 @@ -""" -Generate MDX-ready snippet modules that export raw string constants. - -The generator groups snippets by the originating test file name rather than by -language. For a source file like `tests/py/test_basic_usage.py`, the snippets are -written under `docs/snippets/basic.mdx`. Each exported constant is prefixed with -a language identifier (Py|Ts|Rs) followed by the snippet name converted to -TitleCase so that docs authors can selectively assemble `` blocks. -""" - -from __future__ import annotations - -import argparse -import json -import os -import re -from dataclasses import dataclass -from pathlib import Path -from typing import Dict, Iterable, Iterator, List, Mapping, MutableMapping, Tuple - -# Supported languages and relevant metadata. -LANG_ORDER = ("py", "ts", "rs") -LANG_PREFIX = {"py": "Py", "ts": "Ts", "rs": "Rs"} -LANG_EXTENSIONS = { - ".py": "py", - ".ts": "ts", - ".tsx": "ts", - ".rs": "rs", -} -LANG_MARKERS = {"py": ["#"], "ts": ["//"], "rs": ["//"]} -# Match each export block; we rely on the blank line we emit after each export to -# avoid accidentally truncating on semicolons inside the JSON string literal. -EXPORT_RE = re.compile(r"export const (\w+)\s*=\s*(.+?);\s*\n(?:\s*\n|$)", re.DOTALL) - -DEFAULT_SOURCE_DIRS = ( - Path("tests") / "py", - Path("tests") / "ts", - Path("tests") / "rs", -) - - -@dataclass(frozen=True) -class SnippetRecord: - lang: str - snippet_name: str - export_name: str - text: str - source_rel: str - - -def build_markers( - comment_markers: Iterable[str], -) -> Tuple[re.Pattern[str], re.Pattern[str]]: - escaped = "|".join(re.escape(c) for c in comment_markers) - start_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[start:([^\]]+)\]") - end_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[end:([^\]]+)\]") - return start_re, end_re - - -def parse_snippets( - lines: List[str], start_re: re.Pattern[str], end_re: re.Pattern[str] -) -> Dict[str, List[str]]: - snippets: Dict[str, List[str]] = {} - stack: List[Tuple[str, int]] = [] - - for idx, line in enumerate(lines): - m_start = start_re.match(line) - if m_start: - name = m_start.group(1).strip() - stack.append((name, idx)) - continue - - m_end = end_re.match(line) - if m_end: - name = m_end.group(1).strip() - if not stack: - raise ValueError( - f"End marker for '{name}' found at line {idx + 1} without matching start" - ) - start_name, start_idx = stack.pop() - if start_name != name: - raise ValueError( - f"Mismatched markers: start '{start_name}' at line {start_idx + 1}, " - f"end '{name}' at line {idx + 1}" - ) - snippets[name] = lines[start_idx + 1 : idx] - continue - - if stack: - open_names = ", ".join(f"{n}@{i + 1}" for n, i in stack) - raise ValueError(f"Unclosed snippet markers for: {open_names}") - - return snippets - - -def dedent(lines: List[str]) -> List[str]: - min_indent = None - for ln in lines: - if ln.strip() == "": - continue - expanded = ln.expandtabs(4) - indent = len(expanded) - len(expanded.lstrip(" ")) - if min_indent is None or indent < min_indent: - min_indent = indent - - if not lines: - return [] - if min_indent in (None, 0): - return [ln.rstrip("\n") for ln in lines] - - dedented: List[str] = [] - for ln in lines: - expanded = ln.expandtabs(4) - if expanded.strip() == "": - dedented.append(expanded.rstrip("\n")) - else: - dedented.append(expanded[min_indent:].rstrip("\n")) - return dedented - - -def ensure_dir(path: Path) -> None: - path.mkdir(parents=True, exist_ok=True) - - -def detect_language(file_path: Path) -> str: - lang = LANG_EXTENSIONS.get(file_path.suffix.lower()) - if not lang: - raise ValueError(f"Unsupported file extension for {file_path}") - return lang - - -def markers_for_lang(lang: str) -> Tuple[re.Pattern[str], re.Pattern[str]]: - markers = LANG_MARKERS.get(lang) - if not markers: - raise ValueError(f"Unsupported language for markers: {lang}") - return build_markers(markers) - - -def normalize_target_name(file_path: Path) -> str: - stem = file_path.stem - if stem.startswith("test_"): - stem = stem[len("test_") :] - if stem.endswith("_test"): - stem = stem[: -len("_test")] - if stem.endswith(".test"): - stem = stem[: -len(".test")] - if stem.endswith(".spec"): - stem = stem[: -len(".spec")] - if stem == "": - stem = file_path.stem - return stem.replace(" ", "_") - - -def to_title_case(name: str) -> str: - parts = re.split(r"[^0-9a-zA-Z]+", name) - filtered = [p for p in parts if p] - return "".join(p[:1].upper() + p[1:].lower() for p in filtered) or "Snippet" - - -def format_export_name(lang: str, snippet_name: str) -> str: - prefix = LANG_PREFIX[lang] - return f"{prefix}{to_title_case(snippet_name)}" - - -def snippet_text(lines: List[str]) -> str: - if not lines: - return "" - joined = "\n".join(lines).rstrip("\n") - return (joined + "\n") if joined else "" - - -def to_js_literal(text: str) -> str: - return json.dumps(text, ensure_ascii=False) - - -def split_export_prefix(export_name: str) -> Tuple[str | None, str]: - for lang, prefix in LANG_PREFIX.items(): - if export_name.startswith(prefix): - suffix = export_name[len(prefix) :] - return lang, suffix or export_name - return None, export_name - - -def parse_existing_module(path: Path) -> Mapping[str, Mapping[str, SnippetRecord]]: - try: - text = path.read_text(encoding="utf-8") - except FileNotFoundError: - return {} - - existing: Dict[str, Dict[str, SnippetRecord]] = {} - for match in EXPORT_RE.finditer(text): - export_name, literal = match.group(1), match.group(2).strip() - lang, snippet_suffix = split_export_prefix(export_name) - if not lang: - continue - try: - decoded = json.loads(literal) - except Exception: - decoded = literal.strip().strip(";") - - record = SnippetRecord( - lang=lang, - snippet_name=snippet_suffix or export_name, - export_name=export_name, - text=decoded, - source_rel="(existing module)", - ) - existing.setdefault(lang, {})[record.snippet_name] = record - - return existing - - -def iter_source_files(source_dirs: Iterable[Path]) -> Iterator[Path]: - for root in source_dirs: - for path in root.rglob("*"): - if path.is_file() and path.suffix.lower() in LANG_EXTENSIONS: - yield path - - -def write_if_changed(path: Path, content: str) -> bool: - if path.exists(): - try: - current = path.read_text(encoding="utf-8") - if current == content: - return False - except Exception: - pass - ensure_dir(path.parent) - path.write_text(content, encoding="utf-8") - return True - - -def collect_snippets( - source_dirs: Iterable[Path], -) -> Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]]: - result: Dict[str, Dict[str, Dict[str, SnippetRecord]]] = {} - for file_path in iter_source_files(source_dirs): - lang = detect_language(file_path) - start_re, end_re = markers_for_lang(lang) - try: - text = file_path.read_text(encoding="utf-8") - except Exception as exc: # pragma: no cover - raise RuntimeError(f"Failed to read {file_path}: {exc}") from exc - lines = text.splitlines(keepends=True) - snippets = parse_snippets(lines, start_re, end_re) - if not snippets: - continue - - target = normalize_target_name(file_path) - lang_map = result.setdefault(target, {}).setdefault(lang, {}) - for snippet_name, content_lines in snippets.items(): - if snippet_name in lang_map: - raise ValueError( - f"Duplicate snippet '{snippet_name}' in {file_path} and {lang_map[snippet_name].source_rel}" - ) - dedented = dedent(content_lines) - export_name = format_export_name(lang, snippet_name) - source_rel = os.path.relpath(file_path, Path.cwd()) - lang_map[snippet_name] = SnippetRecord( - lang=lang, - snippet_name=snippet_name, - export_name=export_name, - text=snippet_text(dedented), - source_rel=source_rel, - ) - return result - - -def render_module( - target: str, lang_map: Mapping[str, Mapping[str, SnippetRecord]] -) -> str: - parts: List[str] = [] - parts.append( - "{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}\n\n" - ) - for lang in LANG_ORDER: - snippets = lang_map.get(lang) - if not snippets: - continue - for snippet_name in sorted(snippets.keys()): - record = snippets[snippet_name] - literal = to_js_literal(record.text) - parts.append(f"export const {record.export_name} = {literal};\n\n") - return "".join(parts) - - -def generate_modules( - snippets_by_target: Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]], - output_root: Path, -) -> None: - modules_written = 0 - total_exports = 0 - - for target in sorted(snippets_by_target.keys()): - module_path = output_root / f"{target}.mdx" - lang_map = snippets_by_target[target] - existing_langs = parse_existing_module(module_path) - for lang, snippets in existing_langs.items(): - if lang in lang_map: - continue - print(module_path, " -- ", lang) - merged_lang = lang_map.setdefault(lang, {}) - existing_export_names = {rec.export_name for rec in merged_lang.values()} - for snippet_name, record in snippets.items(): - # If an export with the same name already exists (e.g., regenerated), - # prefer the newly generated snippet and skip the old one to avoid duplicates. - if record.export_name in existing_export_names: - continue - merged_lang.setdefault(snippet_name, record) - - total_exports += sum(len(snippets) for snippets in lang_map.values()) - module_content = render_module(target, lang_map) - if write_if_changed(module_path, module_content): - modules_written += 1 - print( - f"\nGenerated {total_exports} snippets\n---" - ) - - -def resolve_source_dirs(args_dirs: List[str] | None) -> List[Path]: - if args_dirs: - dirs = [Path(p) for p in args_dirs] - else: - dirs = list(DEFAULT_SOURCE_DIRS) - - missing = [str(p) for p in dirs if not p.exists()] - if missing: - raise FileNotFoundError(f"Source directories not found: {', '.join(missing)}") - return dirs - - -def main() -> int: - parser = argparse.ArgumentParser( - description="Generate MDX snippet modules grouped by test filename." - ) - parser.add_argument( - "-s", - "--source-dir", - action="append", - help="Path(s) to language test directories (default: tests/py, ts, rs)", - ) - parser.add_argument( - "-o", - "--output-dir", - default=str(Path("docs") / "snippets"), - help="Directory where snippet modules are written (default: docs/snippets)", - ) - args = parser.parse_args() - - source_dirs = resolve_source_dirs(args.source_dir) - snippets = collect_snippets(source_dirs) - if not snippets: - print("No snippets found.") - return 0 - - output_root = Path(args.output_dir) - generate_modules(snippets, output_root) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/split_nav.py b/scripts/split_nav.py new file mode 100644 index 0000000..057e8f1 --- /dev/null +++ b/scripts/split_nav.py @@ -0,0 +1,127 @@ +""" +Split one navigation into a base owned by the reference root and a fragment +contributed by this repository. + +The published navigation is authored once, in the root that owns the pages. When +a page lives somewhere else, its entry has to be contributed by that root instead +— and put back in the same place, because sidebar order is what a reader +navigates by. + +This walks the original navigation and, for every entry naming a page this +repository still holds, records where it sat: the chain of group names above it +and the sibling it followed. The assembler replays those, so the merged +navigation is identical to the one it replaced. + +Run once when a set of pages moves between roots: + + python scripts/split_nav.py --nav --moved +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def split(nav: dict, owned: set[str], owned_files: set[str]) -> tuple[dict, list[dict], list[dict]]: + """Return (base navigation, insertions, settings) — `owned` is what the base keeps.""" + inserts: list[dict] = [] + settings: list[dict] = [] + + def walk(items: list, path: list[str]) -> list: + kept: list = [] + for item in items: + if isinstance(item, str): + if item.lstrip("/") in owned: + kept.append(item) + else: + inserts.append( + {"into": list(path), "after": kept[-1] if kept else None, + "entry": item} + ) + continue + name = item.get("tab") or item.get("group") + # An `openapi` block names a spec file. If the base root does not + # hold that file, Mintlify refuses to build at all — so the key is + # lifted out and restored by whichever root does own the spec. + spec = (item.get("openapi") or {}).get("source") if isinstance(item.get("openapi"), dict) else None + if spec and spec.lstrip("/").removesuffix(".yml") not in owned_files: + settings.append( + {"into": path + [name], "key": "openapi", "value": item["openapi"]} + ) + item = {k: v for k, v in item.items() if k != "openapi"} + child_key = "groups" if "groups" in item else "pages" + # Remember where this container's own insertions start: if it turns + # out to move wholesale, they are redundant and must be dropped, or + # they would try to fill a container that does not exist yet. + mark = len(inserts) + children = walk(item.get(child_key, []), path + [name]) + if children or child_key not in item: + kept.append({**item, child_key: children} if child_key in item else item) + else: + del inserts[mark:] + # Nothing left in this container, so the whole thing belongs to + # the other root — carried across intact rather than rebuilt. + previous = kept[-1] if kept else None + after = ( + (previous.get("tab") or previous.get("group")) + if isinstance(previous, dict) + else previous + ) + inserts.append({"into": list(path), "after": after, "entry": item}) + return kept + + base = {**nav, "tabs": walk(nav["tabs"], [])} + return base, inserts, settings + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + parser.add_argument("--nav", type=Path, required=True) + parser.add_argument("--owned", type=Path, required=True, + help="directory whose .mdx files the base root owns") + parser.add_argument("--base-out", type=Path, required=True) + parser.add_argument("--fragment-out", type=Path, required=True) + args = parser.parse_args() + + source = json.loads(args.nav.read_text()) + owned = { + str(p.relative_to(args.owned)).removesuffix(".mdx") + for p in args.owned.rglob("*.mdx") + } + owned_files = { + str(p.relative_to(args.owned)).removesuffix(".yml") + for p in args.owned.rglob("*.yml") + } + base_nav, inserts, settings = split(source["navigation"], owned, owned_files) + + # Key order is preserved, not rebuilt: the assembled file is compared byte + # for byte against the one it replaces, and Mintlify hashes its bundles from + # these bytes, so a reordered key renames every CSS and JS asset. + base = {} + for key, value in source.items(): + if key == "navigation": + base[key] = base_nav + elif key == "redirects": + continue # contributed by the fragment, which owns the full list + else: + base[key] = value + args.base_out.write_text(json.dumps(base, indent=2, ensure_ascii=False) + "\n") + + fragment = { + "insert": inserts, + "set": settings, + "redirects": source.get("redirects", []), + } + args.fragment_out.write_text(json.dumps(fragment, indent=2, ensure_ascii=False) + "\n") + + print( + f"base keeps {len(owned)} pages; fragment contributes {len(inserts)} entries " + f"and {len(settings)} settings" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/camelot.json b/tests/camelot.json deleted file mode 100644 index e41bd45..0000000 --- a/tests/camelot.json +++ /dev/null @@ -1,66 +0,0 @@ -[ - { - "id": 1, - "name": "King Arthur", - "role": "King of Camelot", - "description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.", - "vector": [0.72, -0.28, 0.60, 0.86], - "stats": { "strength": 2, "courage": 5, "magic": 1, "wisdom": 4 } - }, - { - "id": 2, - "name": "Merlin", - "role": "Wizard and Advisor", - "description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.", - "vector": [0.05, 0.88, 0.62, 0.85], - "stats": { "strength": 2, "courage": 4, "magic": 5, "wisdom": 5 } - }, - { - "id": 3, - "name": "Queen Guinevere", - "role": "Queen of Camelot", - "description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.", - "vector": [0.22, -0.22, 0.42, 0.82], - "stats": { "strength": 1, "courage": 3, "magic": 1, "wisdom": 4 } - }, - { - "id": 4, - "name": "Sir Lancelot", - "role": "Knight of the Round Table", - "description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.", - "vector": [0.86, -0.35, 0.38, 0.55], - "stats": { "strength": 5, "courage": 5, "magic": 1, "wisdom": 3 } - }, - { - "id": 5, - "name": "Sir Gawain", - "role": "Knight of the Round Table", - "description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.", - "vector": [0.82, -0.32, 0.52, 0.60], - "stats": { "strength": 4, "courage": 5, "magic": 1, "wisdom": 4 } - }, - { - "id": 6, - "name": "Sir Galahad", - "role": "Knight of the Round Table", - "description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.", - "vector": [0.80, -0.20, 0.70, 0.78], - "stats": { "strength": 4, "courage": 5, "magic": 2, "wisdom": 5 } - }, - { - "id": 7, - "name": "Sir Percival", - "role": "Knight of the Round Table", - "description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.", - "vector": [0.78, -0.36, 0.48, 0.52], - "stats": { "strength": 4, "courage": 4, "magic": 1, "wisdom": 3 } - }, - { - "id": 8, - "name": "Mordred", - "role": "Traitor Knight", - "description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.", - "vector": [0.68, -0.30, -0.65, 0.20], - "stats": { "strength": 4, "courage": 2, "magic": 1, "wisdom": 2 } - } -] diff --git a/tests/py/conftest.py b/tests/py/conftest.py deleted file mode 100644 index ebd2c81..0000000 --- a/tests/py/conftest.py +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import shutil -from pathlib import Path - -import lancedb -import pytest -import pytest_asyncio -from lancedb.db import AsyncConnection, DBConnection - - -class DatabasePathFactory: - """Create per-test database directories and ensure they get removed.""" - - def __init__(self, base_dir: Path) -> None: - self._base_dir = base_dir - self._created: list[Path] = [] - - def __call__(self, name: str) -> Path: - safe_name = name.replace("/", "_") - path = self._base_dir / safe_name - if path.exists(): - shutil.rmtree(path, ignore_errors=True) - path.mkdir(parents=True, exist_ok=True) - self._created.append(path) - return path - - def cleanup(self) -> None: - for path in reversed(self._created): - shutil.rmtree(path, ignore_errors=True) - self._created.clear() - - -@pytest.fixture -def db_path_factory(tmp_path: Path): - factory = DatabasePathFactory(tmp_path) - yield factory - factory.cleanup() - - -@pytest.fixture -def mem_db() -> DBConnection: - return lancedb.connect("memory://") - - -@pytest.fixture -def tmp_db(db_path_factory) -> DBConnection: - """Create a temporary database connection for testing.""" - db_path = db_path_factory("tmp_db") - return lancedb.connect(str(db_path)) - - -@pytest_asyncio.fixture -async def mem_db_async() -> AsyncConnection: - return await lancedb.connect_async("memory://") diff --git a/tests/py/test_basic_usage.py b/tests/py/test_basic_usage.py deleted file mode 100644 index 260ebb6..0000000 --- a/tests/py/test_basic_usage.py +++ /dev/null @@ -1,205 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -# --8<-- [start:basic_imports] -import json - -import lancedb -import pandas as pd -import polars as pl -import pyarrow as pa -# --8<-- [end:basic_imports] -import pytest - -data_path = "tests/camelot.json" - -def test_basic_usage(db_path_factory): - uri = "./basic_usage_db" - uri = db_path_factory("basic_usage_db") - db = lancedb.connect(uri) - - # --8<-- [start:data_load] - with open(data_path, "r") as f: - data = json.load(f) - # --8<-- [end:data_load] - - # --8<-- [start:basic_create_table] - table = db.create_table("camelot", data=data, mode="overwrite") - # --8<-- [end:basic_create_table] - assert len(table) == 8 - - # --8<-- [start:basic_open_table] - table = db.open_table("camelot") - # --8<-- [end:basic_open_table] - - # --8<-- [start:basic_create_table_pandas] - pandas_df = pd.DataFrame(data) - table_pd = db.create_table("camelot_pd", data=pandas_df, mode="overwrite") - # --8<-- [end:basic_create_table_pandas] - assert len(table_pd) == 8 - db.drop_table("camelot_pd") - - # --8<-- [start:basic_create_table_polars] - polars_df = pl.DataFrame(data) - table_pl = db.create_table("camelot_pl", data=polars_df, mode="overwrite") - # --8<-- [end:basic_create_table_polars] - assert len(table_pl) == 8 - db.drop_table("camelot_pl") - - # --8<-- [start:basic_create_empty_table] - schema = pa.schema( - [ - pa.field("id", pa.uint16()), - pa.field("name", pa.string()), - pa.field("role", pa.string()), - pa.field("description", pa.string()), - pa.field("vector", pa.list_(pa.float32(), 4)), - pa.field( - "stats", - pa.struct( - [ - pa.field("strength", pa.int8()), - pa.field("courage", pa.int8()), - pa.field("magic", pa.int8()), - pa.field("wisdom", pa.int8()), - ] - ), - ), - ] - ) - db.create_table("camelot_pa", schema=schema, mode="overwrite") - # --8<-- [end:basic_create_empty_table] - assert "camelot_pa" in db.list_tables().tables - db.drop_table("camelot_pa") - - # --8<-- [start:basic_add_data] - magical_characters = [ - { - "id": 9, - "name": "Morgan le Fay", - "role": "Sorceress", - "description": "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.", - "vector": [0.10, 0.84, 0.25, 0.70], - "stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 4 } - }, - { - "id": 10, - "name": "The Lady of the Lake", - "role": "Mystical Guardian", - "description": "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.", - "vector": [0.00, 0.90, 0.58, 0.88], - "stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 5 } - } - ] - table.add(magical_characters) - # --8<-- [end:basic_add_data] - assert len(table) == 10 - - # --8<-- [start:basic_vector_search] - query_vector = [0.03, 0.85, 0.61, 0.90] - result = table.search(query_vector).limit(5).to_polars() - print(result) - # --8<-- [end:basic_vector_search] - - # --8<-- [start:basic_add_columns] - table.add_columns( - { - "power": "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)" - } - ) - # --8<-- [end:basic_add_columns] - assert "power" in table.schema.names - - # Run examples to illustrate search - # --8<-- [start:basic_vector_search_q1] - # Who are the characters similar to "wizard"? - query_vector_1 = [0.03, 0.85, 0.61, 0.90] - r1 = ( - table.search(query_vector_1) - .limit(5) - .select(["name", "role", "description"]) - .to_polars() - ) - print(r1) - # --8<-- [end:basic_vector_search_q1] - - # --8<-- [start:basic_vector_search_q2] - # Who are the characters with high magic stats? - query_vector_2 = [0.03, 0.85, 0.61, 0.90] - r2 = ( - table.search(query_vector_2) - .where("stats.magic > 3") - .select(["name", "role", "description"]) - .limit(5) - .to_polars() - ) - print(r2) - # --8<-- [end:basic_vector_search_q2] - - # --8<-- [start:basic_vector_search_q3] - # Who are the strongest characters? - r3 = ( - table.search() - .where("stats.strength > 3") - .select(["name", "role", "description"]) - .limit(5) - .to_polars() - ) - print(r3) - # --8<-- [end:basic_vector_search_q3] - - # --8<-- [start:basic_vector_search_q4] - # Who are the strongest characters? - r4 = ( - table.search() - .select(["name", "role", "description", "power"]) - .to_polars() - ) - print(r4) - # --8<-- [end:basic_vector_search_q4] - - # --8<-- [start:basic_sort_polars] - # Sort Polars DataFrame by power in descending order - print(r4.sort("power", descending=True).limit(5)) - # --8<-- [end:basic_sort_polars] - sorted_power = r4.sort("power", descending=True)["power"].to_list() - assert sorted_power == sorted(sorted_power, reverse=True) - assert sorted_power[0] == 4.0 - - # --8<-- [start:basic_drop_columns] - table.drop_columns(["power"]) - # --8<-- [end:basic_drop_columns] - # --8<-- [start:basic_delete_rows] - table.delete('role = "Traitor Knight"') - # --8<-- [end:basic_delete_rows] - assert len(table) == 9 - # --8<-- [start:basic_drop_table] - db.drop_table("camelot") - # --8<-- [end:basic_drop_table] - assert "camelot" not in db.list_tables().tables - - -@pytest.mark.asyncio -async def test_basic_usage_async_api(db_path_factory): - uri = db_path_factory("basic_usage_async_db") - with open(data_path, "r") as f: - data = json.load(f) - - # --8<-- [start:basic_async_api] - import lancedb - - async_db = await lancedb.connect_async(uri) - async_table = await async_db.create_table( - "camelot_async", - data=data, - mode="overwrite", - ) - - query_vector = [0.03, 0.85, 0.61, 0.90] - async_results = await ( - await async_table.search(query_vector) - ).limit(5).select(["name", "role", "description"]).to_polars() - print(async_results) - # --8<-- [end:basic_async_api] - - assert async_results.height >= 1 diff --git a/tests/py/test_build_with_ai_agents.py b/tests/py/test_build_with_ai_agents.py deleted file mode 100644 index 5b9091e..0000000 --- a/tests/py/test_build_with_ai_agents.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -# --8<-- [start:camelot_schema] -from lancedb.pydantic import LanceModel -from pydantic import ConfigDict - - -class Stats(LanceModel): - model_config = ConfigDict(strict=True, extra="forbid") - - strength: int - courage: int - magic: int - wisdom: int - - -class Character(LanceModel): - model_config = ConfigDict(strict=True, extra="forbid") - - id: int - name: str - role: str - description: str - stats: Stats - image_filename: str - image: bytes -# --8<-- [end:camelot_schema] - - -# --8<-- [start:camelot_batches] -import json -from collections.abc import Iterator -from pathlib import Path - - -def validated_batches( - input_path: Path, batch_size: int -) -> Iterator[list[dict]]: - raw_records = json.loads(input_path.read_text()) - asset_root = input_path.parent.parent - batch: list[dict] = [] - - for raw in raw_records: - payload = dict(raw) - image_path = asset_root / payload.pop("img") - payload["image_filename"] = image_path.name - payload["image"] = image_path.read_bytes() - - character = Character.model_validate(payload) - batch.append(character.model_dump(mode="python")) - - if len(batch) == batch_size: - yield batch - batch = [] - - if batch: - yield batch -# --8<-- [end:camelot_batches] - - -# --8<-- [start:camelot_oss_ingestion] -import lancedb - - -def ingest_oss( - input_path: Path, - uri: str = "data/camelot.lancedb", - table_name: str = "camelot_multimodal", - batch_size: int = 64, -): - db = lancedb.connect(uri) - if table_name in db.list_tables(): - raise ValueError( - f"Table {table_name!r} already exists. Choose a fresh table name." - ) - - table = db.create_table(table_name, schema=Character) - for batch in validated_batches(input_path, batch_size): - table.add(batch) - - table.optimize() - return table - - -if __name__ == "__main__": - ingest_oss(Path("data/camelot.json")) -# --8<-- [end:camelot_oss_ingestion] - - -def test_camelot_oss_ingestion(tmp_path): - source = Path( - "docs/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json" - ) - table = ingest_oss( - source, - uri=str(tmp_path / "camelot.lancedb"), - table_name="camelot", - ) - - rows = ( - table.search() - .select(["id", "name", "image", "image_filename"]) - .limit(8) - .to_list() - ) - assert len(rows) == 8 - assert all(row["image"] for row in rows) - assert all(row["image_filename"].endswith(".jpg") for row in rows) diff --git a/tests/py/test_connection.py b/tests/py/test_connection.py deleted file mode 100644 index 783ac2d..0000000 --- a/tests/py/test_connection.py +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -from pathlib import Path -import shutil - - -def test_connection(): - # --8<-- [start:connect] - import lancedb - - uri = "ex_lancedb" - db = lancedb.connect(uri) - # --8<-- [end:connect] - assert db is not None - shutil.rmtree(uri, ignore_errors=True) - assert not Path(uri).exists() - - -async def connect_async_example(): - # --8<-- [start:connect_async] - import lancedb - - uri = "ex_lancedb" - async_db = await lancedb.connect_async(uri) - # --8<-- [end:connect_async] - - return async_db - - -def connect_enterprise_quickstart_config(): - import lancedb - - # --8<-- [start:connect_enterprise_quickstart] - uri = "db://your-database-uri" - api_key = "your-api-key" - region = "us-east-1" - host_override = "https://your-enterprise-endpoint.com" - - db = lancedb.connect( - uri=uri, - api_key=api_key, - region=region, - host_override=host_override, - ) - # --8<-- [end:connect_enterprise_quickstart] - return db - - -def test_connect_enterprise_quickstart(monkeypatch): - import lancedb - - captured = {} - - def fake_connect(**kwargs): - captured.update(kwargs) - return object() - - monkeypatch.setattr(lancedb, "connect", fake_connect) - - db = connect_enterprise_quickstart_config() - assert db is not None - assert captured == { - "uri": "db://your-database-uri", - "api_key": "your-api-key", - "region": "us-east-1", - "host_override": "https://your-enterprise-endpoint.com", - } - - -def connect_object_storage_config(): - # --8<-- [start:connect_object_storage] - import lancedb - - uri = "s3://your-bucket/path" - # You can also use "gs://your-bucket/path" or "az://your-container/path". - db = lancedb.connect(uri) - # --8<-- [end:connect_object_storage] - - return db - - -def namespace_table_ops_example(): - # --8<-- [start:namespace_table_ops] - import lancedb - - db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"}) - - # Create namespace tree: prod/search - db.create_namespace(["prod"], mode="exist_ok") - db.create_namespace(["prod", "search"], mode="exist_ok") - db.create_namespace(["prod", "recommendations"], mode="exist_ok") - - db.create_table( - "user", - data=[{"id": 1, "vector": [0.1, 0.2], "name": "alice"}], - namespace_path=["prod", "search"], - mode="create", # use "overwrite" only if you want to replace existing table - ) - - db.create_table( - "user", - data=[{"id": 2, "vector": [0.3, 0.4], "name": "bob"}], - namespace_path=["prod", "recommendations"], - mode="create", # use "overwrite" only if you want to replace existing table - ) - - # Verify - print(db.list_namespaces()) # ['prod'] - print(db.list_namespaces(namespace_path=["prod"])) # ['recommendations', 'search'] - print(db.list_tables(namespace_path=["prod", "search"])) # ['user'] - print(db.list_tables(namespace_path=["prod", "recommendations"])) # ['user'] - # --8<-- [end:namespace_table_ops] - - -def namespace_admin_ops_example(): - # --8<-- [start:namespace_admin_ops] - import lancedb - - db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"}) - namespace = ["prod", "search"] - - db.create_namespace(["prod"]) - db.create_namespace(["prod", "search"]) - - child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces - print(f"Child namespaces under {namespace}: {child_namespaces}") - # Child namespaces under ['prod', 'search']: ['search'] - - metadata = db.describe_namespace(["prod", "search"]) - print(f"Metadata for namespace {namespace}: {metadata}") - # Metadata for namespace ['prod', 'search']: properties=None - - db.drop_namespace(["prod", "search"], mode="skip") - db.drop_namespace(["prod"], mode="skip") - # --8<-- [end:namespace_admin_ops] - return child_namespaces, metadata - -async def connect_object_storage_config_async(): - # --8<-- [start:connect_object_storage_async] - import lancedb - - uri = "s3://your-bucket/path" - # You can also use "gs://your-bucket/path" or "az://your-container/path". - async_db = await lancedb.connect_async(uri) - # --8<-- [end:connect_object_storage_async] - - return async_db diff --git a/tests/py/test_embedding.py b/tests/py/test_embedding.py deleted file mode 100644 index 490be26..0000000 --- a/tests/py/test_embedding.py +++ /dev/null @@ -1,141 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import lancedb - -# --8<-- [start:imports] -from lancedb.pydantic import LanceModel, Vector -from lancedb.embeddings import get_registry - -# --8<-- [end:imports] -import pytest - - -@pytest.mark.skip(reason="OpenAI is not available in the test environment") -def test_create_embedding_function(): - # --8<-- [start:create_embedding_function] - func = get_registry().get("openai").create( - name="text-embedding-3-small", - max_retries=7, - ) - # --8<-- [end:create_embedding_function] - assert func is not None - - -@pytest.mark.skip(reason="OpenAI is not available in the test environment") -def test_embeddings_openai(): - # --8<-- [start:openai_embeddings] - db = lancedb.connect("/tmp/db") - func = get_registry().get("openai").create(name="text-embedding-ada-002") - - class Words(LanceModel): - text: str = func.SourceField() - vector: Vector(func.ndims()) = func.VectorField() - - table = db.create_table("words", schema=Words, mode="overwrite") - table.add([{"text": "hello world"}, {"text": "goodbye world"}]) - - query = "greetings" - actual = table.search(query).limit(1).to_pydantic(Words)[0] - print(actual.text) - # --8<-- [end:openai_embeddings] - - -@pytest.mark.skip(reason="OpenAI is not available in the test environment") -@pytest.mark.asyncio -async def test_embeddings_openai_async(): - uri = "memory://" - # --8<-- [start:async_openai_embeddings] - db = await lancedb.connect_async(uri) - func = get_registry().get("openai").create(name="text-embedding-ada-002") - - class Words(LanceModel): - text: str = func.SourceField() - vector: Vector(func.ndims()) = func.VectorField() - - table = await db.create_table("words", schema=Words, mode="overwrite") - await table.add([{"text": "hello world"}, {"text": "goodbye world"}]) - - query = "greetings" - actual = await (await table.search(query)).limit(1).to_pydantic(Words)[0] - print(actual.text) - # --8<-- [end:async_openai_embeddings] - - -@pytest.mark.skip(reason="OpenAI is not available in the test environment") -def test_embeddings_manual_query(): - # --8<-- [start:manual_query_embeddings] - db = lancedb.connect("/tmp/db") - func = get_registry().get("openai").create(name="text-embedding-ada-002") - - class Words(LanceModel): - text: str = func.SourceField() - vector: Vector(func.ndims()) = func.VectorField() - - table = db.create_table("words", schema=Words, mode="overwrite") - table.add([{"text": "hello world"}, {"text": "goodbye world"}]) - - query_vector = func.generate_embeddings(["greetings"])[0] - # --8<-- [start:manual_query_search] - # query_vector is assumed to already be generated by your embedding function - actual = table.search(query_vector).limit(1).to_pydantic(Words)[0] - print(actual.text) - # --8<-- [end:manual_query_search] - # --8<-- [end:manual_query_embeddings] - - -def test_custom_embedding_function(): - # --8<-- [start:embedding_function] - from functools import cached_property - - from lancedb.embeddings import TextEmbeddingFunction, register - - class MyEmbeddingModel: - def __init__(self, model_name: str): - self.model_name = model_name - - def encode(self, texts: list[str]) -> list[list[float]]: - return [[1.0, 2.0, 3.0] for _ in texts] - - @register("my-embedder") - class MyTextEmbedder(TextEmbeddingFunction): - model_name: str = "my-model" - - def generate_embeddings(self, texts: list[str]) -> list[list[float]]: - # Your embedding logic here - return self._model.encode(texts) - - def ndims(self) -> int: - # Return the dimensionality of the embeddings - return len(self.generate_embeddings(["test"])[0]) - - @cached_property - def _model(self) -> MyEmbeddingModel: - # Initialize your model once - return MyEmbeddingModel(self.model_name) - # --8<-- [end:embedding_function] - - -def test_embeddings_secret(): - # --8<-- [start:register_secret] - registry = get_registry() - registry.set_var("api_key", "sk-...") - - func = registry.get("openai").create(api_key="$var:api_key") - # --8<-- [end:register_secret] - - try: - import torch - except ImportError: - pytest.skip("torch not installed") - - # --8<-- [start:register_device] - import torch - - registry = get_registry() - if torch.cuda.is_available(): - registry.set_var("device", "cuda") - - func = registry.get("huggingface").create(device="$var:device:cpu") - # --8<-- [end:register_device] - assert func.device == "cuda" if torch.cuda.is_available() else "cpu" diff --git a/tests/py/test_geneva_defaults.py b/tests/py/test_geneva_defaults.py deleted file mode 100644 index af4a42b..0000000 --- a/tests/py/test_geneva_defaults.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -# These tests assert the default resource values for Geneva KubeRay cluster nodes. -# If any of these values change, update the tables in: -# docs/geneva/jobs/performance.mdx (the "Geneva defaults" section) - -import pytest - - -def test_head_node_defaults(): - # If these change, update the Head Node table in performance.mdx - from geneva.cluster.builder import KubeRayClusterBuilder - - builder = KubeRayClusterBuilder() - assert builder._head_cpus == 4 - assert builder._head_memory == "8Gi" - assert builder._head_gpus == 0 - assert builder._head_node_selector == {"geneva.lancedb.com/ray-head": "true"} - assert builder._head_service_account == "geneva-service-account" - - -def test_cpu_worker_defaults(): - # If these change, update the CPU Workers table in performance.mdx - from geneva.cluster.builder import CpuWorkerBuilder - - worker = CpuWorkerBuilder() - assert worker._num_cpus == 4 - assert worker._memory == "8Gi" - assert worker._node_selector == {"geneva.lancedb.com/ray-worker-cpu": "true"} - assert worker._replicas == 1 - assert worker._min_replicas == 0 - assert worker._max_replicas == 100 - assert worker._idle_timeout_seconds == 60 - - # Confirm build produces 0 GPUs - config = worker.build() - assert config.num_gpus == 0 - - -def test_gpu_worker_defaults(): - # If these change, update the GPU Workers table in performance.mdx - from geneva.cluster.builder import GpuWorkerBuilder - - worker = GpuWorkerBuilder() - assert worker._num_cpus == 8 - assert worker._memory == "16Gi" - assert worker._num_gpus == 1 - assert worker._node_selector == {"geneva.lancedb.com/ray-worker-gpu": "true"} - assert worker._replicas == 1 - assert worker._min_replicas == 0 - assert worker._max_replicas == 100 - assert worker._idle_timeout_seconds == 60 diff --git a/tests/py/test_geneva_dependency_verification.py b/tests/py/test_geneva_dependency_verification.py deleted file mode 100644 index fc00aba..0000000 --- a/tests/py/test_geneva_dependency_verification.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import os - -import pytest - - -def test_quick_fix_manifest(): - # --8<-- [start:quick_fix_manifest] - from geneva.manifest.builder import PipManifestBuilder - - manifest = PipManifestBuilder.create("fix").pip(["numpy==1.26.4"]).build() - # --8<-- [end:quick_fix_manifest] - assert manifest.pip == ["numpy==1.26.4"] - - -def test_env_vars_via_cluster(monkeypatch): - monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key-id") - monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") - - # --8<-- [start:env_vars_via_cluster] - from geneva.cluster.builder import KubeRayClusterBuilder - import os - - cluster = ( - KubeRayClusterBuilder.create("my-cluster") - .ray_init_kwargs({ - "runtime_env": { - "env_vars": { - "AWS_ACCESS_KEY_ID": os.environ["AWS_ACCESS_KEY_ID"], - "AWS_SECRET_ACCESS_KEY": os.environ["AWS_SECRET_ACCESS_KEY"], - } - } - }) - .build() - ) - # --8<-- [end:env_vars_via_cluster] - assert cluster.kuberay.ray_init_kwargs["runtime_env"]["env_vars"]["AWS_ACCESS_KEY_ID"] == "test-key-id" - - -def test_pip_manifest(monkeypatch): - import geneva - from unittest.mock import MagicMock, create_autospec - mock_conn = create_autospec(geneva.db.Connection, instance=True) - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_conn)) - - # --8<-- [start:pip_manifest] - import geneva - from geneva.manifest.builder import PipManifestBuilder - - manifest = ( - PipManifestBuilder.create("my-manifest") - .pip([ - "numpy==1.26.4", - "torch==2.0.1", - "attrs==23.2.0", - ]) - .build() - ) - - conn = geneva.connect("s3://my-bucket/my-db") - conn.define_manifest("my-manifest", manifest) - with conn.context(cluster="my-cluster", manifest="my-manifest"): - conn.open_table("my-table").backfill("my-column") - # --8<-- [end:pip_manifest] - assert "numpy==1.26.4" in manifest.pip - - -def test_conda_cluster_path(): - # --8<-- [start:conda_cluster_path] - from geneva.cluster.builder import KubeRayClusterBuilder - - cluster = ( - KubeRayClusterBuilder.create("my-cluster") - .ray_init_kwargs({ - "runtime_env": {"conda": "environment.yml"} - }) - .build() - ) - # --8<-- [end:conda_cluster_path] - assert cluster.kuberay.ray_init_kwargs["runtime_env"]["conda"] == "environment.yml" - - -def test_conda_cluster_inline(): - # --8<-- [start:conda_cluster_inline] - from geneva.cluster.builder import KubeRayClusterBuilder - - cluster = ( - KubeRayClusterBuilder.create("my-cluster") - .ray_init_kwargs({ - "runtime_env": { - "conda": { - "channels": ["conda-forge"], - "dependencies": [ - "python=3.10", - "ffmpeg<8", - "torchvision=0.22.1", - ], - }, - "config": {"eager_install": True}, - } - }) - .build() - ) - # --8<-- [end:conda_cluster_inline] - assert cluster.kuberay.ray_init_kwargs["runtime_env"]["conda"]["channels"] == ["conda-forge"] diff --git a/tests/py/test_geneva_profiling_memory.py b/tests/py/test_geneva_profiling_memory.py deleted file mode 100644 index 7ca45db..0000000 --- a/tests/py/test_geneva_profiling_memory.py +++ /dev/null @@ -1,248 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -"""Snippets for docs/geneva/udfs/profiling-memory.mdx.""" - -from unittest.mock import MagicMock - - -def test_stateful_udf_class(): - import geneva - import pyarrow as pa - - load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1] * 512))) - - # --8<-- [start:stateful_udf_class] - @geneva.udf(data_type=pa.list_(pa.float32(), 512)) - class MyEmbedding: - def __init__(self): - self.model = None - - def setup(self): - self.model = load_model() # allocated once per actor - - def __call__(self, text: str) -> list[float]: - if self.model is None: - self.setup() - return self.model.embed(text) - # --8<-- [end:stateful_udf_class] - - assert MyEmbedding is not None - - -def test_memray_tracker_udf(monkeypatch): - import sys - - load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1] * 512))) - monkeypatch.setitem(sys.modules, "memray", MagicMock()) - - # --8<-- [start:memray_tracker_udf] - import os, pathlib, uuid - from typing import Any - import memray - import geneva - import pyarrow as pa - - _MEMRAY_OUT_DIR_ENV = "MY_UDF_MEMRAY_OUT_DIR" - - - @geneva.udf(data_type=pa.list_(pa.float32(), 512)) - class MyEmbedding: - def __init__(self): - self.model = None - self._tracker: Any = None # memray.Tracker, when profiling is on - - def setup(self): - # Open a memray tracker per worker process, if requested. Each - # worker writes its own .bin file so traces don't collide. - out_dir = os.environ.get(_MEMRAY_OUT_DIR_ENV) - if out_dir: - pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True) - bin_path = pathlib.Path(out_dir) / ( - f"memray-{os.getpid()}-{uuid.uuid4().hex}.bin" - ) - self._tracker = memray.Tracker( - str(bin_path), native_traces=False, follow_fork=False - ) - self._tracker.__enter__() - self.model = load_model() - - def __call__(self, text: str) -> list[float]: - if self.model is None: - self.setup() - return self.model.embed(text) - # --8<-- [end:memray_tracker_udf] - - assert MyEmbedding is not None - - -def test_ray_cluster_profile(monkeypatch): - from contextlib import contextmanager - - table = MagicMock() - - @contextmanager - def _mock_cluster(*args, **kwargs): - yield - - monkeypatch.setattr("geneva.runners.ray._mgr.ray_cluster", _mock_cluster) - - # --8<-- [start:ray_cluster_profile] - from geneva.runners.ray._mgr import ray_cluster - - with ray_cluster( - local=True, - extra_env={"MY_UDF_MEMRAY_OUT_DIR": "/tmp/my-udf-profile"}, - ): - table.backfill("embedding", concurrency=1) - # --8<-- [end:ray_cluster_profile] - - table.backfill.assert_called_once_with("embedding", concurrency=1) - - -def test_log_memory(capsys): - # --8<-- [start:log_memory] - import resource, pyarrow as pa - - def log_memory(seq: int) -> None: - rss_bytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - # ru_maxrss is bytes on macOS, KiB on Linux: - import sys - if sys.platform != "darwin": - rss_bytes *= 1024 - arrow_live = pa.total_allocated_bytes() - print( - f"seq={seq} " - f"rss_mb={rss_bytes // 1024**2} " - f"arrow_live_mb={arrow_live // 1024**2} " - f"gap_mb={(rss_bytes - arrow_live) // 1024**2}", - flush=True, - ) - # --8<-- [end:log_memory] - - log_memory(0) - captured = capsys.readouterr() - assert "seq=0" in captured.out - assert "rss_mb=" in captured.out - - -def test_leaky_cache(): - # --8<-- [start:leaky_cache] - class BadEmbedding: - def __init__(self): - self.cache: dict[str, list[float]] = {} - - def __call__(self, text: str) -> list[float]: - if text not in self.cache: - self.cache[text] = self.model.embed(text) - return self.cache[text] - # --8<-- [end:leaky_cache] - - obj = BadEmbedding() - obj.model = MagicMock(embed=MagicMock(return_value=[0.1, 0.2])) - obj("hello") - obj("hello") - assert obj.model.embed.call_count == 1 # second call served from cache - assert len(obj.cache) == 1 - - -def test_bounded_cache(): - load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1, 0.2]))) - - # --8<-- [start:bounded_cache] - from functools import lru_cache - - class GoodEmbedding: - def __init__(self): - self._embed = None - - def setup(self): - model = load_model() - self._embed = lru_cache(maxsize=1024)(model.embed) - - def __call__(self, text: str) -> list[float]: - if self._embed is None: - self.setup() - return self._embed(text) - # --8<-- [end:bounded_cache] - - obj = GoodEmbedding() - assert obj("hello") == [0.1, 0.2] - load_model.assert_called_once() - - -def test_leaky_aggregator(): - import pyarrow as pa - - # --8<-- [start:leaky_aggregator] - class BadAggregator: - def __init__(self): - self.history = [] - - def __call__(self, batch: pa.RecordBatch) -> pa.Array: - self.history.append(batch) # holds every batch ever processed - ... - # --8<-- [end:leaky_aggregator] - - obj = BadAggregator() - assert isinstance(obj.history, list) - - -def test_leaky_closure(): - import pyarrow as pa - - def expensive(x): - return x - - # --8<-- [start:leaky_closure] - class BadDeferred: - def __init__(self): - self.work_queue = [] - - def __call__(self, x: pa.Array) -> pa.Array: - # Lambda captures `x` by reference — the whole Array stays alive - self.work_queue.append(lambda: expensive(x)) - ... - # --8<-- [end:leaky_closure] - - obj = BadDeferred() - assert isinstance(obj.work_queue, list) - - -def test_torch_inference_mode(monkeypatch): - import sys - - mock_torch = MagicMock() - monkeypatch.setitem(sys.modules, "torch", mock_torch) - - class TorchUDF: - def __init__(self): - self.model = MagicMock(encode=MagicMock(return_value=[0.1])) - - # --8<-- [start:torch_inference_mode] - def __call__(self, text: str) -> list[float]: - with torch.inference_mode(): # <-- prevents autograd graph retention - return self.model.encode(text) - # --8<-- [end:torch_inference_mode] - - import torch # resolves to monkeypatched mock above - obj = TorchUDF() - assert obj("hello") == [0.1] - - -def test_confidence_check(): - class DebuggingUDF: - def __init__(self): - self._scratches = [] - - # --8<-- [start:confidence_check] - def __call__(self, x): - scratch = bytearray(8 * 1024 * 1024) # 8 MiB - self._scratches.append(scratch) # <-- deliberate leak - return ... - # --8<-- [end:confidence_check] - - obj = DebuggingUDF() - obj(None) - assert len(obj._scratches) == 1 - assert len(obj._scratches[0]) == 8 * 1024 * 1024 diff --git a/tests/py/test_geneva_scalar_udtfs.py b/tests/py/test_geneva_scalar_udtfs.py deleted file mode 100644 index 30aec52..0000000 --- a/tests/py/test_geneva_scalar_udtfs.py +++ /dev/null @@ -1,311 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import pytest -from unittest.mock import MagicMock - - -def test_scalar_udtf_iterator(): - def extract_video_segment(path, start, end): - return bytes(f"clip:{start}-{end}", "utf-8") - - # --8<-- [start:scalar_udtf_iterator] - from geneva import chunker - from typing import Iterator, NamedTuple - - class Clip(NamedTuple): - clip_start: float - clip_end: float - clip_bytes: bytes - - @chunker - def extract_clips(video_path: str, duration: float) -> Iterator[Clip]: - """Yields multiple clips per video.""" - clip_length = 10.0 - for start in range(0, int(duration), int(clip_length)): - end = min(start + clip_length, duration) - clip_data = extract_video_segment(video_path, start, end) - yield Clip(clip_start=start, clip_end=end, clip_bytes=clip_data) - # --8<-- [end:scalar_udtf_iterator] - - from geneva.transformer import Chunker - assert isinstance(extract_clips, Chunker) - assert extract_clips.input_columns == ["video_path", "duration"] - assert set(extract_clips.output_schema.names) == {"clip_start", "clip_end", "clip_bytes"} - - clips = list(extract_clips.func("/v/a.mp4", 30.0)) - assert len(clips) == 3 - assert clips[0].clip_start == 0.0 - assert clips[2].clip_end == 30.0 - - -def test_scalar_udtf_list_return(): - from geneva import chunker - from typing import NamedTuple - - class Clip(NamedTuple): - clip_start: float - clip_end: float - clip_bytes: bytes - - # --8<-- [start:scalar_udtf_list] - @chunker - def extract_clips(video_path: str, duration: float) -> list[Clip]: - clips = [] - for start in range(0, int(duration), 10): - end = min(start + 10, duration) - clips.append(Clip(clip_start=start, clip_end=end, clip_bytes=b"...")) - return clips - # --8<-- [end:scalar_udtf_list] - - from geneva.transformer import Chunker - assert isinstance(extract_clips, Chunker) - clips = extract_clips.func("/v/a.mp4", 30.0) - assert len(clips) == 3 - assert all(c.clip_bytes == b"..." for c in clips) - - -def test_scalar_udtf_batch(): - import pyarrow as pa - from geneva import chunker - - clip_schema = pa.schema([ - ("clip_start", pa.float64()), - ("clip_end", pa.float64()), - ]) - - # --8<-- [start:scalar_udtf_batch] - @chunker(batch=True, output_schema=clip_schema) - def extract_clips(batch: pa.RecordBatch) -> pa.RecordBatch: - """Process rows in batches. Same 1:N semantic per row.""" - ... - # --8<-- [end:scalar_udtf_batch] - - from geneva.transformer import Chunker - assert isinstance(extract_clips, Chunker) - assert extract_clips.batch is True - - -def test_create_scalar_udtf_view(monkeypatch): - from typing import Iterator, NamedTuple - from geneva.transformer import chunker - - class Clip(NamedTuple): - clip_start: float - clip_end: float - clip_bytes: bytes - - @chunker - def extract_clips(video_path: str, duration: float) -> Iterator[Clip]: - for start in range(0, int(duration), 10): - yield Clip(clip_start=start, clip_end=min(start + 10.0, duration), clip_bytes=b"") - - import geneva - from unittest.mock import create_autospec - mock_clips = MagicMock() - mock_db = create_autospec(geneva.db.Connection, instance=True) - mock_db.create_udtf_view.return_value = mock_clips - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - - # --8<-- [start:create_scalar_udtf_view] - import geneva - - db = geneva.connect("/data/mydb") - videos = db.open_table("videos") - - # Create the 1:N materialized view - clips = db.create_udtf_view( - "clips", - source=videos.search(None).select(["video_path", "metadata"]), - udtf=extract_clips, - ) - - # Populate — runs the UDTF on every source row - clips.refresh() - # --8<-- [end:create_scalar_udtf_view] - - call_kwargs = mock_db.create_udtf_view.call_args - assert call_kwargs.args[0] == "clips" - assert call_kwargs.kwargs["udtf"] is extract_clips - mock_clips.refresh.assert_called_once() - - -def test_add_columns_scalar_udtf(): - import pyarrow as pa - from geneva.transformer import udf - - clips = MagicMock() - embed_model = MagicMock() - embed_model.encode.return_value = [0.1] * 512 - - # --8<-- [start:add_columns_scalar_udtf] - @udf(data_type=pa.list_(pa.float32(), 512)) - def clip_embedding(clip_bytes: bytes) -> list[float]: - return embed_model.encode(clip_bytes) - - # Add an embedding column to the clips table - clips.add_columns({"embedding": clip_embedding}) - - # Backfill computes embeddings for all existing clips - clips.backfill("embedding") - # --8<-- [end:add_columns_scalar_udtf] - - clips.add_columns.assert_called_once_with({"embedding": clip_embedding}) - clips.backfill.assert_called_once_with("embedding") - - -def test_incremental_refresh(): - videos = MagicMock() - clips = MagicMock() - new_video_data = [{"video_path": "/v/c.mp4", "duration": 45.0}] - - # --8<-- [start:incremental_refresh] - # Add new videos to the source table - videos.add(new_video_data) - - # Incremental refresh — only processes the new videos - clips.refresh() - # --8<-- [end:incremental_refresh] - - videos.add.assert_called_once_with(new_video_data) - clips.refresh.assert_called_once() - - -def test_chaining_udtf_views(monkeypatch): - from typing import Iterator, NamedTuple - from geneva.transformer import chunker - - class Clip(NamedTuple): - clip_start: float - clip_end: float - - class Frame(NamedTuple): - frame_index: int - frame_bytes: bytes - - @chunker - def extract_clips(video_path: str, duration: float) -> Iterator[Clip]: - for start in range(0, int(duration), 10): - yield Clip(clip_start=start, clip_end=min(start + 10.0, duration)) - - @chunker - def extract_frames(clip_start: float, clip_end: float) -> Iterator[Frame]: - yield Frame(frame_index=0, frame_bytes=b"") - - import geneva - from unittest.mock import create_autospec - mock_db = create_autospec(geneva.db.Connection, instance=True) - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - - db = geneva.connect("/data/mydb") - videos = db.open_table("videos") - - # --8<-- [start:chaining_udtf_views] - # videos → clips (1:N) - clips = db.create_udtf_view( - "clips", source=videos.search(None), udtf=extract_clips - ) - - # clips → frames (1:N) - frames = db.create_udtf_view( - "frames", source=clips.search(None), udtf=extract_frames - ) - # --8<-- [end:chaining_udtf_views] - - assert mock_db.create_udtf_view.call_count == 2 - first_call = mock_db.create_udtf_view.call_args_list[0] - second_call = mock_db.create_udtf_view.call_args_list[1] - assert first_call.kwargs["udtf"] is extract_clips - assert second_call.kwargs["udtf"] is extract_frames - - -def test_document_chunking_udtf(): - # --8<-- [start:document_chunking_udtf] - from geneva import chunker - from typing import Iterator, NamedTuple - - class Chunk(NamedTuple): - chunk_index: int - chunk_text: str - - @chunker - def chunk_document(text: str) -> Iterator[Chunk]: - """Split a document into overlapping chunks.""" - words = text.split() - chunk_size = 500 - overlap = 50 - for i, start in enumerate(range(0, len(words), chunk_size - overlap)): - chunk_words = words[start:start + chunk_size] - yield Chunk(chunk_index=i, chunk_text=" ".join(chunk_words)) - # --8<-- [end:document_chunking_udtf] - - sample_text = " ".join(["word"] * 600) - chunks = list(chunk_document.func(sample_text)) - assert len(chunks) == 2 - assert chunks[0].chunk_index == 0 - assert chunks[1].chunk_index == 1 - assert len(chunks[0].chunk_text.split()) == 500 - - -def test_document_chunking_full(monkeypatch): - import pyarrow as pa - from unittest.mock import MagicMock - - import geneva - from unittest.mock import create_autospec - mock_chunks_table = MagicMock() - mock_db = create_autospec(geneva.db.Connection, instance=True) - mock_db.create_udtf_view.return_value = mock_chunks_table - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - - embedding_model = MagicMock() - embedding_model.encode.return_value = [0.1] * 1536 - - # --8<-- [start:document_chunking_full] - from geneva import connect, chunker, udf - from typing import Iterator, NamedTuple - import pyarrow as pa - - class Chunk(NamedTuple): - chunk_index: int - chunk_text: str - - @chunker - def chunk_document(text: str) -> Iterator[Chunk]: - """Split a document into overlapping chunks.""" - words = text.split() - chunk_size = 500 - overlap = 50 - for i, start in enumerate(range(0, len(words), chunk_size - overlap)): - chunk_words = words[start:start + chunk_size] - yield Chunk(chunk_index=i, chunk_text=" ".join(chunk_words)) - - db = connect("/data/mydb") - docs = db.open_table("documents") - - # Create chunked view — inherits doc_id, title, etc. from source - chunks = db.create_udtf_view( - "doc_chunks", - source=docs.search(None).select(["doc_id", "title", "text"]), - udtf=chunk_document, - ) - chunks.refresh() - - # Add embeddings to chunks for semantic search - @udf(data_type=pa.list_(pa.float32(), 1536)) - def embed_text(chunk_text: str) -> list[float]: - return embedding_model.encode(chunk_text) - - chunks.add_columns({"embedding": embed_text}) - chunks.backfill("embedding") # Backfills embeddings on all existing chunks - - # Query — parent columns available alongside chunk columns - chunks.search(None).select(["doc_id", "title", "chunk_text", "embedding"]).to_pandas() - # --8<-- [end:document_chunking_full] - - call_kwargs = mock_db.create_udtf_view.call_args - assert call_kwargs.args[0] == "doc_chunks" - assert call_kwargs.kwargs["udtf"] is chunk_document - mock_chunks_table.refresh.assert_called_once() - mock_chunks_table.add_columns.assert_called_once() - mock_chunks_table.backfill.assert_called_once_with("embedding") diff --git a/tests/py/test_geneva_udfs_index.py b/tests/py/test_geneva_udfs_index.py deleted file mode 100644 index dffe00d..0000000 --- a/tests/py/test_geneva_udfs_index.py +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -from unittest.mock import MagicMock, create_autospec - - -def test_registration_udf(monkeypatch): - # Verifies that Table.add_columns exists with the expected signature. - # If this fails, update the Registration row in docs/geneva/udfs/index.mdx. - import geneva - mock_table = create_autospec(geneva.table.Table, instance=True) - my_udf = MagicMock() - - # --8<-- [start:registration_udf] - mock_table.add_columns({"col": my_udf}) - # --8<-- [end:registration_udf] - - mock_table.add_columns.assert_called_once() - - -def test_registration_scalar_udtf(monkeypatch): - # Verifies that Connection.create_udtf_view accepts a chunker. - # If this fails, update the Registration row in docs/geneva/udfs/index.mdx. - import geneva - mock_db = create_autospec(geneva.db.Connection, instance=True) - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - my_source = MagicMock() - my_chunker = MagicMock() - - # --8<-- [start:registration_scalar_udtf] - db = geneva.connect("/data/mydb") - db.create_udtf_view("my_view", source=my_source, udtf=my_chunker) - # --8<-- [end:registration_scalar_udtf] - - mock_db.create_udtf_view.assert_called_once() - - -def test_registration_udtf(monkeypatch): - # Verifies that Connection.create_udtf_view exists with the expected signature. - # If this fails, update the Registration row in docs/geneva/udfs/index.mdx. - import geneva - mock_db = create_autospec(geneva.db.Connection, instance=True) - monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db)) - my_source = MagicMock() - my_udtf = MagicMock() - - # --8<-- [start:registration_udtf] - db = geneva.connect("/data/mydb") - db.create_udtf_view("my_view", source=my_source, udtf=my_udtf) - # --8<-- [end:registration_udtf] - - mock_db.create_udtf_view.assert_called_once() diff --git a/tests/py/test_indexing.py b/tests/py/test_indexing.py deleted file mode 100644 index 7f63e8f..0000000 --- a/tests/py/test_indexing.py +++ /dev/null @@ -1,738 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import random -import string -import uuid - -import numpy as np -import pyarrow as pa -import pytest - - -def _make_vector_rows(count: int, dim: int, column: str = "vector"): - rows = [ - {column: np.random.random(dim).astype(np.float32).tolist(), "id": i} - for i in range(count) - ] - assert len(rows) == count - assert all(len(row[column]) == dim for row in rows) - return rows - - -def test_vector_index_configure_ivf(tmp_db): - table = tmp_db.create_table( - "vector_index_configure_ivf", - _make_vector_rows(512, 4), - mode="overwrite", - ) - - # --8<-- [start:vector_index_configure_ivf] - table.create_index(metric="l2", num_partitions=16, num_sub_vectors=4) - # --8<-- [end:vector_index_configure_ivf] - - assert table.list_indices() - - -def test_vector_index_setup(tmp_db): - tmp_db.create_table( - "vector-index-tbl", - _make_vector_rows(8, 4), - mode="overwrite", - ) - - db = tmp_db - # --8<-- [start:vector_index_setup] - table_name = "vector-index-tbl" - table = db.open_table(table_name) - # --8<-- [end:vector_index_setup] - - assert table.name == table_name - - -def test_vector_index_build_ivf(tmp_db): - table = tmp_db.create_table( - "vector-index-build-ivf", - _make_vector_rows(512, 4, column="keywords_embeddings"), - mode="overwrite", - ) - db = tmp_db - # --8<-- [start:vector_index_build_ivf] - table_name = "vector-index-build-ivf" - table = db.open_table(table_name) - table.create_index( - metric="cosine", - vector_column_name="keywords_embeddings", - ) - # --8<-- [end:vector_index_build_ivf] - - assert table.list_indices() - - -def test_vector_index_nested_field(tmp_db): - dim = 2 - schema = pa.schema( - [ - pa.field("id", pa.int32()), - pa.field( - "image", - pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]), - ), - ] - ) - data = [ - { - "id": i, - "image": {"embedding": np.random.random(dim).astype(np.float32).tolist()}, - } - for i in range(512) - ] - table = tmp_db.create_table( - "vector_index_nested_field", data=data, schema=schema, mode="overwrite" - ) - - # --8<-- [start:vector_index_nested_field] - # The vector column `embedding` is nested inside the `image` struct. - # Pass its full dotted path as `vector_column_name`; the same path is used - # at query time and is what `list_indices()` reports under `columns`. - table.create_index( - vector_column_name="image.embedding", - num_partitions=1, - num_sub_vectors=1, - name="image_embedding_idx", - ) - - results = ( - table.search([0.0, 1.0], vector_column_name="image.embedding") - .limit(1) - .to_list() - ) - # --8<-- [end:vector_index_nested_field] - - assert table.index_stats("image_embedding_idx") - assert len(results) == 1 - - -@pytest.mark.asyncio -async def test_vector_index_async_config(tmp_path, monkeypatch): - monkeypatch.chdir(tmp_path) - - # --8<-- [start:vector_index_async_config] - import lancedb - import numpy as np - from lancedb.index import IvfPq - - async def main(): - data = [ - {"id": i, "vector": np.random.random(8).astype(np.float32).tolist()} - for i in range(512) - ] - - db = await lancedb.connect_async("ex_lancedb") - table = await db.create_table( - "vector_index_async", data=data, mode="overwrite" - ) - - await table.create_index( - "vector", - config=IvfPq( - distance_type="cosine", - num_partitions=16, - num_sub_vectors=4, - ), - ) - return await table.list_indices() - # --8<-- [end:vector_index_async_config] - - assert await main() - - -def test_vector_index_query_ivf(tmp_db): - dim = 1536 - data = [ - {"id": i, "keywords_embeddings": np.random.random(dim).tolist()} - for i in range(512) - ] - table = tmp_db.create_table("vector_index_query_ivf", data, mode="overwrite") - table.create_index( - metric="cosine", - vector_column_name="keywords_embeddings", - ) - - # --8<-- [start:vector_index_query_ivf] - tbl = table - tbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor( - 10 - ).to_pandas() - # --8<-- [end:vector_index_query_ivf] - - df = ( - table.search(np.random.random((1536))) - .limit(2) - .nprobes(20) - .refine_factor(10) - .to_pandas() - ) - assert len(df) == 2 - - -def test_vector_index_nprobes(tmp_db): - dim = 128 - data = [ - {"id": i, "keywords_embeddings": np.random.random(dim).tolist()} - for i in range(512) - ] - table = tmp_db.create_table("vector_index_nprobes", data, mode="overwrite") - table.create_index( - metric="cosine", - vector_column_name="keywords_embeddings", - ) - - # --8<-- [start:vector_index_nprobes] - # Always scan 10 partitions; scan up to 50 only if the initial pass - # returns fewer than `limit` results (common with narrow filters). - ( - table.search(np.random.random(128)) - .minimum_nprobes(10) - .maximum_nprobes(50) - .where("id > 100") - .limit(5) - .to_pandas() - ) - # --8<-- [end:vector_index_nprobes] - - -def test_vector_index_distance_range(tmp_db): - dim = 128 - data = [ - {"id": i, "keywords_embeddings": np.random.random(dim).tolist()} - for i in range(256) - ] - table = tmp_db.create_table("vector_index_distance_range", data, mode="overwrite") - table.create_index( - metric="cosine", - vector_column_name="keywords_embeddings", - ) - - # --8<-- [start:vector_index_distance_range] - # Only return results whose distance falls within [0.0, 0.5). - # Useful for near-duplicate detection or thresholded similarity search. - ( - table.search(np.random.random(128)) - .distance_range(lower_bound=0.0, upper_bound=0.5) - .limit(10) - .to_pandas() - ) - # --8<-- [end:vector_index_distance_range] - - -def test_vector_index_bypass_recall(tmp_db): - dim = 128 - data = [ - {"id": i, "keywords_embeddings": np.random.random(dim).tolist()} - for i in range(256) - ] - table = tmp_db.create_table("vector_index_bypass_recall", data, mode="overwrite") - table.create_index( - metric="cosine", - vector_column_name="keywords_embeddings", - ) - - # --8<-- [start:vector_index_bypass_recall] - query = np.random.random(128) - k = 10 - - # Ground truth: flat (exhaustive) scan, ignoring the ANN index. - truth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()["id"]) - - # ANN results with the current nprobes setting. - ann = set(table.search(query).nprobes(20).limit(k).to_pandas()["id"]) - - recall_at_k = len(truth & ann) / k - # --8<-- [end:vector_index_bypass_recall] - assert 0.0 <= recall_at_k <= 1.0 - - -def test_vector_index_custom_name(tmp_db): - table = tmp_db.create_table( - "vector_index_custom_name", - _make_vector_rows(512, 8, column="keywords_embeddings"), - mode="overwrite", - ) - - # --8<-- [start:vector_index_custom_name] - # Override the default `{column}_idx` convention by passing `name=...`. - table.create_index( - metric="cosine", - vector_column_name="keywords_embeddings", - name="my_custom_index", - ) - table.wait_for_index(["my_custom_index"]) - print(table.index_stats("my_custom_index")) - # --8<-- [end:vector_index_custom_name] - - assert table.index_stats("my_custom_index") - - -def test_vector_index_hnsw(tmp_db): - table = tmp_db.create_table( - "vector_index_hnsw", - _make_vector_rows(64, 16), - mode="overwrite", - ) - - # --8<-- [start:vector_index_build_hnsw] - table.create_index(index_type="IVF_HNSW_SQ") - # --8<-- [end:vector_index_build_hnsw] - - # --8<-- [start:vector_index_query_hnsw] - tbl = table - tbl.search(np.random.random((16))).limit(2).to_pandas() - # --8<-- [end:vector_index_query_hnsw] - - df = table.search(np.random.random((16))).limit(2).to_pandas() - assert len(df) == 2 - - -def test_vector_index_binary(tmp_db): - table_name = "hamming-index-tbl" - ndim = 256 - schema = pa.schema( - [ - pa.field("id", pa.int64()), - pa.field("vector", pa.list_(pa.uint8(), ndim // 8)), - ] - ) - - # --8<-- [start:vector_index_binary_schema] - table = tmp_db.create_table(table_name, schema=schema, mode="overwrite") - # --8<-- [end:vector_index_binary_schema] - - data = [] - for i in range(64): - vector = np.random.randint(0, 2, size=ndim) - vector = np.packbits(vector) - data.append({"id": i, "vector": vector}) - - # --8<-- [start:vector_index_binary_add_data] - table.add(data) - # --8<-- [end:vector_index_binary_add_data] - - # --8<-- [start:vector_index_binary_build_index] - table.create_index( - metric="hamming", - vector_column_name="vector", - index_type="IVF_FLAT", - ) - # --8<-- [end:vector_index_binary_build_index] - - # --8<-- [start:vector_index_binary_search] - query = np.random.randint(0, 2, size=ndim) - query = np.packbits(query) - df = table.search(query).metric("hamming").limit(10).to_pandas() - df.vector = df.vector.apply(np.unpackbits) - # --8<-- [end:vector_index_binary_search] - - assert not df.empty - - -def test_vector_index_check_status(tmp_db): - table = tmp_db.create_table( - "vector_index_check_status", - _make_vector_rows(512, 8, column="keywords_embeddings"), - mode="overwrite", - ) - table.create_index( - metric="cosine", - vector_column_name="keywords_embeddings", - ) - - # --8<-- [start:vector_index_check_status] - index_name = "keywords_embeddings_idx" - table.wait_for_index([index_name]) - print(table.index_stats(index_name)) - # --8<-- [end:vector_index_check_status] - - assert table.index_stats(index_name) - - -def test_scalar_index_build(tmp_db): - table = tmp_db.create_table( - "scalar_index_build", - [ - {"book_id": 1, "publisher": "A", "vector": [0.1, 0.2]}, - {"book_id": 2, "publisher": "B", "vector": [0.2, 0.3]}, - ], - mode="overwrite", - ) - db = tmp_db - # --8<-- [start:scalar_index_build] - tbl = db.open_table("scalar_index_build") - tbl.create_scalar_index("book_id") - tbl.create_scalar_index("publisher", index_type="BITMAP") - # --8<-- [end:scalar_index_build] - - assert tbl.list_indices() - - -def test_scalar_index_wait(tmp_db): - table = tmp_db.create_table( - "scalar_index_wait", - [{"label": "fiction"}], - mode="overwrite", - ) - table.create_scalar_index("label") - - # --8<-- [start:scalar_index_wait] - index_name = "label_idx" - table.wait_for_index([index_name]) - # --8<-- [end:scalar_index_wait] - - assert table.list_indices() - - -def test_scalar_index_optimize(tmp_db): - table = tmp_db.create_table( - "scalar_index_optimize", - [{"vector": [7.0, 8.0], "book_id": 3}], - mode="overwrite", - ) - - # --8<-- [start:scalar_index_optimize] - table.add([{"vector": [7, 8], "book_id": 4}]) - table.optimize() - # --8<-- [end:scalar_index_optimize] - - result = table.search().where("book_id = 4").limit(10).to_pandas() - assert len(result) == 1 - - -def test_scalar_index_filter(tmp_db): - table = tmp_db.create_table( - "books", - [ - {"vector": [1.1, 1.2], "book_id": 1}, - {"vector": [2.1, 2.2], "book_id": 2}, - ], - mode="overwrite", - ) - db = tmp_db - # --8<-- [start:scalar_index_filter] - table = db.open_table("books") - result = table.search().where("book_id = 2").limit(10).to_pandas() - # --8<-- [end:scalar_index_filter] - - assert len(result) == 1 - - -def test_scalar_index_prefilter(tmp_db): - table = tmp_db.create_table( - "book_with_embeddings", - [ - {"vector": [1.2, 1.3], "book_id": 1}, - {"vector": [4.2, 4.3], "book_id": 2}, - ], - mode="overwrite", - ) - db = tmp_db - # --8<-- [start:scalar_index_prefilter] - table = db.open_table("book_with_embeddings") - table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas() - # --8<-- [end:scalar_index_prefilter] - - result = table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas() - assert len(result) == 2 - - -def test_scalar_index_uuid(tmp_db): - # --8<-- [start:scalar_index_uuid_type] - import pyarrow as pa - # --8<-- [end:scalar_index_uuid_type] - - # --8<-- [start:scalar_index_uuid_data] - def generate_random_names(): - base_names = ["Alice", "Bob", "Carla", "David", "Eve", "Frank", "Grace"] - letter = random.choice(string.ascii_uppercase) - return f"{random.choice(base_names)} {letter}." - - def generate_uuids(num_items): - return [uuid.uuid4().bytes for _ in range(num_items)] - - # Generate some UUIDs and random names - n = 7 - uuids = generate_uuids(n) - names = [generate_random_names() for _ in range(n)] - # --8<-- [end:scalar_index_uuid_data] - - db = tmp_db - # --8<-- [start:scalar_index_uuid_table] - table_name = "index-on-uuid" - - uuid_array = pa.array(uuids, pa.uuid()) - name_array = pa.array(names, pa.string()) - schema = pa.schema( - [ - pa.field("id", pa.uuid()), - pa.field("name", pa.string()), - ] - ) - data_table = pa.Table.from_arrays([uuid_array, name_array], schema=schema) - table = db.create_table(table_name, data=data_table, mode="overwrite") - # --8<-- [end:scalar_index_uuid_table] - - # --8<-- [start:scalar_index_uuid_wait] - index_name = "id_idx" - table.create_scalar_index("id") - table.wait_for_index([index_name]) - # --8<-- [end:scalar_index_uuid_wait] - - # --8<-- [start:scalar_index_uuid_upsert] - new_users = [ - {"id": uuid.uuid4().bytes, "name": "Hannah D."}, - {"id": uuid.uuid4().bytes, "name": "Ian B."}, - ] - # Insert or update using the UUID index - table.merge_insert( - "id" - ).when_matched_update_all().when_not_matched_insert_all().execute(new_users) - # --8<-- [end:scalar_index_uuid_upsert] - - assert table.list_indices() - result = table.search().limit(100).to_pandas() - assert len(result) == n + len(new_users) - - -@pytest.mark.asyncio -async def test_scalar_index_nested_fields(mem_db_async): - db = mem_db_async - - # --8<-- [start:scalar_index_nested_fields] - import pyarrow as pa - from lancedb.index import BTree - - metadata_type = pa.struct( - [ - pa.field("user_id", pa.int32()), - pa.field("user.id", pa.int32()), - ] - ) - data = pa.Table.from_arrays( - [ - pa.array([1, 2, 3], type=pa.int32()), - pa.array( - [ - {"user_id": 10, "user.id": 100}, - {"user_id": 20, "user.id": 200}, - {"user_id": 30, "user.id": 300}, - ], - type=metadata_type, - ), - ], - names=["user_id", "metadata"], - ) - table = await db.create_table("nested_scalar_index", data) - - # Index a nested struct field. - await table.create_index( - "metadata.user_id", config=BTree(), name="nested_user_id_idx" - ) - - # Escape literal dots inside a segment with backticks. - await table.create_index( - "metadata.`user.id`", config=BTree(), name="escaped_user_id_idx" - ) - - # `columns` is returned as the canonical path you passed in. - for index in await table.list_indices(): - print(index.name, index.columns) - # nested_user_id_idx ['metadata.user_id'] - # escaped_user_id_idx ['metadata.`user.id`'] - # --8<-- [end:scalar_index_nested_fields] - - index_columns = {index.name: index.columns for index in await table.list_indices()} - assert index_columns["nested_user_id_idx"] == ["metadata.user_id"] - assert index_columns["escaped_user_id_idx"] == ["metadata.`user.id`"] - - -def test_fts_index_create(tmp_db): - table = tmp_db.create_table( - "fts-index-create", - [{"text": "hello world", "vector": [0.1, 0.2]}], - mode="overwrite", - ) - - db = tmp_db - # --8<-- [start:fts_index_create] - table_name = "fts-index-create" - table = db.open_table(table_name) - table.create_fts_index("text") - # --8<-- [end:fts_index_create] - - assert table.list_indices() - - -def test_fts_index_wait(tmp_db): - table = tmp_db.create_table( - "fts-index-wait", - [{"text": "full text search"}], - mode="overwrite", - ) - - db = tmp_db - # --8<-- [start:fts_index_wait] - table_name = "fts-index-wait" - - table = db.open_table(table_name) - table.create_fts_index("text") - - index_name = "text_idx" - table.wait_for_index([index_name]) - # --8<-- [end:fts_index_wait] - - assert table.list_indices() - - -def test_fts_index_nested_field(tmp_db): - nested_schema = pa.struct([ - pa.field("text", pa.string()), - pa.field("count", pa.int32()), - ]) - schema = pa.schema([ - pa.field("id", pa.int64()), - pa.field("payload", nested_schema), - ]) - tmp_db.create_table( - "fts-index-nested", - pa.table( - { - "id": pa.array([1, 2], pa.int64()), - "payload": pa.array( - [ - {"text": "Frodo was a happy puppy", "count": 1}, - {"text": "puppy runs through the meadow", "count": 2}, - ], - type=nested_schema, - ), - }, - schema=schema, - ), - mode="overwrite", - ) - - db = tmp_db - # --8<-- [start:fts_index_nested] - from lancedb.query import MatchQuery, PhraseQuery - - table = db.open_table("fts-index-nested") - - # Index a text leaf inside a struct column using a dotted path. - table.create_fts_index("payload.text", with_position=True) - - # The same dotted path works in MatchQuery and PhraseQuery. - matches = ( - table.search(MatchQuery("puppy", "payload.text")).limit(5).to_list() - ) - phrases = ( - table.search(PhraseQuery("puppy runs", "payload.text")) - .limit(5) - .to_list() - ) - # --8<-- [end:fts_index_nested] - - assert len(matches) > 0 - assert all("puppy" in row["payload"]["text"] for row in matches) - assert len(phrases) > 0 - assert all("puppy runs" in row["payload"]["text"] for row in phrases) - - -@pytest.mark.asyncio -async def test_fts_index_async(tmp_path, monkeypatch): - monkeypatch.chdir(tmp_path) - - # --8<-- [start:fts_index_async] - import asyncio - - import lancedb - import polars as pl - from lancedb.index import FTS - - data = pl.DataFrame( - { - "id": [1, 2], - "text": [ - "His first language is spanish", - "Her first language is english", - ], - } - ) - - async def main(data: pl.DataFrame): - uri = "ex_lancedb" - db = await lancedb.connect_async(uri) - tbl = await db.create_table("my_text", data=data, mode="overwrite") - - await tbl.create_index("text", config=FTS(language="English")) - - response = await tbl.search("spanish", query_type="fts") - result = await response.limit(1).to_polars() - print(result) - return result - - if __name__ == "__main__": - asyncio.run(main(data)) - # --8<-- [end:fts_index_async] - - result = await main(data) - assert result.height == 1 - - -def test_gpu_index_snippets(tmp_db, monkeypatch): - table = tmp_db.create_table( - "gpu_index", - _make_vector_rows(32, 8), - mode="overwrite", - ) - - calls = [] - - def fake_create_index(*args, **kwargs): - calls.append(kwargs) - return None - - monkeypatch.setattr(table, "create_index", fake_create_index) - - # --8<-- [start:gpu_index_cuda] - table.create_index( - num_partitions=256, - num_sub_vectors=96, - accelerator="cuda", - ) - # --8<-- [end:gpu_index_cuda] - - # --8<-- [start:gpu_index_mps] - table.create_index( - num_partitions=256, - num_sub_vectors=96, - accelerator="mps", - ) - # --8<-- [end:gpu_index_mps] - - assert calls[0]["accelerator"] == "cuda" - assert calls[1]["accelerator"] == "mps" - - -def test_reindexing_incremental(tmp_db): - table = tmp_db.create_table( - "reindexing_incremental", - [{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}], - mode="overwrite", - ) - db = tmp_db - # --8<-- [start:reindexing_incremental] - table = db.open_table("reindexing_incremental") - table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}]) - table.optimize() - # --8<-- [end:reindexing_incremental] - - result = table.search().limit(10).to_pandas() - assert len(result) == 2 diff --git a/tests/py/test_integrations.py b/tests/py/test_integrations.py deleted file mode 100644 index 20c3170..0000000 --- a/tests/py/test_integrations.py +++ /dev/null @@ -1,1767 +0,0 @@ -import os - -import pytest - - -def require_env(var_name: str) -> str: - """Skip the test unless the required environment variable is provided.""" - value = os.environ.get(var_name) - if not value: - pytest.skip(f"Set {var_name} to run this integration snippet") - return value - - -def require_flag(flag_name: str) -> None: - """Skip unless a generic feature flag is set to a truthy value.""" - value = os.environ.get(flag_name, "").lower() - if value not in {"1", "true", "yes", "on"}: - pytest.skip(f"Enable {flag_name} to run this integration snippet") - - -def test_embedding_openai_basic() -> None: - require_env("OPENAI_API_KEY") - - # --8<-- [start:embedding_openai_basic] - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - db_path = Path(tempfile.mkdtemp()) / "openai-embeddings" - db = lancedb.connect(str(db_path)) - func = get_registry().get("openai").create(name="text-embedding-ada-002") - - class Words(LanceModel): - text: str = func.SourceField() - vector: Vector(func.ndims()) = func.VectorField() - - table = db.create_table("words", schema=Words, mode="overwrite") - table.add( - [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - ) - - query = "greetings" - actual = table.search(query).limit(1).to_pydantic(Words)[0] - print(actual.text) - # --8<-- [end:embedding_openai_basic] - - -def test_embedding_aws_usage() -> None: - require_flag("RUN_AWS_BEDROCK_SNIPPETS") - - # --8<-- [start:embedding_aws_usage] - import tempfile - from pathlib import Path - - import lancedb - import pandas as pd - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - model = get_registry().get("bedrock-text").create() - - class TextModel(LanceModel): - text: str = model.SourceField() - vector: Vector(model.ndims()) = model.VectorField() - - df = pd.DataFrame({"text": ["hello world", "goodbye world"]}) - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "bedrock-demo")) - tbl = db.create_table("test", schema=TextModel, mode="overwrite") - - tbl.add(df) - rs = tbl.search("hello").limit(1).to_pandas() - print(rs.head()) - # --8<-- [end:embedding_aws_usage] - - -def test_embedding_cohere_usage() -> None: - require_env("COHERE_API_KEY") - - # --8<-- [start:embedding_cohere_usage] - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import EmbeddingFunctionRegistry - from lancedb.pydantic import LanceModel, Vector - - cohere = ( - EmbeddingFunctionRegistry.get_instance() - .get("cohere") - .create(name="embed-multilingual-v2.0") - ) - - class TextModel(LanceModel): - text: str = cohere.SourceField() - vector: Vector(cohere.ndims()) = cohere.VectorField() - - data = [{"text": "hello world"}, {"text": "goodbye world"}] - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cohere-demo")) - tbl = db.create_table("test", schema=TextModel, mode="overwrite") - tbl.add(data) - # --8<-- [end:embedding_cohere_usage] - - -def test_embedding_gemini_usage() -> None: - require_flag("RUN_GEMINI_SNIPPETS") - - # --8<-- [start:embedding_gemini_usage] - import tempfile - from pathlib import Path - - import lancedb - import pandas as pd - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - model = get_registry().get("gemini-text").create() - - class TextModel(LanceModel): - text: str = model.SourceField() - vector: Vector(model.ndims()) = model.VectorField() - - df = pd.DataFrame({"text": ["hello world", "goodbye world"]}) - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "gemini-demo")) - tbl = db.create_table("test", schema=TextModel, mode="overwrite") - - tbl.add(df) - rs = tbl.search("hello").limit(1).to_pandas() - print(rs.head()) - # --8<-- [end:embedding_gemini_usage] - - -def test_embedding_huggingface_usage() -> None: - require_flag("RUN_HUGGINGFACE_SNIPPETS") - - # --8<-- [start:embedding_huggingface_usage] - import tempfile - from pathlib import Path - - import lancedb - import pandas as pd - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "huggingface-demo")) - model = get_registry().get("huggingface").create(name="facebook/bart-base") - - class Words(LanceModel): - text: str = model.SourceField() - vector: Vector(model.ndims()) = model.VectorField() - - df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]}) - table = db.create_table("greets", schema=Words) - table.add(df) - query = "old greeting" - actual = table.search(query).limit(1).to_pydantic(Words)[0] - print(actual.text) - # --8<-- [end:embedding_huggingface_usage] - - -def test_frameworks_lerobot_lancedb_usage() -> None: - require_flag("RUN_LEROBOT_LANCEDB_SNIPPETS") - - # --8<-- [start:frameworks_lerobot_lancedb_image_dataset] - from lerobot_lancedb import LeRobotLanceDataset - - dataset = LeRobotLanceDataset( - repo_id="your-org/your-lerobot-lance-images", - delta_timestamps={ - "observation.images.front": [-0.2, -0.1, 0.0], - }, - return_uint8=True, - ) - - sample = dataset[0] - print(sample["observation.state"].shape) - print(sample["action"].shape) - # --8<-- [end:frameworks_lerobot_lancedb_image_dataset] - - # --8<-- [start:frameworks_lerobot_lancedb_video_dataset] - from lerobot_lancedb import LeRobotLanceVideoDataset - - video_dataset = LeRobotLanceVideoDataset( - repo_id="lance-format/lerobot-pusht-lance", - delta_timestamps={ - "observation.images.image": [-0.2, -0.1, 0.0], - }, - return_uint8=True, - ) - - video_sample = video_dataset[0] - print(video_sample["observation.images.image"].shape) - # --8<-- [end:frameworks_lerobot_lancedb_video_dataset] - - # --8<-- [start:frameworks_lerobot_open_lance_tables] - import lancedb - - db = lancedb.connect("hf://datasets/lance-format/lerobot-pusht-lance/data") - frames = db.open_table("frames") - episodes = db.open_table("episodes") - videos = db.open_table("videos") - - print(len(frames), len(episodes), len(videos)) - print(frames.schema) - # --8<-- [end:frameworks_lerobot_open_lance_tables] - - # --8<-- [start:frameworks_lerobot_filter_frames] - frame_rows = ( - frames.search() - .where("episode_index = 0 AND frame_index < 10", prefilter=True) - .select(["episode_index", "frame_index", "timestamp", "action"]) - .limit(10) - .to_list() - ) - - for row in frame_rows: - print(row["episode_index"], row["frame_index"], row["timestamp"]) - # --8<-- [end:frameworks_lerobot_filter_frames] - - -def test_frameworks_stable_worldmodel_usage() -> None: - require_flag("RUN_STABLE_WORLDMODEL_SNIPPETS") - - # --8<-- [start:frameworks_stable_worldmodel_collect_lance] - import stable_worldmodel as swm - - world = swm.World("swm/PushT-v1", num_envs=8) - world.set_policy(your_expert_policy) - world.collect("data/pusht_demo.lance", episodes=100, seed=0) - # --8<-- [end:frameworks_stable_worldmodel_collect_lance] - - # --8<-- [start:frameworks_stable_worldmodel_load_lance] - dataset = swm.data.load_dataset("data/pusht_demo.lance", num_steps=16) - - batch = dataset[0] - print(batch.keys()) - # --8<-- [end:frameworks_stable_worldmodel_load_lance] - - # --8<-- [start:frameworks_stable_worldmodel_convert] - swm.data.convert( - "data/pusht_demo.lance", - "data/pusht_video", - dest_format="video", - fps=30, - ) - # --8<-- [end:frameworks_stable_worldmodel_convert] - - # --8<-- [start:frameworks_stable_worldmodel_evaluate] - from stable_worldmodel.policy import PlanConfig, WorldModelPolicy - from stable_worldmodel.solver import CEMSolver - - solver = CEMSolver(model=world_model, num_samples=300) - policy = WorldModelPolicy(solver=solver, config=PlanConfig(horizon=10)) - - world.set_policy(policy) - results = world.evaluate(episodes=50) - print(f"Success Rate: {results['success_rate']:.1f}%") - # --8<-- [end:frameworks_stable_worldmodel_evaluate] - - -def test_embedding_ibm_usage() -> None: - require_flag("RUN_IBM_WATSONX_SNIPPETS") - - # --8<-- [start:embedding_ibm_usage] - import os - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import EmbeddingFunctionRegistry - from lancedb.pydantic import LanceModel, Vector - - watsonx_embed = ( - EmbeddingFunctionRegistry.get_instance() - .get("watsonx") - .create( - name="ibm/slate-125m-english-rtrvr", - api_key=os.environ.get("WATSONX_API_KEY"), - project_id=os.environ.get("WATSONX_PROJECT_ID"), - ) - ) - - class TextModel(LanceModel): - text: str = watsonx_embed.SourceField() - vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-demo")) - tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite") - tbl.add(data) - - rs = tbl.search("hello").limit(1).to_pandas() - print(rs.head()) - # --8<-- [end:embedding_ibm_usage] - - -def test_embedding_imagebind_examples() -> None: - require_flag("RUN_IMAGEBIND_SNIPPETS") - pytest.importorskip("imagebind") - - # --8<-- [start:embedding_imagebind_setup] - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - db = lancedb.connect("/tmp/imagebind-db") - func = get_registry().get("imagebind").create() - - class ImageBindModel(LanceModel): - text: str - image_uri: str = func.SourceField() - audio_path: str - vector: Vector(func.ndims()) = func.VectorField() - - text_list = ["A dog.", "A car", "A bird"] - image_paths = [ - "./assets/dog_image.jpg", - "./assets/car_image.jpg", - "./assets/bird_image.jpg", - ] - audio_paths = [ - "./assets/dog_audio.wav", - "./assets/car_audio.wav", - "./assets/bird_audio.wav", - ] - - inputs = [ - {"text": a, "audio_path": b, "image_uri": c} - for a, b, c in zip(text_list, audio_paths, image_paths) - ] - - table = db.create_table("img_bind", schema=ImageBindModel) - table.add(inputs) - # --8<-- [end:embedding_imagebind_setup] - - # --8<-- [start:embedding_imagebind_image_search] - query_image = "./assets/dog_image2.jpg" - actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0] - print(actual.text == "dog") - # --8<-- [end:embedding_imagebind_image_search] - - # --8<-- [start:embedding_imagebind_audio_search] - query_audio = "./assets/car_audio2.wav" - actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0] - print(actual.text == "car") - # --8<-- [end:embedding_imagebind_audio_search] - - # --8<-- [start:embedding_imagebind_text_search] - query = "an animal which flies and tweets" - actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0] - print(actual.text == "bird") - # --8<-- [end:embedding_imagebind_text_search] - - -def test_embedding_colpali_examples() -> None: - require_flag("RUN_COLPALI_SNIPPETS") - pytest.importorskip("colpali_engine") - - # --8<-- [start:embedding_colpali_setup] - import tempfile - from pathlib import Path - - import lancedb - import pandas as pd - import requests - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, MultiVector - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "colpali-demo")) - func = get_registry().get("colpali").create() - - class Images(LanceModel): - label: str - image_uri: str = func.SourceField() - image_bytes: bytes = func.SourceField() - vector: MultiVector(func.ndims()) = func.VectorField() - vec_from_bytes: MultiVector(func.ndims()) = func.VectorField() - - table = db.create_table("images", schema=Images) - labels = ["cat", "dog", "horse"] - uris = [ - "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg", - "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg", - "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg", - ] - image_bytes = [requests.get(uri).content for uri in uris] - table.add( - pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes}) - ) - # --8<-- [end:embedding_colpali_setup] - - # --8<-- [start:embedding_colpali_text_search] - actual = ( - table.search("a furry pet", vector_column_name="vector") - .limit(1) - .to_pydantic(Images)[0] - ) - print(actual.label) - # --8<-- [end:embedding_colpali_text_search] - - -def test_embedding_instructor_usage() -> None: - require_flag("RUN_INSTRUCTOR_SNIPPETS") - - # --8<-- [start:embedding_instructor_usage] - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - instructor = ( - get_registry() - .get("instructor") - .create( - source_instruction="represent the document for retrieval", - query_instruction="represent the document for retrieving the most similar documents", - ) - ) - - class Schema(LanceModel): - vector: Vector(instructor.ndims()) = instructor.VectorField() - text: str = instructor.SourceField() - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "instructor-demo")) - tbl = db.create_table("test", schema=Schema, mode="overwrite") - - texts = [ - { - "text": "Capitalism has been dominant in the Western world since the end of feudalism." - }, - { - "text": "The disparate impact theory is especially controversial under the Fair Housing Act." - }, - { - "text": "Disparate impact in United States labor law refers to practices in employment." - }, - ] - - tbl.add(texts) - # --8<-- [end:embedding_instructor_usage] - - -def test_embedding_jina_text() -> None: - require_env("JINA_API_KEY") - - # --8<-- [start:embedding_jina_text] - import os - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import EmbeddingFunctionRegistry - from lancedb.pydantic import LanceModel, Vector - - os.environ["JINA_API_KEY"] = os.environ["JINA_API_KEY"] - - jina_embed = ( - EmbeddingFunctionRegistry.get_instance() - .get("jina") - .create(name="jina-embeddings-v2-base-en") - ) - - class TextModel(LanceModel): - text: str = jina_embed.SourceField() - vector: Vector(jina_embed.ndims()) = jina_embed.VectorField() - - data = [{"text": "hello world"}, {"text": "goodbye world"}] - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-text")) - tbl = db.create_table("test", schema=TextModel, mode="overwrite") - - tbl.add(data) - # --8<-- [end:embedding_jina_text] - - -def test_embedding_jina_multimodal() -> None: - require_flag("RUN_JINA_MULTIMODAL_SNIPPETS") - - # --8<-- [start:embedding_jina_multimodal] - import os - import tempfile - from pathlib import Path - - import lancedb - import pandas as pd - import requests - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - os.environ["JINA_API_KEY"] = os.environ.get("JINA_API_KEY", "jina_*") - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-images")) - func = get_registry().get("jina").create() - - class Images(LanceModel): - label: str - image_uri: str = func.SourceField() - image_bytes: bytes = func.SourceField() - vector: Vector(func.ndims()) = func.VectorField() - vec_from_bytes: Vector(func.ndims()) = func.VectorField() - - table = db.create_table("images", schema=Images) - labels = ["cat", "cat", "dog", "dog", "horse", "horse"] - uris = [ - "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg", - "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg", - "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg", - "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg", - "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg", - "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg", - ] - image_bytes = [requests.get(uri).content for uri in uris] - table.add( - pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes}) - ) - # --8<-- [end:embedding_jina_multimodal] - - -def test_embedding_ollama_usage() -> None: - require_flag("RUN_OLLAMA_SNIPPETS") - - # --8<-- [start:embedding_ollama_usage] - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "ollama-demo")) - func = get_registry().get("ollama").create(name="nomic-embed-text") - - class Words(LanceModel): - text: str = func.SourceField() - vector: Vector(func.ndims()) = func.VectorField() - - table = db.create_table("words", schema=Words, mode="overwrite") - table.add( - [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - ) - - query = "greetings" - actual = table.search(query).limit(1).to_pydantic(Words)[0] - print(actual.text) - # --8<-- [end:embedding_ollama_usage] - - -def test_embedding_openclip_examples() -> None: - require_flag("RUN_OPENCLIP_SNIPPETS") - - # --8<-- [start:embedding_openclip_setup] - import tempfile - from pathlib import Path - - import lancedb - import pandas as pd - import requests - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "openclip-demo")) - func = get_registry().get("open-clip").create() - - class Images(LanceModel): - label: str - image_uri: str = func.SourceField() - image_bytes: bytes = func.SourceField() - vector: Vector(func.ndims()) = func.VectorField() - vec_from_bytes: Vector(func.ndims()) = func.VectorField() - - table = db.create_table("images", schema=Images) - labels = ["cat", "cat", "dog", "dog", "horse", "horse"] - uris = [ - "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg", - "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg", - "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg", - "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg", - "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg", - "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg", - ] - image_bytes = [requests.get(uri).content for uri in uris] - table.add( - pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes}) - ) - # --8<-- [end:embedding_openclip_setup] - - # --8<-- [start:embedding_openclip_text_search] - actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0] - print(actual.label) - - frombytes = ( - table.search("man's best friend", vector_column_name="vec_from_bytes") - .limit(1) - .to_pydantic(Images)[0] - ) - print(frombytes.label) - # --8<-- [end:embedding_openclip_text_search] - - # --8<-- [start:embedding_openclip_image_search] - import io - - from PIL import Image - - query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg" - image_bytes = requests.get(query_image_uri).content - query_image = Image.open(io.BytesIO(image_bytes)) - actual = table.search(query_image).limit(1).to_pydantic(Images)[0] - print(actual.label == "dog") - - other = ( - table.search(query_image, vector_column_name="vec_from_bytes") - .limit(1) - .to_pydantic(Images)[0] - ) - print(other.label) - # --8<-- [end:embedding_openclip_image_search] - - -def test_embedding_sentence_transformers_baai() -> None: - require_flag("RUN_SENTENCE_TRANSFORMERS_SNIPPETS") - - # --8<-- [start:embedding_sentence_transformers_baai] - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "sentence-transformers")) - model = ( - get_registry() - .get("sentence-transformers") - .create(name="BAAI/bge-small-en-v1.5", device="cpu") - ) - - class Words(LanceModel): - text: str = model.SourceField() - vector: Vector(model.ndims()) = model.VectorField() - - table = db.create_table("words", schema=Words) - table.add( - [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - ) - - query = "greetings" - actual = table.search(query).limit(1).to_pydantic(Words)[0] - print(actual.text) - # --8<-- [end:embedding_sentence_transformers_baai] - - -def test_embedding_voyageai_usage() -> None: - require_env("VOYAGE_API_KEY") - - # --8<-- [start:embedding_voyageai_usage] - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import EmbeddingFunctionRegistry - from lancedb.pydantic import LanceModel, Vector - - voyageai = ( - EmbeddingFunctionRegistry.get_instance().get("voyageai").create(name="voyage-3") - ) - - class TextModel(LanceModel): - text: str = voyageai.SourceField() - vector: Vector(voyageai.ndims()) = voyageai.VectorField() - - data = [{"text": "hello world"}, {"text": "goodbye world"}] - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-demo")) - tbl = db.create_table("test", schema=TextModel, mode="overwrite") - - tbl.add(data) - # --8<-- [end:embedding_voyageai_usage] - - -def test_embedding_voyageai_multimodal() -> None: - require_env("VOYAGE_API_KEY") - - # --8<-- [start:embedding_voyageai_multimodal] - import tempfile - from pathlib import Path - - import lancedb - from lancedb.embeddings import EmbeddingFunctionRegistry - from lancedb.pydantic import LanceModel, Vector - - # Create multimodal embedding function with custom dimension - voyageai = ( - EmbeddingFunctionRegistry.get_instance() - .get("voyageai") - .create(name="voyage-multimodal-3.5", output_dimension=512) - ) - - class ImageModel(LanceModel): - image_uri: str = voyageai.SourceField() - vector: Vector(voyageai.ndims()) = voyageai.VectorField() - - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-multimodal")) - tbl = db.create_table("images", schema=ImageModel, mode="overwrite") - - # Add images using URLs - tbl.add( - [ - {"image_uri": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/300px-PNG_transparency_demonstration_1.png"}, - ] - ) - - # Search with text query - results = tbl.search("dice").limit(1).to_list() - print(results) - # --8<-- [end:embedding_voyageai_multimodal] - - -# Reranking integrations - - -def test_reranking_answerdotai_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - - # --8<-- [start:reranking_answerdotai_usage] - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import AnswerdotaiRerankers - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = AnswerdotaiRerankers() - - # Run vector search with a reranker - result = tbl.search("hello").rerank(reranker=reranker).to_list() - - # Run FTS search with a reranker - result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_answerdotai_usage] - - -def test_reranking_cohere_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - os.environ["COHERE_API_KEY"] = require_env("COHERE_API_KEY") - - # --8<-- [start:reranking_cohere_usage] - import os - - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import CohereReranker - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = CohereReranker(api_key=os.environ["COHERE_API_KEY"]) - - # Run vector search with a reranker - result = tbl.search("hello").rerank(reranker=reranker).to_list() - - # Run FTS search with a reranker - result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_cohere_usage] - - -def test_reranking_colbert_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - - # --8<-- [start:reranking_colbert_usage] - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import ColbertReranker - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = ColbertReranker() - - # Run vector search with a reranker - result = tbl.search("hello").rerank(reranker=reranker).to_list() - - # Run FTS search with a reranker - result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_colbert_usage] - - -def test_reranking_cross_encoder_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - - # --8<-- [start:reranking_cross_encoder_usage] - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import CrossEncoderReranker - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = CrossEncoderReranker() - - # Run vector search with a reranker - result = tbl.search("hello").rerank(reranker=reranker).to_list() - - # Run FTS search with a reranker - result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_cross_encoder_usage] - - -def test_reranking_jina_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - os.environ["JINA_API_KEY"] = require_env("JINA_API_KEY") - - # --8<-- [start:reranking_jina_usage] - import os - - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import JinaReranker - - embedder = get_registry().get("jina").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = JinaReranker(api_key=os.environ["JINA_API_KEY"]) - - # Run vector search with a reranker - result = tbl.search("hello").rerank(reranker=reranker).to_list() - - # Run FTS search with a reranker - result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_jina_usage] - - -def test_reranking_linear_combination_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - - # --8<-- [start:reranking_linear_combination_usage] - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import LinearCombinationReranker - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = LinearCombinationReranker() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_linear_combination_usage] - - -def test_reranking_openai_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - os.environ["OPENAI_API_KEY"] = require_env("OPENAI_API_KEY") - - # --8<-- [start:reranking_openai_usage] - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import OpenaiReranker - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = OpenaiReranker() - - # Run vector search with a reranker - result = tbl.search("hello").rerank(reranker=reranker).to_list() - - # Run FTS search with a reranker - result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_openai_usage] - - -def test_reranking_rrf_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - - # --8<-- [start:reranking_rrf_usage] - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import RRFReranker - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = RRFReranker() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_rrf_usage] - - -def test_reranking_mrr_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - - # --8<-- [start:reranking_mrr_usage] - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import MRRReranker - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = MRRReranker(weight_vector=0.7, weight_fts=0.3) - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - - # Run multivector search across multiple vector columns - rs1 = tbl.search("hello").limit(10).with_row_id(True).to_arrow() - rs2 = tbl.search("greeting").limit(10).with_row_id(True).to_arrow() - combined = MRRReranker().rerank_multivector([rs1, rs2]) - # --8<-- [end:reranking_mrr_usage] - - -def test_reranking_voyageai_usage() -> None: - require_flag("RUN_RERANKER_SNIPPETS") - os.environ["VOYAGE_API_KEY"] = require_env("VOYAGE_API_KEY") - - # --8<-- [start:reranking_voyageai_usage] - import os - - import lancedb - from lancedb.embeddings import get_registry - from lancedb.pydantic import LanceModel, Vector - from lancedb.rerankers import VoyageAIReranker - - embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") - - class Schema(LanceModel): - text: str = embedder.SourceField() - vector: Vector(embedder.ndims()) = embedder.VectorField() - - data = [ - {"text": "hello world"}, - {"text": "goodbye world"}, - ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") - tbl.add(data) - reranker = VoyageAIReranker(model_name="rerank-2") - - # Run vector search with a reranker - result = tbl.search("hello").rerank(reranker=reranker).to_list() - - # Run FTS search with a reranker - result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() - - # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) - result = ( - tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() - ) - # --8<-- [end:reranking_voyageai_usage] - - -# Framework integrations - - -def test_frameworks_langchain_examples() -> None: - require_flag("RUN_LANGCHAIN_SNIPPETS") - pytest.importorskip("langchain") - pytest.importorskip("langchain_openai") - pytest.importorskip("langchain_text_splitters") - - # --8<-- [start:frameworks_langchain_quick_start] - import os - - from langchain.document_loaders import TextLoader - from langchain.vectorstores import LanceDB - from langchain_openai import OpenAIEmbeddings - from langchain_text_splitters import CharacterTextSplitter - - os.environ["OPENAI_API_KEY"] = "sk-..." - - loader = TextLoader( - "../../modules/state_of_the_union.txt" - ) # Replace with your data path - documents = loader.load() - - documents = CharacterTextSplitter().split_documents(documents) - embeddings = OpenAIEmbeddings() - - docsearch = LanceDB.from_documents(documents, embeddings) - query = "What did the president say about Ketanji Brown Jackson" - docs = docsearch.similarity_search(query) - print(docs[0].page_content) - # --8<-- [end:frameworks_langchain_quick_start] - - # --8<-- [start:frameworks_langchain_vector_store_config] - db_url = "db://lang_test" # url of db you created - api_key = "xxxxx" # your API key - region = "us-east-1-dev" # your selected region - - vector_store = LanceDB( - uri=db_url, - api_key=api_key, # (dont include for local API) - region=region, # (dont include for local API) - embedding=embeddings, - table_name="langchain_test", # Optional - ) - # --8<-- [end:frameworks_langchain_vector_store_config] - - # --8<-- [start:frameworks_langchain_add_texts] - vector_store.add_texts(texts=["test_123"], metadatas=[{"source": "wiki"}]) - - # Additionaly, to explore the table you can load it into a df or save it in a csv file: - - tbl = vector_store.get_table() - print("tbl:", tbl) - pd_df = tbl.to_pandas() - pd_df.to_csv("docsearch.csv", index=False) - - # you can also create a new vector store object using an older connection object: - vector_store = LanceDB(connection=tbl, embedding=embeddings) - # --8<-- [end:frameworks_langchain_add_texts] - - # --8<-- [start:frameworks_langchain_create_index] - # for creating vector index - vector_store.create_index(vector_col="vector", metric="cosine") - - # for creating scalar index(for non-vector columns) - vector_store.create_index(col_name="text") - # --8<-- [end:frameworks_langchain_create_index] - - # --8<-- [start:frameworks_langchain_similarity_search] - docs = docsearch.similarity_search(query) - print(docs[0].page_content) - # --8<-- [end:frameworks_langchain_similarity_search] - - # --8<-- [start:frameworks_langchain_similarity_search_by_vector] - docs = docsearch.similarity_search_by_vector(query) - print(docs[0].page_content) - # --8<-- [end:frameworks_langchain_similarity_search_by_vector] - - # --8<-- [start:frameworks_langchain_similarity_search_with_scores] - docs = docsearch.similarity_search_with_relevance_scores(query) - print("relevance score - ", docs[0][1]) - print("text- ", docs[0][0].page_content[:1000]) - # --8<-- [end:frameworks_langchain_similarity_search_with_scores] - - # --8<-- [start:frameworks_langchain_similarity_search_by_vector_with_scores] - query_embedding = embeddings.embed_query("text") - docs = docsearch.similarity_search_by_vector_with_relevance_scores(query_embedding) - print("relevance score - ", docs[0][1]) - print("text- ", docs[0][0].page_content[:1000]) - # --8<-- [end:frameworks_langchain_similarity_search_by_vector_with_scores] - - # --8<-- [start:frameworks_langchain_max_marginal_relevance] - result = docsearch.max_marginal_relevance_search(query="text") - result_texts = [doc.page_content for doc in result] - print(result_texts) - - # search by vector : - result = docsearch.max_marginal_relevance_search_by_vector( - embeddings.embed_query("text") - ) - result_texts = [doc.page_content for doc in result] - print(result_texts) - # --8<-- [end:frameworks_langchain_max_marginal_relevance] - - # --8<-- [start:frameworks_langchain_add_images] - image_uris = ["./assets/image-1.png", "./assets/image-2.png"] - vector_store.add_images(uris=image_uris) - # here image_uris are local fs paths to the images. - # --8<-- [end:frameworks_langchain_add_images] - - -def test_frameworks_llamaindex_examples() -> None: - require_flag("RUN_LLAMAINDEX_SNIPPETS") - pytest.importorskip("llama_index") - pytest.importorskip("llama_index.vector_stores.lancedb") - - # --8<-- [start:frameworks_llamaindex_quick_start] - import logging - import sys - import textwrap - - import openai - - # Uncomment to see debug logs - # logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - # logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout)) - from llama_index.core import ( - Document, - SimpleDirectoryReader, - StorageContext, - VectorStoreIndex, - ) - from llama_index.vector_stores.lancedb import LanceDBVectorStore - - openai.api_key = "sk-..." - - documents = SimpleDirectoryReader("./data/your-data-dir/").load_data() - print("Document ID:", documents[0].doc_id, "Document Hash:", documents[0].hash) - - ## For LanceDB Enterprise : - # vector_store = LanceDBVectorStore( - # uri="db://db_name", # your remote DB URI - # api_key="sk_..", # lancedb enterprise api key - # region="your-region" # the region you configured - # host_override="https://your-host.com" # if you have a custom host, otherwise omit this - # ) - - vector_store = LanceDBVectorStore( - uri="./lancedb", mode="overwrite", query_type="vector" - ) - storage_context = StorageContext.from_defaults(vector_store=vector_store) - - index = VectorStoreIndex.from_documents(documents, storage_context=storage_context) - lance_filter = "metadata.file_name = 'paul_graham_essay.txt' " - retriever = index.as_retriever(vector_store_kwargs={"where": lance_filter}) - response = retriever.retrieve("What did the author do growing up?") - # --8<-- [end:frameworks_llamaindex_quick_start] - - # --8<-- [start:frameworks_llamaindex_filtering] - from llama_index.core.vector_stores import ( - FilterCondition, - FilterOperator, - MetadataFilter, - MetadataFilters, - ) - - query_filters = MetadataFilters( - filters=[ - MetadataFilter( - key="creation_date", operator=FilterOperator.EQ, value="2024-05-23" - ), - MetadataFilter(key="file_size", value=75040, operator=FilterOperator.GT), - ], - condition=FilterCondition.AND, - ) - # --8<-- [end:frameworks_llamaindex_filtering] - - # --8<-- [start:frameworks_llamaindex_hybrid_search] - from lancedb.rerankers import ColbertReranker - - reranker = ColbertReranker() - vector_store._add_reranker(reranker) - - query_engine = index.as_query_engine( - filters=query_filters, - vector_store_kwargs={ - "query_type": "hybrid", - }, - ) - - response = query_engine.query("How much did Viaweb charge per month?") - # --8<-- [end:frameworks_llamaindex_hybrid_search] - - # --8<-- [start:frameworks_llamaindex_add_reranker] - from lancedb.rerankers import ColbertReranker - - reranker = ColbertReranker() - vector_store._add_reranker(reranker) - # --8<-- [end:frameworks_llamaindex_add_reranker] - - -def test_frameworks_pydantic_examples() -> None: - require_flag("RUN_PYDANTIC_SNIPPETS") - pytest.importorskip("pyarrow") - - # --8<-- [start:frameworks_pydantic_imports] - import tempfile - from pathlib import Path - - import lancedb - from lancedb.pydantic import LanceModel, Vector - - # --8<-- [end:frameworks_pydantic_imports] - # --8<-- [start:frameworks_pydantic_base_model] - class LanceDocs(LanceModel): - text: str - vector: Vector(2) - - # --8<-- [end:frameworks_pydantic_base_model] - - # --8<-- [start:frameworks_pydantic_set_url] - db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "pydantic-docs")) - # --8<-- [end:frameworks_pydantic_set_url] - - # --8<-- [start:frameworks_pydantic_vector_field] - import pyarrow as pa - import pydantic - from lancedb.pydantic import Vector, pydantic_to_schema - - class MyModel(pydantic.BaseModel): - id: int - url: str - embeddings: Vector(768) - - schema = pydantic_to_schema(MyModel) - assert schema == pa.schema( - [ - pa.field("id", pa.int64(), False), - pa.field("url", pa.utf8(), False), - pa.field("embeddings", pa.list_(pa.float32(), 768)), - ] - ) - # --8<-- [end:frameworks_pydantic_vector_field] - - # --8<-- [start:frameworks_pydantic_type_conversion] - from typing import List, Optional - - import pyarrow as pa - import pydantic - from lancedb.pydantic import Vector, pydantic_to_schema - - class FooModel(pydantic.BaseModel): - id: int - s: str - vec: Vector(1536) # fixed_size_list[1536] - li: List[int] - - schema = pydantic_to_schema(FooModel) - assert schema == pa.schema( - [ - pa.field("id", pa.int64(), False), - pa.field("s", pa.utf8(), False), - pa.field("vec", pa.list_(pa.float32(), 1536)), - pa.field("li", pa.list_(pa.int64()), False), - ] - ) - # --8<-- [end:frameworks_pydantic_type_conversion] - - # --8<-- [start:frameworks_pydantic_base_example] - table = db.create_table("docs", schema=LanceDocs, mode="overwrite") - table.add( - [ - {"text": "hello world", "vector": [1.0, 0.0]}, - {"text": "goodbye world", "vector": [0.0, 1.0]}, - ] - ) - results = table.search("hello world").limit(1).to_pydantic(LanceDocs) - print(results[0].text) - # --8<-- [end:frameworks_pydantic_base_example] - - -# Platform integrations - - -def test_platforms_dlt_examples() -> None: - require_flag("RUN_DLT_SNIPPETS") - pytest.importorskip("dlt") - - # --8<-- [start:platforms_dlt_pipeline] - # Import necessary modules - import dlt - from rest_api import rest_api_source - - # Configure the REST API source - movies_source = rest_api_source( - { - "client": { - "base_url": "https://www.omdbapi.com/", - "auth": { # authentication strategy for the OMDb API - "type": "api_key", - "name": "apikey", - "api_key": dlt.secrets[ - "sources.rest_api.api_token" - ], # read API credentials directly from secrets.toml - "location": "query", - }, - "paginator": { # pagination strategy for the OMDb API - "type": "page_number", - "base_page": 1, - "total_path": "totalResults", - "maximum_page": 5, - }, - }, - "resources": [ # list of API endpoints to request - { - "name": "movie_search", - "endpoint": { - "path": "/", - "params": { - "s": "godzilla", - "type": "movie", - }, - }, - } - ], - } - ) - - if __name__ == "__main__": - # Create a pipeline object - pipeline = dlt.pipeline( - pipeline_name="movies_pipeline", - destination="lancedb", # this tells dlt to load the data into LanceDB - dataset_name="movies_data_pipeline", - ) - - # Run the pipeline - load_info = pipeline.run(movies_source) - - # pretty print the information on data that was loaded - print(load_info) - # --8<-- [end:platforms_dlt_pipeline] - - # --8<-- [start:platforms_dlt_adapter_import] - from dlt.destinations.adapters import lancedb_adapter - - # --8<-- [end:platforms_dlt_adapter_import] - # --8<-- [start:platforms_dlt_adapter_usage] - load_info = pipeline.run( - lancedb_adapter( - movies_source, - embed="Title", - ) - ) - # --8<-- [end:platforms_dlt_adapter_usage] - - -def test_platforms_duckdb_examples() -> None: - require_flag("RUN_DUCKDB_SNIPPETS") - pytest.importorskip("duckdb") - - # --8<-- [start:platforms_duckdb_create_table] - import lancedb - - db = lancedb.connect("data/sample-lancedb") - data = [ - {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, - {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, - ] - table = db.create_table("pd_table", data=data) - # --8<-- [end:platforms_duckdb_create_table] - - # --8<-- [start:platforms_duckdb_query_table] - import duckdb - - arrow_table = table.to_lance() - - duckdb.query("SELECT * FROM arrow_table") - # --8<-- [end:platforms_duckdb_query_table] - - # --8<-- [start:platforms_duckdb_mean_price] - duckdb.query("SELECT mean(price) FROM arrow_table") - # --8<-- [end:platforms_duckdb_mean_price] - - -def test_frameworks_agno_openai_examples() -> None: - require_flag("RUN_AGNO_SNIPPETS") - pytest.importorskip("agno") - pytest.importorskip("youtube_transcript_api") - - # --8<-- [start:frameworks_agno_setup] - import os - import re - - from agno.agent import Agent - from agno.knowledge.embedder.openai import OpenAIEmbedder - from agno.knowledge.knowledge import Knowledge - from agno.models.openai import OpenAIResponses - from agno.vectordb.lancedb import LanceDb, SearchType - from youtube_transcript_api import YouTubeTranscriptApi - - if "OPENAI_API_KEY" not in os.environ: - os.environ["OPENAI_API_KEY"] = "sk-..." - - def extract_video_id(youtube_url: str) -> str: - match = re.search(r"(?<=v=)[\w-]+", youtube_url) or re.search( - r"(?<=be/)[\w-]+", youtube_url - ) - if not match: - raise ValueError("Could not parse YouTube video ID from URL") - return match.group(0) - - knowledge = Knowledge( - vector_db=LanceDb( - uri="./tmp/lancedb", - table_name="youtube_transcripts", - search_type=SearchType.hybrid, - embedder=OpenAIEmbedder(id="text-embedding-3-small"), - ), - ) - # --8<-- [end:frameworks_agno_setup] - - # --8<-- [start:frameworks_agno_ingest_youtube] - youtube_url = "https://www.youtube.com/watch?v=wl6mFyXoxos" - video_id = extract_video_id(youtube_url) - ytt = YouTubeTranscriptApi() - transcript_segments = ytt.fetch(video_id, languages=["en", "en-US"]).to_raw_data() - transcript_text = " ".join(segment["text"] for segment in transcript_segments) - - knowledge.insert( - name=f"YouTube Transcript ({video_id})", - text_content=transcript_text, - metadata={"source": "youtube", "video_id": video_id, "video_url": youtube_url}, - ) - # --8<-- [end:frameworks_agno_ingest_youtube] - - # --8<-- [start:frameworks_agno_agent] - agent = Agent( - model=OpenAIResponses(id="gpt-5-mini"), - knowledge=knowledge, - search_knowledge=True, - instructions="Search the transcript and answer only from retrieved context.", - markdown=True, - ) - # --8<-- [end:frameworks_agno_agent] - - # --8<-- [start:frameworks_agno_cli_chat] - agent.print_response( - "Summarize the loaded video transcript in 5 concise bullet points.", - stream=True, - ) - while True: - question = input("You: ").strip() - if question.lower() in {"exit", "quit", "bye"}: - break - agent.print_response(question, stream=True) - # --8<-- [end:frameworks_agno_cli_chat] - - -def test_platforms_voxel51_examples() -> None: - require_flag("RUN_VOXEL51_SNIPPETS") - pytest.importorskip("fiftyone") - - # --8<-- [start:platforms_voxel51_load_dataset] - import fiftyone as fo - import fiftyone.brain as fob - import fiftyone.zoo as foz - - # Step 1: Load your data into FiftyOne - dataset = foz.load_zoo_dataset("quickstart") - # --8<-- [end:platforms_voxel51_load_dataset] - - # --8<-- [start:platforms_voxel51_compute_similarity] - # Steps 2 and 3: Compute embeddings and create a similarity index - lancedb_index = fob.compute_similarity( - dataset, - model="clip-vit-base32-torch", - brain_key="lancedb_index", - backend="lancedb", - ) - # --8<-- [end:platforms_voxel51_compute_similarity] - - # --8<-- [start:platforms_voxel51_sort_by_similarity] - # Step 4: Query your data - query = dataset.first().id # query by sample ID - view = dataset.sort_by_similarity( - query, - brain_key="lancedb_index", - k=10, # limit to 10 most similar samples - ) - # --8<-- [end:platforms_voxel51_sort_by_similarity] - - # --8<-- [start:platforms_voxel51_cleanup] - # Step 5 (optional): Cleanup - - # Delete the LanceDB table - lancedb_index.cleanup() - - # Delete run record from FiftyOne - dataset.delete_brain_run("lancedb_index") - # --8<-- [end:platforms_voxel51_cleanup] - - if False: - # --8<-- [start:platforms_voxel51_backend_flag] - import fiftyone.brain as fob - - # Re-run similarity creation using the LanceDB backend explicitly - fob.compute_similarity( - dataset, - model="clip-vit-base32-torch", - brain_key="lancedb_index", - backend="lancedb", - ) - # --8<-- [end:platforms_voxel51_backend_flag] - - # --8<-- [start:platforms_voxel51_brain_config] - import fiftyone.brain as fob - - # Print your current brain config - print(fob.brain_config) - # --8<-- [end:platforms_voxel51_brain_config] - - if False: - # --8<-- [start:platforms_voxel51_backend_params] - lancedb_index = fob.compute_similarity( - dataset, - model="clip-vit-base32-torch", - backend="lancedb", - brain_key="lancedb_index", - table_name="your-table", - metric="euclidean", - uri="/tmp/lancedb", - ) - # --8<-- [end:platforms_voxel51_backend_params] - - -def test_platforms_pandas_examples() -> None: - require_flag("RUN_PANDAS_SNIPPETS") - pytest.importorskip("pandas") - - # --8<-- [start:platforms_pandas_imports] - import asyncio - import tempfile - from pathlib import Path - - import lancedb - import pandas as pd - - # --8<-- [end:platforms_pandas_imports] - # --8<-- [start:platforms_pandas_create_table] - pandas_df = pd.DataFrame( - [ - {"id": "1", "text": "dragon", "vector": [0.9, 0.1, 0.3]}, - {"id": "2", "text": "griffin", "vector": [0.4, 0.5, 0.2]}, - {"id": "3", "text": "phoenix", "vector": [0.7, 0.3, 0.6]}, - ] - ) - pandas_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "pandas-demo")) - pandas_table = pandas_db.create_table("creatures", data=pandas_df, mode="overwrite") - # --8<-- [end:platforms_pandas_create_table] - - # --8<-- [start:platforms_pandas_vector_search] - pandas_results = ( - pandas_table.search([0.9, 0.1, 0.3]) - .select(["text", "_distance"]) - .limit(1) - .to_pandas() - ) - print(pandas_results) - # --8<-- [end:platforms_pandas_vector_search] - - # --8<-- [start:platforms_pandas_async_example] - async def run_pandas_async_example() -> None: - async_db = await lancedb.connect_async( - str(Path(tempfile.mkdtemp()) / "pandas-async") - ) - async_df = pd.DataFrame( - [ - {"id": "10", "text": "sage", "vector": [0.6, 0.4, 0.8]}, - {"id": "11", "text": "bard", "vector": [0.2, 0.7, 0.3]}, - ] - ) - async_table = await async_db.create_table( - "creatures_async", data=async_df, mode="overwrite" - ) - async_results = await ( - async_table.search([0.6, 0.4, 0.8]) - .select(["text", "_distance"]) - .limit(1) - .to_pandas() - ) - print(async_results) - - asyncio.run(run_pandas_async_example()) - # --8<-- [end:platforms_pandas_async_example] - - -def test_platforms_polars_examples() -> None: - require_flag("RUN_POLARS_SNIPPETS") - pytest.importorskip("polars") - - # --8<-- [start:platforms_polars_imports] - import tempfile - from pathlib import Path - - import lancedb - import polars as pl - from lancedb.pydantic import LanceModel, Vector - - # --8<-- [end:platforms_polars_imports] - # --8<-- [start:platforms_polars_create_table] - birds = pl.DataFrame( - { - "text": ["phoenix", "sparrow"], - "vector": [ - [0.1, 0.2, 0.3], - [0.8, 0.6, 0.5], - ], - } - ) - polars_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "polars-demo")) - polars_table = polars_db.create_table( - "birds", data=birds.to_arrow(), mode="overwrite" - ) - # --8<-- [end:platforms_polars_create_table] - - # --8<-- [start:platforms_polars_vector_search] - polars_results = ( - polars_table.search([0.1, 0.2, 0.3]) - .select(["text", "_distance"]) - .limit(1) - .to_polars() - ) - print(polars_results) - # --8<-- [end:platforms_polars_vector_search] - - # --8<-- [start:platforms_polars_lazyframe] - lazy_frame = polars_table.to_polars().lazy() - print(lazy_frame.select(["text"]).collect()) - # --8<-- [end:platforms_polars_lazyframe] - - # --8<-- [start:platforms_polars_pydantic] - class BirdModel(LanceModel): - text: str - vector: Vector(3) - - schema_table = polars_db.create_table( - "birds_schema", schema=BirdModel, mode="overwrite" - ) - schema_table.add(birds.to_dicts()) - # --8<-- [end:platforms_polars_pydantic] diff --git a/tests/py/test_multimodal.py b/tests/py/test_multimodal.py deleted file mode 100644 index a2656bd..0000000 --- a/tests/py/test_multimodal.py +++ /dev/null @@ -1,232 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import pytest -try: - import lancedb - import numpy as np - import pyarrow as pa - import io - from PIL import Image -except ImportError: - pass - -# --8<-- [start:multimodal_imports] -import lancedb -import pyarrow as pa -import pandas as pd -import numpy as np -import io -from PIL import Image -# --8<-- [end:multimodal_imports] - -def test_multimodal_ingestion(db_path_factory): - # Ensure dependencies are available - pytest.importorskip("PIL") - pytest.importorskip("lancedb") - pytest.importorskip("numpy") - - # --8<-- [start:create_dummy_data] - # Create some dummy images - def create_dummy_image(color): - img = Image.new('RGB', (100, 100), color=color) - buf = io.BytesIO() - img.save(buf, format='PNG') - return buf.getvalue() - - # Create dataset with metadata, vectors, and image blobs - data = [ - { - "id": 1, - "filename": "red_square.png", - "vector": np.random.rand(128).astype(np.float32), - "image_blob": create_dummy_image('red'), - "label": "red" - }, - { - "id": 2, - "filename": "blue_square.png", - "vector": np.random.rand(128).astype(np.float32), - "image_blob": create_dummy_image('blue'), - "label": "blue" - } - ] - # --8<-- [end:create_dummy_data] - - # --8<-- [start:define_schema] - # Define schema explictly to ensure image_blob is treated as binary - schema = pa.schema([ - pa.field("id", pa.int32()), - pa.field("filename", pa.string()), - pa.field("vector", pa.list_(pa.float32(), 128)), - pa.field("image_blob", pa.binary()), # Important: Use pa.binary() for blobs - pa.field("label", pa.string()) - ]) - # --8<-- [end:define_schema] - - db_uri = db_path_factory("multimodal_db") - db = lancedb.connect(db_uri) - - # --8<-- [start:ingest_data] - tbl = db.create_table("images", data=data, schema=schema, mode="overwrite") - # --8<-- [end:ingest_data] - - assert len(tbl) == 2 - - # --8<-- [start:search_data] - # Search for similar images - query_vector = np.random.rand(128).astype(np.float32) - results = tbl.search(query_vector).limit(1).to_pandas() - # --8<-- [end:search_data] - - # --8<-- [start:process_results] - # Convert back to PIL Image - for _, row in results.iterrows(): - image_bytes = row['image_blob'] - image = Image.open(io.BytesIO(image_bytes)) - print(f"Retrieved image: {row['filename']}, Size: {image.size}") - # You can now use 'image' with other libraries or display it - # --8<-- [end:process_results] - - assert len(results) == 1 - -def test_blob_api_definition(db_path_factory): - # --8<-- [start:blob_api_schema] - import pyarrow as pa - - # Define schema with Blob API metadata for lazy loading - schema = pa.schema([ - pa.field("id", pa.int64()), - pa.field( - "video", - pa.large_binary(), - metadata={"lance-encoding:blob": "true"} # Enable Blob API - ), - ]) - # --8<-- [end:blob_api_schema] - - # --8<-- [start:blob_api_ingest] - import lancedb - - db = lancedb.connect(db_path_factory("blob_db")) - - # Create sample data - data = [ - {"id": 1, "video": b"fake_video_bytes_1"}, - {"id": 2, "video": b"fake_video_bytes_2"} - ] - - # Create the table - tbl = db.create_table("videos", data=data, schema=schema) - # --8<-- [end:blob_api_ingest] - assert len(tbl) == 2 - - -def test_blob_api_to_pandas(db_path_factory): - db = lancedb.connect(db_path_factory("blob_to_pandas_db")) - schema = pa.schema([ - pa.field("id", pa.int64()), - pa.field( - "video", - pa.large_binary(), - metadata={"lance-encoding:blob": "true"}, - ), - ]) - tbl = db.create_table( - "videos", - data=[ - {"id": 1, "video": b"fake_video_bytes_1"}, - {"id": 2, "video": b"fake_video_bytes_2"}, - ], - schema=schema, - mode="overwrite", - ) - - # --8<-- [start:blob_api_to_pandas] - # Default: blob columns come back lazily - df_lazy = tbl.to_pandas() - - # Materialize blob bytes eagerly - df_bytes = tbl.to_pandas(blob_mode="bytes") - - # Return descriptors instead of payloads - df_desc = tbl.to_pandas(blob_mode="descriptions") - - # Forward extra kwargs to PyArrow's to_pandas - df_typed = tbl.to_pandas(split_blocks=True, self_destruct=True) - # --8<-- [end:blob_api_to_pandas] - - assert len(df_lazy) == 2 - assert isinstance(df_bytes["video"].iloc[0], bytes) - assert df_bytes["video"].tolist() == [ - b"fake_video_bytes_1", - b"fake_video_bytes_2", - ] - assert len(df_desc) == 2 - assert len(df_typed) == 2 - - -@pytest.mark.asyncio -async def test_query_to_pandas_kwargs(db_path_factory): - schema = pa.schema([ - pa.field("id", pa.int64()), - pa.field("vector", pa.list_(pa.float32(), 128)), - pa.field( - "video", - pa.large_binary(), - metadata={"lance-encoding:blob": "true"}, - ), - ]) - data = [ - { - "id": i, - "vector": np.random.rand(128).astype(np.float32), - "video": f"fake_video_bytes_{i}".encode(), - } - for i in range(10) - ] - - db = lancedb.connect(db_path_factory("query_to_pandas_db")) - tbl = db.create_table("search_demo", data=data, schema=schema, mode="overwrite") - - async_db = await lancedb.connect_async( - str(db_path_factory("query_to_pandas_async_db")) - ) - tbl_async = await async_db.create_table( - "search_demo", data=data, schema=schema, mode="overwrite" - ) - - query_vector = np.random.rand(128).astype(np.float32) - - # --8<-- [start:query_to_pandas_kwargs] - # Plain scan query: blob_mode is supported end to end - df_lazy = ( - tbl.search() - .where("id = 1") - .select(["id", "video"]) - .to_pandas(blob_mode="lazy") - ) - - # Same call shape works on async query builders - df_bytes = await ( - tbl_async.query() - .where("id = 1") - .select(["id", "video"]) - .to_pandas(blob_mode="bytes") - ) - - # Vector / FTS / hybrid queries can't materialize blob columns, - # so omit them from the projection - df_vec = ( - tbl.search(query_vector) - .limit(10) - .select(["id", "vector"]) - .to_pandas(split_blocks=True, self_destruct=True) - ) - # --8<-- [end:query_to_pandas_kwargs] - - assert len(df_lazy) == 1 - assert len(df_bytes) == 1 - assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1" - assert len(df_vec) == 10 - assert "video" not in df_vec.columns diff --git a/tests/py/test_quickstart.py b/tests/py/test_quickstart.py deleted file mode 100644 index 207e654..0000000 --- a/tests/py/test_quickstart.py +++ /dev/null @@ -1,208 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import lancedb -import pytest - - -def test_quickstart(db_path_factory): - uri = db_path_factory("quickstart_db") - db = lancedb.connect(uri) - - # --8<-- [start:quickstart_data] - data = [ - { - "id": "1", - "name": "King Arthur", - "role": "King", - "description": "Leader of Camelot and wielder of Excalibur.", - "stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4}, - "vector": [0.7, 0.1, 0.9, 0.7], - }, - { - "id": "2", - "name": "Merlin", - "role": "Wizard", - "description": "Advisor and prophet with deep magical knowledge.", - "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5}, - "vector": [0.2, 0.9, 0.4, 0.9], - }, - { - "id": "3", - "name": "Sir Lancelot", - "role": "Knight", - "description": "Legendary knight known for courage and combat skill.", - "stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3}, - "vector": [0.9, 0.1, 0.5, 0.4], - }, - ] - # --8<-- [end:quickstart_data] - - # --8<-- [start:quickstart_create_table] - table = db.create_table("characters", data=data, mode="overwrite") - # --8<-- [end:quickstart_create_table] - assert len(table) == 3 - # Drop the table to test create without overwrite - db.drop_table("characters") - - # --8<-- [start:quickstart_create_table_no_overwrite] - table = db.create_table("characters", data=data) - # --8<-- [end:quickstart_create_table_no_overwrite] - assert len(table) == 3 - # --8<-- [start:quickstart_vector_search_1] - # Search for examples similar to a "wise magical advisor" - query_vector = [0.2, 0.8, 0.4, 0.9] - - # Ensure you run `pip install polars` beforehand - result = ( - table.search(query_vector) - .select(["name", "role", "description", "_distance"]) - .limit(2) - .to_polars() - ) - print(result) - # --8<-- [end:quickstart_vector_search_1] - assert result.head(1)["name"][0] == "Merlin" - - # --8<-- [start:quickstart_curate_with_metadata] - curated = ( - table.search(query_vector) - .where("stats.magic >= 4") - .select(["name", "role", "description", "_distance"]) - .limit(2) - .to_polars() - ) - print(curated) - # --8<-- [end:quickstart_curate_with_metadata] - assert curated.head(1)["name"][0] == "Merlin" - - # --8<-- [start:quickstart_output_pandas] - # Ensure you run `pip install pandas` beforehand - result = table.search(query_vector).limit(2).to_pandas() - print(result) - # --8<-- [end:quickstart_output_pandas] - assert result.iloc[0]["name"] == "Merlin" - - # --8<-- [start:quickstart_add_feature] - table.add_columns( - { - "power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)" - } - ) - # --8<-- [end:quickstart_add_feature] - assert "power_score" in table.schema.names - - # --8<-- [start:quickstart_query_feature] - features = table.search().select(["name", "role", "power_score"]).to_polars() - print(features) - # --8<-- [end:quickstart_query_feature] - assert "power_score" in features.columns - - # --8<-- [start:quickstart_multimodal_bytes] - from pathlib import Path - - image_path = Path("docs/static/assets/images/quickstart/sir-lancelot.jpg") - image_bytes = image_path.read_bytes() - - multimodal_table = db.create_table( - "character_images", - data=[ - { - "id": "lancelot", - "description": "Portrait of Sir Lancelot", - "image": image_bytes, - "vector": [0.9, 0.1, 0.5, 0.4], - } - ], - mode="overwrite", - ) - # --8<-- [end:quickstart_multimodal_bytes] - assert len(multimodal_table) == 1 - - # --8<-- [start:quickstart_open_table] - table = db.open_table("characters") - # --8<-- [end:quickstart_open_table] - - # --8<-- [start:quickstart_add_data] - more_data = [ - { - "id": "4", - "name": "Morgana", - "role": "Sorceress", - "description": "Powerful sorceress of Avalon.", - "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4}, - "vector": [0.3, 0.9, 0.6, 0.8], - "power_score": 3.75, - }, - ] - - # Add data to table - table.add(more_data) - # --8<-- [end:quickstart_add_data] - assert len(table) == 4 - - # --8<-- [start:quickstart_vector_search_2] - # Search for examples similar to a "powerful sorceress" - query_vector = [0.3, 0.9, 0.6, 0.8] - - results = table.search(query_vector).limit(2).to_polars() - print(results) - # --8<-- [end:quickstart_vector_search_2] - assert results.head(1)["name"][0] == "Morgana" - - -@pytest.mark.asyncio -async def test_quickstart_async_api(db_path_factory): - db_uri = db_path_factory("quickstart_async_db") - import lancedb - async_db = await lancedb.connect_async(db_uri) - - # --8<-- [start:quickstart_data_async] - data = [ - { - "id": "1", - "name": "King Arthur", - "role": "King", - "description": "Leader of Camelot and wielder of Excalibur.", - "stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4}, - "vector": [0.7, 0.1, 0.9, 0.7], - }, - { - "id": "2", - "name": "Merlin", - "role": "Wizard", - "description": "Advisor and prophet with deep magical knowledge.", - "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5}, - "vector": [0.2, 0.9, 0.4, 0.9], - }, - { - "id": "3", - "name": "Sir Lancelot", - "role": "Knight", - "description": "Legendary knight known for courage and combat skill.", - "stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3}, - "vector": [0.9, 0.1, 0.5, 0.4], - }, - ] - # --8<-- [end:quickstart_data_async] - - # --8<-- [start:quickstart_create_table_async] - async_table = await async_db.create_table( - "characters", - data=data, - mode="overwrite", - ) - # --8<-- [end:quickstart_create_table_async] - - # --8<-- [start:quickstart_vector_search_1_async] - # Search for examples similar to a "wise magical advisor" - query_vector = [0.2, 0.8, 0.4, 0.9] - - # Ensure you run `pip install polars` beforehand - async_result = await ( - await async_table.search(query_vector) - ).select(["name", "role", "description", "_distance"]).limit(2).to_polars() - print(async_result) - # --8<-- [end:quickstart_vector_search_1_async] - - assert async_result.head(1)["name"][0] == "Merlin" diff --git a/tests/py/test_search.py b/tests/py/test_search.py deleted file mode 100644 index 4b7e8a9..0000000 --- a/tests/py/test_search.py +++ /dev/null @@ -1,730 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -# --8<-- [start:import-lancedb] -import lancedb - -# --8<-- [end:import-lancedb] -# --8<-- [start:import-numpy] -from lancedb.query import BoostQuery, MatchQuery -import numpy as np -import pyarrow as pa - -# --8<-- [end:import-numpy] -# --8<-- [start:import-datetime] -from datetime import datetime - -# --8<-- [end:import-datetime] -# --8<-- [start:import-lancedb-pydantic] -from lancedb.pydantic import Vector, LanceModel - -# --8<-- [end:import-lancedb-pydantic] -# --8<-- [start:import-pydantic-base-model] -from pydantic import BaseModel - -# --8<-- [end:import-pydantic-base-model] -# --8<-- [start:import-lancedb-fts] -from lancedb.index import FTS - -# --8<-- [end:import-lancedb-fts] -# --8<-- [start:import-os] -import os - -# --8<-- [end:import-os] -# --8<-- [start:import-embeddings] -from lancedb.embeddings import get_registry - -# --8<-- [end:import-embeddings] -import pytest - - -# --8<-- [start:class-definition] -class Metadata(BaseModel): - source: str - timestamp: datetime - - -class Document(BaseModel): - content: str - meta: Metadata - - -class LanceSchema(LanceModel): - id: str - vector: Vector(1536) - payload: Document - - -# --8<-- [end:class-definition] - - -def test_vector_search(): - # --8<-- [start:exhaustive_search] - uri = "data/sample-lancedb" - db = lancedb.connect(uri) - data = [ - {"vector": row, "item": f"item {i}"} - for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32")) - ] - tbl = db.create_table("vector_search", data=data, mode="overwrite") - tbl.search(np.random.random((1536))).limit(10).to_list() - # --8<-- [end:exhaustive_search] - # --8<-- [start:exhaustive_search_cosine] - tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list() - # --8<-- [end:exhaustive_search_cosine] - # --8<-- [start:create_table_with_nested_schema] - # Let's add 100 sample rows to our dataset - data = [ - LanceSchema( - id=f"id{i}", - vector=np.random.randn(1536), - payload=Document( - content=f"document{i}", - meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()), - ), - ) - for i in range(100) - ] - - # Synchronous client - tbl = db.create_table("documents", data=data, mode="overwrite") - # --8<-- [end:create_table_with_nested_schema] - # --8<-- [start:search_result_as_pyarrow] - tbl.search(np.random.randn(1536)).to_arrow() - # --8<-- [end:search_result_as_pyarrow] - # --8<-- [start:search_result_as_pandas] - tbl.search(np.random.randn(1536)).to_pandas() - # --8<-- [end:search_result_as_pandas] - # --8<-- [start:search_result_as_pandas_flatten_true] - tbl.search(np.random.randn(1536)).to_pandas(flatten=True) - # --8<-- [end:search_result_as_pandas_flatten_true] - # --8<-- [start:search_result_as_pandas_flatten_1] - tbl.search(np.random.randn(1536)).to_pandas(flatten=1) - # --8<-- [end:search_result_as_pandas_flatten_1] - # --8<-- [start:search_result_as_list] - tbl.search(np.random.randn(1536)).to_list() - # --8<-- [end:search_result_as_list] - # --8<-- [start:search_result_as_pydantic] - tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema) - # --8<-- [end:search_result_as_pydantic] - - -@pytest.mark.asyncio -async def test_vector_search_async(): - # --8<-- [start:exhaustive_search_async] - uri = "data/sample-lancedb" - async_db = await lancedb.connect_async(uri) - data = [ - {"vector": row, "item": f"item {i}"} - for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32")) - ] - async_tbl = await async_db.create_table( - "vector_search_async", data=data, mode="overwrite" - ) - (await (await async_tbl.search(np.random.random((1536)))).limit(10).to_list()) - # --8<-- [end:exhaustive_search_async] - # --8<-- [start:exhaustive_search_async_cosine] - ( - await (await async_tbl.search(np.random.random((1536)))) - .distance_type("cosine") - .limit(10) - .to_list() - ) - # --8<-- [end:exhaustive_search_async_cosine] - # --8<-- [start:create_table_async_with_nested_schema] - # Let's add 100 sample rows to our dataset - data = [ - LanceSchema( - id=f"id{i}", - vector=np.random.randn(1536), - payload=Document( - content=f"document{i}", - meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()), - ), - ) - for i in range(100) - ] - - async_tbl = await async_db.create_table( - "documents_async", data=data, mode="overwrite" - ) - # --8<-- [end:create_table_async_with_nested_schema] - # --8<-- [start:search_result_async_as_pyarrow] - await (await async_tbl.search(np.random.randn(1536))).to_arrow() - # --8<-- [end:search_result_async_as_pyarrow] - # --8<-- [start:search_result_async_as_pandas] - await (await async_tbl.search(np.random.randn(1536))).to_pandas() - # --8<-- [end:search_result_async_as_pandas] - # --8<-- [start:search_result_async_as_list] - await (await async_tbl.search(np.random.randn(1536))).to_list() - # --8<-- [end:search_result_async_as_list] - - -def test_fts_fuzzy_query(): - uri = "data/fuzzy-example" - db = lancedb.connect(uri) - - table = db.create_table( - "my_table_fts_fuzzy", - data=pa.table( - { - "text": [ - "fa", - "fo", # spellchecker:disable-line - "fob", - "focus", - "foo", - "food", - "foul", - ] - } - ), - mode="overwrite", - ) - table.create_fts_index("text", replace=True) - - results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas() - assert len(results) == 4 - assert set(results["text"].to_list()) == { - "foo", - "fo", # 1 deletion # spellchecker:disable-line - "fob", # 1 substitution - "food", # 1 insertion - } - - -def test_fts_boost_query(): - uri = "data/boost-example" - db = lancedb.connect(uri) - - table = db.create_table( - "my_table_fts_boost", - data=pa.table( - { - "title": [ - "The Hidden Gems of Travel", - "Exploring Nature's Wonders", - "Cultural Treasures Unveiled", - "The Nightlife Chronicles", - "Scenic Escapes and Challenges", - ], - "desc": [ - "A vibrant city with occasional traffic jams.", - "Beautiful landscapes but overpriced tourist spots.", - "Rich cultural heritage but humid summers.", - "Bustling nightlife but noisy streets.", - "Scenic views but limited public transport options.", - ], - } - ), - mode="overwrite", - ) - table.create_fts_index("desc", replace=True) - - results = table.search( - BoostQuery( - MatchQuery("beautiful, cultural, nightlife", "desc"), - MatchQuery("bad traffic jams, overpriced", "desc"), - ), - ).to_pandas() - - # we will hit 3 results because the positive query has 3 hits - assert len(results) == 3 - # the one containing "overpriced" will be negatively boosted, - # so it will be the last one - assert ( - results["desc"].to_list()[2] - == "Beautiful landscapes but overpriced tourist spots." - ) - - -def test_fts_native(): - # --8<-- [start:basic_fts] - uri = "data/sample-lancedb" - db = lancedb.connect(uri) - - table = db.create_table( - "my_table_fts", - data=[ - {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}, - {"vector": [5.9, 26.5], "text": "There are several kittens playing"}, - ], - mode="overwrite", - ) - - table.create_fts_index("text") - table.search("puppy").limit(10).select(["text"]).to_list() - # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}] - # ... - # --8<-- [end:basic_fts] - # --8<-- [start:fts_config_stem] - table.create_fts_index("text", tokenizer_name="en_stem", replace=True) - # --8<-- [end:fts_config_stem] - # --8<-- [start:fts_config_folding] - table.create_fts_index( - "text", - language="French", - stem=True, - ascii_folding=True, - replace=True, - ) - # --8<-- [end:fts_config_folding] - # --8<-- [start:fts_prefiltering] - table.search("puppy").limit(10).where("text='foo'", prefilter=True).to_list() - # --8<-- [end:fts_prefiltering] - # --8<-- [start:fts_postfiltering] - table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list() - # --8<-- [end:fts_postfiltering] - # --8<-- [start:fts_with_position] - table.create_fts_index("text", with_position=True, replace=True) - # --8<-- [end:fts_with_position] - # --8<-- [start:fts_incremental_index] - table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}]) - table.optimize() - # --8<-- [end:fts_incremental_index] - - -@pytest.mark.asyncio -async def test_fts_native_async(): - # --8<-- [start:basic_fts_async] - uri = "data/sample-lancedb" - async_db = await lancedb.connect_async(uri) - - async_tbl = await async_db.create_table( - "my_table_fts_async", - data=[ - {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}, - {"vector": [5.9, 26.5], "text": "There are several kittens playing"}, - ], - mode="overwrite", - ) - - # async API uses our native FTS algorithm - await async_tbl.create_index("text", config=FTS()) - await (await async_tbl.search("puppy")).select(["text"]).limit(10).to_list() - # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}] - # ... - # --8<-- [end:basic_fts_async] - # --8<-- [start:fts_config_stem_async] - await async_tbl.create_index( - "text", config=FTS(language="English", stem=True, remove_stop_words=True) - ) - # --8<-- [end:fts_config_stem_async] - # --8<-- [start:fts_config_folding_async] - await async_tbl.create_index( - "text", config=FTS(language="French", stem=True, ascii_folding=True) - ) - # --8<-- [end:fts_config_folding_async] - # --8<-- [start:fts_prefiltering_async] - await (await async_tbl.search("puppy")).limit(10).where("text='foo'").to_list() - # --8<-- [end:fts_prefiltering_async] - # --8<-- [start:fts_postfiltering_async] - await ( - (await async_tbl.search("puppy")) - .limit(10) - .where("text='foo'") - .postfilter() - .to_list() - ) - # --8<-- [end:fts_postfiltering_async] - # --8<-- [start:fts_with_position_async] - await async_tbl.create_index("text", config=FTS(with_position=True)) - # --8<-- [end:fts_with_position_async] - # --8<-- [start:fts_incremental_index_async] - await async_tbl.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}]) - await async_tbl.optimize() - # --8<-- [end:fts_incremental_index_async] - - -def _vectors(n, dim, seed=0, column="vector"): - rng = np.random.default_rng(seed) - return [ - {column: rng.random(dim).astype("float32"), "id": i} for i in range(n) - ] - - -def _build_indexed_table(db, name, dim=128, n=512): - tbl = db.create_table(name, _vectors(n, dim), mode="overwrite") - tbl.create_index(metric="cosine", num_partitions=4, num_sub_vectors=8) - return tbl - - -def test_vs_distance_metric_and_brute_force(tmp_db): - tbl = tmp_db.create_table("vs_metric", _vectors(64, 1536), mode="overwrite") - - # --8<-- [start:configure_distance_metric] - tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list() - # --8<-- [end:configure_distance_metric] - - # --8<-- [start:brute_force_search] - tbl.search(np.random.random((1536))).limit(3).to_list() - # --8<-- [end:brute_force_search] - - -def test_vs_select_vector_column(tmp_db): - db = tmp_db - - # --8<-- [start:select_vector_column] - import pyarrow as pa - - schema = pa.schema([ - pa.field("id", pa.int32()), - pa.field( - "image", - pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]), - ), - ]) - table = db.create_table( - "nested", - data=[{"id": 0, "image": {"embedding": [0.0, 1.0]}}], - schema=schema, - ) - - # Inferred: the only vector leaf is `image.embedding`. - table.search([0.0, 1.0]).limit(1).to_list() - - # Explicit: required when more than one vector column matches. - table.search([0.0, 1.0], vector_column_name="image.embedding").limit(1).to_list() - # --8<-- [end:select_vector_column] - - -def test_vs_index_nested_column(tmp_db): - dim = 16 - schema = pa.schema( - [ - pa.field("id", pa.int32()), - pa.field( - "image", - pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]), - ), - ] - ) - rng = np.random.default_rng(0) - data = [ - {"id": i, "image": {"embedding": rng.random(dim).astype("float32").tolist()}} - for i in range(512) - ] - table = tmp_db.create_table( - "nested_index", data=data, schema=schema, mode="overwrite" - ) - - # --8<-- [start:index_nested_column] - table.create_index(vector_column_name="image.embedding") - # --8<-- [end:index_nested_column] - - assert table.list_indices() - - -def test_vs_indexed_queries(tmp_db): - table = _build_indexed_table(tmp_db, "vs_indexed", dim=128, n=512) - embedding = np.random.random(128) - - # --8<-- [start:exact_vs_approximate_distances] - # Indexed ANN search without refinement (fast, approximate `_distance`) - fast_results = ( - table.search(embedding) - .limit(10) - .to_pandas() - ) - - # Recompute distances on full vectors for reranked candidates - exact_distance_results = ( - table.search(embedding) - .limit(10) - .refine_factor(1) - .to_pandas() - ) - - # Rerank a larger candidate set for better recall (higher latency) - higher_recall_results = ( - table.search(embedding) - .limit(10) - .refine_factor(20) - .to_pandas() - ) - # --8<-- [end:exact_vs_approximate_distances] - - # --8<-- [start:bypass_vector_index] - table.search(embedding).bypass_vector_index().limit(5).to_pandas() - # --8<-- [end:bypass_vector_index] - - assert len(fast_results) == 10 - assert len(exact_distance_results) == 10 - assert len(higher_recall_results) == 10 - - -def test_vs_fast_search(tmp_db, monkeypatch): - table = _build_indexed_table(tmp_db, "vs_fast", dim=128, n=512) - embedding = np.random.random(128) - - # `fast_search` is an Enterprise/async query flag; strip it so the snippet - # runs against a local OSS table without changing what readers see. - real_search = table.search - monkeypatch.setattr( - table, - "search", - lambda *a, **k: real_search( - *a, **{key: val for key, val in k.items() if key != "fast_search"} - ), - ) - - # --8<-- [start:fast_search] - table.search(embedding, fast_search=True).limit(5).to_pandas() - # --8<-- [end:fast_search] - - -def test_vs_distance_range(tmp_db): - tbl = tmp_db.create_table("vs_distance_range", _vectors(256, 256), mode="overwrite") - - # --8<-- [start:search_distance_range] - query = np.random.random(256) - - # Search for the vectors within the range of [0.1, 0.5) - tbl.search(query).distance_range(0.1, 0.5).to_arrow() - - # Search for the vectors with the distance less than 0.5 - tbl.search(query).distance_range(upper_bound=0.5).to_arrow() - - # Search for the vectors with the distance greater or equal to 0.1 - tbl.search(query).distance_range(lower_bound=0.1).to_arrow() - # --8<-- [end:search_distance_range] - - -def test_vs_multivector_search(tmp_db): - schema = pa.schema( - [ - pa.field("id", pa.int64()), - pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))), - ] - ) - rng = np.random.default_rng(0) - data = [{"id": i, "vector": rng.random(size=(2, 256)).tolist()} for i in range(64)] - tbl = tmp_db.create_table( - "vs_multivector", data=data, schema=schema, mode="overwrite" - ) - - # --8<-- [start:multivector_search] - query_multi = np.random.random(size=(2, 256)) - results_multi = tbl.search(query_multi).limit(5).to_pandas() - # --8<-- [end:multivector_search] - - assert len(results_multi) <= 5 - - -def test_vs_binary_search(tmp_db): - db = tmp_db - - # --8<-- [start:search_binary_vectors] - import numpy as np - import pyarrow as pa - - schema = pa.schema( - [ - pa.field("id", pa.int64()), - # for dim=256, lance stores every 8 bits in a byte - # so the vector field should be a list of 256 / 8 = 32 bytes - pa.field("vector", pa.list_(pa.uint8(), 32)), - ] - ) - tbl = db.create_table("my_binary_vectors", schema=schema) - - data = [] - for i in range(1024): - vector = np.random.randint(0, 2, size=256) - # pack the binary vector into bytes to save space - packed_vector = np.packbits(vector) - data.append( - { - "id": i, - "vector": packed_vector, - } - ) - tbl.add(data) - - query = np.random.randint(0, 2, size=256) - packed_query = np.packbits(query) - tbl.search(packed_query).distance_type("hamming").to_arrow() - # --8<-- [end:search_binary_vectors] - - -def test_vs_enterprise_filtering(tmp_db, monkeypatch): - import sys - import types - - dim = 384 - rng = np.random.default_rng(0) - rows = [ - { - "vector": rng.random(dim).astype("float32"), - "text": f"story {i}", - "keywords": f"kw{i}", - "label": i % 4, - } - for i in range(50) - ] - tmp_db.create_table("lancedb-enterprise-quickstart", data=rows, mode="overwrite") - db = tmp_db - - # Mock the Hugging Face dataset loader to avoid network downloads. - class _FakeDataset: - def __init__(self, n): - self._emb = [rng.random(dim).astype("float32") for _ in range(n)] - self._kw = [f"kw{i}" for i in range(n)] - - def __getitem__(self, key): - if isinstance(key, str): - return {"keywords_embeddings": self._emb, "keywords": self._kw}[key] - return {"keywords": self._kw[key], "keywords_embeddings": self._emb[key]} - - fake_datasets = types.ModuleType("datasets") - fake_datasets.load_dataset = lambda *a, **k: _FakeDataset(10) - monkeypatch.setitem(sys.modules, "datasets", fake_datasets) - - # --8<-- [start:vector_search_prefilter] - from datasets import load_dataset - - # Load query vector from dataset - query_dataset = load_dataset("sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5001]") - print(f"Query keywords: {query_dataset[0]['keywords']}") - query_embed = query_dataset["keywords_embeddings"][0] - - # Open table and perform search - table_name = "lancedb-enterprise-quickstart" - table = db.open_table(table_name) - - # Vector search with filters (pre-filtering is the default) - search_results = ( - table.search(query_embed) - .where("label > 2") - .select(["text", "keywords", "label"]) - .limit(5) - .to_pandas() - ) - - print("Search results (with pre-filtering):") - print(search_results) - # --8<-- [end:vector_search_prefilter] - - # --8<-- [start:vector_search_postfilter] - results_post_filtered = ( - table.search(query_embed) - .where("label > 1", prefilter=False) - .select(["text", "keywords", "label"]) - .limit(5) - .to_pandas() - ) - - print("Vector search results with post-filter:") - print(results_post_filtered) - # --8<-- [end:vector_search_postfilter] - - # --8<-- [start:batch_search] - # Load a batch of query embeddings - query_dataset = load_dataset( - "sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5005]" - ) - query_embeds = query_dataset["keywords_embeddings"] - batch_results = table.search(query_embeds).limit(5).to_pandas() - print(batch_results) - # --8<-- [end:batch_search] - - -@pytest.mark.skip() -def test_hybrid_search(): - # --8<-- [start:import-openai] - import openai - - # --8<-- [end:import-openai] - # --8<-- [start:openai-embeddings] - # Ingest embedding function in LanceDB table - # Configuring the environment variable OPENAI_API_KEY - if "OPENAI_API_KEY" not in os.environ: - # OR set the key here as a variable - openai.api_key = "sk-..." - embeddings = get_registry().get("openai").create() - - # --8<-- [end:openai-embeddings] - # --8<-- [start:class-Documents] - class Documents(LanceModel): - vector: Vector(embeddings.ndims()) = embeddings.VectorField() - text: str = embeddings.SourceField() - - # --8<-- [end:class-Documents] - # --8<-- [start:basic_hybrid_search] - data = [ - {"text": "rebel spaceships striking from a hidden base"}, - {"text": "have won their first victory against the evil Galactic Empire"}, - {"text": "during the battle rebel spies managed to steal secret plans"}, - {"text": "to the Empire's ultimate weapon the Death Star"}, - ] - uri = "data/sample-lancedb" - db = lancedb.connect(uri) - table = db.create_table("documents", schema=Documents) - # ingest docs with auto-vectorization - table.add(data) - # Create a fts index before the hybrid search - table.create_fts_index("text") - # hybrid search with default re-ranker - table.search("flower moon", query_type="hybrid").to_pandas() - # --8<-- [end:basic_hybrid_search] - # --8<-- [start:hybrid_search_pass_vector_text] - vector_query = [0.1, 0.2, 0.3, 0.4, 0.5] - text_query = "flower moon" - ( - table.search(query_type="hybrid") - .vector(vector_query) - .text(text_query) - .limit(5) - .to_pandas() - ) - # --8<-- [end:hybrid_search_pass_vector_text] - - -@pytest.mark.skip -async def test_hybrid_search_async(): - import openai - - # --8<-- [start:openai-embeddings] - # Ingest embedding function in LanceDB table - # Configuring the environment variable OPENAI_API_KEY - if "OPENAI_API_KEY" not in os.environ: - # OR set the key here as a variable - openai.api_key = "sk-..." - embeddings = get_registry().get("openai").create() - - # --8<-- [end:openai-embeddings] - # --8<-- [start:class-Documents] - class Documents(LanceModel): - vector: Vector(embeddings.ndims()) = embeddings.VectorField() - text: str = embeddings.SourceField() - - # --8<-- [end:class-Documents] - # --8<-- [start:basic_hybrid_search_async] - uri = "data/sample-lancedb" - async_db = await lancedb.connect_async(uri) - data = [ - {"text": "rebel spaceships striking from a hidden base"}, - {"text": "have won their first victory against the evil Galactic Empire"}, - {"text": "during the battle rebel spies managed to steal secret plans"}, - {"text": "to the Empire's ultimate weapon the Death Star"}, - ] - async_tbl = await async_db.create_table("documents_async", schema=Documents) - # ingest docs with auto-vectorization - await async_tbl.add(data) - # Create a fts index before the hybrid search - await async_tbl.create_index("text", config=FTS()) - text_query = "flower moon" - # hybrid search with default re-ranker - await (await async_tbl.search("flower moon", query_type="hybrid")).to_pandas() - # --8<-- [end:basic_hybrid_search_async] - # --8<-- [start:hybrid_search_pass_vector_text_async] - vector_query = [0.1, 0.2, 0.3, 0.4, 0.5] - text_query = "flower moon" - await ( - async_tbl.query() - .nearest_to(vector_query) - .nearest_to_text(text_query) - .limit(5) - .to_pandas() - ) - # --8<-- [end:hybrid_search_pass_vector_text_async] diff --git a/tests/py/test_storage.py b/tests/py/test_storage.py deleted file mode 100644 index f08ab07..0000000 --- a/tests/py/test_storage.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import lancedb -import pytest - - -class DummyTable: - def __init__(self, name: str, storage_options: dict | None = None): - self.name = name - self.storage_options = storage_options - - -class DummyConnection: - def __init__(self, uri: str, options: dict): - self.uri = uri - self.options = options - self.created_tables: list[DummyTable] = [] - - def create_table( - self, - name: str, - data, - storage_options: dict | None = None, - ): - table = DummyTable(name, storage_options=storage_options) - self.created_tables.append(table) - return table - - -@pytest.fixture -def fake_connect(monkeypatch): - calls: list[DummyConnection] = [] - - def _fake_connect(uri: str, **kwargs): - conn = DummyConnection(uri, kwargs) - calls.append(conn) - return conn - - monkeypatch.setattr(lancedb, "connect", _fake_connect) - return calls - - -def test_storage_snippets(fake_connect): - # --8<-- [start:storage_connect_s3] - db = lancedb.connect("s3://bucket/path") - # --8<-- [end:storage_connect_s3] - - # --8<-- [start:storage_connect_gcs] - db = lancedb.connect("gs://bucket/path") - # --8<-- [end:storage_connect_gcs] - - # --8<-- [start:storage_connect_azure] - db = lancedb.connect("az://bucket/path") - # --8<-- [end:storage_connect_azure] - - # --8<-- [start:storage_connect_timeout] - db = lancedb.connect( - "s3://bucket/path", - storage_options={"timeout": "60s"}, - ) - # --8<-- [end:storage_connect_timeout] - - # --8<-- [start:storage_table_timeout] - table = db.create_table( - "table", - [{"a": 1, "b": 2}], - storage_options={"timeout": "60s"}, - ) - # --8<-- [end:storage_table_timeout] - - # --8<-- [start:storage_s3_ddb] - db = lancedb.connect( - "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table", - ) - # --8<-- [end:storage_s3_ddb] - - # --8<-- [start:storage_s3_ddb_local] - db = lancedb.connect( - "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table", - storage_options={ - "endpoint": "http://localhost:4566", - "dynamodb_endpoint": "http://localhost:4566", - "allow_http": "true", - }, - ) - # --8<-- [end:storage_s3_ddb_local] - - # --8<-- [start:storage_s3_sse_kms] - db = lancedb.connect( - "s3://bucket/path", - storage_options={ - "aws_server_side_encryption": "aws:kms", - "aws_sse_kms_key_id": "", - }, - ) - # --8<-- [end:storage_s3_sse_kms] - - # --8<-- [start:storage_azure_sas] - db = lancedb.connect( - "az://my-container/my-database", - storage_options={ - "azure_storage_account_name": "some-account", - "azure_storage_sas_token": "", - }, - ) - # --8<-- [end:storage_azure_sas] - - # --8<-- [start:storage_s3_minio] - db = lancedb.connect( - "s3://bucket/path", - storage_options={ - "region": "us-east-1", - "endpoint": "http://minio:9000", - }, - ) - # --8<-- [end:storage_s3_minio] - - # --8<-- [start:storage_s3_express] - db = lancedb.connect( - "s3://my-bucket--use1-az4--x-s3/path", - storage_options={ - "region": "us-east-1", - "s3_express": "true", - }, - ) - # --8<-- [end:storage_s3_express] - - # --8<-- [start:storage_gcs_service_account] - db = lancedb.connect( - "gs://my-bucket/my-database", - storage_options={ - "service_account": "path/to/service-account.json", - }, - ) - # --8<-- [end:storage_gcs_service_account] - - # --8<-- [start:storage_azure_account] - db = lancedb.connect( - "az://my-container/my-database", - storage_options={ - "account_name": "some-account", - "account_key": "some-key", - }, - ) - # --8<-- [end:storage_azure_account] - - # --8<-- [start:storage_tigris_connect] - db = lancedb.connect( - "s3://your-bucket/path", - storage_options={ - "endpoint": "https://t3.storage.dev", - "region": "auto", - }, - ) - # --8<-- [end:storage_tigris_connect] - - assert len(fake_connect) == 13 - assert all( - conn.uri.startswith(("s3://", "gs://", "az://", "s3+ddb://")) - for conn in fake_connect - ) diff --git a/tests/py/test_tables.py b/tests/py/test_tables.py deleted file mode 100644 index b3a456a..0000000 --- a/tests/py/test_tables.py +++ /dev/null @@ -1,1546 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The LanceDB Authors - -# --8<-- [start:tables_imports] -import lancedb -import numpy as np -import pandas as pd -import pyarrow as pa -import pytest -from numpy.random import randint, random - -# --8<-- [end:tables_imports] - - -# ============================================================================ -# Table Creation Examples -# ============================================================================ - - -def test_tables_basic_connect_snippet(monkeypatch): - calls = {} - - class DummyDB: - pass - - def fake_connect(uri): - calls["uri"] = uri - return DummyDB() - - import lancedb as _lancedb - - # Monkey patch is used only in this test. - # Maybe can just actually connect, and disconnect instead? - monkeypatch.setattr(_lancedb, "connect", fake_connect) - - # --8<-- [start:tables_basic_connect] - import lancedb - - uri = "data/sample-lancedb" - db = lancedb.connect(uri) - # --8<-- [end:tables_basic_connect] - - assert calls["uri"] == "data/sample-lancedb" - assert isinstance(db, DummyDB) - - -def test_update_connect_enterprise_snippet(monkeypatch): - calls = {} - - class DummyDB: - pass - - def fake_connect(**kwargs): - calls.update(kwargs) - return DummyDB() - - import lancedb as _lancedb - - monkeypatch.setattr(_lancedb, "connect", fake_connect) - - # --8<-- [start:update_connect_enterprise] - import lancedb - - db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1", - ) - # --8<-- [end:update_connect_enterprise] - - assert calls["uri"] == "db://your-project-slug" - assert calls["api_key"] == "your-api-key" - assert calls["region"] == "us-east-1" - assert isinstance(db, DummyDB) - - -def test_update_connect_local_snippet(monkeypatch): - calls = {} - - class DummyDB: - pass - - def fake_connect(uri): - calls["uri"] = uri - return DummyDB() - - import lancedb as _lancedb - - monkeypatch.setattr(_lancedb, "connect", fake_connect) - - # --8<-- [start:update_connect_local] - import lancedb - - db = lancedb.connect("./data") - # --8<-- [end:update_connect_local] - - assert calls["uri"] == "./data" - assert isinstance(db, DummyDB) - - -def test_table_creation_from_dicts(tmp_db): - # --8<-- [start:create_table_from_dicts] - data = [ - {"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}, - {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}, - ] - db = tmp_db - db.create_table("test_table", data, mode="overwrite") - tbl = db["test_table"] - tbl.head() - # --8<-- [end:create_table_from_dicts] - - -def test_create_table_conflict_handling(tmp_db): - db = tmp_db - data = [ - {"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}, - {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}, - ] - db.create_table("conflict_table", data) - - # --8<-- [start:create_table_conflict_handling] - # Idempotent open: reuse the existing table if it exists. - # The provided data is ignored; the schema is validated against the - # existing table and a mismatch raises an error. - tbl = db.create_table("conflict_table", data, exist_ok=True) - - # Overwrite: drop the existing table and create a new one with the - # provided data. This permanently discards the old table's data. - tbl = db.create_table("conflict_table", data, mode="overwrite") - # --8<-- [end:create_table_conflict_handling] - assert tbl.count_rows() == 2 - - -def test_table_creation_from_pandas(tmp_db): - # --8<-- [start:create_table_from_pandas] - import pandas as pd - - data = pd.DataFrame( - { - "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]], - "lat": [45.5, 40.1], - "long": [-122.7, -74.1], - } - ) - db = tmp_db - db.create_table("my_table_pandas", data, mode="overwrite") - db["my_table_pandas"].head() - # --8<-- [end:create_table_from_pandas] - - -def test_table_creation_with_custom_schema(tmp_db): - # --8<-- [start:create_table_custom_schema] - import pyarrow as pa - - custom_schema = pa.schema( - [ - pa.field("vector", pa.list_(pa.float32(), 4)), - pa.field("lat", pa.float32()), - pa.field("long", pa.float32()), - ] - ) - - data = [ - {"vector": [1.1, 1.2, 1.3, 1.4], "lat": 45.5, "long": -122.7}, - {"vector": [0.2, 1.8, 0.4, 3.6], "lat": 40.1, "long": -74.1}, - ] - db = tmp_db - tbl = db.create_table( - "my_table_custom_schema", data, schema=custom_schema, mode="overwrite" - ) - # --8<-- [end:create_table_custom_schema] - - -def test_table_creation_from_polars(tmp_db): - # --8<-- [start:create_table_from_polars] - import polars as pl - - data = pl.DataFrame( - { - "vector": [[3.1, 4.1], [5.9, 26.5]], - "item": ["foo", "bar"], - "price": [10.0, 20.0], - } - ) - db = tmp_db - tbl = db.create_table("my_table_pl", data, mode="overwrite") - # --8<-- [end:create_table_from_polars] - - -def test_table_creation_from_arrow(tmp_db): - # --8<-- [start:create_table_from_arrow] - import numpy as np - import pyarrow as pa - - dim = 16 - total = 2 - schema = pa.schema( - [pa.field("vector", pa.list_(pa.float16(), dim)), pa.field("text", pa.string())] - ) - data = pa.Table.from_arrays( - [ - pa.array( - [np.random.randn(dim).astype(np.float16) for _ in range(total)], - pa.list_(pa.float16(), dim), - ), - pa.array(["foo", "bar"]), - ], - ["vector", "text"], - ) - db = tmp_db - tbl = db.create_table("f16_tbl", data, schema=schema, mode="overwrite") - # --8<-- [end:create_table_from_arrow] - - -def test_table_creation_from_pydantic(tmp_db): - # --8<-- [start:create_table_from_pydantic] - from lancedb.pydantic import LanceModel, Vector - - class Content(LanceModel): - movie_id: int - vector: Vector(128) - genres: str - title: str - imdb_id: int - - @property - def imdb_url(self) -> str: - return f"https://www.imdb.com/title/tt{self.imdb_id}" - - db = tmp_db - tbl = db.create_table("movielens_small", schema=Content, mode="overwrite") - # --8<-- [end:create_table_from_pydantic] - - -def test_table_creation_nested_schema(tmp_db): - # --8<-- [start:create_table_nested_schema] - from lancedb.pydantic import LanceModel, Vector - - # --8<-- [start:tables_document_model] - from pydantic import BaseModel - - class Document(BaseModel): - content: str - source: str - - # --8<-- [end:tables_document_model] - - class NestedSchema(LanceModel): - id: str - vector: Vector(1536) - document: Document - - db = tmp_db - tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite") - # --8<-- [end:create_table_nested_schema] - - -def test_tables_tz_validator_snippet(): - # --8<-- [start:tables_tz_validator] - from datetime import datetime - from zoneinfo import ZoneInfo - - from lancedb.pydantic import LanceModel - from pydantic import Field, ValidationError, ValidationInfo, field_validator - - tzname = "America/New_York" - tz = ZoneInfo(tzname) - - class TestModel(LanceModel): - dt_with_tz: datetime = Field(json_schema_extra={"tz": tzname}) - - @field_validator("dt_with_tz") - @classmethod - def tz_must_match(cls, dt: datetime) -> datetime: - assert dt.tzinfo == tz - return dt - - ok = TestModel(dt_with_tz=datetime.now(tz)) - - try: - TestModel(dt_with_tz=datetime.now(ZoneInfo("Asia/Shanghai"))) - assert 0 == 1, "this should raise ValidationError" - except ValidationError: - print("A ValidationError was raised.") - pass - # --8<-- [end:tables_tz_validator] - - assert ok is not None - - -def test_add_from_dataset(tmp_db, tmp_path): - import pyarrow as pa - import pyarrow.dataset as ds - import pyarrow.parquet as pq - - schema = pa.schema( - [ - pa.field("vector", pa.list_(pa.float32(), 4)), - pa.field("item", pa.utf8()), - pa.field("price", pa.float32()), - ] - ) - for i in range(3): - batch = pa.table( - { - "vector": [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], - "item": [f"item{i * 2}", f"item{i * 2 + 1}"], - "price": [float(i * 2), float(i * 2 + 1)], - }, - schema=schema, - ) - pq.write_table(batch, tmp_path / f"part-{i}.parquet") - - data_path = str(tmp_path) - - # --8<-- [start:add_from_dataset] - import pyarrow.dataset as ds - - dataset = ds.dataset(data_path, format="parquet") - db = tmp_db - table = db.create_table("my_table", schema=dataset.schema, mode="overwrite") - table.add(dataset) - # --8<-- [end:add_from_dataset] - assert table.count_rows() == 6 - - -def test_table_creation_from_iterator(tmp_db): - # --8<-- [start:create_table_from_iterator] - import pyarrow as pa - - schema = pa.schema( - [ - pa.field("vector", pa.list_(pa.float32(), 4)), - pa.field("item", pa.utf8()), - pa.field("price", pa.float32()), - ] - ) - - def make_batches(): - for i in range(5): - yield pa.RecordBatch.from_arrays( - [ - pa.array( - [[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]], - pa.list_(pa.float32(), 4), - ), - pa.array(["foo", "bar"]), - pa.array([10.0, 20.0]), - ], - ["vector", "item", "price"], - ) - - db = tmp_db - db.create_table("batched_table", make_batches(), schema=schema, mode="overwrite") - # --8<-- [end:create_table_from_iterator] - -def test_open_existing_table(tmp_db): - # --8<-- [start:open_existing_table] - db = tmp_db - # Create a table first - data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}] - db.create_table("test_table", data, mode="overwrite") - - # List table names - print(db.list_tables().tables) - - # Open existing table - tbl = db.open_table("test_table") - # --8<-- [end:open_existing_table] - - -def test_create_empty_table(tmp_db): - # --8<-- [start:create_empty_table] - import pyarrow as pa - - schema = pa.schema( - [ - pa.field("vector", pa.list_(pa.float32(), 2)), - pa.field("item", pa.string()), - pa.field("price", pa.float32()), - ] - ) - db = tmp_db - tbl = db.create_table("test_empty_table", schema=schema, mode="overwrite") - # --8<-- [end:create_empty_table] - - -def test_create_empty_table_pydantic(tmp_db): - # --8<-- [start:create_empty_table_pydantic] - from lancedb.pydantic import LanceModel, Vector - - class Item(LanceModel): - vector: Vector(2) - item: str - price: float - - db = tmp_db - tbl = db.create_table( - "test_empty_table_new", schema=Item.to_arrow_schema(), mode="overwrite" - ) - # --8<-- [end:create_empty_table_pydantic] - - -def test_drop_table(tmp_db): - # --8<-- [start:drop_table] - db = tmp_db - # Create a table first - data = [{"vector": [1.1, 1.2], "lat": 45.5}] - db.create_table("my_table", data, mode="overwrite") - - # Drop the table - db.drop_table("my_table") - # --8<-- [end:drop_table] - - -# ============================================================================ -# Data Update Examples -# ============================================================================ - - -def test_add_data_to_table(tmp_db): - db = tmp_db - - # --8<-- [start:add_data_to_table] - import pyarrow as pa - - # create an empty table with schema - data = [ - {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, - {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, - {"vector": [10.2, 100.8], "item": "baz", "price": 30.0}, - {"vector": [1.4, 9.5], "item": "fred", "price": 40.0}, - ] - - schema = pa.schema( - [ - pa.field("vector", pa.list_(pa.float32(), 2)), - pa.field("item", pa.utf8()), - pa.field("price", pa.float32()), - ] - ) - - table_name = "basic_ingestion_example" - table = db.create_table(table_name, schema=schema, mode="overwrite") - # Add data - table.add(data) - # --8<-- [end:add_data_to_table] - assert table.count_rows() == len(data) - - -def test_add_data_pydantic_model(tmp_db): - db = tmp_db - - # --8<-- [start:add_data_pydantic_model] - from lancedb.pydantic import LanceModel, Vector - - # Define a Pydantic model - class Content(LanceModel): - movie_id: int - vector: Vector(128) - genres: str - title: str - imdb_id: int - - @property - def imdb_url(self) -> str: - return f"https://www.imdb.com/title/tt{self.imdb_id}" - - # Create table with Pydantic model schema - table_name = "pydantic_example" - table = db.create_table(table_name, schema=Content, mode="overwrite") - # --8<-- [end:add_data_pydantic_model] - assert table.count_rows() == 0 - - -def test_add_data_nested_model(tmp_db): - db = tmp_db - - # --8<-- [start:add_data_nested_model] - from lancedb.pydantic import LanceModel, Vector - from pydantic import BaseModel - - class Document(BaseModel): - content: str - source: str - - class NestedSchema(LanceModel): - id: str - vector: Vector(128) - document: Document - - # Create table with nested schema - table_name = "nested_model_example" - table = db.create_table(table_name, schema=NestedSchema, mode="overwrite") - # --8<-- [end:add_data_nested_model] - assert table.count_rows() == 0 - - -def test_batch_data_insertion(tmp_db): - db = tmp_db - - # --8<-- [start:batch_data_insertion] - import pyarrow as pa - - def make_batches(): - for i in range(5): # Create 5 batches - yield pa.RecordBatch.from_arrays( - [ - pa.array([[3.1, 4.1], [5.9, 26.5]], pa.list_(pa.float32(), 2)), - pa.array([f"item{i * 2 + 1}", f"item{i * 2 + 2}"]), - pa.array([float((i * 2 + 1) * 10), float((i * 2 + 2) * 10)]), - ], - ["vector", "item", "price"], - ) - - schema = pa.schema( - [ - pa.field("vector", pa.list_(pa.float32(), 2)), - pa.field("item", pa.utf8()), - pa.field("price", pa.float32()), - ] - ) - # Create table with batches - table_name = "batch_ingestion_example" - table = db.create_table(table_name, make_batches(), schema=schema, mode="overwrite") - # --8<-- [end:batch_data_insertion] - assert table.count_rows() == 10 - - -def _create_users_example_table(db, table_name="users_example"): - return db.create_table( - table_name, - data=pa.table( - { - "id": [1, 2, 3], - "name": ["Alice", "Bob", "Charlie"], - "login_count": [10, 20, 5], - } - ), - mode="overwrite", - ) - - -def test_update_example_table_setup(tmp_db): - db = tmp_db - - # --8<-- [start:update_example_table_setup] - import pyarrow as pa - - table = db.create_table( - "users_example", - data=pa.table( - { - "id": [1, 2], - "name": ["Alice", "Bob"], - "login_count": [10, 20], - } - ), - mode="overwrite", - ) - # --8<-- [end:update_example_table_setup] - assert table.count_rows() == 2 - - -def test_update_operation(tmp_db): - db = tmp_db - - # --8<-- [start:update_operation] - import pyarrow as pa - - table = db.create_table( - "users_example", - data=pa.table( - { - "id": [1, 2], - "name": ["Alice", "Bob"], - "login_count": [10, 20], - } - ), - mode="overwrite", - ) - table.update(where="id = 2", values={"name": "Bobby"}) - # --8<-- [end:update_operation] - rows = table.to_arrow().sort_by("id").to_pylist() - assert rows == [ - {"id": 1, "name": "Alice", "login_count": 10}, - {"id": 2, "name": "Bobby", "login_count": 20}, - ] - - -def test_update_using_sql(tmp_db): - db = tmp_db - - # --8<-- [start:update_using_sql] - import pyarrow as pa - - table = db.create_table( - "users_example", - data=pa.table( - { - "id": [1, 2], - "name": ["Alice", "Bob"], - "login_count": [10, 20], - } - ), - mode="overwrite", - ) - table.update(where="id = 2", values_sql={"login_count": "login_count + 1"}) - # --8<-- [end:update_using_sql] - rows = table.to_arrow().sort_by("id").to_pylist() - assert rows == [ - {"id": 1, "name": "Alice", "login_count": 10}, - {"id": 2, "name": "Bob", "login_count": 21}, - ] - - -def test_merge_matched_update_only(tmp_db): - db = tmp_db - - # --8<-- [start:merge_matched_update_only] - import pyarrow as pa - - table = db.create_table( - "users_example", - data=pa.table( - { - "id": [1, 2], - "name": ["Alice", "Bob"], - "login_count": [10, 20], - } - ), - mode="overwrite", - ) - - incoming_users = pa.table( - { - "id": [2, 3], - "name": ["Bobby", "Charlie"], - "login_count": [21, 5], - } - ) - - (table.merge_insert("id").when_matched_update_all().execute(incoming_users)) - # --8<-- [end:merge_matched_update_only] - rows = table.to_arrow().sort_by("id").to_pylist() - assert rows == [ - {"id": 1, "name": "Alice", "login_count": 10}, - {"id": 2, "name": "Bobby", "login_count": 21}, - ] - - -def test_insert_if_not_exists(tmp_db): - db = tmp_db - - # --8<-- [start:insert_if_not_exists] - import pyarrow as pa - - table = db.create_table( - "users_example", - data=pa.table( - { - "id": [1, 2], - "name": ["Alice", "Bob"], - "login_count": [10, 20], - } - ), - mode="overwrite", - ) - - incoming_users = pa.table( - { - "id": [2, 3], - "name": ["Bobby", "Charlie"], - "login_count": [21, 5], - } - ) - - (table.merge_insert("id").when_not_matched_insert_all().execute(incoming_users)) - # --8<-- [end:insert_if_not_exists] - rows = table.to_arrow().sort_by("id").to_pylist() - assert rows == [ - {"id": 1, "name": "Alice", "login_count": 10}, - {"id": 2, "name": "Bob", "login_count": 20}, - {"id": 3, "name": "Charlie", "login_count": 5}, - ] - - -def test_merge_update_insert(tmp_db): - db = tmp_db - - # --8<-- [start:merge_update_insert] - import pyarrow as pa - - table = db.create_table( - "users_example", - data=pa.table( - { - "id": [1, 2], - "name": ["Alice", "Bob"], - "login_count": [10, 20], - } - ), - mode="overwrite", - ) - - incoming_users = pa.table( - { - "id": [2, 3], - "name": ["Bobby", "Charlie"], - "login_count": [21, 5], - } - ) - - ( - table.merge_insert("id") - .when_matched_update_all() - .when_not_matched_insert_all() - .execute(incoming_users) - ) - # --8<-- [end:merge_update_insert] - rows = table.to_arrow().sort_by("id").to_pylist() - assert rows == [ - {"id": 1, "name": "Alice", "login_count": 10}, - {"id": 2, "name": "Bobby", "login_count": 21}, - {"id": 3, "name": "Charlie", "login_count": 5}, - ] - - -def test_merge_delete_missing_by_source(tmp_db): - db = tmp_db - - # --8<-- [start:merge_delete_missing_by_source] - import pyarrow as pa - - table = db.create_table( - "users_example", - data=pa.table( - { - "id": [1, 2, 3], - "name": ["Alice", "Bob", "Charlie"], - "login_count": [10, 20, 5], - } - ), - mode="overwrite", - ) - - incoming_users = pa.table( - { - "id": [2, 3], - "name": ["Bobby", "Charlie"], - "login_count": [21, 5], - } - ) - - ( - table.merge_insert("id") - .when_matched_update_all() - .when_not_matched_insert_all() - .when_not_matched_by_source_delete() - .execute(incoming_users) - ) - # --8<-- [end:merge_delete_missing_by_source] - rows = table.to_arrow().sort_by("id").to_pylist() - assert rows == [ - {"id": 2, "name": "Bobby", "login_count": 21}, - {"id": 3, "name": "Charlie", "login_count": 5}, - ] - - -def test_merge_partial_columns(tmp_db): - db = tmp_db - - # --8<-- [start:merge_partial_columns] - import pyarrow as pa - - table = db.create_table( - "users_example", - data=pa.table( - { - "id": [1, 2], - "name": ["Alice", "Bob"], - "login_count": [10, 20], - } - ), - mode="overwrite", - ) - - incoming_users = pa.table( - { - "id": [2, 3], - "name": ["Bobby", "Charlie"], - } - ) - - ( - table.merge_insert("id") - .when_matched_update_all() - .when_not_matched_insert_all() - .execute(incoming_users) - ) - # --8<-- [end:merge_partial_columns] - rows = table.to_arrow().sort_by("id").to_pylist() - assert rows == [ - {"id": 1, "name": "Alice", "login_count": 10}, - {"id": 2, "name": "Bobby", "login_count": 20}, - {"id": 3, "name": "Charlie", "login_count": None}, - ] - - -def test_delete_operation(tmp_db): - db = tmp_db - table = _create_users_example_table(db) - - # --8<-- [start:delete_operation] - # delete data - predicate = "id = 3" - table.delete(predicate) - # --8<-- [end:delete_operation] - assert table.count_rows() == 2 - - -def test_update_optimize_cleanup_snippet(tmp_db): - table = _create_users_example_table(tmp_db, table_name="users_cleanup_example") - - # --8<-- [start:update_optimize_cleanup] - from datetime import timedelta - - table.optimize(cleanup_older_than=timedelta(days=1)) - # --8<-- [end:update_optimize_cleanup] - - -# ============================================================================ -# Schema Evolution Examples -# ============================================================================ - - -def _setup_schema_add_table(tmp_db, data=None): - # --8<-- [start:schema_add_setup] - table_name = "schema_evolution_add_example" - if data is None: - data = [ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 2, - "name": "Smartphone", - "price": 800.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 3, - "name": "Headphones", - "price": 150.00, - "vector": np.random.random(128).tolist(), - }, - ] - table = tmp_db.create_table(table_name, data, mode="overwrite") - # --8<-- [end:schema_add_setup] - return table - - -def _setup_schema_alter_table(tmp_db, data=None): - # --8<-- [start:schema_alter_setup] - table_name = "schema_evolution_alter_example" - if data is None: - data = [ - { - "id": 1, - "name": "Laptop", - "price": 1200, - "discount_price": 1080.0, - "vector": np.random.random(128).tolist(), - }, - { - "id": 2, - "name": "Smartphone", - "price": 800, - "discount_price": 720.0, - "vector": np.random.random(128).tolist(), - }, - ] - schema = pa.schema( - { - "id": pa.int64(), - "name": pa.string(), - "price": pa.int32(), - "discount_price": pa.float64(), - "vector": pa.list_(pa.float32(), 128), - } - ) - table = tmp_db.create_table(table_name, data, schema=schema, mode="overwrite") - # --8<-- [end:schema_alter_setup] - return table - - -def _setup_schema_drop_table(tmp_db, data=None): - # --8<-- [start:schema_drop_setup] - if data is None: - data = [ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "temp_col1": "X", - "temp_col2": 100, - "vector": np.random.random(128).tolist(), - }, - { - "id": 2, - "name": "Smartphone", - "price": 800.00, - "temp_col1": "Y", - "temp_col2": 200, - "vector": np.random.random(128).tolist(), - }, - { - "id": 3, - "name": "Headphones", - "price": 150.00, - "temp_col1": "Z", - "temp_col2": 300, - "vector": np.random.random(128).tolist(), - }, - ] - table = tmp_db.create_table("schema_evolution_drop_example", data, mode="overwrite") - # --8<-- [end:schema_drop_setup] - return table - - -def test_add_columns_calculated(tmp_db): - table = _setup_schema_add_table(tmp_db) - - # --8<-- [start:add_columns_calculated] - # Add a discounted price column (10% discount) - table.add_columns({"discounted_price": "cast((price * 0.9) as float)"}) - # --8<-- [end:add_columns_calculated] - assert "discounted_price" in table.schema.names - - -def test_add_columns_default_values(tmp_db): - table = _setup_schema_add_table(tmp_db) - - # --8<-- [start:add_columns_default_values] - # Add a stock status column with default value - table.add_columns({"in_stock": "cast(true as boolean)"}) - # --8<-- [end:add_columns_default_values] - assert "in_stock" in table.schema.names - - -def test_add_columns_nullable(tmp_db): - table = _setup_schema_add_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - } - ], - ) - - # --8<-- [start:add_columns_nullable] - # Add a nullable timestamp column - table.add_columns({"last_ordered": "cast(NULL as timestamp)"}) - # --8<-- [end:add_columns_nullable] - assert "last_ordered" in table.schema.names - - -def test_add_feature_columns_sql(tmp_db): - table = _setup_schema_add_table(tmp_db) - - # --8<-- [start:add_feature_columns_sql] - table.add_columns( - { - "price_per_id": "cast(price / id as float)", - "price_log": "ln(price)", - "price_score": "cast(price / (price + 100.0) as float)", - } - ) - # --8<-- [end:add_feature_columns_sql] - assert {"price_per_id", "price_log", "price_score"}.issubset(table.schema.names) - - -def test_alter_columns_rename(tmp_db): - table = _setup_schema_alter_table(tmp_db) - - # --8<-- [start:alter_columns_rename] - # Rename discount_price to sale_price - table.alter_columns({"path": "discount_price", "rename": "sale_price"}) - # --8<-- [end:alter_columns_rename] - assert "sale_price" in table.schema.names - assert "discount_price" not in table.schema.names - - -def test_alter_columns_data_type(tmp_db): - table = _setup_schema_alter_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200, - "discount_price": 1080.0, - "vector": np.random.random(128).tolist(), - } - ], - ) - - # --8<-- [start:alter_columns_data_type] - # Change price from int32 to int64 for larger numbers - table.alter_columns({"path": "price", "data_type": pa.int64()}) - # --8<-- [end:alter_columns_data_type] - assert table.schema.field("price").type == pa.int64() - - -def test_alter_columns_nullable(tmp_db): - table = _setup_schema_alter_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200, - "discount_price": 1080.0, - "vector": np.random.random(128).tolist(), - } - ], - ) - - # --8<-- [start:alter_columns_nullable] - # Make the name column nullable - table.alter_columns({"path": "name", "nullable": True}) - # --8<-- [end:alter_columns_nullable] - assert table.schema.field("name").nullable is True - - -def test_alter_columns_multiple(tmp_db): - table = _setup_schema_alter_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200, - "discount_price": 1080.0, - "vector": np.random.random(128).tolist(), - } - ], - ) - table.alter_columns({"path": "discount_price", "rename": "sale_price"}) - - # --8<-- [start:alter_columns_multiple] - # Rename, change type, and make nullable in one operation - table.alter_columns( - { - "path": "sale_price", - "rename": "final_price", - "data_type": pa.float64(), - "nullable": True, - } - ) - # --8<-- [end:alter_columns_multiple] - assert "final_price" in table.schema.names - assert table.schema.field("final_price").nullable is True - - -def test_alter_columns_with_expression(tmp_db): - # --8<-- [start:alter_columns_with_expression] - # For custom transforms, create a new column from a SQL expression. - expression_table = tmp_db.create_table( - "schema_evolution_expression_example", - [{"id": 1, "price_text": "$100"}], - mode="overwrite", - ) - - expression_table.add_columns( - {"price_numeric": "cast(replace(price_text, '$', '') as int)"} - ) - expression_table.drop_columns(["price_text"]) - expression_table.alter_columns({"path": "price_numeric", "rename": "price"}) - # --8<-- [end:alter_columns_with_expression] - assert "price" in expression_table.schema.names - - -def test_drop_columns_single(tmp_db): - table = _setup_schema_drop_table(tmp_db) - - # --8<-- [start:drop_columns_single] - # Remove the first temporary column - table.drop_columns(["temp_col1"]) - # --8<-- [end:drop_columns_single] - assert "temp_col1" not in table.schema.names - - -def test_drop_columns_multiple(tmp_db): - table = _setup_schema_drop_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "temp_col1": "X", - "temp_col2": 100, - "vector": np.random.random(128).tolist(), - }, - ], - ) - - # --8<-- [start:drop_columns_multiple] - # Remove the second temporary column - table.drop_columns(["temp_col2"]) - # --8<-- [end:drop_columns_multiple] - assert "temp_col2" not in table.schema.names - - -def test_alter_vector_column(tmp_db): - # --8<-- [start:alter_vector_column] - vector_dim = 768 # Your embedding dimension - table_name = "vector_alter_example" - db = tmp_db - data = [ - { - "id": 1, - "embedding": np.random.random(vector_dim).tolist(), - }, - ] - table = db.create_table(table_name, data, mode="overwrite") - - table.alter_columns( - dict(path="embedding", data_type=pa.list_(pa.float32(), vector_dim)) - ) - # --8<-- [end:alter_vector_column] - - -def test_schema_field_metadata(tmp_db): - table = tmp_db.create_table( - "schema_field_metadata_example", - pa.table({"id": [0, 1], "category": ["a", "b"]}), - mode="overwrite", - ) - - # --8<-- [start:schema_field_metadata_merge] - # Set two metadata keys on the `category` field. - res = table.update_field_metadata( - {"path": "category", "metadata": {"unit": "label", "pii": "false"}} - ) - print(res.version) - - # Merge: add a new key, delete one with None, keep the rest. - table.update_field_metadata( - {"path": "category", "metadata": {"source": "import", "pii": None}} - ) - - # Arrow stores field metadata as bytes. - assert table.schema.field("category").metadata == { - b"unit": b"label", - b"source": b"import", - } - # --8<-- [end:schema_field_metadata_merge] - - # --8<-- [start:schema_field_metadata_replace] - table.update_field_metadata( - { - "path": "category", - "metadata": {"owner": "search-team"}, - "replace": True, - } - ) - # --8<-- [end:schema_field_metadata_replace] - - assert table.schema.field("category").metadata == {b"owner": b"search-team"} - - -# ============================================================================ -# Versioning Examples -# ============================================================================ - - -def _setup_versioning_table(tmp_db, data=None, table_name="quotes_versioning_example"): - import pyarrow as pa - - if data is None: - data = [ - {"id": 1, "author": "Richard", "quote": "Wubba Lubba Dub Dub!"}, - {"id": 2, "author": "Morty", "quote": "Rick, what's going on?"}, - { - "id": 3, - "author": "Richard", - "quote": "I turned myself into a pickle, Morty!", - }, - ] - schema = pa.schema( - [ - pa.field("id", pa.int64()), - pa.field("author", pa.string()), - pa.field("quote", pa.string()), - ] - ) - return tmp_db.create_table(table_name, data, schema=schema, mode="overwrite") - - -def test_versioning_basic_setup(tmp_db): - # --8<-- [start:versioning_basic_setup] - import pyarrow as pa - - db = tmp_db - - table_name = "quotes_versioning_example" - data = [ - {"id": 1, "author": "Richard", "quote": "Wubba Lubba Dub Dub!"}, - {"id": 2, "author": "Morty", "quote": "Rick, what's going on?"}, - { - "id": 3, - "author": "Richard", - "quote": "I turned myself into a pickle, Morty!", - }, - ] - - # Define schema - schema = pa.schema( - [ - pa.field("id", pa.int64()), - pa.field("author", pa.string()), - pa.field("quote", pa.string()), - ] - ) - - table = db.create_table(table_name, data, schema=schema, mode="overwrite") - # --8<-- [end:versioning_basic_setup] - assert table.count_rows() == 3 - - -def test_versioning_check_initial_version(tmp_db): - table = _setup_versioning_table(tmp_db) - - # --8<-- [start:versioning_check_initial_version] - versions = table.list_versions() - current_version = table.version - print(f"Number of versions after creation: {len(versions)}") - print(f"Current version: {current_version}") - # --8<-- [end:versioning_check_initial_version] - assert len(versions) == 1 - assert current_version == versions[-1]["version"] - - -def test_versioning_flow(tmp_db): - table = _setup_versioning_table(tmp_db) - - # --8<-- [start:versioning_update_data] - table.update(where="author='Richard'", values={"author": "Richard Daniel Sanchez"}) - rows_after_update = table.count_rows("author = 'Richard Daniel Sanchez'") - print(f"Rows updated to Richard Daniel Sanchez: {rows_after_update}") - # --8<-- [end:versioning_update_data] - assert rows_after_update == 2 - - # --8<-- [start:versioning_add_data] - more_data = [ - { - "id": 4, - "author": "Richard Daniel Sanchez", - "quote": "That's the way the news goes!", - }, - {"id": 5, "author": "Morty", "quote": "Aww geez, Rick!"}, - ] - table.add(more_data) - # --8<-- [end:versioning_add_data] - assert table.count_rows() == 5 - - # --8<-- [start:versioning_check_versions_after_mod] - versions = table.list_versions() - version_count_after_mod = len(versions) - version_after_mod = table.version - print(f"Number of versions after modifications: {version_count_after_mod}") - print(f"Current version: {version_after_mod}") - # --8<-- [end:versioning_check_versions_after_mod] - assert version_count_after_mod >= 2 - assert version_after_mod == versions[-1]["version"] - - # --8<-- [start:versioning_list_all_versions] - versions = table.list_versions() - for v in versions: - print(f"Version {v['version']}, created at {v['timestamp']}") - # --8<-- [end:versioning_list_all_versions] - assert len(versions) >= 1 - - # --8<-- [start:versioning_rollback] - table.restore(version_after_mod) - versions = table.list_versions() - version_count_after_rollback = len(versions) - print(f"Total number of versions after rollback: {version_count_after_rollback}") - # --8<-- [end:versioning_rollback] - assert version_count_after_rollback == version_count_after_mod + 1 - assert table.version == versions[-1]["version"] - assert table.count_rows() == 5 - - # --8<-- [start:versioning_checkout_latest] - table.checkout_latest() - # --8<-- [end:versioning_checkout_latest] - assert table.version == table.list_versions()[-1]["version"] - - # --8<-- [start:versioning_delete_data] - table.delete("author = 'Morty'") - rows_after_deletion = table.count_rows() - print(f"Number of rows after deletion: {rows_after_deletion}") - # --8<-- [end:versioning_delete_data] - assert rows_after_deletion == 3 - - -def test_versioning_tags(tmp_db): - import pyarrow as pa - - schema = pa.schema( - [ - pa.field("id", pa.int64()), - pa.field("author", pa.string()), - pa.field("quote", pa.string()), - ] - ) - table = tmp_db.create_table( - "quotes_tags_example", - [{"id": 1, "author": "Richard", "quote": "Wubba Lubba Dub Dub!"}], - schema=schema, - mode="overwrite", - ) # v1 - table.add([{"id": 2, "author": "Morty", "quote": "Aww geez, Rick!"}]) # v2 - table.add([{"id": 3, "author": "Summer", "quote": "Whatever, Grandpa"}]) # v3 - - # --8<-- [start:versioning_tags] - # Create a tag pointing at a specific version - table.tags.create("baseline", 1) - table.tags.create("with-edits", table.version) - - # List all tags on this table - print(table.tags.list()) - - # Look up the version a tag points at - print(table.tags.get_version("baseline")) - - # Move an existing tag to a different version - table.tags.update("baseline", 2) - - # Check out a version by tag name - table.checkout("baseline") - print(table.version) - - # Delete a tag (does not delete the underlying version) - table.tags.delete("with-edits") - - # Return to the latest version - table.checkout_latest() - # --8<-- [end:versioning_tags] - assert table.version == 3 - assert "baseline" in table.tags.list() - assert "with-edits" not in table.tags.list() - - -# ============================================================================ -# Branches Examples -# ============================================================================ - - -def test_branches(tmp_db): - import numpy as np - import pyarrow as pa - from lancedb.index import FTS, IvfPq - - db = tmp_db - schema = pa.schema( - [ - pa.field("id", pa.int64()), - pa.field("author", pa.string()), - pa.field("quote", pa.string()), - ] - ) - table = db.create_table( - "quotes_branches_example", - [ - {"id": 1, "author": "Lancelot", "quote": "My lance never fails."}, - {"id": 2, "author": "Arthur", "quote": "Long live Camelot!"}, - {"id": 3, "author": "Merlin", "quote": "Magic always has a price."}, - ], - schema=schema, - mode="overwrite", - ) - - # --8<-- [start:branch_create] - # Fork an isolated, writable branch from main's latest version. - # `create` returns a table handle scoped to the new branch. - branch = table.branches.create("exp") - # --8<-- [end:branch_create] - - # --8<-- [start:branch_write] - # Writes land on the branch handle only; main is left untouched. - branch.add([{"id": 4, "author": "Lancelot", "quote": "For the realm!"}]) - print(branch.count_rows()) # 4 rows on the branch - print(table.count_rows()) # 3 rows; main is unaffected - - # List every branch, each mapped to its metadata (including its fork point). - print(table.branches.list()) - # --8<-- [end:branch_write] - - # --8<-- [start:branch_reopen] - # Reopen an existing branch by name from the table handle... - checked_out = table.branches.checkout("exp") - # ...or open it directly from the database connection. - branch_handle = db.open_table("quotes_branches_example", branch="exp") - print(checked_out.count_rows(), branch_handle.count_rows()) # both 4 - # --8<-- [end:branch_reopen] - - # --8<-- [start:branch_delete] - # Delete the branch and its branch-local history. Data on main is safe. - table.branches.delete("exp") - # --8<-- [end:branch_delete] - - assert table.count_rows() == 3 - assert "exp" not in table.branches.list() - - # Setup: a branch with row results that we want to apply to main. - candidate = table.branches.create("candidate") - candidate.update(where="id = 1", values={"quote": "Revised on the branch"}) - candidate.add( - [{"id": 4, "author": "Galahad", "quote": "The grail awaits."}] - ) - - # --8<-- [start:branch_upsert_to_main] - # This is a row-level upsert, not a merge of branch histories. - # `merge_insert` updates matching rows and inserts new rows using a stable - # unique key. Filter the branch read if you only want to apply some results. - rows_to_apply = candidate.to_arrow() - ( - table.merge_insert("id") - .when_matched_update_all() # update rows that already exist on main - .when_not_matched_insert_all() # insert rows that are new on the branch - .execute(rows_to_apply) - ) - # --8<-- [end:branch_upsert_to_main] - - main_rows = { - row["id"]: row["quote"] for row in table.to_arrow().to_pylist() - } - assert table.count_rows() == 4 - assert main_rows[1] == "Revised on the branch" - assert 4 in main_rows - table.branches.delete("candidate") - - # Setup: a larger table with a vector and a text column to index. - rng = np.random.default_rng(0) - products = db.create_table( - "products_branch_index", - [ - {"id": i, "vector": rng.random(4).tolist(), "text": f"product number {i}"} - for i in range(512) - ], - mode="overwrite", - ) - - # --8<-- [start:branch_index] - # Build and validate indexes on a branch before using the configuration on - # main. - dev = products.branches.create("index-dev") - - # A vector (ANN) index and a full-text search index, both branch-scoped. - dev.create_index( - "vector", - config=IvfPq(distance_type="cosine", num_partitions=1, num_sub_vectors=2), - ) - dev.create_index("text", config=FTS()) - - # Both indexes live only on the branch; main still has none. - print([ix.name for ix in dev.list_indices()]) # branch: two indexes - print([ix.name for ix in products.list_indices()]) # main: [] (untouched) - # --8<-- [end:branch_index] - - assert len(dev.list_indices()) == 2 - assert len(products.list_indices()) == 0 - products.branches.delete("index-dev") - - -# ============================================================================ -# Consistency Examples -# ============================================================================ - - -def test_consistency_strong(tmp_db): - # --8<-- [start:consistency_strong] - from datetime import timedelta - - uri = str(tmp_db.uri) - writer_db = lancedb.connect(uri) - reader_db = lancedb.connect(uri, read_consistency_interval=timedelta(0)) - writer_table = writer_db.create_table( - "consistency_strong_table", [{"id": 1}], mode="overwrite" - ) - reader_table = reader_db.open_table("consistency_strong_table") - writer_table.add([{"id": 2}]) - rows_after_write = reader_table.count_rows() - print(f"Rows visible with strong consistency: {rows_after_write}") - # --8<-- [end:consistency_strong] - assert rows_after_write == 2 - - -def test_consistency_eventual(tmp_db): - # --8<-- [start:consistency_eventual] - from datetime import timedelta - - uri = str(tmp_db.uri) - writer_db = lancedb.connect(uri) - reader_db = lancedb.connect(uri, read_consistency_interval=timedelta(seconds=3600)) - writer_table = writer_db.create_table( - "consistency_eventual_table", [{"id": 1}], mode="overwrite" - ) - reader_table = reader_db.open_table("consistency_eventual_table") - writer_table.add([{"id": 2}]) - rows_after_write = reader_table.count_rows() - print(f"Rows visible before eventual refresh interval: {rows_after_write}") - # --8<-- [end:consistency_eventual] - assert rows_after_write == 1 - - -def test_consistency_checkout_latest(tmp_db): - # --8<-- [start:consistency_checkout_latest] - uri = str(tmp_db.uri) - writer_db = lancedb.connect(uri) - reader_db = lancedb.connect(uri) - writer_table = writer_db.create_table( - "consistency_checkout_latest_table", [{"id": 1}], mode="overwrite" - ) - reader_table = reader_db.open_table("consistency_checkout_latest_table") - - writer_table.add([{"id": 2}]) - rows_before_refresh = reader_table.count_rows() - print(f"Rows before checkout_latest: {rows_before_refresh}") - - reader_table.checkout_latest() - rows_after_refresh = reader_table.count_rows() - print(f"Rows after checkout_latest: {rows_after_refresh}") - # --8<-- [end:consistency_checkout_latest] - assert rows_before_refresh == 1 - assert rows_after_refresh == 2 diff --git a/tests/rs/Cargo.lock b/tests/rs/Cargo.lock deleted file mode 100644 index f646196..0000000 --- a/tests/rs/Cargo.lock +++ /dev/null @@ -1,7395 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "const-random", - "getrandom 0.3.4", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "aligned-vec" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" -dependencies = [ - "equator", -] - -[[package]] -name = "alloca" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" -dependencies = [ - "cc", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - -[[package]] -name = "ar_archive_writer" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" -dependencies = [ - "object 0.32.2", -] - -[[package]] -name = "arc-swap" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" - -[[package]] -name = "argminmax" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f13d10a41ac8d2ec79ee34178d61e6f47a29c2edfe7ef1721c7383b0359e65" -dependencies = [ - "num-traits", -] - -[[package]] -name = "array-init-cursor" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed51fe0f224d1d4ea768be38c51f9f831dee9d05c163c11fba0b8c44387b1fc3" - -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - -[[package]] -name = "arrow" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", -] - -[[package]] -name = "arrow-arith" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "num-traits", -] - -[[package]] -name = "arrow-array" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" -dependencies = [ - "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "chrono-tz 0.10.4", - "half", - "hashbrown 0.17.1", - "num-complex", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-buffer" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" -dependencies = [ - "bytes", - "half", - "num-bigint", - "num-traits", -] - -[[package]] -name = "arrow-cast" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "atoi", - "base64 0.22.1", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num-traits", - "ryu", -] - -[[package]] -name = "arrow-csv" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" -dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", - "chrono", - "csv", - "csv-core", - "regex", -] - -[[package]] -name = "arrow-data" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" -dependencies = [ - "arrow-buffer", - "arrow-schema", - "half", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-ipc" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "flatbuffers", - "lz4_flex", - "zstd", -] - -[[package]] -name = "arrow-json" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "chrono", - "half", - "indexmap 2.14.0", - "itoa", - "lexical-core", - "memchr", - "num-traits", - "ryu", - "serde_core", - "serde_json", - "simdutf8", -] - -[[package]] -name = "arrow-ord" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", -] - -[[package]] -name = "arrow-row" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "half", -] - -[[package]] -name = "arrow-schema" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" -dependencies = [ - "bitflags 2.10.0", - "serde_core", - "serde_json", -] - -[[package]] -name = "arrow-select" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" -dependencies = [ - "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "num-traits", -] - -[[package]] -name = "arrow-string" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "memchr", - "num-traits", - "regex", - "regex-syntax", -] - -[[package]] -name = "async-channel" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-compression" -version = "0.4.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98ec5f6c2f8bc326c994cb9e241cc257ddaba9afa8555a43cffbb5dd86efaa37" -dependencies = [ - "compression-codecs", - "compression-core", - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "async-lock" -version = "3.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-recursion" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "async_cell" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447ab28afbb345f5408b120702a44e5529ebf90b1796ec76e9528df8e288e6c2" -dependencies = [ - "loom", -] - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - -[[package]] -name = "atoi_simd" -version = "0.15.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae037714f313c1353189ead58ef9eec30a8e8dc101b2622d461418fd59e28a9" - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object 0.37.3", - "rustc-demangle", - "windows-link 0.2.1", -] - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bigdecimal" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" -dependencies = [ - "autocfg", - "libm", - "num-bigint", - "num-integer", - "num-traits", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" - -[[package]] -name = "bitpacking" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" -dependencies = [ - "crunchy", -] - -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest", -] - -[[package]] -name = "blake3" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bumpalo" -version = "3.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - -[[package]] -name = "bytecheck" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "rancor", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "bytecount" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" - -[[package]] -name = "bytemuck" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" - -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - -[[package]] -name = "cc" -version = "1.2.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cedarwood" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55" -dependencies = [ - "smallvec", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "chrono" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link 0.2.1", -] - -[[package]] -name = "chrono-tz" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" -dependencies = [ - "chrono", - "chrono-tz-build", - "phf 0.11.3", -] - -[[package]] -name = "chrono-tz" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" -dependencies = [ - "chrono", - "phf 0.12.1", -] - -[[package]] -name = "chrono-tz-build" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" -dependencies = [ - "parse-zoneinfo", - "phf 0.11.3", - "phf_codegen 0.11.3", -] - -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" -dependencies = [ - "anstyle", - "clap_lex", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - -[[package]] -name = "comfy-table" -version = "7.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" -dependencies = [ - "crossterm 0.27.0", - "crossterm 0.28.1", - "strum 0.26.3", - "strum_macros 0.26.4", - "unicode-width", -] - -[[package]] -name = "compression-codecs" -version = "0.4.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f7ac3e5b97fdce45e8922fb05cae2c37f7bbd63d30dd94821dacfd8f3f2bf2" -dependencies = [ - "compression-core", - "flate2", - "memchr", -] - -[[package]] -name = "compression-core" -version = "0.4.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.16", - "once_cell", - "tiny-keccak", -] - -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpp_demangle" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "criterion" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" -dependencies = [ - "alloca", - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "itertools 0.13.0", - "num-traits", - "oorandom", - "page_size", - "plotters", - "rayon", - "regex", - "serde", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" -dependencies = [ - "cast", - "itertools 0.13.0", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-skiplist" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crossterm" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" -dependencies = [ - "bitflags 2.10.0", - "crossterm_winapi", - "libc", - "parking_lot", - "winapi", -] - -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags 2.10.0", - "parking_lot", - "rustix 0.38.44", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "csv" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde_core", -] - -[[package]] -name = "csv-core" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" -dependencies = [ - "memchr", -] - -[[package]] -name = "daachorse" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db756b5eb7d81d31f31f660f4132f8cf5698de52fca144c143d0ae0cbb5f2e06" - -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - -[[package]] -name = "darling" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" -dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", -] - -[[package]] -name = "darling_core" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", -] - -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core 0.20.11", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "darling_macro" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" -dependencies = [ - "darling_core 0.21.3", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "datafusion" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" -dependencies = [ - "arrow", - "arrow-schema", - "async-trait", - "bytes", - "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "rand 0.9.2", - "regex", - "sqlparser 0.61.0", - "tempfile", - "tokio", - "url", - "uuid", -] - -[[package]] -name = "datafusion-catalog" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" -dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "tokio", -] - -[[package]] -name = "datafusion-catalog-listing" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" -dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "futures", - "itertools 0.14.0", - "log", - "object_store", -] - -[[package]] -name = "datafusion-common" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" -dependencies = [ - "ahash", - "arrow", - "arrow-ipc", - "chrono", - "half", - "hashbrown 0.16.1", - "indexmap 2.14.0", - "itertools 0.14.0", - "libc", - "log", - "object_store", - "paste", - "sqlparser 0.61.0", - "tokio", - "web-time", -] - -[[package]] -name = "datafusion-common-runtime" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" -dependencies = [ - "futures", - "log", - "tokio", -] - -[[package]] -name = "datafusion-datasource" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "glob", - "itertools 0.14.0", - "log", - "object_store", - "rand 0.9.2", - "tokio", - "url", -] - -[[package]] -name = "datafusion-datasource-arrow" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" -dependencies = [ - "arrow", - "arrow-ipc", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.14.0", - "object_store", - "tokio", -] - -[[package]] -name = "datafusion-datasource-csv" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "regex", - "tokio", -] - -[[package]] -name = "datafusion-datasource-json" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "serde_json", - "tokio", - "tokio-stream", -] - -[[package]] -name = "datafusion-doc" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" - -[[package]] -name = "datafusion-execution" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" -dependencies = [ - "arrow", - "arrow-buffer", - "async-trait", - "chrono", - "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", - "futures", - "log", - "object_store", - "parking_lot", - "rand 0.9.2", - "tempfile", - "url", -] - -[[package]] -name = "datafusion-expr" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" -dependencies = [ - "arrow", - "async-trait", - "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", - "indexmap 2.14.0", - "itertools 0.14.0", - "paste", - "serde_json", - "sqlparser 0.61.0", -] - -[[package]] -name = "datafusion-expr-common" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" -dependencies = [ - "arrow", - "datafusion-common", - "indexmap 2.14.0", - "itertools 0.14.0", - "paste", -] - -[[package]] -name = "datafusion-functions" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" -dependencies = [ - "arrow", - "arrow-buffer", - "base64 0.22.1", - "blake2", - "blake3", - "chrono", - "chrono-tz 0.10.4", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", - "hex", - "itertools 0.14.0", - "log", - "md-5", - "memchr", - "num-traits", - "rand 0.9.2", - "regex", - "sha2", - "unicode-segmentation", - "uuid", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" -dependencies = [ - "ahash", - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "half", - "log", - "num-traits", - "paste", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" -dependencies = [ - "ahash", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-functions-nested" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", - "hashbrown 0.16.1", - "itertools 0.14.0", - "itoa", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-table" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" -dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", - "parking_lot", - "paste", -] - -[[package]] -name = "datafusion-functions-window" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" -dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-macros" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" -dependencies = [ - "datafusion-doc", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "datafusion-optimizer" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "indexmap 2.14.0", - "itertools 0.14.0", - "log", - "regex", - "regex-syntax", -] - -[[package]] -name = "datafusion-physical-expr" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" -dependencies = [ - "ahash", - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", - "half", - "hashbrown 0.16.1", - "indexmap 2.14.0", - "itertools 0.14.0", - "parking_lot", - "paste", - "petgraph", - "tokio", -] - -[[package]] -name = "datafusion-physical-expr-adapter" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "itertools 0.14.0", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" -dependencies = [ - "ahash", - "arrow", - "chrono", - "datafusion-common", - "datafusion-expr-common", - "hashbrown 0.16.1", - "indexmap 2.14.0", - "itertools 0.14.0", - "parking_lot", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "itertools 0.14.0", -] - -[[package]] -name = "datafusion-physical-plan" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" -dependencies = [ - "ahash", - "arrow", - "arrow-ord", - "arrow-schema", - "async-trait", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "futures", - "half", - "hashbrown 0.16.1", - "indexmap 2.14.0", - "itertools 0.14.0", - "log", - "num-traits", - "parking_lot", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "datafusion-pruning" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "itertools 0.14.0", - "log", -] - -[[package]] -name = "datafusion-session" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" -dependencies = [ - "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", - "parking_lot", -] - -[[package]] -name = "datafusion-sql" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" -dependencies = [ - "arrow", - "bigdecimal", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", - "indexmap 2.14.0", - "log", - "regex", - "sqlparser 0.61.0", -] - -[[package]] -name = "debugid" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" -dependencies = [ - "uuid", -] - -[[package]] -name = "deepsize" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c" -dependencies = [ - "deepsize_derive", -] - -[[package]] -name = "deepsize_derive" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "deranged" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" -dependencies = [ - "powerfmt", - "serde_core", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.117", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.61.2", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "encoding_rs_io" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" -dependencies = [ - "encoding_rs", -] - -[[package]] -name = "enum_dispatch" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "equator" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" -dependencies = [ - "equator-macro", -] - -[[package]] -name = "equator-macro" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "ethnum" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" - -[[package]] -name = "event-listener" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] -name = "fast-float" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" - -[[package]] -name = "fast-float2" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "find-msvc-tools" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" - -[[package]] -name = "findshlibs" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" -dependencies = [ - "cc", - "lazy_static", - "libc", - "winapi", -] - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "flatbuffers" -version = "25.9.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" -dependencies = [ - "bitflags 2.10.0", - "rustc_version", -] - -[[package]] -name = "flate2" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "foreign_vec" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fsst" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af6e24ed12cf382082d5a7f365df2b8e3d6b1518f615d54c85165e9b9718250" -dependencies = [ - "arrow-array", - "rand 0.9.2", -] - -[[package]] -name = "fst" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" -dependencies = [ - "utf8-ranges", -] - -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generator" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" -dependencies = [ - "cc", - "cfg-if", - "libc", - "log", - "rustversion", - "windows 0.61.3", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 5.3.0", - "wasip2", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" -dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", - "wasip2", - "wasip3", -] - -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - -[[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - -[[package]] -name = "h2" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "num-traits", - "zerocopy", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", - "rayon", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", -] - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "http" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "humantime" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" - -[[package]] -name = "hyper" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "pin-utils", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "hyperloglogplus" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3" -dependencies = [ - "serde", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core 0.62.2", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5a396343c7208121dc86e35623d3dfe19814a7613cfd14964994cdc9c9a2e26" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_locale_data", - "icu_provider", - "potential_utf", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "serde", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locale_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fdcc9ac77c6d74ff5cf6e65ef3181d6af32003b16fce3a77fb451d2f695993" - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "serde", - "stable_deref_trait", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_segmenter" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c0794db0b1a86193ac9c48768d0e6c52c54448e0870ad87907d456ee0dac964" -dependencies = [ - "icu_collections", - "icu_locale", - "icu_provider", - "icu_segmenter_data", - "potential_utf", - "utf8_iter", - "zerovec", -] - -[[package]] -name = "icu_segmenter_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a2c462a4d927d512f5f882a033ddd62f33a05bb9f230d98f736ac3dc85938f" - -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", -] - -[[package]] -name = "inferno" -version = "0.11.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" -dependencies = [ - "ahash", - "indexmap 2.14.0", - "is-terminal", - "itoa", - "log", - "num-format", - "once_cell", - "quick-xml", - "rgb", - "str_stack", -] - -[[package]] -name = "io-uring" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" -dependencies = [ - "bitflags 2.10.0", - "cfg-if", - "libc", -] - -[[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "itoap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" - -[[package]] -name = "jieba-macros" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" -dependencies = [ - "phf_codegen 0.13.1", -] - -[[package]] -name = "jieba-rs" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" -dependencies = [ - "bytecount", - "cedarwood", - "jieba-macros", - "phf 0.13.1", - "regex", - "rustc-hash", -] - -[[package]] -name = "jiff" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" -dependencies = [ - "jiff-static", - "jiff-tzdb-platform", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", - "windows-sys 0.61.2", -] - -[[package]] -name = "jiff-static" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "jiff-tzdb" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" - -[[package]] -name = "jiff-tzdb-platform" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" -dependencies = [ - "jiff-tzdb", -] - -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "jsonb" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6" -dependencies = [ - "byteorder", - "ethnum", - "fast-float2", - "itoa", - "jiff", - "nom", - "num-traits", - "ordered-float", - "rand 0.9.2", - "ryu", - "serde", - "serde_json", -] - -[[package]] -name = "kanaria" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f9d9652540055ac4fded998a73aca97d965899077ab1212587437da44196ff" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "lance" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2122e9f5f5f4b38bb9f0c4991c8d5171696402b3cc6187885c8385875f6df2e" -dependencies = [ - "arc-swap", - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ipc", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "async-recursion", - "async-trait", - "async_cell", - "bitpacking", - "byteorder", - "bytes", - "chrono", - "crossbeam-queue", - "crossbeam-skiplist", - "dashmap", - "datafusion", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-plan", - "either", - "fst", - "futures", - "half", - "humantime", - "itertools 0.13.0", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "lance-datafusion", - "lance-encoding", - "lance-file", - "lance-index", - "lance-io", - "lance-linalg", - "lance-namespace 8.0.0", - "lance-select", - "lance-table", - "lance-tokenizer", - "log", - "moka", - "object_store", - "permutation", - "pin-project", - "prost", - "prost-build", - "prost-types", - "rand 0.9.2", - "rayon", - "roaring", - "rustc-hash", - "semver", - "serde", - "serde_json", - "snafu 0.9.1", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "lance-arrow" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "253f4a0a70580c985b91e65e9ca6cad644825a4078de28d8efbacf3ffbd7ecdc" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", - "arrow-select", - "bytes", - "futures", - "getrandom 0.2.16", - "half", - "jsonb", - "num-traits", - "rand 0.9.2", -] - -[[package]] -name = "lance-arrow" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95f45fb7e0822cc0233d686222ab451f6ec58879620029e0b7bae8192c2f7c7e" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", - "arrow-select", - "bytes", - "futures", - "getrandom 0.2.16", - "half", - "jsonb", - "num-traits", - "rand 0.9.2", -] - -[[package]] -name = "lance-arrow-scalar" -version = "58.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-row", - "arrow-schema", - "half", -] - -[[package]] -name = "lance-arrow-stats" -version = "58.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f" -dependencies = [ - "arrow-array", - "arrow-schema", - "half", - "lance-arrow-scalar", -] - -[[package]] -name = "lance-bitpacking" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56c21a39860ca7bae712b4e24b9fd066029c570a72428a579faf697de5b8822" -dependencies = [ - "arrayref", - "paste", - "seq-macro", -] - -[[package]] -name = "lance-core" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "async-trait", - "byteorder", - "bytes", - "deepsize", - "futures", - "itertools 0.13.0", - "lance-arrow 7.0.0", - "libc", - "log", - "moka", - "num_cpus", - "object_store", - "pin-project", - "prost", - "rand 0.9.2", - "roaring", - "serde_json", - "snafu 0.9.1", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "url", -] - -[[package]] -name = "lance-core" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ccc7b0b61c11bdb74f929111214553b36b1ee43c88445edda17bc9a2bda75e9" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "async-trait", - "byteorder", - "bytes", - "datafusion-common", - "datafusion-sql", - "futures", - "itertools 0.13.0", - "lance-arrow 8.0.0", - "lance-derive", - "libc", - "libm", - "log", - "moka", - "num_cpus", - "object_store", - "pin-project", - "prost", - "rand 0.9.2", - "roaring", - "serde_json", - "snafu 0.9.1", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "twox-hash", - "url", -] - -[[package]] -name = "lance-datafusion" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0932140306faa6c3c4e17f0478c031e2be659ea2721311a530fb7c664e1b9a0" -dependencies = [ - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "async-trait", - "chrono", - "datafusion", - "datafusion-common", - "datafusion-functions", - "datafusion-physical-expr", - "futures", - "jsonb", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "lance-datagen", - "log", - "pin-project", - "prost", - "prost-build", - "tokio", - "tracing", -] - -[[package]] -name = "lance-datagen" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "075c8154e6b27ff6859f90886efd8cbac348cca255c8b855da630994b4fed714" -dependencies = [ - "arrow", - "arrow-array", - "arrow-cast", - "arrow-schema", - "chrono", - "futures", - "half", - "hex", - "rand 0.9.2", - "rand_distr 0.5.1", - "rand_xoshiro", -] - -[[package]] -name = "lance-derive" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7056da2e742fd466f6bdfaa876c91d3e0e99bb4f756be058e374ceae2630ec2f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "lance-encoding" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e165c668ae61fce336d5f6f9a999ee99b963bb395a3fb818737c533fc9528d1" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", - "bytemuck", - "byteorder", - "bytes", - "fsst", - "futures", - "hex", - "hyperloglogplus", - "itertools 0.13.0", - "lance-arrow 8.0.0", - "lance-bitpacking", - "lance-core 8.0.0", - "log", - "lz4", - "num-traits", - "prost", - "prost-build", - "rand 0.9.2", - "strum 0.26.3", - "tokio", - "tracing", - "xxhash-rust", - "zstd", -] - -[[package]] -name = "lance-file" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1691bd5c37bc5e07bde00e32950b5526c2e5653bd2493a3773712574366a37a1" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "async-recursion", - "async-trait", - "byteorder", - "bytes", - "datafusion-common", - "futures", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "lance-encoding", - "lance-io", - "log", - "num-traits", - "object_store", - "prost", - "prost-build", - "prost-types", - "tokio", - "tracing", -] - -[[package]] -name = "lance-index" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4792b56f0e047eed706d05a1eedc5f549090913fb4527585bb3a331b83416b" -dependencies = [ - "arc-swap", - "arrow", - "arrow-arith", - "arrow-array", - "arrow-ord", - "arrow-schema", - "arrow-select", - "async-channel", - "async-recursion", - "async-trait", - "bitpacking", - "bitvec", - "bytes", - "chrono", - "crossbeam-queue", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "dirs", - "fst", - "futures", - "half", - "itertools 0.13.0", - "jieba-rs", - "jsonb", - "lance-arrow 8.0.0", - "lance-arrow-stats", - "lance-core 8.0.0", - "lance-datafusion", - "lance-datagen", - "lance-encoding", - "lance-file", - "lance-io", - "lance-linalg", - "lance-select", - "lance-table", - "lance-tokenizer", - "libsais-rs", - "log", - "ndarray", - "num-traits", - "object_store", - "prost", - "prost-build", - "prost-types", - "rand 0.9.2", - "rand_distr 0.5.1", - "rangemap", - "rayon", - "regex-syntax", - "roaring", - "serde", - "serde_json", - "smallvec", - "tempfile", - "tokio", - "tracing", - "uuid", -] - -[[package]] -name = "lance-io" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c81dda99d5b8c8741f413d65650a28ff203d94eda3cbbdda6fd3af3ea9b2a69" -dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", - "async-recursion", - "async-trait", - "byteorder", - "bytes", - "chrono", - "futures", - "http", - "io-uring", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "lance-namespace 8.0.0", - "log", - "moka", - "object_store", - "path_abs", - "pin-project", - "prost", - "rand 0.9.2", - "serde", - "tempfile", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "lance-linalg" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c0a8d2a2bc7e6b6229abe320ec6d9e2049a0e65e084684cba01cf032fc71896" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "cc", - "half", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "num-traits", - "rand 0.9.2", -] - -[[package]] -name = "lance-namespace" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014e8332ca0615506342e0d3af608639864b68396973be14239f09c9f21f1fc2" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "lance-core 7.0.0", - "lance-namespace-reqwest-client 0.7.7", - "snafu 0.9.1", -] - -[[package]] -name = "lance-namespace" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee2d75929caec41747e58ce090b1a061b650a16cb10d09998128d7912203b1d" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "lance-core 8.0.0", - "lance-namespace-reqwest-client 0.8.6", - "snafu 0.9.1", -] - -[[package]] -name = "lance-namespace-impls" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92dd5dcb98562a91650da0b5fa63726d435fad8b03327625cc3de445b7e191a" -dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-physical-plan", - "futures", - "lance", - "lance-core 8.0.0", - "lance-index", - "lance-io", - "lance-linalg", - "lance-namespace 8.0.0", - "lance-table", - "log", - "object_store", - "rand 0.9.2", - "roaring", - "serde_json", - "time", - "tokio", - "url", - "uuid", -] - -[[package]] -name = "lance-namespace-reqwest-client" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6369eee4682fb11edf538388b43c61ce288b8302fe89bb40944d7daa7faaae99" -dependencies = [ - "reqwest", - "serde", - "serde_json", - "serde_repr", - "serde_with", - "url", -] - -[[package]] -name = "lance-namespace-reqwest-client" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" -dependencies = [ - "reqwest", - "serde", - "serde_json", - "serde_repr", - "serde_with", - "url", -] - -[[package]] -name = "lance-select" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3a2162385a4392376ed7a732e070afd1432bb6f86b0ebcb7c4304c7196953b" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "byteorder", - "bytes", - "itertools 0.13.0", - "lance-core 8.0.0", - "roaring", - "tracing", -] - -[[package]] -name = "lance-table" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73da3932a85489e80d5458d4bbc1d340e15c72b89bab529723f575d4d8fda5eb" -dependencies = [ - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ipc", - "arrow-schema", - "async-trait", - "byteorder", - "bytes", - "chrono", - "futures", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "lance-file", - "lance-io", - "lance-select", - "log", - "object_store", - "prost", - "prost-build", - "prost-types", - "rand 0.9.2", - "rangemap", - "roaring", - "semver", - "serde", - "serde_json", - "snafu 0.9.1", - "tokio", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "lance-testing" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a5a7b5dbda917170b705301d0c6a8753a8c02f501b20d8b7067b5463ba071d" -dependencies = [ - "arrow-array", - "arrow-schema", - "criterion", - "lance-arrow 8.0.0", - "num-traits", - "pprof", - "rand 0.9.2", -] - -[[package]] -name = "lance-tokenizer" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16326f6b6d3b736aac40a0ffa78d8aa17d3a53a3766cd0af6d23db87472bd5e" -dependencies = [ - "icu_segmenter", - "jieba-rs", - "lindera", - "rust-stemmers", - "serde", - "stop-words", - "unicode-normalization", -] - -[[package]] -name = "lancedb" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd0b54bb1cdd075efa5a8827ec16dcf5c0781253cd88e63988c174915c53fe2" -dependencies = [ - "ahash", - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", - "arrow-select", - "async-trait", - "bytes", - "chrono", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-sql", - "futures", - "half", - "lance", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "lance-datafusion", - "lance-datagen", - "lance-encoding", - "lance-file", - "lance-index", - "lance-io", - "lance-linalg", - "lance-namespace 8.0.0", - "lance-namespace-impls", - "lance-table", - "lance-testing", - "lazy_static", - "log", - "moka", - "num-traits", - "object_store", - "pin-project", - "polars", - "polars-arrow", - "rand 0.9.2", - "regex", - "semver", - "serde", - "serde_json", - "serde_with", - "snafu 0.8.9", - "tempfile", - "tokio", - "url", - "uuid", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - -[[package]] -name = "lexical-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" -dependencies = [ - "lexical-parse-integer", - "lexical-util", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "lexical-util" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" - -[[package]] -name = "lexical-write-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" -dependencies = [ - "lexical-util", - "lexical-write-integer", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "libc" -version = "0.2.186" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" - -[[package]] -name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - -[[package]] -name = "libredox" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" -dependencies = [ - "bitflags 2.10.0", - "libc", -] - -[[package]] -name = "libsais-rs" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40fe164dbd47ea0c20e78a121c980ef673326905f1d4fba55e3645a20ef6717f" -dependencies = [ - "rayon", -] - -[[package]] -name = "lindera" -version = "3.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74cda79d7161e99b414e4d292ff673cc3f8d22f070d8be3b6185c033363a9216" -dependencies = [ - "anyhow", - "byteorder", - "csv", - "daachorse", - "kanaria", - "lindera-dictionary", - "log", - "once_cell", - "percent-encoding", - "regex", - "serde", - "serde_json", - "serde_yaml_ng", - "strum 0.28.0", - "strum_macros 0.28.0", - "unicode-blocks", - "unicode-normalization", - "unicode-segmentation", - "url", -] - -[[package]] -name = "lindera-dictionary" -version = "3.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2385456ca9fe87c29072c5f156b52fdd5e28d5b5738ddfb3979501dbd736530" -dependencies = [ - "anyhow", - "byteorder", - "csv", - "daachorse", - "derive_builder", - "encoding_rs", - "encoding_rs_io", - "glob", - "log", - "memmap2 0.9.10", - "num_cpus", - "once_cell", - "regex", - "rkyv", - "serde", - "serde_json", - "strum 0.28.0", - "strum_macros 0.28.0", - "thiserror 2.0.18", -] - -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - -[[package]] -name = "linux-raw-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" - -[[package]] -name = "litemap" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" - -[[package]] -name = "loom" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "lz4" -version = "1.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" -dependencies = [ - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.11.1+lz4-1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "lz4_flex" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" -dependencies = [ - "twox-hash", -] - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "matrixmultiply" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" -dependencies = [ - "autocfg", - "num_cpus", - "once_cell", - "rawpointer", - "thread-tree", -] - -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - -[[package]] -name = "memchr" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" - -[[package]] -name = "memmap2" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" -dependencies = [ - "libc", -] - -[[package]] -name = "memmap2" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" -dependencies = [ - "libc", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - -[[package]] -name = "mio" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "moka" -version = "0.12.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8261cd88c312e0004c1d51baad2980c66528dfdb2bee62003e643a4d8f86b077" -dependencies = [ - "async-lock", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "equivalent", - "event-listener", - "futures-util", - "parking_lot", - "portable-atomic", - "rustc_version", - "smallvec", - "tagptr", - "uuid", -] - -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - -[[package]] -name = "multiversion" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4851161a11d3ad0bf9402d90ffc3967bf231768bfd7aeb61755ad06dbf1a142" -dependencies = [ - "multiversion-macros", - "target-features", -] - -[[package]] -name = "multiversion-macros" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a74ddee9e0c27d2578323c13905793e91622148f138ba29738f9dddb835e90" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "target-features", -] - -[[package]] -name = "munge" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" -dependencies = [ - "munge_macro", -] - -[[package]] -name = "munge_macro" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "ndarray" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - -[[package]] -name = "nix" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", -] - -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - -[[package]] -name = "now" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d89e9874397a1f0a52fc1f197a8effd9735223cb2390e9dcc83ac6cd02923d0" -dependencies = [ - "chrono", -] - -[[package]] -name = "ntapi" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" -dependencies = [ - "winapi", -] - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "object" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" -dependencies = [ - "memchr", -] - -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - -[[package]] -name = "object_store" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures-channel", - "futures-core", - "futures-util", - "http", - "humantime", - "itertools 0.14.0", - "parking_lot", - "percent-encoding", - "thiserror 2.0.18", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - -[[package]] -name = "ordered-float" -version = "5.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" -dependencies = [ - "num-traits", -] - -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link 0.2.1", -] - -[[package]] -name = "parquet-format-safe" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" - -[[package]] -name = "parse-zoneinfo" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" -dependencies = [ - "regex", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "path_abs" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3" -dependencies = [ - "serde", - "serde_derive", - "std_prelude", - "stfu8", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "permutation" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7" - -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "serde", -] - -[[package]] -name = "phf" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_shared 0.11.3", -] - -[[package]] -name = "phf" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" -dependencies = [ - "phf_shared 0.12.1", -] - -[[package]] -name = "phf" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" -dependencies = [ - "phf_shared 0.13.1", - "serde", -] - -[[package]] -name = "phf_codegen" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" -dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", -] - -[[package]] -name = "phf_codegen" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" -dependencies = [ - "phf_generator 0.13.1", - "phf_shared 0.13.1", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared 0.11.3", - "rand 0.8.5", -] - -[[package]] -name = "phf_generator" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" -dependencies = [ - "fastrand", - "phf_shared 0.13.1", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher", -] - -[[package]] -name = "phf_shared" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" -dependencies = [ - "siphasher", -] - -[[package]] -name = "phf_shared" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "planus" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1691dd09e82f428ce8d6310bd6d5da2557c82ff17694d2a32cad7242aea89f" -dependencies = [ - "array-init-cursor", -] - -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "polars" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea21b858b16b9c0e17a12db2800d11aa5b4bd182be6b3022eb537bbfc1f2db5" -dependencies = [ - "getrandom 0.2.16", - "polars-arrow", - "polars-core", - "polars-error", - "polars-io", - "polars-lazy", - "polars-ops", - "polars-parquet", - "polars-sql", - "polars-time", - "polars-utils", - "version_check", -] - -[[package]] -name = "polars-arrow" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "725b09f2b5ef31279b66e27bbab63c58d49d8f6696b66b1f46c7eaab95e80f75" -dependencies = [ - "ahash", - "atoi", - "atoi_simd", - "bytemuck", - "chrono", - "chrono-tz 0.8.6", - "dyn-clone", - "either", - "ethnum", - "fast-float", - "foreign_vec", - "getrandom 0.2.16", - "hashbrown 0.14.5", - "itoa", - "itoap", - "lz4", - "multiversion", - "num-traits", - "polars-arrow-format", - "polars-error", - "polars-utils", - "ryu", - "simdutf8", - "streaming-iterator", - "strength_reduce", - "version_check", - "zstd", -] - -[[package]] -name = "polars-arrow-format" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b0ef2474af9396b19025b189d96e992311e6a47f90c53cd998b36c4c64b84c" -dependencies = [ - "planus", - "serde", -] - -[[package]] -name = "polars-compute" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a796945b14b14fbb79b91ef0406e6fddca2be636e889f81ea5d6ee7d36efb4fe" -dependencies = [ - "bytemuck", - "either", - "num-traits", - "polars-arrow", - "polars-error", - "polars-utils", - "strength_reduce", - "version_check", -] - -[[package]] -name = "polars-core" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465f70d3e96b6d0b1a43c358ba451286b8c8bd56696feff020d65702aa33e35c" -dependencies = [ - "ahash", - "bitflags 2.10.0", - "bytemuck", - "chrono", - "chrono-tz 0.8.6", - "comfy-table", - "either", - "hashbrown 0.14.5", - "indexmap 2.14.0", - "num-traits", - "once_cell", - "polars-arrow", - "polars-compute", - "polars-error", - "polars-row", - "polars-utils", - "rand 0.8.5", - "rand_distr 0.4.3", - "rayon", - "regex", - "smartstring", - "thiserror 1.0.69", - "version_check", - "xxhash-rust", -] - -[[package]] -name = "polars-error" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5224d5d05e6b8a6f78b75951ae1b5f82c8ab1979e11ffaf5fd41941e3d5b0757" -dependencies = [ - "polars-arrow-format", - "regex", - "simdutf8", - "thiserror 1.0.69", -] - -[[package]] -name = "polars-io" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2c8589e418cbe4a48228d64b2a8a40284a82ec3c98817c0c2bcc0267701338b" -dependencies = [ - "ahash", - "atoi_simd", - "bytes", - "chrono", - "fast-float", - "home", - "itoa", - "memchr", - "memmap2 0.7.1", - "num-traits", - "once_cell", - "percent-encoding", - "polars-arrow", - "polars-core", - "polars-error", - "polars-time", - "polars-utils", - "rayon", - "regex", - "ryu", - "simdutf8", - "smartstring", -] - -[[package]] -name = "polars-lazy" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2632b1af668e2058d5f8f916d8fbde3cac63d03ae29a705f598e41dcfeb7f" -dependencies = [ - "ahash", - "bitflags 2.10.0", - "glob", - "once_cell", - "polars-arrow", - "polars-core", - "polars-io", - "polars-ops", - "polars-pipe", - "polars-plan", - "polars-time", - "polars-utils", - "rayon", - "smartstring", - "version_check", -] - -[[package]] -name = "polars-ops" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efdbdb4d9a92109bc2e0ce8e17af5ae8ab643bb5b7ee9d1d74f0aeffd1fbc95f" -dependencies = [ - "ahash", - "argminmax", - "base64 0.21.7", - "bytemuck", - "chrono", - "chrono-tz 0.8.6", - "either", - "hashbrown 0.14.5", - "hex", - "indexmap 2.14.0", - "memchr", - "num-traits", - "polars-arrow", - "polars-compute", - "polars-core", - "polars-error", - "polars-utils", - "rayon", - "regex", - "smartstring", - "unicode-reverse", - "version_check", -] - -[[package]] -name = "polars-parquet" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b421d2196f786fdfe162db614c8485f8308fe41575d4de634a39bbe460d1eb6a" -dependencies = [ - "ahash", - "base64 0.21.7", - "ethnum", - "num-traits", - "parquet-format-safe", - "polars-arrow", - "polars-error", - "polars-utils", - "seq-macro", - "simdutf8", - "streaming-decompression", -] - -[[package]] -name = "polars-pipe" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48700f1d5bd56a15451e581f465c09541492750360f18637b196f995470a015c" -dependencies = [ - "crossbeam-channel", - "crossbeam-queue", - "enum_dispatch", - "hashbrown 0.14.5", - "num-traits", - "polars-arrow", - "polars-compute", - "polars-core", - "polars-io", - "polars-ops", - "polars-plan", - "polars-row", - "polars-utils", - "rayon", - "smartstring", - "uuid", - "version_check", -] - -[[package]] -name = "polars-plan" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fb8e2302e20c44defd5be8cad9c96e75face63c3a5f609aced8c4ec3b3ac97d" -dependencies = [ - "ahash", - "bytemuck", - "chrono-tz 0.8.6", - "hashbrown 0.14.5", - "once_cell", - "percent-encoding", - "polars-arrow", - "polars-core", - "polars-io", - "polars-ops", - "polars-time", - "polars-utils", - "rayon", - "recursive", - "regex", - "smartstring", - "strum_macros 0.25.3", - "version_check", -] - -[[package]] -name = "polars-row" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a515bdc68c2ae3702e3de70d89601f3b71ca8137e282a226dddb53ee4bacfa2e" -dependencies = [ - "bytemuck", - "polars-arrow", - "polars-error", - "polars-utils", -] - -[[package]] -name = "polars-sql" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4bb7cc1c04c3023d1953b2f1dec50515e8fd8169a5a2bf4967b3b082232db7" -dependencies = [ - "hex", - "polars-arrow", - "polars-core", - "polars-error", - "polars-lazy", - "polars-plan", - "rand 0.8.5", - "serde", - "serde_json", - "sqlparser 0.39.0", -] - -[[package]] -name = "polars-time" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc18e3ad92eec55db89d88f16c22d436559ba7030cf76f86f6ed7a754b673f1" -dependencies = [ - "atoi", - "chrono", - "chrono-tz 0.8.6", - "now", - "once_cell", - "polars-arrow", - "polars-core", - "polars-error", - "polars-ops", - "polars-utils", - "regex", - "smartstring", -] - -[[package]] -name = "polars-utils" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c760b6c698cfe2fbbbd93d6cfb408db14ececfe1d92445dae2229ce1b5b21ae8" -dependencies = [ - "ahash", - "bytemuck", - "hashbrown 0.14.5", - "indexmap 2.14.0", - "num-traits", - "once_cell", - "polars-error", - "raw-cpuid", - "rayon", - "smartstring", - "stacker", - "sysinfo", - "version_check", -] - -[[package]] -name = "portable-atomic" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" - -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - -[[package]] -name = "potential_utf" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" -dependencies = [ - "serde_core", - "writeable", - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "pprof" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" -dependencies = [ - "aligned-vec", - "backtrace", - "cfg-if", - "findshlibs", - "inferno", - "libc", - "log", - "nix", - "once_cell", - "smallvec", - "spin", - "symbolic-demangle", - "tempfile", - "thiserror 2.0.18", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.117", -] - -[[package]] -name = "proc-macro2" -version = "1.0.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" -dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.117", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" -dependencies = [ - "anyhow", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "prost-types" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" -dependencies = [ - "prost", -] - -[[package]] -name = "psm" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" -dependencies = [ - "ar_archive_writer", - "cc", -] - -[[package]] -name = "ptr_meta" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "quick-xml" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" -dependencies = [ - "memchr", -] - -[[package]] -name = "quinn" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.18", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" -dependencies = [ - "bytes", - "getrandom 0.3.4", - "lru-slab", - "rand 0.9.2", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.18", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.60.2", -] - -[[package]] -name = "quote" -version = "1.0.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "rancor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" -dependencies = [ - "ptr_meta", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand 0.8.5", -] - -[[package]] -name = "rand_distr" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" -dependencies = [ - "num-traits", - "rand 0.9.2", -] - -[[package]] -name = "rand_xoshiro" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" -dependencies = [ - "rand_core 0.9.3", -] - -[[package]] -name = "rangemap" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbbbbea733ec66275512d0b9694f34102e7d5406fdbe2ad8d21b28dce92887c" - -[[package]] -name = "raw-cpuid" -version = "11.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" -dependencies = [ - "bitflags 2.10.0", -] - -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - -[[package]] -name = "rayon" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "recursive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" -dependencies = [ - "recursive-proc-macro-impl", - "stacker", -] - -[[package]] -name = "recursive-proc-macro-impl" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" -dependencies = [ - "quote", - "syn 2.0.117", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags 2.10.0", -] - -[[package]] -name = "redox_users" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" -dependencies = [ - "getrandom 0.2.16", - "libredox", - "thiserror 2.0.18", -] - -[[package]] -name = "ref-cast" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "regex" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" - -[[package]] -name = "rend" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" -dependencies = [ - "bytecheck", -] - -[[package]] -name = "reqwest" -version = "0.12.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6eff9328d40131d43bd911d42d79eb6a47312002a4daefc9e37f17e74a7701a" -dependencies = [ - "base64 0.22.1", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "mime", - "mime_guess", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", -] - -[[package]] -name = "rgb" -version = "0.8.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" -dependencies = [ - "bytemuck", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.16", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rkyv" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" -dependencies = [ - "bytecheck", - "bytes", - "hashbrown 0.17.1", - "indexmap 2.14.0", - "munge", - "ptr_meta", - "rancor", - "rend", - "rkyv_derive", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "roaring" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" -dependencies = [ - "bytemuck", - "byteorder", -] - -[[package]] -name = "rs" -version = "0.1.0" -dependencies = [ - "arrow-array", - "arrow-json", - "arrow-schema", - "futures", - "futures-util", - "lance-namespace 7.0.0", - "lancedb", - "polars", - "polars-arrow", - "serde", - "serde_json", - "tempfile", - "tokio", -] - -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" - -[[package]] -name = "rustc-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.10.0", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" -dependencies = [ - "bitflags 2.10.0", - "errno", - "libc", - "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls" -version = "0.23.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" -dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pki-types" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "schemars" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - -[[package]] -name = "schemars" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "security-framework" -version = "3.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" -dependencies = [ - "bitflags 2.10.0", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" - -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "serde_json" -version = "1.0.150" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_repr" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "3.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" -dependencies = [ - "base64 0.22.1", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.14.0", - "schemars 0.9.0", - "schemars 1.1.0", - "serde_core", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" -dependencies = [ - "darling 0.21.3", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "serde_yaml_ng" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" -dependencies = [ - "indexmap 2.14.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha1_smol" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" -dependencies = [ - "libc", -] - -[[package]] -name = "simd-adler32" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" - -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - -[[package]] -name = "siphasher" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" - -[[package]] -name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "smartstring" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" -dependencies = [ - "autocfg", - "static_assertions", - "version_check", -] - -[[package]] -name = "snafu" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" -dependencies = [ - "snafu-derive 0.8.9", -] - -[[package]] -name = "snafu" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" -dependencies = [ - "snafu-derive 0.9.1", -] - -[[package]] -name = "snafu-derive" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "snafu-derive" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "socket2" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" -dependencies = [ - "libc", - "windows-sys 0.60.2", -] - -[[package]] -name = "spin" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" -dependencies = [ - "lock_api", -] - -[[package]] -name = "sqlparser" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" -dependencies = [ - "log", -] - -[[package]] -name = "sqlparser" -version = "0.61.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" -dependencies = [ - "log", - "sqlparser_derive", -] - -[[package]] -name = "sqlparser_derive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "stacker" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" -dependencies = [ - "cc", - "cfg-if", - "libc", - "psm", - "windows-sys 0.59.0", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "std_prelude" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe" - -[[package]] -name = "stfu8" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978" - -[[package]] -name = "stop-words" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68df56303396bcfb639455b3c166804aeb7994005010aab5e9e8a1277b8871d" -dependencies = [ - "serde_json", -] - -[[package]] -name = "str_stack" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f446288b699d66d0fd2e30d1cfe7869194312524b3b9252594868ed26ef056a" - -[[package]] -name = "streaming-decompression" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" -dependencies = [ - "fallible-streaming-iterator", -] - -[[package]] -name = "streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" - -[[package]] -name = "strength_reduce" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - -[[package]] -name = "strum" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" -dependencies = [ - "strum_macros 0.28.0", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - -[[package]] -name = "strum_macros" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "symbolic-common" -version = "12.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" -dependencies = [ - "debugid", - "memmap2 0.9.10", - "stable_deref_trait", - "uuid", -] - -[[package]] -name = "symbolic-demangle" -version = "12.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" -dependencies = [ - "cpp_demangle", - "rustc-demangle", - "symbolic-common", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "sysinfo" -version = "0.30.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" -dependencies = [ - "cfg-if", - "core-foundation-sys", - "libc", - "ntapi", - "once_cell", - "windows 0.52.0", -] - -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - -[[package]] -name = "target-features" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" - -[[package]] -name = "tempfile" -version = "3.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" -dependencies = [ - "fastrand", - "getrandom 0.3.4", - "once_cell", - "rustix 1.1.2", - "windows-sys 0.61.2", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "thread-tree" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbd370cb847953a25954d9f63e14824a36113f8c72eecf6eccef5dc4b45d630" -dependencies = [ - "crossbeam-channel", -] - -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "time" -version = "0.3.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" -dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" - -[[package]] -name = "time-macros" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "serde_core", - "zerovec", -] - -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "tinyvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" -dependencies = [ - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-macros" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", -] - -[[package]] -name = "tokio-util" -version = "0.7.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" -dependencies = [ - "async-compression", - "bitflags 2.10.0", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "iri-string", - "pin-project-lite", - "tokio", - "tokio-util", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "tracing-core" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "twox-hash" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -dependencies = [ - "rand 0.9.2", -] - -[[package]] -name = "typenum" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" - -[[package]] -name = "unicase" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" - -[[package]] -name = "unicode-blocks" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b12e05d9e06373163a9bb6bb8c263c261b396643a99445fe6b9811fd376581b" - -[[package]] -name = "unicode-ident" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" - -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-reverse" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6f4888ebc23094adfb574fdca9fdc891826287a6397d2cd28802ffd6f20c76" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "unicode-segmentation" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8-ranges" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "uuid" -version = "1.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" -dependencies = [ - "getrandom 0.4.2", - "js-sys", - "serde_core", - "sha1_smol", - "wasm-bindgen", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" -dependencies = [ - "wit-bindgen 0.46.0", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 2.0.117", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.10.0", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - -[[package]] -name = "web-sys" -version = "0.3.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" -dependencies = [ - "windows-core 0.52.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections", - "windows-core 0.61.2", - "windows-future", - "windows-link 0.1.3", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core 0.61.2", -] - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", - "windows-threading", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", -] - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link 0.2.1", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link 0.2.1", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link 0.2.1", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link 0.2.1", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - -[[package]] -name = "windows-threading" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" -dependencies = [ - "windows-link 0.1.3", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - -[[package]] -name = "wit-bindgen" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck 0.5.0", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck 0.5.0", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.10.0", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - -[[package]] -name = "writeable" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" - -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - -[[package]] -name = "xxhash-rust" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "serde", - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/tests/rs/Cargo.toml b/tests/rs/Cargo.toml deleted file mode 100644 index c8a4cc7..0000000 --- a/tests/rs/Cargo.toml +++ /dev/null @@ -1,43 +0,0 @@ -[package] -name = "rs" -version = "0.1.0" -edition = "2024" - -[dependencies] -arrow-array = "58.3.0" -arrow-json = "58.3.0" -arrow-schema = "58.3.0" -futures = "0.3.31" -futures-util = "0.3.31" -lance-namespace = "7.0.0" -lancedb = { version = "0.31.0", features = ["polars"] } -polars = ">=0.37,<0.40.0" -polars-arrow = ">=0.37,<0.40.0" -serde = { version = "1", features = ["derive"] } -serde_json = "1.0.145" -tempfile = "3.23.0" -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } - -[[example]] -name = "basic_usage" -path = "basic_usage.rs" - -[[example]] -name = "connection" -path = "connection.rs" - -[[example]] -name = "quickstart" -path = "quickstart.rs" - -[[example]] -name = "tables" -path = "tables.rs" - -[[example]] -name = "multimodal" -path = "multimodal.rs" - -[[example]] -name = "search" -path = "search.rs" diff --git a/tests/rs/basic_usage.rs b/tests/rs/basic_usage.rs deleted file mode 100644 index 0adcef4..0000000 --- a/tests/rs/basic_usage.rs +++ /dev/null @@ -1,346 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -use std::fs; -use std::path::PathBuf; -use std::sync::Arc; - -use serde::Deserialize; - -type BatchIter = Box; - -// --8<-- [start:basic_imports] -use arrow_array::types::Float32Type; -use arrow_array::{ - FixedSizeListArray, Int8Array, Int16Array, RecordBatch, RecordBatchIterator, StringArray, - StructArray, -}; -use arrow_schema::{DataType, Field, FieldRef, Schema}; -use futures_util::TryStreamExt; -use lancedb::database::CreateTableMode; -use lancedb::query::{ExecutableQuery, QueryBase, Select}; -use lancedb::{connect, table::NewColumnTransform}; -// --8<-- [end:basic_imports] - -#[derive(Debug, Clone, Deserialize)] -struct Stats { - strength: i8, - courage: i8, - magic: i8, - wisdom: i8, -} - -#[derive(Debug, Clone, Deserialize)] -struct Character { - id: i16, - name: String, - role: String, - description: String, - vector: [f32; 4], - stats: Stats, -} - -fn camelot_schema() -> Arc { - Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int16, false), - Field::new("name", DataType::Utf8, false), - Field::new("role", DataType::Utf8, false), - Field::new("description", DataType::Utf8, false), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), - false, - ), - Field::new( - "stats", - DataType::Struct(arrow_schema::Fields::from(vec![ - Arc::new(Field::new("strength", DataType::Int8, false)), - Arc::new(Field::new("courage", DataType::Int8, false)), - Arc::new(Field::new("magic", DataType::Int8, false)), - Arc::new(Field::new("wisdom", DataType::Int8, false)), - ])), - false, - ), - ])) -} - -fn characters_to_record_batch(schema: Arc, characters: &[Character]) -> RecordBatch { - let ids = Int16Array::from_iter_values(characters.iter().map(|c| c.id)); - let names = StringArray::from_iter_values(characters.iter().map(|c| c.name.as_str())); - let roles = StringArray::from_iter_values(characters.iter().map(|c| c.role.as_str())); - let descriptions = - StringArray::from_iter_values(characters.iter().map(|c| c.description.as_str())); - - let vectors = FixedSizeListArray::from_iter_primitive::( - characters - .iter() - .map(|c| Some(c.vector.iter().copied().map(Some).collect::>())), - 4, - ); - - let strength = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.strength)); - let courage = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.courage)); - let magic = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.magic)); - let wisdom = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.wisdom)); - - let stats_fields: Vec = vec![ - Arc::new(Field::new("strength", DataType::Int8, false)), - Arc::new(Field::new("courage", DataType::Int8, false)), - Arc::new(Field::new("magic", DataType::Int8, false)), - Arc::new(Field::new("wisdom", DataType::Int8, false)), - ]; - let stats = StructArray::new( - stats_fields.into(), - vec![ - Arc::new(strength), - Arc::new(courage), - Arc::new(magic), - Arc::new(wisdom), - ], - None, - ); - - RecordBatch::try_new( - schema, - vec![ - Arc::new(ids), - Arc::new(names), - Arc::new(roles), - Arc::new(descriptions), - Arc::new(vectors), - Arc::new(stats), - ], - ) - .unwrap() -} - -fn characters_to_reader(schema: Arc, characters: &[Character]) -> BatchIter { - let batch = characters_to_record_batch(schema.clone(), characters); - Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema)) -} - -fn camelot_json_path() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join("camelot.json") -} - -#[tokio::main] -async fn main() { - let temp_dir = tempfile::tempdir().unwrap(); - let uri = temp_dir.path().to_str().unwrap(); - let db = connect(uri).execute().await.unwrap(); - - // --8<-- [start:data_load] - let data: Vec = - serde_json::from_str(&fs::read_to_string(camelot_json_path()).unwrap()).unwrap(); - // --8<-- [end:data_load] - - let schema = camelot_schema(); - - // --8<-- [start:basic_create_table] - let mut table = db - .create_table("camelot", characters_to_reader(schema.clone(), &data)) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:basic_create_table] - assert_eq!(table.count_rows(None).await.unwrap(), 8); - - // --8<-- [start:basic_open_table] - table = db.open_table("camelot").execute().await.unwrap(); - // --8<-- [end:basic_open_table] - - // --8<-- [start:basic_create_empty_table] - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int16, false), - Field::new("name", DataType::Utf8, false), - Field::new("role", DataType::Utf8, false), - Field::new("description", DataType::Utf8, false), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), - false, - ), - Field::new( - "stats", - DataType::Struct(arrow_schema::Fields::from(vec![ - Arc::new(Field::new("strength", DataType::Int8, false)), - Arc::new(Field::new("courage", DataType::Int8, false)), - Arc::new(Field::new("magic", DataType::Int8, false)), - Arc::new(Field::new("wisdom", DataType::Int8, false)), - ])), - false, - ), - ])); - db.create_empty_table("camelot_empty", schema) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:basic_create_empty_table] - db.drop_table("camelot_empty", &[]).await.unwrap(); - - // --8<-- [start:basic_add_data] - let magical_characters = vec![ - Character { - id: 9, - name: "Morgan le Fay".to_string(), - role: "Sorceress".to_string(), - description: "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.".to_string(), - vector: [0.10, 0.84, 0.25, 0.70], - stats: Stats { - strength: 2, - courage: 3, - magic: 5, - wisdom: 4, - }, - }, - Character { - id: 10, - name: "The Lady of the Lake".to_string(), - role: "Mystical Guardian".to_string(), - description: "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.".to_string(), - vector: [0.00, 0.90, 0.58, 0.88], - stats: Stats { - strength: 2, - courage: 3, - magic: 5, - wisdom: 5, - }, - }, - ]; - table - .add(characters_to_reader(camelot_schema(), &magical_characters)) - .execute() - .await - .unwrap(); - // --8<-- [end:basic_add_data] - - // --8<-- [start:basic_vector_search] - let query_vector = [0.03, 0.85, 0.61, 0.90]; - let result = table - .query() - .nearest_to(&query_vector) - .unwrap() - .limit(5) - .execute() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - println!("{result:?}"); - // --8<-- [end:basic_vector_search] - - // --8<-- [start:basic_add_columns] - table - .add_columns( - NewColumnTransform::SqlExpressions(vec![( - "power".to_string(), - "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)" - .to_string(), - )]), - None, - ) - .await - .unwrap(); - // --8<-- [end:basic_add_columns] - - // --8<-- [start:basic_vector_search_q1] - // Who are the characters similar to "wizard"? - let query_vector_1 = [0.03, 0.85, 0.61, 0.90]; - let r1 = table - .query() - .nearest_to(&query_vector_1) - .unwrap() - .limit(5) - .select(Select::Columns(vec![ - "name".to_string(), - "role".to_string(), - "description".to_string(), - ])) - .execute() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - println!("{r1:?}"); - // --8<-- [end:basic_vector_search_q1] - - // --8<-- [start:basic_vector_search_q2] - // Who are the characters similar to "wizard" with high magic stats? - let query_vector_2 = [0.03, 0.85, 0.61, 0.90]; - let r2 = table - .query() - .nearest_to(&query_vector_2) - .unwrap() - .only_if("stats.magic > 3") - .select(Select::Columns(vec![ - "name".to_string(), - "role".to_string(), - "description".to_string(), - ])) - .limit(5) - .execute() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - println!("{r2:?}"); - // --8<-- [end:basic_vector_search_q2] - - // --8<-- [start:basic_vector_search_q3] - // Who are the strongest characters? - let r3 = table - .query() - .only_if("stats.strength > 3") - .select(Select::Columns(vec![ - "name".to_string(), - "role".to_string(), - "description".to_string(), - ])) - .limit(5) - .execute() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - println!("{r3:?}"); - // --8<-- [end:basic_vector_search_q3] - - // --8<-- [start:basic_vector_search_q4] - // Who are the strongest characters? - let r4 = table - .query() - .select(Select::Columns(vec![ - "name".to_string(), - "role".to_string(), - "description".to_string(), - "power".to_string(), - ])) - .execute() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - println!("{r4:?}"); - // --8<-- [end:basic_vector_search_q4] - - // --8<-- [start:basic_drop_columns] - table.drop_columns(&["power"]).await.unwrap(); - // --8<-- [end:basic_drop_columns] - - // --8<-- [start:basic_delete_rows] - table.delete("role = 'Traitor Knight'").await.unwrap(); - // --8<-- [end:basic_delete_rows] - - // --8<-- [start:basic_drop_table] - db.drop_table("camelot", &[]).await.unwrap(); - // --8<-- [end:basic_drop_table] -} diff --git a/tests/rs/bedrock.rs b/tests/rs/bedrock.rs deleted file mode 100644 index 5cc7e0c..0000000 --- a/tests/rs/bedrock.rs +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -use std::{iter::once, sync::Arc}; - -use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray}; -use arrow_schema::{DataType, Field, Schema}; -use aws_config::Region; -use aws_sdk_bedrockruntime::Client; -use futures::StreamExt; -use lancedb::{ - arrow::IntoArrow, - connect, - embeddings::{bedrock::BedrockEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction}, - query::{ExecutableQuery, QueryBase}, - Result, -}; - -#[tokio::main] -async fn main() -> Result<()> { - let tempdir = tempfile::tempdir().unwrap(); - let tempdir = tempdir.path().to_str().unwrap(); - - // create Bedrock embedding function - let region: String = "us-east-1".to_string(); - let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) - .region(Region::new(region)) - .load() - .await; - - let embedding = Arc::new(BedrockEmbeddingFunction::new( - Client::new(&config), // AWS Region - )); - - let db = connect(tempdir).execute().await?; - db.embedding_registry() - .register("bedrock", embedding.clone())?; - - let table = db - .create_table("vectors", make_data()) - .add_embedding(EmbeddingDefinition::new( - "text", - "bedrock", - Some("embeddings"), - ))? - .execute() - .await?; - - // execute vector search - let query = Arc::new(StringArray::from_iter_values(once("something warm"))); - let query_vector = embedding.compute_query_embeddings(query)?; - let mut results = table - .vector_search(query_vector)? - .limit(1) - .execute() - .await?; - - let rb = results.next().await.unwrap()?; - let out = rb - .column_by_name("text") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - let text = out.iter().next().unwrap().unwrap(); - println!("Closest match: {}", text); - Ok(()) -} - -fn make_data() -> impl IntoArrow { - let schema = Schema::new(vec![ - Field::new("id", DataType::Int32, true), - Field::new("text", DataType::Utf8, false), - Field::new("price", DataType::Float64, false), - ]); - - let id = Int32Array::from(vec![1, 2, 3, 4]); - let text = StringArray::from_iter_values(vec![ - "Black T-Shirt", - "Leather Jacket", - "Winter Parka", - "Hooded Sweatshirt", - ]); - let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]); - let schema = Arc::new(schema); - let rb = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(id), Arc::new(text), Arc::new(price)], - ) - .unwrap(); - Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema)) -} diff --git a/tests/rs/connection.rs b/tests/rs/connection.rs deleted file mode 100644 index 4f7212e..0000000 --- a/tests/rs/connection.rs +++ /dev/null @@ -1,134 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -use std::path::PathBuf; - -use lancedb::connect; - -// --8<-- [start:connect] -async fn connect_example(uri: &str) { - let db = connect(uri).execute().await.unwrap(); - let _ = db; -} -// --8<-- [end:connect] - -#[tokio::main] -async fn main() { - let temp_dir = tempfile::tempdir().unwrap(); - let uri = temp_dir.path().join("ex_lancedb"); - connect_example(uri.to_str().unwrap()).await; - - // Keep the enterprise snippet in this file, but don't run it in CI. - let _ = connect_enterprise_quickstart_config(); - let _ = connect_object_storage_config(); -} - -fn connect_enterprise_quickstart_config() -> (String, String, String, String) { - // --8<-- [start:connect_enterprise_quickstart] - let uri = "db://your-database-uri"; - let api_key = "your-api-key"; - let region = "us-east-1"; - let host_override = "https://your-enterprise-endpoint.com"; - // --8<-- [end:connect_enterprise_quickstart] - ( - uri.to_string(), - api_key.to_string(), - region.to_string(), - host_override.to_string(), - ) -} - -fn connect_object_storage_config() -> &'static str { - // --8<-- [start:connect_object_storage] - let uri = "s3://your-bucket/path"; - // You can also use "gs://your-bucket/path" or "az://your-container/path". - // --8<-- [end:connect_object_storage] - - uri -} - -async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> { - // --8<-- [start:namespace_table_ops] - let conn = connect(uri).execute().await?; - let search_namespace = vec!["prod".to_string(), "search".to_string()]; - let recommendations_namespace = vec!["prod".to_string(), "recommendations".to_string()]; - - let schema = std::sync::Arc::new(arrow_schema::Schema::new(vec![ - arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false), - ])); - - conn.create_empty_table("user", schema.clone()) - .namespace(search_namespace.clone()) - .execute() - .await?; - - conn.create_empty_table("user", schema) - .namespace(recommendations_namespace.clone()) - .execute() - .await?; - - let search_table_names = conn - .table_names() - .namespace(search_namespace) - .execute() - .await?; - let recommendation_table_names = conn - .table_names() - .namespace(recommendations_namespace) - .execute() - .await?; - - println!("{search_table_names:?}"); // ["user"] - println!("{recommendation_table_names:?}"); // ["user"] - // --8<-- [end:namespace_table_ops] - Ok(()) -} - -async fn namespace_admin_ops_example() -> lancedb::Result<()> { - // --8<-- [start:namespace_admin_ops] - let mut properties = std::collections::HashMap::new(); - properties.insert("root".to_string(), "./local_lancedb".to_string()); - let db = lancedb::connect_namespace("dir", properties).execute().await?; - let namespace = vec!["prod".to_string(), "search".to_string()]; - - db.create_namespace(lance_namespace::models::CreateNamespaceRequest { - id: Some(vec!["prod".to_string()]), - ..Default::default() - }) - .await?; - db.create_namespace(lance_namespace::models::CreateNamespaceRequest { - id: Some(namespace.clone()), - ..Default::default() - }) - .await?; - - let child_namespaces = db - .list_namespaces(lance_namespace::models::ListNamespacesRequest { - id: Some(vec!["prod".to_string()]), - ..Default::default() - }) - .await?; - println!( - "Child namespaces under {:?}: {:?}", - namespace, child_namespaces - ); - // Child namespaces under ["prod", "search"]: ["search"] - - db.drop_namespace(lance_namespace::models::DropNamespaceRequest { - id: Some(namespace.clone()), - ..Default::default() - }) - .await?; - db.drop_namespace(lance_namespace::models::DropNamespaceRequest { - id: Some(vec!["prod".to_string()]), - ..Default::default() - }) - .await?; - // --8<-- [end:namespace_admin_ops] - Ok(()) -} - -#[allow(dead_code)] -fn repo_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..") -} diff --git a/tests/rs/embedding.rs b/tests/rs/embedding.rs deleted file mode 100644 index 33bdec9..0000000 --- a/tests/rs/embedding.rs +++ /dev/null @@ -1,182 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -// --8<-- [start:openai_embeddings] -use std::{iter::once, sync::Arc}; - -use arrow_array::{record_batch, StringArray}; -use arrow_schema::{DataType, Field, Schema}; -use futures::StreamExt; -use lancedb::{ - connect, - embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction}, - query::{ExecutableQuery, QueryBase}, - Result, -}; - -#[tokio::main] -async fn main() -> Result<()> { - let db = connect("./mydb").execute().await?; - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); - let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( - api_key, - "text-embedding-3-large", - )?); - - db.embedding_registry().register("openai", embedding.clone())?; - - let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)])); - let table = db - .create_empty_table("mytable", schema) - .add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))? - .execute() - .await?; - - table - .add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?) - .execute() - .await?; - - let query = Arc::new(StringArray::from_iter_values(once("test example"))); - let query_vector = embedding.compute_query_embeddings(query)?; - let mut results = table.vector_search(query_vector)?.limit(5).execute().await?; - - while let Some(batch) = results.next().await { - println!("{:?}", batch?); - } - - Ok(()) -} -// --8<-- [end:openai_embeddings] - -// --8<-- [start:create_embedding_function] -use std::sync::Arc; - -use lancedb::embeddings::openai::OpenAIEmbeddingFunction; - -let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); -let embedding = Arc::new( - OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small") - .expect("failed to create OpenAI embedding function"), -); -// --8<-- [end:create_embedding_function] - -// --8<-- [start:manual_query_embeddings] -use std::{iter::once, sync::Arc}; - -use arrow_array::{record_batch, StringArray}; -use arrow_schema::{DataType, Field, Schema}; -use futures::StreamExt; -use lancedb::{ - connect, - embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction}, - query::{ExecutableQuery, QueryBase}, - Result, -}; - -#[tokio::main] -async fn main() -> Result<()> { - let db = connect("./mydb").execute().await?; - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); - let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( - api_key, - "text-embedding-3-large", - )?); - db.embedding_registry().register("openai", embedding.clone())?; - - let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)])); - let table = db - .create_empty_table("mytable", schema) - .add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))? - .execute() - .await?; - - table - .add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?) - .execute() - .await?; - - // Manually generate embeddings for the query (Enterprise path) - let query = Arc::new(StringArray::from_iter_values(once("test example"))); - let query_vector = embedding.compute_query_embeddings(query)?; - // --8<-- [start:manual_query_search] - // query_vector is assumed to already be generated by your embedding function - let mut results = table.vector_search(query_vector)?.limit(5).execute().await?; - - while let Some(batch) = results.next().await { - println!("{:?}", batch?); - } - // --8<-- [end:manual_query_search] - - Ok(()) -} -// --8<-- [end:manual_query_embeddings] - -// --8<-- [start:embedding_function] -use std::{borrow::Cow, sync::Arc}; - -use arrow_array::{Array, FixedSizeListArray, Float32Array}; -use arrow_schema::{DataType, Field, Schema}; -use lancedb::{ - connect, - embeddings::{EmbeddingDefinition, EmbeddingFunction}, - Result, -}; - -#[derive(Debug, Clone)] -struct MyTextEmbedder { - dim: usize, -} - -impl EmbeddingFunction for MyTextEmbedder { - fn name(&self) -> &str { - "my-embedder" - } - - fn source_type(&self) -> Result> { - Ok(Cow::Owned(DataType::Utf8)) - } - - fn dest_type(&self) -> Result> { - Ok(Cow::Owned(DataType::new_fixed_size_list( - DataType::Float32, - self.dim as i32, - true, - ))) - } - - fn compute_source_embeddings(&self, source: Arc) -> Result> { - let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim])); - let field = Arc::new(Field::new("item", DataType::Float32, true)); - Ok(Arc::new(FixedSizeListArray::new( - field, - self.dim as i32, - values, - None, - ))) - } - - fn compute_query_embeddings(&self, _input: Arc) -> Result> { - unimplemented!() - } -} - -#[tokio::main] -async fn main() -> Result<()> { - let db = connect("./mydb").execute().await?; - db.embedding_registry() - .register("my-embedder", Arc::new(MyTextEmbedder { dim: 3 }))?; - - let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)])); - db.create_empty_table("mytable", schema) - .add_embedding(EmbeddingDefinition::new( - "text", - "my-embedder", - Some("vector"), - ))? - .execute() - .await?; - - Ok(()) -} -// --8<-- [end:embedding_function] diff --git a/tests/rs/full_text_search.rs b/tests/rs/full_text_search.rs deleted file mode 100644 index b79400a..0000000 --- a/tests/rs/full_text_search.rs +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -use std::sync::Arc; - -use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; -use arrow_schema::{DataType, Field, Schema}; - -use futures::TryStreamExt; -use lance_index::scalar::FullTextSearchQuery; -use lancedb::connection::Connection; -use lancedb::index::scalar::FtsIndexBuilder; -use lancedb::index::Index; -use lancedb::query::{ExecutableQuery, QueryBase}; -use lancedb::{connect, Result, Table}; -use rand::random; - -#[tokio::main] -async fn main() -> Result<()> { - if std::path::Path::new("data").exists() { - std::fs::remove_dir_all("data").unwrap(); - } - let uri = "data/sample-lancedb"; - let db = connect(uri).execute().await?; - let tbl = create_table(&db).await?; - - create_index(&tbl).await?; - search_index(&tbl).await?; - Ok(()) -} - -fn create_some_records() -> Result> { - const TOTAL: usize = 1000; - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("doc", DataType::Utf8, true), - ])); - - let words = random_word::all(random_word::Lang::En) - .iter() - .step_by(1024) - .take(500) - .copied() - .collect::>(); - let n_terms = 3; - let batches = RecordBatchIterator::new( - vec![RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)), - Arc::new(StringArray::from_iter_values((0..TOTAL).map(|_| { - (0..n_terms) - .map(|_| words[random::() % words.len()]) - .collect::>() - .join(" ") - }))), - ], - ) - .unwrap()] - .into_iter() - .map(Ok), - schema.clone(), - ); - Ok(Box::new(batches)) -} - -async fn create_table(db: &Connection) -> Result { - let initial_data: Box = create_some_records()?; - let tbl = db.create_table("my_table", initial_data).execute().await?; - Ok(tbl) -} - -async fn create_index(table: &Table) -> Result<()> { - table - .create_index(&["doc"], Index::FTS(FtsIndexBuilder::default())) - .execute() - .await?; - Ok(()) -} - -async fn search_index(table: &Table) -> Result<()> { - let words = random_word::all(random_word::Lang::En) - .iter() - .step_by(1024) - .take(500) - .copied() - .collect::>(); - let query = words[0].to_owned(); - println!("Searching for: {}", query); - - let mut results = table - .query() - .full_text_search(FullTextSearchQuery::new(words[0].to_owned())) - .select(lancedb::query::Select::Columns(vec!["doc".to_owned()])) - .limit(10) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - Ok(()) -} diff --git a/tests/rs/ivf_pq.rs b/tests/rs/ivf_pq.rs deleted file mode 100644 index ccecc22..0000000 --- a/tests/rs/ivf_pq.rs +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -//! This example demonstrates setting advanced parameters when building an IVF PQ index -//! -//! Snippets from this example are used in the documentation on ANN indices. - -use std::sync::Arc; - -use arrow_array::types::Float32Type; -use arrow_array::{ - FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, -}; -use arrow_schema::{DataType, Field, Schema}; - -use futures::TryStreamExt; -use lancedb::connection::Connection; -use lancedb::index::vector::IvfPqIndexBuilder; -use lancedb::index::Index; -use lancedb::query::{ExecutableQuery, QueryBase}; -use lancedb::{connect, DistanceType, Result, Table}; - -#[tokio::main] -async fn main() -> Result<()> { - if std::path::Path::new("data").exists() { - std::fs::remove_dir_all("data").unwrap(); - } - let uri = "data/sample-lancedb"; - let db = connect(uri).execute().await?; - let tbl = create_table(&db).await?; - - create_index(&tbl).await?; - search_index(&tbl).await?; - Ok(()) -} - -fn create_some_records() -> Result> { - const TOTAL: usize = 1000; - const DIM: usize = 128; - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vector", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Float32, true)), - DIM as i32, - ), - true, - ), - ])); - - // Create a RecordBatch stream. - let batches = RecordBatchIterator::new( - vec![RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])), - DIM as i32, - ), - ), - ], - ) - .unwrap()] - .into_iter() - .map(Ok), - schema.clone(), - ); - Ok(Box::new(batches)) -} - -async fn create_table(db: &Connection) -> Result
{ - let initial_data: Box = create_some_records()?; - let tbl = db - .create_table("my_table", initial_data) - .execute() - .await - .unwrap(); - Ok(tbl) -} - -async fn create_index(table: &Table) -> Result<()> { - // --8<-- [start:create_index] - // For this example, `table` is a lancedb::Table with a column named - // "vector" that is a vector column with dimension 128. - - // By default, if the column "vector" appears to be a vector column, - // then an IVF_PQ index with reasonable defaults is created. - table - .create_index(&["vector"], Index::Auto) - .execute() - .await?; - // For advanced cases, it is also possible to specifically request an - // IVF_PQ index and provide custom parameters. - table - .create_index( - &["vector"], - Index::IvfPq( - // Here we specify advanced indexing parameters. In this case - // we are creating an index that my have better recall than the - // default but is also larger and slower. - IvfPqIndexBuilder::default() - // This overrides the default distance type of l2 - .distance_type(DistanceType::Cosine) - // With 1000 rows this have been ~31 by default - .num_partitions(50) - // With dimension 128 this would have been 8 by default - .num_sub_vectors(16), - ), - ) - .execute() - .await?; - // --8<-- [end:create_index] - Ok(()) -} - -async fn search_index(table: &Table) -> Result<()> { - // --8<-- [start:search1] - let query_vector = [1.0; 128]; - // By default the index will find the 10 closest results using default - // search parameters that give a reasonable tradeoff between accuracy - // and search latency - let mut results = table - .vector_search(&query_vector)? - // Note: you should always set the distance_type to match the value used - // to train the index - .distance_type(DistanceType::Cosine) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // We can also provide custom search parameters. Here we perform a - // slower but more accurate search - let mut results = table - .vector_search(&query_vector)? - .distance_type(DistanceType::Cosine) - // Override the default of 10 to get more rows - .limit(15) - // Override the default of 20 to search more partitions - .nprobes(30) - // Override the default of None to apply a refine step - .refine_factor(1) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - Ok(()) - // --8<-- [end:search1] -} diff --git a/tests/rs/multimodal.rs b/tests/rs/multimodal.rs deleted file mode 100644 index 8cae02e..0000000 --- a/tests/rs/multimodal.rs +++ /dev/null @@ -1,178 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -// --8<-- [start:multimodal_imports] -use std::collections::HashMap; -use std::sync::Arc; - -use arrow_array::types::Float32Type; -use arrow_array::{ - BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch, - RecordBatchIterator, StringArray, -}; -use arrow_schema::{DataType, Field, Schema}; -use futures_util::TryStreamExt; -use lancedb::connect; -use lancedb::database::CreateTableMode; -use lancedb::query::{ExecutableQuery, QueryBase}; -// --8<-- [end:multimodal_imports] - -#[tokio::main] -async fn main() { - let temp_dir = tempfile::tempdir().unwrap(); - let db_uri = temp_dir.path().to_str().unwrap().to_string(); - let db = connect(&db_uri).execute().await.unwrap(); - - // --8<-- [start:create_dummy_data] - let create_dummy_image = |color: u8| -> Vec { - let mut png_like = vec![137, 80, 78, 71, 13, 10, 26, 10]; - png_like.push(color); - png_like - }; - - let data = vec![ - ( - 1_i32, - "red_square.png", - vec![0.1_f32; 128], - create_dummy_image(1), - "red", - ), - ( - 2_i32, - "blue_square.png", - vec![0.2_f32; 128], - create_dummy_image(2), - "blue", - ), - ]; - // --8<-- [end:create_dummy_data] - - // --8<-- [start:define_schema] - let schema = Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("filename", DataType::Utf8, false), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 128), - false, - ), - Field::new("image_blob", DataType::Binary, false), - Field::new("label", DataType::Utf8, false), - ]); - // --8<-- [end:define_schema] - - // --8<-- [start:ingest_data] - let schema = Arc::new(schema); - let image_batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(data.iter().map(|row| row.0))), - Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.1))), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - data.iter() - .map(|row| Some(row.2.iter().copied().map(Some).collect::>())), - 128, - ), - ), - Arc::new(BinaryArray::from_iter_values( - data.iter().map(|row| row.3.as_slice()), - )), - Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.4))), - ], - ) - .unwrap(); - let image_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone())); - let table = db - .create_table("images", image_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:ingest_data] - assert_eq!(table.count_rows(None).await.unwrap(), 2); - - // --8<-- [start:search_data] - let query_vector = vec![0.1_f32; 128]; - let results = table - .query() - .nearest_to(query_vector) - .unwrap() - .limit(1) - .execute() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - // --8<-- [end:search_data] - - // --8<-- [start:process_results] - for batch in &results { - let filenames = batch - .column_by_name("filename") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - let images = batch - .column_by_name("image_blob") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - - for row in 0..batch.num_rows() { - let image_bytes = images.value(row); - println!( - "Retrieved image: {}, Byte length: {}", - filenames.value(row), - image_bytes.len() - ); - } - } - // --8<-- [end:process_results] - let search_rows: usize = results.iter().map(|batch| batch.num_rows()).sum(); - assert_eq!(search_rows, 1); - - // --8<-- [start:blob_api_schema] - let blob_metadata = HashMap::from([( - "lance-encoding:blob".to_string(), - "true".to_string(), - )]); - let blob_schema = Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("video", DataType::LargeBinary, true).with_metadata(blob_metadata), - ]); - // --8<-- [end:blob_api_schema] - - // --8<-- [start:blob_api_ingest] - let blob_rows = vec![ - (1_i64, b"fake_video_bytes_1".to_vec()), - (2_i64, b"fake_video_bytes_2".to_vec()), - ]; - - let blob_schema = Arc::new(blob_schema); - let blob_batch = RecordBatch::try_new( - blob_schema.clone(), - vec![ - Arc::new(Int64Array::from_iter_values(blob_rows.iter().map(|row| row.0))), - Arc::new(LargeBinaryArray::from_iter_values( - blob_rows.iter().map(|row| row.1.as_slice()), - )), - ], - ) - .unwrap(); - let blob_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(blob_batch)].into_iter(), blob_schema)); - let blob_table = db - .create_table("videos", blob_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:blob_api_ingest] - assert_eq!(blob_table.count_rows(None).await.unwrap(), 2); -} diff --git a/tests/rs/openai.rs b/tests/rs/openai.rs deleted file mode 100644 index 73954f3..0000000 --- a/tests/rs/openai.rs +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -// --8<-- [start:imports] - -use std::{iter::once, sync::Arc}; - -use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray}; -use arrow_schema::{DataType, Field, Schema}; -use futures::StreamExt; -use lancedb::{ - arrow::IntoArrow, - connect, - embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction}, - query::{ExecutableQuery, QueryBase}, - Result, -}; - -// --8<-- [end:imports] - -// --8<-- [start:openai_embeddings] -#[tokio::main] -async fn main() -> Result<()> { - let tempdir = tempfile::tempdir().unwrap(); - let tempdir = tempdir.path().to_str().unwrap(); - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); - let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( - api_key, - "text-embedding-3-large", - )?); - - let db = connect(tempdir).execute().await?; - db.embedding_registry() - .register("openai", embedding.clone())?; - - let table = db - .create_table("vectors", make_data()) - .add_embedding(EmbeddingDefinition::new( - "text", - "openai", - Some("embeddings"), - ))? - .execute() - .await?; - - let query = Arc::new(StringArray::from_iter_values(once("something warm"))); - let query_vector = embedding.compute_query_embeddings(query)?; - let mut results = table - .vector_search(query_vector)? - .limit(1) - .execute() - .await?; - - let rb = results.next().await.unwrap()?; - let out = rb - .column_by_name("text") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - let text = out.iter().next().unwrap().unwrap(); - println!("Closest match: {}", text); - Ok(()) -} -// --8<-- [end:openai_embeddings] - -fn make_data() -> impl IntoArrow { - let schema = Schema::new(vec![ - Field::new("id", DataType::Int32, true), - Field::new("text", DataType::Utf8, false), - Field::new("price", DataType::Float64, false), - ]); - - let id = Int32Array::from(vec![1, 2, 3, 4]); - let text = StringArray::from_iter_values(vec![ - "Black T-Shirt", - "Leather Jacket", - "Winter Parka", - "Hooded Sweatshirt", - ]); - let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]); - let schema = Arc::new(schema); - let rb = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(id), Arc::new(text), Arc::new(price)], - ) - .unwrap(); - Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema)) -} diff --git a/tests/rs/quickstart.rs b/tests/rs/quickstart.rs deleted file mode 100644 index bc966aa..0000000 --- a/tests/rs/quickstart.rs +++ /dev/null @@ -1,347 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -use std::sync::Arc; - -use arrow_array::types::Float32Type; -use arrow_array::{ - FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, StructArray, -}; -use arrow_schema::{DataType, Field, FieldRef, Schema}; -use lancedb::arrow::IntoPolars; -use lancedb::database::CreateTableMode; -use lancedb::query::{ExecutableQuery, QueryBase, Select}; -use lancedb::{connect, table::NewColumnTransform}; -use polars::prelude::DataFrame; -use serde::{Deserialize, Serialize}; - -// --8<-- [start:quickstart_define_struct] -// Define structs representing the data schema -#[derive(Debug, Clone, Serialize, Deserialize)] -struct Stats { - strength: i8, - magic: i8, - leadership: i8, - wisdom: i8, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct Character { - id: String, - name: String, - role: String, - description: String, - stats: Stats, - vector: [f32; 4], -} - -fn characters_schema() -> Arc { - Arc::new(Schema::new(vec![ - Field::new("id", DataType::LargeUtf8, false), - Field::new("name", DataType::LargeUtf8, false), - Field::new("role", DataType::LargeUtf8, false), - Field::new("description", DataType::LargeUtf8, false), - Field::new( - "stats", - DataType::Struct(arrow_schema::Fields::from(vec![ - Arc::new(Field::new("strength", DataType::Int8, false)), - Arc::new(Field::new("magic", DataType::Int8, false)), - Arc::new(Field::new("leadership", DataType::Int8, false)), - Arc::new(Field::new("wisdom", DataType::Int8, false)), - ])), - false, - ), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), - false, - ), - ])) -} -// --8<-- [end:quickstart_define_struct] - -type BatchIter = Box; - -fn characters_to_reader(schema: Arc, rows: &[Character]) -> BatchIter { - let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str())); - let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str())); - let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str())); - let descriptions = - LargeStringArray::from_iter_values(rows.iter().map(|row| row.description.as_str())); - - let strength = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.strength)); - let magic = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.magic)); - let leadership = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.leadership)); - let wisdom = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.wisdom)); - let stats_fields: Vec = vec![ - Arc::new(Field::new("strength", DataType::Int8, false)), - Arc::new(Field::new("magic", DataType::Int8, false)), - Arc::new(Field::new("leadership", DataType::Int8, false)), - Arc::new(Field::new("wisdom", DataType::Int8, false)), - ]; - let stats = StructArray::new( - stats_fields.into(), - vec![ - Arc::new(strength), - Arc::new(magic), - Arc::new(leadership), - Arc::new(wisdom), - ], - None, - ); - - let vectors = FixedSizeListArray::from_iter_primitive::( - rows.iter() - .map(|row| Some(row.vector.iter().copied().map(Some).collect::>())), - 4, - ); - - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(ids), - Arc::new(names), - Arc::new(roles), - Arc::new(descriptions), - Arc::new(stats), - Arc::new(vectors), - ], - ) - .unwrap(); - - Box::new(RecordBatchIterator::new( - vec![Ok(batch)].into_iter(), - schema, - )) -} - -#[tokio::main] -async fn main() { - let temp_dir = tempfile::tempdir().unwrap(); - let uri = temp_dir.path().to_str().unwrap(); - let db = connect(uri).execute().await.unwrap(); - - // --8<-- [start:quickstart_data] - let data = vec![ - Character { - id: "1".to_string(), - name: "King Arthur".to_string(), - role: "King".to_string(), - description: "Leader of Camelot and wielder of Excalibur.".to_string(), - stats: Stats { - strength: 4, - magic: 1, - leadership: 5, - wisdom: 4, - }, - vector: [0.7, 0.1, 0.9, 0.7], - }, - Character { - id: "2".to_string(), - name: "Merlin".to_string(), - role: "Wizard".to_string(), - description: "Advisor and prophet with deep magical knowledge.".to_string(), - stats: Stats { - strength: 2, - magic: 5, - leadership: 4, - wisdom: 5, - }, - vector: [0.2, 0.9, 0.4, 0.9], - }, - Character { - id: "3".to_string(), - name: "Sir Lancelot".to_string(), - role: "Knight".to_string(), - description: "Legendary knight known for courage and combat skill.".to_string(), - stats: Stats { - strength: 5, - magic: 1, - leadership: 3, - wisdom: 3, - }, - vector: [0.9, 0.1, 0.5, 0.4], - }, - ]; - // --8<-- [end:quickstart_data] - - // --8<-- [start:quickstart_create_table] - let schema = characters_schema(); - let table = db - .create_table("characters", characters_to_reader(schema.clone(), &data)) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:quickstart_create_table] - assert_eq!(table.count_rows(None).await.unwrap(), 3); - db.drop_table("characters", &[]).await.unwrap(); - - // --8<-- [start:quickstart_create_table_no_overwrite] - let table = db - .create_table("characters", characters_to_reader(schema.clone(), &data)) - .execute() - .await - .unwrap(); - // --8<-- [end:quickstart_create_table_no_overwrite] - assert_eq!(table.count_rows(None).await.unwrap(), 3); - - // --8<-- [start:quickstart_vector_search_1] - // Search for examples similar to a "wise magical advisor" - let query_vector = [0.2, 0.8, 0.4, 0.9]; - - let result: DataFrame = table - .query() - .nearest_to(&query_vector) - .unwrap() - .select(Select::Columns(vec![ - "name".to_string(), - "role".to_string(), - "description".to_string(), - "_distance".to_string(), - ])) - .limit(2) - .execute() - .await - .unwrap() - .into_polars() - .await - .unwrap(); - println!("{result:?}"); - // --8<-- [end:quickstart_vector_search_1] - let name_col = result.column("name").unwrap().str().unwrap(); - assert_eq!(name_col.get(0).unwrap(), "Merlin"); - - // --8<-- [start:quickstart_curate_with_metadata] - let curated: DataFrame = table - .query() - .nearest_to(&query_vector) - .unwrap() - .only_if("stats.magic >= 4") - .select(Select::Columns(vec![ - "name".to_string(), - "role".to_string(), - "description".to_string(), - "_distance".to_string(), - ])) - .limit(2) - .execute() - .await - .unwrap() - .into_polars() - .await - .unwrap(); - println!("{curated:?}"); - // --8<-- [end:quickstart_curate_with_metadata] - let curated_name_col = curated.column("name").unwrap().str().unwrap(); - assert_eq!(curated_name_col.get(0).unwrap(), "Merlin"); - - // --8<-- [start:quickstart_output_array] - let result: DataFrame = table - .query() - .nearest_to(&query_vector) - .unwrap() - .select(Select::Columns(vec![ - "name".to_string(), - "role".to_string(), - "description".to_string(), - "_distance".to_string(), - ])) - .limit(2) - .execute() - .await - .unwrap() - .into_polars() - .await - .unwrap(); - println!("{result:?}"); - // --8<-- [end:quickstart_output_array] - let name_col = result.column("name").unwrap().str().unwrap(); - assert_eq!(name_col.get(0).unwrap(), "Merlin"); - - // --8<-- [start:quickstart_add_feature] - table - .add_columns( - NewColumnTransform::SqlExpressions(vec![( - "power_score".to_string(), - "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)" - .to_string(), - )]), - None, - ) - .await - .unwrap(); - // --8<-- [end:quickstart_add_feature] - - // --8<-- [start:quickstart_query_feature] - let features: DataFrame = table - .query() - .select(Select::Columns(vec![ - "name".to_string(), - "role".to_string(), - "power_score".to_string(), - ])) - .execute() - .await - .unwrap() - .into_polars() - .await - .unwrap(); - println!("{features:?}"); - // --8<-- [end:quickstart_query_feature] - assert!(features.column("power_score").is_ok()); - - // --8<-- [start:quickstart_multimodal_bytes] - use std::sync::Arc; - - use arrow_array::{ - BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator, - }; - use arrow_schema::{DataType, Field, Schema}; - - let image_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../docs/static/assets/images/quickstart/sir-lancelot.jpg"); - let image_bytes = std::fs::read(image_path).unwrap(); - - let image_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::LargeUtf8, false), - Field::new("description", DataType::LargeUtf8, false), - Field::new("image", DataType::Binary, false), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), - false, - ), - ])); - let image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]]; - let image_batch = RecordBatch::try_new( - image_schema.clone(), - vec![ - Arc::new(LargeStringArray::from_iter_values(["lancelot"])), - Arc::new(LargeStringArray::from_iter_values([ - "Portrait of Sir Lancelot", - ])), - Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - image_vectors - .iter() - .map(|vector| Some(vector.iter().copied().map(Some).collect::>())), - 4, - ), - ), - ], - ) - .unwrap(); - let image_reader: Box = Box::new( - RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema), - ); - let multimodal_table = db - .create_table("character_images", image_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:quickstart_multimodal_bytes] - assert_eq!(multimodal_table.count_rows(None).await.unwrap(), 1); -} diff --git a/tests/rs/search.rs b/tests/rs/search.rs deleted file mode 100644 index bf510f0..0000000 --- a/tests/rs/search.rs +++ /dev/null @@ -1,311 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -//! This example demonstrates vector search query options. -//! -//! Snippets from this example are used in the documentation on vector search. - -use std::sync::Arc; - -use arrow_array::types::Float32Type; -use arrow_array::{ - Array, FixedSizeListArray, Int32Array, Int64Array, RecordBatch, RecordBatchIterator, - RecordBatchReader, UInt8Array, -}; -use arrow_schema::{DataType, Field, Schema}; - -use futures::TryStreamExt; -use lancedb::connection::Connection; -use lancedb::index::vector::IvfFlatIndexBuilder; -use lancedb::index::Index; -use lancedb::query::{ExecutableQuery, QueryBase, Select}; -use lancedb::{connect, DistanceType, Result, Table}; - -const DIM: usize = 128; - -#[tokio::main] -async fn main() -> Result<()> { - if std::path::Path::new("data").exists() { - std::fs::remove_dir_all("data").unwrap(); - } - let uri = "data/sample-lancedb"; - let db = connect(uri).execute().await?; - let tbl = create_table(&db).await?; - tbl.create_index(&["vector"], Index::Auto).execute().await?; - - configure_distance_metric(&tbl).await?; - exact_vs_approximate(&tbl).await?; - search_distance_range(&tbl).await?; - vector_search_prefilter(&tbl).await?; - vector_search_postfilter(&tbl).await?; - fast_search(&tbl).await?; - brute_force_search(&tbl).await?; - bypass_vector_index(&tbl).await?; - batch_search(&tbl).await?; - binary_search(&db).await?; - Ok(()) -} - -fn create_some_records() -> Result> { - const TOTAL: usize = 1000; - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vector", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Float32, true)), - DIM as i32, - ), - true, - ), - ])); - - let batches = RecordBatchIterator::new( - vec![RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])), - DIM as i32, - ), - ), - ], - ) - .unwrap()] - .into_iter() - .map(Ok), - schema.clone(), - ); - Ok(Box::new(batches)) -} - -async fn create_table(db: &Connection) -> Result
{ - let initial_data: Box = create_some_records()?; - let tbl = db - .create_table("my_vectors", initial_data) - .execute() - .await - .unwrap(); - Ok(tbl) -} - -async fn configure_distance_metric(table: &Table) -> Result<()> { - let query_vector = [1.0; DIM]; - // --8<-- [start:configure_distance_metric] - // Use the same distance metric the index was trained with. - let mut results = table - .vector_search(&query_vector)? - .distance_type(DistanceType::Cosine) - .limit(10) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:configure_distance_metric] - Ok(()) -} - -async fn exact_vs_approximate(table: &Table) -> Result<()> { - let query_vector = [1.0; DIM]; - // --8<-- [start:exact_vs_approximate] - // Approximate ANN search (fast, distances may come from the index representation) - let mut fast_results = table.vector_search(&query_vector)?.limit(10).execute().await?; - while let Some(batch) = fast_results.try_next().await? { - println!("{:?}", batch); - } - - // Rerank a larger candidate set on full vectors for better recall - let mut refined_results = table - .vector_search(&query_vector)? - .limit(10) - .refine_factor(20) - .execute() - .await?; - while let Some(batch) = refined_results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:exact_vs_approximate] - Ok(()) -} - -async fn search_distance_range(table: &Table) -> Result<()> { - let query_vector = [1.0; DIM]; - // --8<-- [start:search_distance_range] - // Only return rows whose distance falls within [0.1, 0.5). - let mut results = table - .vector_search(&query_vector)? - .distance_range(Some(0.1), Some(0.5)) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:search_distance_range] - Ok(()) -} - -async fn vector_search_prefilter(table: &Table) -> Result<()> { - let query_vector = [1.0; DIM]; - // --8<-- [start:vector_search_prefilter] - // Prefiltering is the default: the filter is applied before vector search. - let mut results = table - .vector_search(&query_vector)? - .only_if("id > 100") - .select(Select::columns(&["id"])) - .limit(5) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:vector_search_prefilter] - Ok(()) -} - -async fn vector_search_postfilter(table: &Table) -> Result<()> { - let query_vector = [1.0; DIM]; - // --8<-- [start:vector_search_postfilter] - // Apply the filter after vector search by calling postfilter(). - let mut results = table - .vector_search(&query_vector)? - .only_if("id > 100") - .postfilter() - .select(Select::columns(&["id"])) - .limit(5) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:vector_search_postfilter] - Ok(()) -} - -async fn fast_search(table: &Table) -> Result<()> { - let query_vector = [1.0; DIM]; - // --8<-- [start:fast_search] - // Skip unindexed data for lower latency. - let mut results = table - .vector_search(&query_vector)? - .fast_search() - .limit(5) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:fast_search] - Ok(()) -} - -async fn brute_force_search(table: &Table) -> Result<()> { - let query_vector = [1.0; DIM]; - // --8<-- [start:brute_force_search] - // A plain vector search returns the top-k closest rows. - let mut results = table.vector_search(&query_vector)?.limit(3).execute().await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:brute_force_search] - Ok(()) -} - -async fn bypass_vector_index(table: &Table) -> Result<()> { - let query_vector = [1.0; DIM]; - // --8<-- [start:bypass_vector_index] - // Force an exhaustive (flat) scan for exact, ground-truth results. - let mut results = table - .vector_search(&query_vector)? - .bypass_vector_index() - .limit(5) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:bypass_vector_index] - Ok(()) -} - -async fn batch_search(table: &Table) -> Result<()> { - let query_1 = [1.0; DIM]; - let query_2 = [0.5; DIM]; - // --8<-- [start:batch_search] - // Search multiple query vectors in one call. Each result row carries a - // `query_index` mapping it back to the query it matched. - let mut results = table - .vector_search(&query_1)? - .add_query_vector(&query_2)? - .limit(5) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:batch_search] - Ok(()) -} - -async fn binary_search(db: &Connection) -> Result<()> { - // A 256-bit binary vector is stored as 256 / 8 = 32 packed uint8 bytes. - const NUM_BYTES: i32 = 32; - const TOTAL: usize = 1024; - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new( - "vector", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::UInt8, true)), - NUM_BYTES, - ), - true, - ), - ])); - - let values = - UInt8Array::from_iter_values((0..TOTAL * NUM_BYTES as usize).map(|i| (i % 256) as u8)); - let vectors = FixedSizeListArray::try_new( - Arc::new(Field::new("item", DataType::UInt8, true)), - NUM_BYTES, - Arc::new(values), - None, - ) - .unwrap(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int64Array::from_iter_values(0..TOTAL as i64)), - Arc::new(vectors), - ], - ) - .unwrap(); - let reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone())); - let tbl = db.create_table("binary_vectors", reader).execute().await?; - tbl.create_index( - &["vector"], - Index::IvfFlat(IvfFlatIndexBuilder::default().distance_type(DistanceType::Hamming)), - ) - .execute() - .await?; - - // --8<-- [start:binary_search] - // Binary vectors use `hamming` distance over the packed uint8 bytes. - let query: Arc = Arc::new(UInt8Array::from(vec![1u8; NUM_BYTES as usize])); - let mut results = tbl - .vector_search(query)? - .distance_type(DistanceType::Hamming) - .limit(10) - .execute() - .await?; - while let Some(batch) = results.try_next().await? { - println!("{:?}", batch); - } - // --8<-- [end:binary_search] - Ok(()) -} diff --git a/tests/rs/sentence_transformers.rs b/tests/rs/sentence_transformers.rs deleted file mode 100644 index 9250430..0000000 --- a/tests/rs/sentence_transformers.rs +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -use std::{iter::once, sync::Arc}; - -use arrow_array::{RecordBatch, RecordBatchIterator, StringArray}; -use arrow_schema::{DataType, Field, Schema}; -use futures::StreamExt; -use lancedb::{ - arrow::IntoArrow, - connect, - embeddings::{ - sentence_transformers::SentenceTransformersEmbeddings, EmbeddingDefinition, - EmbeddingFunction, - }, - query::{ExecutableQuery, QueryBase}, - Result, -}; - -#[tokio::main] -async fn main() -> Result<()> { - let tempdir = tempfile::tempdir().unwrap(); - let tempdir = tempdir.path().to_str().unwrap(); - let embedding = SentenceTransformersEmbeddings::builder().build()?; - let embedding = Arc::new(embedding); - let db = connect(tempdir).execute().await?; - db.embedding_registry() - .register("sentence-transformers", embedding.clone())?; - - let table = db - .create_table("vectors", make_data()) - .add_embedding(EmbeddingDefinition::new( - "facts", - "sentence-transformers", - Some("embeddings"), - ))? - .execute() - .await?; - - let query = Arc::new(StringArray::from_iter_values(once( - "How many bones are in the human body?", - ))); - let query_vector = embedding.compute_query_embeddings(query)?; - let mut results = table - .vector_search(query_vector)? - .limit(1) - .execute() - .await?; - - let rb = results.next().await.unwrap()?; - let out = rb - .column_by_name("facts") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - let text = out.iter().next().unwrap().unwrap(); - println!("Answer: {}", text); - Ok(()) -} - -fn make_data() -> impl IntoArrow { - let schema = Schema::new(vec![Field::new("facts", DataType::Utf8, false)]); - - let facts = StringArray::from_iter_values(vec![ - "Albert Einstein was a theoretical physicist.", - "The capital of France is Paris.", - "The Great Wall of China is one of the Seven Wonders of the World.", - "Python is a popular programming language.", - "Mount Everest is the highest mountain in the world.", - "Leonardo da Vinci painted the Mona Lisa.", - "Shakespeare wrote Hamlet.", - "The human body has 206 bones.", - "The speed of light is approximately 299,792 kilometers per second.", - "Water boils at 100 degrees Celsius.", - "The Earth orbits the Sun.", - "The Pyramids of Giza are located in Egypt.", - "Coffee is one of the most popular beverages in the world.", - "Tokyo is the capital city of Japan.", - "Photosynthesis is the process by which plants make their food.", - "The Pacific Ocean is the largest ocean on Earth.", - "Mozart was a prolific composer of classical music.", - "The Internet is a global network of computers.", - "Basketball is a sport played with a ball and a hoop.", - "The first computer virus was created in 1983.", - "Artificial neural networks are inspired by the human brain.", - "Deep learning is a subset of machine learning.", - "IBM's Watson won Jeopardy! in 2011.", - "The first computer programmer was Ada Lovelace.", - "The first chatbot was ELIZA, created in the 1960s.", - ]); - let schema = Arc::new(schema); - let rb = RecordBatch::try_new(schema.clone(), vec![Arc::new(facts)]).unwrap(); - Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema)) -} diff --git a/tests/rs/simple.rs b/tests/rs/simple.rs deleted file mode 100644 index ac98957..0000000 --- a/tests/rs/simple.rs +++ /dev/null @@ -1,147 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -//! This example demonstrates basic usage of LanceDb. -//! -//! Snippets from this example are used in the quickstart documentation. - -use std::sync::Arc; - -use arrow_array::types::Float32Type; -use arrow_array::{FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator}; -use arrow_schema::{DataType, Field, Schema}; -use futures::TryStreamExt; - -use lancedb::arrow::IntoArrow; -use lancedb::connection::Connection; -use lancedb::index::Index; -use lancedb::query::{ExecutableQuery, QueryBase}; -use lancedb::{connect, Result, Table as LanceDbTable}; - -// --8<-- [start:connect] -#[tokio::main] -async fn main() -> Result<()> { - if std::path::Path::new("data").exists() { - std::fs::remove_dir_all("data").unwrap(); - } - // --8<-- [start:connect_uri] - let uri = "data/sample-lancedb"; - let db = connect(uri).execute().await?; - // --8<-- [end:connect_uri] - - // --8<-- [start:list_names] - println!("{:?}", db.table_names().execute().await?); - // --8<-- [end:list_names] - let tbl = create_table(&db).await?; - create_index(&tbl).await?; - let batches = search(&tbl).await?; - println!("{:?}", batches); - - create_empty_table(&db).await.unwrap(); - - // --8<-- [start:delete] - tbl.delete("id > 24").await.unwrap(); - // --8<-- [end:delete] - - // --8<-- [start:drop_table] - db.drop_table("my_table").await.unwrap(); - // --8<-- [end:drop_table] - Ok(()) -} -// --8<-- [end:connect] - -#[allow(dead_code)] -async fn open_with_existing_tbl() -> Result<()> { - let uri = "data/sample-lancedb"; - let db = connect(uri).execute().await?; - #[allow(unused_variables)] - // --8<-- [start:open_existing_tbl] - let table = db.open_table("my_table").execute().await.unwrap(); - // --8<-- [end:open_existing_tbl] - Ok(()) -} - -fn create_some_records() -> Result { - const TOTAL: usize = 1000; - const DIM: usize = 128; - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vector", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Float32, true)), - DIM as i32, - ), - true, - ), - ])); - - // Create a RecordBatch stream. - let batches = RecordBatchIterator::new( - vec![RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])), - DIM as i32, - ), - ), - ], - ) - .unwrap()] - .into_iter() - .map(Ok), - schema.clone(), - ); - Ok(Box::new(batches)) -} - -async fn create_table(db: &Connection) -> Result { - // --8<-- [start:create_table] - let initial_data = create_some_records()?; - let tbl = db - .create_table("my_table", initial_data) - .execute() - .await - .unwrap(); - // --8<-- [end:create_table] - - // --8<-- [start:add] - let new_data = create_some_records()?; - tbl.add(new_data).execute().await.unwrap(); - // --8<-- [end:add] - - Ok(tbl) -} - -async fn create_empty_table(db: &Connection) -> Result { - // --8<-- [start:create_empty_table] - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("item", DataType::Utf8, true), - ])); - db.create_empty_table("empty_table", schema).execute().await - // --8<-- [end:create_empty_table] -} - -async fn create_index(table: &LanceDbTable) -> Result<()> { - // --8<-- [start:create_index] - table.create_index(&["vector"], Index::Auto).execute().await - // --8<-- [end:create_index] -} - -async fn search(table: &LanceDbTable) -> Result> { - // --8<-- [start:search] - table - .query() - .limit(2) - .nearest_to(&[1.0; 128])? - .execute() - .await? - .try_collect::>() - .await - // --8<-- [end:search] -} diff --git a/tests/rs/src/main.rs b/tests/rs/src/main.rs deleted file mode 100644 index e7a11a9..0000000 --- a/tests/rs/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/tests/rs/tables.rs b/tests/rs/tables.rs deleted file mode 100644 index 0d56853..0000000 --- a/tests/rs/tables.rs +++ /dev/null @@ -1,1516 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -use std::sync::Arc; -use std::time::Duration as StdDuration; - -use arrow_array::types::Float32Type; -use arrow_array::{ - Array, FixedSizeListArray, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, - RecordBatchIterator, RecordBatchReader, StringArray, -}; -use arrow_schema::{DataType, Field, Schema}; -use futures::TryStreamExt; -use lancedb::connect; -use lancedb::query::ExecutableQuery; -use lancedb::database::CreateTableMode; -use lancedb::table::{ - ColumnAlteration, Duration, FieldMetadataUpdate, NewColumnTransform, OptimizeAction, -}; - -// --8<-- [start:update_make_users_reader] -fn make_users_reader( - ids: Vec, - names: Vec<&str>, - login_counts: Option>, -) -> Box { - let mut fields = vec![ - Field::new("id", DataType::Int64, false), - Field::new("name", DataType::Utf8, false), - ]; - let mut columns: Vec> = - vec![Arc::new(Int64Array::from(ids)), Arc::new(StringArray::from(names))]; - - if let Some(login_counts) = login_counts { - fields.push(Field::new("login_count", DataType::Int64, true)); - columns.push(Arc::new(Int64Array::from(login_counts))); - } - - let schema = Arc::new(Schema::new(fields)); - let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); - let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); - Box::new(reader) -} -// --8<-- [end:update_make_users_reader] - -// --8<-- [start:versioning_make_quotes_reader] -fn make_quotes_reader(rows: Vec<(i64, &str, &str)>) -> Box { - let ids: Vec = rows.iter().map(|(id, _, _)| *id).collect(); - let authors: Vec<&str> = rows.iter().map(|(_, author, _)| *author).collect(); - let quotes: Vec<&str> = rows.iter().map(|(_, _, quote)| *quote).collect(); - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("author", DataType::Utf8, false), - Field::new("quote", DataType::Utf8, false), - ])); - - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int64Array::from(ids)), - Arc::new(StringArray::from(authors)), - Arc::new(StringArray::from(quotes)), - ], - ) - .unwrap(); - let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); - Box::new(reader) -} -// --8<-- [end:versioning_make_quotes_reader] - -// Helper: a table with a `vector` and a `text` column, used to demonstrate -// building indexes on a branch. -fn make_products_reader(n: i32) -> Box { - const DIM: usize = 4; - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vector", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Float32, true)), - DIM as i32, - ), - true, - ), - Field::new("text", DataType::Utf8, false), - ])); - - let ids = Int32Array::from_iter_values(0..n); - let vectors = FixedSizeListArray::from_iter_primitive::( - (0..n).map(|i| { - Some( - (0..DIM) - .map(|d| Some((((i as usize * 31 + d * 7) % 97) as f32) / 97.0)) - .collect::>>(), - ) - }), - DIM as i32, - ); - let texts = StringArray::from_iter_values((0..n).map(|i| format!("product number {i}"))); - - let batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(ids), Arc::new(vectors), Arc::new(texts)], - ) - .unwrap(); - let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); - Box::new(reader) -} - -#[allow(dead_code)] -async fn update_connect_enterprise_example() { - // --8<-- [start:update_connect_enterprise] - let uri = "db://your-project-slug"; - let api_key = "your-api-key"; - let region = "us-east-1"; - // --8<-- [end:update_connect_enterprise] - let _ = (uri, api_key, region); -} - -#[allow(dead_code)] -async fn update_connect_local_example() { - // --8<-- [start:update_connect_local] - let db = connect("./data").execute().await.unwrap(); - // --8<-- [end:update_connect_local] - let _ = db; -} - -#[tokio::main] -async fn main() { - let temp_dir = tempfile::tempdir().unwrap(); - let db_uri = temp_dir.path().to_str().unwrap().to_string(); - let db = connect(&db_uri).execute().await.unwrap(); - - // --8<-- [start:create_table_from_dicts] - struct Location { - vector: [f32; 2], - lat: f32, - long: f32, - } - - let data = vec![ - Location { - vector: [1.1, 1.2], - lat: 45.5, - long: -122.7, - }, - Location { - vector: [0.2, 1.8], - lat: 40.1, - long: -74.1, - }, - ]; - - let schema = Arc::new(Schema::new(vec![ - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), - false, - ), - Field::new("lat", DataType::Float32, false), - Field::new("long", DataType::Float32, false), - ])); - - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new( - FixedSizeListArray::from_iter_primitive::( - data.iter() - .map(|row| Some(row.vector.iter().copied().map(Some).collect::>())), - 2, - ), - ), - Arc::new(Float32Array::from_iter_values( - data.iter().map(|row| row.lat), - )), - Arc::new(Float32Array::from_iter_values( - data.iter().map(|row| row.long), - )), - ], - ) - .unwrap(); - let reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone())); - let table = db - .create_table("test_table", reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:create_table_from_dicts] - assert_eq!(table.count_rows(None).await.unwrap(), 2); - - // Seed an existing table so the conflict-handling examples have something to act on. - let conflict_seed = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new( - FixedSizeListArray::from_iter_primitive::( - data.iter() - .map(|row| Some(row.vector.iter().copied().map(Some).collect::>())), - 2, - ), - ), - Arc::new(Float32Array::from_iter_values( - data.iter().map(|row| row.lat), - )), - Arc::new(Float32Array::from_iter_values( - data.iter().map(|row| row.long), - )), - ], - ) - .unwrap(); - let conflict_seed_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(conflict_seed)].into_iter(), schema.clone())); - db.create_table("conflict_table", conflict_seed_reader) - .execute() - .await - .unwrap(); - - // Build readers for the rows we want to ingest. Outside the snippet markers - // because the docs only need to highlight the `.mode(...)` differences. - let make_reader = || { - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new( - FixedSizeListArray::from_iter_primitive::( - data.iter().map(|row| { - Some(row.vector.iter().copied().map(Some).collect::>()) - }), - 2, - ), - ), - Arc::new(Float32Array::from_iter_values( - data.iter().map(|row| row.lat), - )), - Arc::new(Float32Array::from_iter_values( - data.iter().map(|row| row.long), - )), - ], - ) - .unwrap(); - let reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone())); - reader - }; - let exist_ok_reader = make_reader(); - let overwrite_reader = make_reader(); - - // --8<-- [start:create_table_conflict_handling] - // Idempotent open: reuse the existing table if it exists. - // The provided data is ignored; the schema is validated against the - // existing table and a mismatch raises an error. - let _conflict_table = db - .create_table("conflict_table", exist_ok_reader) - .mode(CreateTableMode::exist_ok(|req| req)) - .execute() - .await - .unwrap(); - - // Overwrite: drop the existing table and create a new one with the - // provided data. This permanently discards the old table's data. - let conflict_table = db - .create_table("conflict_table", overwrite_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:create_table_conflict_handling] - assert_eq!(conflict_table.count_rows(None).await.unwrap(), 2); - - // --8<-- [start:create_table_custom_schema] - let custom_schema = Arc::new(Schema::new(vec![ - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), - false, - ), - Field::new("lat", DataType::Float32, false), - Field::new("long", DataType::Float32, false), - ])); - - let custom_batch = RecordBatch::try_new( - custom_schema.clone(), - vec![ - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![ - Some(vec![Some(1.1), Some(1.2), Some(1.3), Some(1.4)]), - Some(vec![Some(0.2), Some(1.8), Some(0.4), Some(3.6)]), - ], - 4, - ), - ), - Arc::new(Float32Array::from(vec![45.5, 40.1])), - Arc::new(Float32Array::from(vec![-122.7, -74.1])), - ], - ) - .unwrap(); - let custom_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(custom_batch)].into_iter(), custom_schema.clone())); - let custom_table = db - .create_table("my_table_custom_schema", custom_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:create_table_custom_schema] - assert_eq!(custom_table.count_rows(None).await.unwrap(), 2); - - // --8<-- [start:create_table_from_arrow] - let arrow_schema = Arc::new(Schema::new(vec![ - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 16), - false, - ), - Field::new("text", DataType::Utf8, false), - ])); - - let arrow_batch = RecordBatch::try_new( - arrow_schema.clone(), - vec![ - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![Some(vec![Some(0.1); 16]), Some(vec![Some(0.2); 16])], - 16, - ), - ), - Arc::new(StringArray::from(vec!["foo", "bar"])), - ], - ) - .unwrap(); - let arrow_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(arrow_batch)].into_iter(), arrow_schema.clone())); - let arrow_table = db - .create_table("arrow_table_example", arrow_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:create_table_from_arrow] - assert_eq!(arrow_table.count_rows(None).await.unwrap(), 2); - - // --8<-- [start:create_table_from_iterator] - let batch_schema = Arc::new(Schema::new(vec![ - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), - false, - ), - Field::new("item", DataType::Utf8, false), - Field::new("price", DataType::Float32, false), - ])); - - let batches = (0..5) - .map(|i| { - RecordBatch::try_new( - batch_schema.clone(), - vec![ - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![ - Some(vec![Some(3.1 + i as f32), Some(4.1), Some(5.1), Some(6.1)]), - Some(vec![ - Some(5.9), - Some(26.5 + i as f32), - Some(4.7), - Some(32.8), - ]), - ], - 4, - ), - ), - Arc::new(StringArray::from(vec![ - format!("item{}", i * 2 + 1), - format!("item{}", i * 2 + 2), - ])), - Arc::new(Float32Array::from(vec![ - ((i * 2 + 1) * 10) as f32, - ((i * 2 + 2) * 10) as f32, - ])), - ], - ) - .unwrap() - }) - .collect::>(); - - let batch_reader: Box = - Box::new(RecordBatchIterator::new(batches.into_iter().map(Ok), batch_schema.clone())); - let batch_table = db - .create_table("batched_table", batch_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:create_table_from_iterator] - assert_eq!(batch_table.count_rows(None).await.unwrap(), 10); - - // --8<-- [start:open_existing_table] - let open_schema = Arc::new(Schema::new(vec![ - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), - false, - ), - Field::new("lat", DataType::Float32, false), - Field::new("long", DataType::Float32, false), - ])); - let open_batch = RecordBatch::try_new( - open_schema.clone(), - vec![ - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![Some(vec![Some(1.1), Some(1.2)])], - 2, - ), - ), - Arc::new(Float32Array::from(vec![45.5])), - Arc::new(Float32Array::from(vec![-122.7])), - ], - ) - .unwrap(); - let open_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(open_batch)].into_iter(), open_schema.clone())); - db.create_table("test_table", open_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - println!("{:?}", db.table_names().execute().await.unwrap()); - - let opened_table = db.open_table("test_table").execute().await.unwrap(); - // --8<-- [end:open_existing_table] - assert_eq!(opened_table.count_rows(None).await.unwrap(), 1); - - // --8<-- [start:create_empty_table] - let empty_schema = Arc::new(Schema::new(vec![ - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), - false, - ), - Field::new("item", DataType::Utf8, false), - Field::new("price", DataType::Float32, false), - ])); - let empty_table = db - .create_empty_table("test_empty_table", empty_schema) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:create_empty_table] - assert_eq!(empty_table.count_rows(None).await.unwrap(), 0); - - // --8<-- [start:drop_table] - let drop_schema = Arc::new(Schema::new(vec![ - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), - false, - ), - Field::new("lat", DataType::Float32, false), - ])); - let drop_batch = RecordBatch::try_new( - drop_schema.clone(), - vec![ - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![Some(vec![Some(1.1), Some(1.2)])], - 2, - ), - ), - Arc::new(Float32Array::from(vec![45.5])), - ], - ) - .unwrap(); - let drop_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(drop_batch)].into_iter(), drop_schema.clone())); - db.create_table("my_table", drop_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - db.drop_table("my_table", &[]).await.unwrap(); - // --8<-- [end:drop_table] - assert!( - !db.table_names() - .execute() - .await - .unwrap() - .contains(&"my_table".to_string()) - ); - - // --8<-- [start:schema_add_setup] - let schema_add_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("name", DataType::Utf8, false), - Field::new("price", DataType::Float64, false), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 128), - false, - ), - ])); - let schema_add_batch = RecordBatch::try_new( - schema_add_schema.clone(), - vec![ - Arc::new(Int64Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["Laptop", "Smartphone", "Headphones"])), - Arc::new(Float64Array::from(vec![1200.0, 800.0, 150.0])), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![ - Some(vec![Some(0.1_f32); 128]), - Some(vec![Some(0.2_f32); 128]), - Some(vec![Some(0.3_f32); 128]), - ], - 128, - ), - ), - ], - ) - .unwrap(); - let schema_add_reader: Box = Box::new(RecordBatchIterator::new( - vec![Ok(schema_add_batch)].into_iter(), - schema_add_schema.clone(), - )); - let schema_add_table = db - .create_table("schema_evolution_add_example", schema_add_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:schema_add_setup] - assert_eq!(schema_add_table.count_rows(None).await.unwrap(), 3); - - // --8<-- [start:add_columns_calculated] - // Add a discounted price column (10% discount) - schema_add_table - .add_columns( - NewColumnTransform::SqlExpressions(vec![( - "discounted_price".to_string(), - "cast((price * 0.9) as float)".to_string(), - )]), - None, - ) - .await - .unwrap(); - // --8<-- [end:add_columns_calculated] - - // --8<-- [start:add_columns_default_values] - // Add a stock status column with default value - schema_add_table - .add_columns( - NewColumnTransform::SqlExpressions(vec![( - "in_stock".to_string(), - "cast(true as boolean)".to_string(), - )]), - None, - ) - .await - .unwrap(); - // --8<-- [end:add_columns_default_values] - - // --8<-- [start:add_columns_nullable] - // Add a nullable timestamp column - schema_add_table - .add_columns( - NewColumnTransform::SqlExpressions(vec![( - "last_ordered".to_string(), - "cast(NULL as timestamp)".to_string(), - )]), - None, - ) - .await - .unwrap(); - // --8<-- [end:add_columns_nullable] - - // --8<-- [start:add_feature_columns_sql] - schema_add_table - .add_columns( - NewColumnTransform::SqlExpressions(vec![ - ( - "price_per_id".to_string(), - "cast(price / id as float)".to_string(), - ), - ("price_log".to_string(), "ln(price)".to_string()), - ( - "price_score".to_string(), - "cast(price / (price + 100.0) as float)".to_string(), - ), - ]), - None, - ) - .await - .unwrap(); - // --8<-- [end:add_feature_columns_sql] - - // --8<-- [start:schema_alter_setup] - let schema_alter_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("name", DataType::Utf8, false), - Field::new("price", DataType::Int32, false), - Field::new("discount_price", DataType::Float64, false), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 128), - false, - ), - ])); - let schema_alter_batch = RecordBatch::try_new( - schema_alter_schema.clone(), - vec![ - Arc::new(Int64Array::from(vec![1, 2])), - Arc::new(StringArray::from(vec!["Laptop", "Smartphone"])), - Arc::new(Int32Array::from(vec![1200, 800])), - Arc::new(Float64Array::from(vec![1080.0, 720.0])), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![Some(vec![Some(0.1_f32); 128]), Some(vec![Some(0.2_f32); 128])], - 128, - ), - ), - ], - ) - .unwrap(); - let schema_alter_reader: Box = Box::new(RecordBatchIterator::new( - vec![Ok(schema_alter_batch)].into_iter(), - schema_alter_schema.clone(), - )); - let schema_alter_table = db - .create_table("schema_evolution_alter_example", schema_alter_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:schema_alter_setup] - assert_eq!(schema_alter_table.count_rows(None).await.unwrap(), 2); - - // --8<-- [start:alter_columns_rename] - // Rename discount_price to sale_price - schema_alter_table - .alter_columns(&[ColumnAlteration::new("discount_price".to_string()) - .rename("sale_price".to_string())]) - .await - .unwrap(); - // --8<-- [end:alter_columns_rename] - - // --8<-- [start:alter_columns_data_type] - // Change price from int32 to int64 for larger numbers - schema_alter_table - .alter_columns(&[ColumnAlteration::new("price".to_string()).cast_to(DataType::Int64)]) - .await - .unwrap(); - // --8<-- [end:alter_columns_data_type] - - // --8<-- [start:alter_columns_nullable] - // Make the name column nullable - schema_alter_table - .alter_columns(&[ColumnAlteration::new("name".to_string()).set_nullable(true)]) - .await - .unwrap(); - // --8<-- [end:alter_columns_nullable] - - // --8<-- [start:alter_columns_multiple] - // Rename, change type, and make nullable in one operation - schema_alter_table - .alter_columns(&[ColumnAlteration::new("sale_price".to_string()) - .rename("final_price".to_string()) - .cast_to(DataType::Float64) - .set_nullable(true)]) - .await - .unwrap(); - // --8<-- [end:alter_columns_multiple] - - // --8<-- [start:alter_columns_with_expression] - // For custom transforms, create a new column from a SQL expression. - let expression_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("price_text", DataType::Utf8, false), - ])); - let expression_batch = RecordBatch::try_new( - expression_schema.clone(), - vec![ - Arc::new(Int64Array::from(vec![1])), - Arc::new(StringArray::from(vec!["$100"])), - ], - ) - .unwrap(); - let expression_reader: Box = Box::new(RecordBatchIterator::new( - vec![Ok(expression_batch)].into_iter(), - expression_schema.clone(), - )); - let expression_table = db - .create_table("schema_evolution_expression_example", expression_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - expression_table - .add_columns( - NewColumnTransform::SqlExpressions(vec![( - "price_numeric".to_string(), - "cast(replace(price_text, '$', '') as int)".to_string(), - )]), - None, - ) - .await - .unwrap(); - expression_table.drop_columns(&["price_text"]).await.unwrap(); - expression_table - .alter_columns(&[ColumnAlteration::new("price_numeric".to_string()) - .rename("price".to_string())]) - .await - .unwrap(); - // --8<-- [end:alter_columns_with_expression] - assert_eq!(expression_table.count_rows(None).await.unwrap(), 1); - - // --8<-- [start:schema_drop_setup] - let schema_drop_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("name", DataType::Utf8, false), - Field::new("price", DataType::Float64, false), - Field::new("temp_col1", DataType::Utf8, false), - Field::new("temp_col2", DataType::Int32, false), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 128), - false, - ), - ])); - let schema_drop_batch = RecordBatch::try_new( - schema_drop_schema.clone(), - vec![ - Arc::new(Int64Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["Laptop", "Smartphone", "Headphones"])), - Arc::new(Float64Array::from(vec![1200.0, 800.0, 150.0])), - Arc::new(StringArray::from(vec!["X", "Y", "Z"])), - Arc::new(Int32Array::from(vec![100, 200, 300])), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![ - Some(vec![Some(0.1_f32); 128]), - Some(vec![Some(0.2_f32); 128]), - Some(vec![Some(0.3_f32); 128]), - ], - 128, - ), - ), - ], - ) - .unwrap(); - let schema_drop_reader: Box = Box::new(RecordBatchIterator::new( - vec![Ok(schema_drop_batch)].into_iter(), - schema_drop_schema.clone(), - )); - let schema_drop_table = db - .create_table("schema_evolution_drop_example", schema_drop_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:schema_drop_setup] - assert_eq!(schema_drop_table.count_rows(None).await.unwrap(), 3); - - // --8<-- [start:drop_columns_single] - // Remove the first temporary column - schema_drop_table.drop_columns(&["temp_col1"]).await.unwrap(); - // --8<-- [end:drop_columns_single] - - // --8<-- [start:drop_columns_multiple] - // Remove the second temporary column - schema_drop_table.drop_columns(&["temp_col2"]).await.unwrap(); - // --8<-- [end:drop_columns_multiple] - - // --8<-- [start:alter_vector_column] - let old_dim = 384; - let new_dim = 1024; - let vector_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new( - "embedding", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Float32, true)), - old_dim, - ), - true, - ), - ])); - let vector_batch = RecordBatch::try_new( - vector_schema.clone(), - vec![ - Arc::new(Int64Array::from(vec![1])), - Arc::new( - FixedSizeListArray::from_iter_primitive::( - vec![Some(vec![Some(0.1_f32); old_dim as usize])], - old_dim, - ), - ), - ], - ) - .unwrap(); - let vector_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(vector_batch)].into_iter(), vector_schema.clone())); - let vector_table = db - .create_table("vector_alter_example", vector_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - // Changing FixedSizeList dimensions (384 -> 1024) is not supported via alter_columns. - // Use add_columns + drop_columns + alter_columns(rename) to replace the column. - vector_table - .add_columns( - NewColumnTransform::SqlExpressions(vec![( - "embedding_v2".to_string(), - format!("arrow_cast(NULL, 'FixedSizeList({}, Float32)')", new_dim), - )]), - None, - ) - .await - .unwrap(); - vector_table.drop_columns(&["embedding"]).await.unwrap(); - vector_table - .alter_columns(&[ColumnAlteration::new("embedding_v2".to_string()) - .rename("embedding".to_string())]) - .await - .unwrap(); - // --8<-- [end:alter_vector_column] - assert_eq!(vector_table.count_rows(None).await.unwrap(), 1); - - let field_metadata_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("category", DataType::Utf8, false), - ])); - let field_metadata_batch = RecordBatch::try_new( - field_metadata_schema.clone(), - vec![ - Arc::new(Int64Array::from(vec![0, 1])), - Arc::new(StringArray::from(vec!["a", "b"])), - ], - ) - .unwrap(); - let field_metadata_reader: Box = Box::new( - RecordBatchIterator::new(vec![Ok(field_metadata_batch)].into_iter(), field_metadata_schema), - ); - let field_metadata_table = db - .create_table("schema_field_metadata_example", field_metadata_reader) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - // --8<-- [start:schema_field_metadata_merge] - // Set two metadata keys on the `category` field. - let res = field_metadata_table - .update_field_metadata(&[FieldMetadataUpdate::new("category") - .set("unit", "label") - .set("pii", "false")]) - .await - .unwrap(); - println!("version: {}", res.version); - - // Merge: add a new key, delete one with `.remove`, keep the rest. - field_metadata_table - .update_field_metadata(&[FieldMetadataUpdate::new("category") - .set("source", "import") - .remove("pii")]) - .await - .unwrap(); - // --8<-- [end:schema_field_metadata_merge] - - // --8<-- [start:schema_field_metadata_replace] - field_metadata_table - .update_field_metadata(&[FieldMetadataUpdate::new("category") - .set("owner", "search-team") - .replace()]) - .await - .unwrap(); - // --8<-- [end:schema_field_metadata_replace] - - // --8<-- [start:update_example_table_setup] - let table = db - .create_table( - "users_example", - make_users_reader(vec![1, 2], vec!["Alice", "Bob"], Some(vec![10, 20])), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:update_example_table_setup] - let _ = table; - - // --8<-- [start:update_operation] - let table = db - .create_table( - "users_example", - make_users_reader(vec![1, 2], vec!["Alice", "Bob"], Some(vec![10, 20])), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - table - .update() - .only_if("id = 2") - .column("name", "'Bobby'") - .execute() - .await - .unwrap(); - // --8<-- [end:update_operation] - - // --8<-- [start:update_using_sql] - let table = db - .create_table( - "users_example", - make_users_reader(vec![1, 2], vec!["Alice", "Bob"], Some(vec![10, 20])), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - table - .update() - .only_if("id = 2") - .column("login_count", "login_count + 1") - .execute() - .await - .unwrap(); - // --8<-- [end:update_using_sql] - - // --8<-- [start:merge_matched_update_only] - let table = db - .create_table( - "users_example", - make_users_reader(vec![1, 2], vec!["Alice", "Bob"], Some(vec![10, 20])), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - let mut merge_insert = table.merge_insert(&["id"]); - merge_insert.when_matched_update_all(None); - merge_insert - .execute(make_users_reader( - vec![2, 3], - vec!["Bobby", "Charlie"], - Some(vec![21, 5]), - )) - .await - .unwrap(); - // --8<-- [end:merge_matched_update_only] - - // --8<-- [start:insert_if_not_exists] - let table = db - .create_table( - "users_example", - make_users_reader(vec![1, 2], vec!["Alice", "Bob"], Some(vec![10, 20])), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - let mut merge_insert = table.merge_insert(&["id"]); - merge_insert.when_not_matched_insert_all(); - merge_insert - .execute(make_users_reader( - vec![2, 3], - vec!["Bobby", "Charlie"], - Some(vec![21, 5]), - )) - .await - .unwrap(); - // --8<-- [end:insert_if_not_exists] - - // --8<-- [start:merge_update_insert] - let table = db - .create_table( - "users_example", - make_users_reader(vec![1, 2], vec!["Alice", "Bob"], Some(vec![10, 20])), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - let mut merge_insert = table.merge_insert(&["id"]); - merge_insert - .when_matched_update_all(None) - .when_not_matched_insert_all(); - merge_insert - .execute(make_users_reader( - vec![2, 3], - vec!["Bobby", "Charlie"], - Some(vec![21, 5]), - )) - .await - .unwrap(); - // --8<-- [end:merge_update_insert] - - // --8<-- [start:merge_delete_missing_by_source] - let table = db - .create_table( - "users_example", - make_users_reader( - vec![1, 2, 3], - vec!["Alice", "Bob", "Charlie"], - Some(vec![10, 20, 5]), - ), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - let mut merge_insert = table.merge_insert(&["id"]); - merge_insert - .when_matched_update_all(None) - .when_not_matched_insert_all() - .when_not_matched_by_source_delete(None); - merge_insert - .execute(make_users_reader( - vec![2, 3], - vec!["Bobby", "Charlie"], - Some(vec![21, 5]), - )) - .await - .unwrap(); - // --8<-- [end:merge_delete_missing_by_source] - - // --8<-- [start:merge_partial_columns] - let table = db - .create_table( - "users_example", - make_users_reader(vec![1, 2], vec!["Alice", "Bob"], Some(vec![10, 20])), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - let mut merge_insert = table.merge_insert(&["id"]); - merge_insert - .when_matched_update_all(None) - .when_not_matched_insert_all(); - merge_insert - .execute(make_users_reader(vec![2, 3], vec!["Bobby", "Charlie"], None)) - .await - .unwrap(); - // --8<-- [end:merge_partial_columns] - - let table = db - .create_table( - "users_example", - make_users_reader( - vec![1, 2, 3], - vec!["Alice", "Bob", "Charlie"], - Some(vec![10, 20, 5]), - ), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - // --8<-- [start:delete_operation] - // delete data - let predicate = "id = 3"; - table.delete(predicate).await.unwrap(); - // --8<-- [end:delete_operation] - - let table = db - .create_table( - "users_cleanup_example", - make_users_reader( - vec![1, 2, 3], - vec!["Alice", "Bob", "Charlie"], - Some(vec![10, 20, 5]), - ), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - // --8<-- [start:update_optimize_cleanup] - table - .optimize(OptimizeAction::Prune { - older_than: Some(Duration::days(1)), - delete_unverified: None, - error_if_tagged_old_versions: None, - }) - .await - .unwrap(); - // --8<-- [end:update_optimize_cleanup] - - // --8<-- [start:consistency_strong] - let strong_writer_db = connect(&db_uri).execute().await.unwrap(); - let strong_reader_db = connect(&db_uri) - .read_consistency_interval(StdDuration::from_secs(0)) - .execute() - .await - .unwrap(); - let strong_writer_table = strong_writer_db - .create_table( - "consistency_strong_table", - make_users_reader(vec![1], vec!["Alice"], None), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - let strong_reader_table = strong_reader_db - .open_table("consistency_strong_table") - .execute() - .await - .unwrap(); - strong_writer_table - .add(make_users_reader(vec![2], vec!["Bob"], None)) - .execute() - .await - .unwrap(); - let strong_rows_after_write = strong_reader_table.count_rows(None).await.unwrap(); - println!( - "Rows visible with strong consistency: {}", - strong_rows_after_write - ); - // --8<-- [end:consistency_strong] - assert_eq!(strong_rows_after_write, 2); - - // --8<-- [start:consistency_eventual] - let eventual_writer_db = connect(&db_uri).execute().await.unwrap(); - let eventual_reader_db = connect(&db_uri) - .read_consistency_interval(StdDuration::from_secs(3600)) - .execute() - .await - .unwrap(); - let eventual_writer_table = eventual_writer_db - .create_table( - "consistency_eventual_table", - make_users_reader(vec![1], vec!["Alice"], None), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - let eventual_reader_table = eventual_reader_db - .open_table("consistency_eventual_table") - .execute() - .await - .unwrap(); - eventual_writer_table - .add(make_users_reader(vec![2], vec!["Bob"], None)) - .execute() - .await - .unwrap(); - let eventual_rows_after_write = eventual_reader_table.count_rows(None).await.unwrap(); - println!( - "Rows visible before eventual refresh interval: {}", - eventual_rows_after_write - ); - // --8<-- [end:consistency_eventual] - assert_eq!(eventual_rows_after_write, 1); - - // --8<-- [start:consistency_checkout_latest] - let checkout_writer_db = connect(&db_uri).execute().await.unwrap(); - let checkout_reader_db = connect(&db_uri).execute().await.unwrap(); - let checkout_writer_table = checkout_writer_db - .create_table( - "consistency_checkout_latest_table", - make_users_reader(vec![1], vec!["Alice"], None), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - let checkout_reader_table = checkout_reader_db - .open_table("consistency_checkout_latest_table") - .execute() - .await - .unwrap(); - checkout_writer_table - .add(make_users_reader(vec![2], vec!["Bob"], None)) - .execute() - .await - .unwrap(); - let rows_before_refresh = checkout_reader_table.count_rows(None).await.unwrap(); - println!("Rows before checkout_latest: {}", rows_before_refresh); - checkout_reader_table.checkout_latest().await.unwrap(); - let rows_after_refresh = checkout_reader_table.count_rows(None).await.unwrap(); - println!("Rows after checkout_latest: {}", rows_after_refresh); - // --8<-- [end:consistency_checkout_latest] - assert_eq!(rows_before_refresh, 1); - assert_eq!(rows_after_refresh, 2); - - // --8<-- [start:versioning_basic_setup] - let table_name = "quotes_versioning_example"; - let data = vec![ - (1, "Richard", "Wubba Lubba Dub Dub!"), - (2, "Morty", "Rick, what's going on?"), - (3, "Richard", "I turned myself into a pickle, Morty!"), - ]; - - let table = db - .create_table(table_name, make_quotes_reader(data)) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - // --8<-- [end:versioning_basic_setup] - assert_eq!(table.count_rows(None).await.unwrap(), 3); - - // --8<-- [start:versioning_check_initial_version] - let versions = table.list_versions().await.unwrap(); - let current_version = table.version().await.unwrap(); - println!("Number of versions after creation: {}", versions.len()); - println!("Current version: {}", current_version); - // --8<-- [end:versioning_check_initial_version] - assert_eq!(versions.len(), 1); - assert_eq!(current_version, versions.last().unwrap().version); - - // --8<-- [start:versioning_update_data] - table - .update() - .only_if("author = 'Richard'") - .column("author", "'Richard Daniel Sanchez'") - .execute() - .await - .unwrap(); - let rows_after_update = table - .count_rows(Some("author = 'Richard Daniel Sanchez'".to_string())) - .await - .unwrap(); - println!( - "Rows updated to Richard Daniel Sanchez: {}", - rows_after_update - ); - // --8<-- [end:versioning_update_data] - assert_eq!(rows_after_update, 2); - - // --8<-- [start:versioning_add_data] - let more_data = vec![ - (4, "Richard Daniel Sanchez", "That's the way the news goes!"), - (5, "Morty", "Aww geez, Rick!"), - ]; - table - .add(make_quotes_reader(more_data)) - .execute() - .await - .unwrap(); - // --8<-- [end:versioning_add_data] - assert_eq!(table.count_rows(None).await.unwrap(), 5); - - // --8<-- [start:versioning_check_versions_after_mod] - let versions_after_mod = table.list_versions().await.unwrap(); - let version_count_after_mod = versions_after_mod.len(); - let version_after_mod = table.version().await.unwrap(); - println!( - "Number of versions after modifications: {}", - version_count_after_mod - ); - println!("Current version: {}", version_after_mod); - // --8<-- [end:versioning_check_versions_after_mod] - assert!(version_count_after_mod >= 2); - assert_eq!(version_after_mod, versions_after_mod.last().unwrap().version); - - // --8<-- [start:versioning_list_all_versions] - let all_versions = table.list_versions().await.unwrap(); - for v in &all_versions { - println!("Version {}, created at {}", v.version, v.timestamp); - } - // --8<-- [end:versioning_list_all_versions] - assert!(!all_versions.is_empty()); - - // --8<-- [start:versioning_rollback] - table.checkout(version_after_mod).await.unwrap(); - table.restore().await.unwrap(); - let versions_after_rollback = table.list_versions().await.unwrap(); - let version_count_after_rollback = versions_after_rollback.len(); - println!( - "Total number of versions after rollback: {}", - version_count_after_rollback - ); - // --8<-- [end:versioning_rollback] - assert_eq!(version_count_after_rollback, version_count_after_mod + 1); - assert_eq!(table.count_rows(None).await.unwrap(), 5); - - // --8<-- [start:versioning_checkout_latest] - table.checkout_latest().await.unwrap(); - // --8<-- [end:versioning_checkout_latest] - let latest_version = table.version().await.unwrap(); - let versions_after_checkout = table.list_versions().await.unwrap(); - assert_eq!(latest_version, versions_after_checkout.last().unwrap().version); - - // --8<-- [start:versioning_delete_data] - table.delete("author = 'Morty'").await.unwrap(); - let rows_after_deletion = table.count_rows(None).await.unwrap(); - println!("Number of rows after deletion: {}", rows_after_deletion); - // --8<-- [end:versioning_delete_data] - assert_eq!(rows_after_deletion, 3); - - // Setup: build a table with three versions to operate on with tags. - let tags_table = db - .create_table( - "quotes_tags_example", - make_quotes_reader(vec![(1, "Richard", "Wubba Lubba Dub Dub!")]), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); // v1 - tags_table - .add(make_quotes_reader(vec![(2, "Morty", "Aww geez, Rick!")])) - .execute() - .await - .unwrap(); // v2 - tags_table - .add(make_quotes_reader(vec![(3, "Summer", "Whatever, Grandpa")])) - .execute() - .await - .unwrap(); // v3 - - // --8<-- [start:versioning_tags] - let mut tags = tags_table.tags().await.unwrap(); - - // Create a tag pointing at a specific version - tags.create("baseline", 1).await.unwrap(); - let current_version = tags_table.version().await.unwrap(); - tags.create("with-edits", current_version).await.unwrap(); - - // List all tags on this table - let all_tags = tags.list().await.unwrap(); - println!("Tags: {:?}", all_tags); - - // Look up the version a tag points at - let baseline_version = tags.get_version("baseline").await.unwrap(); - println!("baseline -> v{}", baseline_version); - - // Move an existing tag to a different version - tags.update("baseline", 2).await.unwrap(); - - // Check out a version by tag name (separate method in Rust) - tags_table.checkout_tag("baseline").await.unwrap(); - println!("Current version: {}", tags_table.version().await.unwrap()); - - // Delete a tag (does not delete the underlying version) - tags.delete("with-edits").await.unwrap(); - - // Return to the latest version - tags_table.checkout_latest().await.unwrap(); - // --8<-- [end:versioning_tags] - assert_eq!(tags_table.version().await.unwrap(), 3); - let remaining = tags.list().await.unwrap(); - assert!(remaining.contains_key("baseline")); - assert!(!remaining.contains_key("with-edits")); - - // Setup: a fresh quotes table to demonstrate branches on. - let branches_table = db - .create_table( - "quotes_branches_example", - make_quotes_reader(vec![ - (1, "Lancelot", "My lance never fails."), - (2, "Arthur", "Long live Camelot!"), - (3, "Merlin", "Magic always has a price."), - ]), - ) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - use lancedb::table::Ref; - - // --8<-- [start:branch_create] - // Fork an isolated, writable branch from main's latest version. - // `create_branch` returns a table handle scoped to the new branch. - let branch = branches_table - .create_branch("exp", Ref::Version(None, None)) - .await - .unwrap(); - // --8<-- [end:branch_create] - - // --8<-- [start:branch_write] - // Writes land on the branch handle only; main is left untouched. - branch - .add(make_quotes_reader(vec![(4, "Lancelot", "For the realm!")])) - .execute() - .await - .unwrap(); - println!("Branch rows: {}", branch.count_rows(None).await.unwrap()); // 4 - println!("Main rows: {}", branches_table.count_rows(None).await.unwrap()); // 3 - - // List every branch, each mapped to its metadata (including its fork point). - println!("Branches: {:?}", branches_table.list_branches().await.unwrap()); - // --8<-- [end:branch_write] - - // --8<-- [start:branch_reopen] - // Reopen an existing branch by name from the table handle... - let checked_out = branches_table.checkout_branch("exp", None).await.unwrap(); - // ...or open it directly via the connection's builder. - let opened = db - .open_table("quotes_branches_example") - .branch("exp") - .execute() - .await - .unwrap(); - println!( - "Reopened rows: {}, {}", - checked_out.count_rows(None).await.unwrap(), - opened.count_rows(None).await.unwrap() - ); // both 4 - // --8<-- [end:branch_reopen] - - // --8<-- [start:branch_delete] - // Delete the branch and its branch-local history. Data on main is safe. - branches_table.delete_branch("exp").await.unwrap(); - // --8<-- [end:branch_delete] - - assert_eq!(branches_table.count_rows(None).await.unwrap(), 3); - assert!(!branches_table.list_branches().await.unwrap().contains_key("exp")); - - // Setup: a branch with row results that we want to apply to main. - let candidate = branches_table - .create_branch("candidate", Ref::Version(None, None)) - .await - .unwrap(); - candidate - .update() - .only_if("id = 1") - .column("quote", "'Revised on the branch'") - .execute() - .await - .unwrap(); - candidate - .add(make_quotes_reader(vec![( - 4, - "Galahad", - "The grail awaits.", - )])) - .execute() - .await - .unwrap(); - - // --8<-- [start:branch_upsert_to_main] - // This is a row-level upsert, not a merge of branch histories. - // `merge_insert` updates matching rows and inserts new rows using a stable - // unique key. Filter the branch read if you only want to apply some results. - let schema = candidate.schema().await.unwrap(); - let batches = candidate - .query() - .execute() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - let rows_to_apply = RecordBatchIterator::new(batches.into_iter().map(Ok), schema); - - let mut merge = branches_table.merge_insert(&["id"]); - merge - .when_matched_update_all(None) // update rows that already exist on main - .when_not_matched_insert_all(); // insert rows that are new on the branch - merge.execute(Box::new(rows_to_apply)).await.unwrap(); - // --8<-- [end:branch_upsert_to_main] - - assert_eq!(branches_table.count_rows(None).await.unwrap(), 4); - branches_table.delete_branch("candidate").await.unwrap(); - - // Setup: a larger table with a vector and a text column to index. - let products = db - .create_table("products_branch_index", make_products_reader(512)) - .mode(CreateTableMode::Overwrite) - .execute() - .await - .unwrap(); - - // --8<-- [start:branch_index] - use lancedb::index::scalar::FtsIndexBuilder; - use lancedb::index::Index; - - // Build and validate indexes on a branch before using the configuration on - // main. - let dev = products - .create_branch("index-dev", Ref::Version(None, None)) - .await - .unwrap(); - - // A vector (ANN) index and a full-text search index, both branch-scoped. - dev.create_index(&["vector"], Index::Auto) - .execute() - .await - .unwrap(); - dev.create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) - .execute() - .await - .unwrap(); - - // Both indexes live only on the branch; main still has none. - println!("Branch indexes: {}", dev.list_indices().await.unwrap().len()); // 2 - println!("Main indexes: {}", products.list_indices().await.unwrap().len()); // 0 - // --8<-- [end:branch_index] - - assert_eq!(dev.list_indices().await.unwrap().len(), 2); - assert_eq!(products.list_indices().await.unwrap().len(), 0); - products.delete_branch("index-dev").await.unwrap(); -} diff --git a/tests/ts/ann_indexes.test.ts b/tests/ts/ann_indexes.test.ts deleted file mode 100644 index a37872c..0000000 --- a/tests/ts/ann_indexes.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -// --8<-- [start:import] -import * as lancedb from "@lancedb/lancedb"; -import type { VectorQuery } from "@lancedb/lancedb"; -// --8<-- [end:import] -import { withTempDirectory } from "./util.ts"; - -test("ann index examples", async () => { - await withTempDirectory(async (databaseDir) => { - // --8<-- [start:ingest] - const db = await lancedb.connect(databaseDir); - - const data = Array.from({ length: 5_000 }, (_, i) => ({ - vector: Array(128).fill(i), - id: `${i}`, - content: "", - longId: `${i}`, - })); - - const table = await db.createTable("my_vectors", data, { - mode: "overwrite", - }); - await table.createIndex("vector", { - config: lancedb.Index.ivfPq({ - numPartitions: 10, - numSubVectors: 16, - }), - }); - // --8<-- [end:ingest] - - // --8<-- [start:search1] - const search = table.search(Array(128).fill(1.2)).limit(2) as VectorQuery; - const results1 = await search.nprobes(20).refineFactor(10).toArray(); - // --8<-- [end:search1] - expect(results1.length).toBe(2); - - // --8<-- [start:search2] - const results2 = await table - .search(Array(128).fill(1.2)) - .where("id != '1141'") - .limit(2) - .toArray(); - // --8<-- [end:search2] - expect(results2.length).toBe(2); - - // --8<-- [start:search3] - const results3 = await table - .search(Array(128).fill(1.2)) - .select(["id"]) - .limit(2) - .toArray(); - // --8<-- [end:search3] - expect(results3.length).toBe(2); - }); -}, 100_000); diff --git a/tests/ts/basic_usage.test.ts b/tests/ts/basic_usage.test.ts deleted file mode 100644 index f00e69b..0000000 --- a/tests/ts/basic_usage.test.ts +++ /dev/null @@ -1,161 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -import * as fs from "node:fs"; -import * as path from "node:path"; -// --8<-- [start:basic_imports] -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; -// --8<-- [end:basic_imports] -import { withTempDirectory } from "./util.ts"; - -const dataPath = path.join(__dirname, "..", "camelot.json"); - -test("basic usage examples (async)", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - // --8<-- [start:data_load] - const data = JSON.parse(fs.readFileSync(dataPath, "utf-8")); - // --8<-- [end:data_load] - - // --8<-- [start:basic_create_table] - let table = await db.createTable("camelot", data, { - mode: "overwrite", - }); - // --8<-- [end:basic_create_table] - expect(await table.countRows()).toBe(8); - - // --8<-- [start:basic_open_table] - table = await db.openTable("camelot"); - // --8<-- [end:basic_open_table] - - // --8<-- [start:basic_create_empty_table] - const schema = new arrow.Schema([ - new arrow.Field("id", new arrow.Int16()), - new arrow.Field("name", new arrow.Utf8()), - new arrow.Field("role", new arrow.Utf8()), - new arrow.Field("description", new arrow.Utf8()), - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 4, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - new arrow.Field( - "stats", - new arrow.Struct([ - new arrow.Field("strength", new arrow.Int8()), - new arrow.Field("courage", new arrow.Int8()), - new arrow.Field("magic", new arrow.Int8()), - new arrow.Field("wisdom", new arrow.Int8()), - ]), - ), - ]); - await db.createEmptyTable("camelot_empty", schema, { mode: "overwrite" }); - // --8<-- [end:basic_create_empty_table] - expect(await db.tableNames()).toContain("camelot_empty"); - await db.dropTable("camelot_empty"); - - // --8<-- [start:basic_add_data] - const magicalCharacters = [ - { - id: 9, - name: "Morgan le Fay", - role: "Sorceress", - description: - "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.", - vector: [0.1, 0.84, 0.25, 0.7], - stats: { strength: 2, courage: 3, magic: 5, wisdom: 4 }, - }, - { - id: 10, - name: "The Lady of the Lake", - role: "Mystical Guardian", - description: - "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.", - vector: [0.0, 0.9, 0.58, 0.88], - stats: { strength: 2, courage: 3, magic: 5, wisdom: 5 }, - }, - ]; - await table.add(magicalCharacters); - // --8<-- [end:basic_add_data] - expect(await table.countRows()).toBe(10); - - // --8<-- [start:basic_vector_search] - const queryVector = [0.03, 0.85, 0.61, 0.9]; - const result = await table.search(queryVector).limit(5).toArray(); - console.log(result); - // --8<-- [end:basic_vector_search] - - // --8<-- [start:basic_add_columns] - await table.addColumns([ - { - name: "power", - valueSql: - "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)", - }, - ]); - // --8<-- [end:basic_add_columns] - const schemaWithPower = await table.schema(); - expect(schemaWithPower.fields.some((f) => f.name === "power")).toBe(true); - - // --8<-- [start:basic_vector_search_q1] - // Who are the characters similar to "wizard"? - const queryVector1 = [0.03, 0.85, 0.61, 0.9]; - const r1 = await table - .search(queryVector1) - .limit(5) - .select(["name", "role", "description"]) - .toArray(); - console.log(r1); - // --8<-- [end:basic_vector_search_q1] - - // --8<-- [start:basic_vector_search_q2] - // Who are the characters similar to "wizard" with high magic stats? - const queryVector2 = [0.03, 0.85, 0.61, 0.9]; - const r2 = await table - .search(queryVector2) - .where("stats.magic > 3") - .select(["name", "role", "description"]) - .limit(5) - .toArray(); - console.log(r2); - // --8<-- [end:basic_vector_search_q2] - - // --8<-- [start:basic_vector_search_q3] - // Who are the strongest characters? - const r3 = await table - .query() - .where("stats.strength > 3") - .select(["name", "role", "description"]) - .limit(5) - .toArray(); - console.log(r3); - // --8<-- [end:basic_vector_search_q3] - - // --8<-- [start:basic_vector_search_q4] - // Who are the strongest characters? - const r4 = await table - .query() - .select(["name", "role", "description", "power"]) - .toArray(); - console.log(r4); - // --8<-- [end:basic_vector_search_q4] - - // --8<-- [start:basic_drop_columns] - await table.dropColumns(["power"]); - // --8<-- [end:basic_drop_columns] - - // --8<-- [start:basic_delete_rows] - await table.delete('role = "Traitor Knight"'); - // --8<-- [end:basic_delete_rows] - expect(await table.countRows()).toBe(9); - - // --8<-- [start:basic_drop_table] - await db.dropTable("camelot"); - // --8<-- [end:basic_drop_table] - expect(await db.tableNames()).not.toContain("camelot"); - }); -}); diff --git a/tests/ts/biome.json b/tests/ts/biome.json deleted file mode 100644 index f8fb5b2..0000000 --- a/tests/ts/biome.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json", - "vcs": { - "enabled": false, - "clientKind": "git", - "useIgnoreFile": false - }, - "files": { - "ignoreUnknown": false, - "ignore": [] - }, - "formatter": { - "enabled": true, - "indentStyle": "space" - }, - "organizeImports": { - "enabled": true - }, - "linter": { - "enabled": true, - "rules": { - "recommended": true - } - }, - "javascript": { - "formatter": { - "quoteStyle": "double" - } - }, - "overrides": [ - { - "include": ["*"], - "linter": { - "rules": { - "style": { - "noNonNullAssertion": "off" - } - } - } - }, - { - "include": ["merge_insert.test.ts"], - "linter": { - "rules": { - "style": { - "useNamingConvention": "off" - } - } - } - } - ] -} diff --git a/tests/ts/connection.test.ts b/tests/ts/connection.test.ts deleted file mode 100644 index f522654..0000000 --- a/tests/ts/connection.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, jest, test } from "@jest/globals"; -import * as path from "node:path"; -import { withTempDirectory } from "./util.ts"; -// --8<-- [start:connect] -import * as lancedb from "@lancedb/lancedb"; - -async function connectExample(uri: string) { - const db = await lancedb.connect(uri); - return db; -} -// --8<-- [end:connect] - -test("connect to a local database", async () => { - await withTempDirectory(async (tempDir) => { - const uri = path.join(tempDir, "ex_lancedb"); - const db = await connectExample(uri); - expect(db).toBeDefined(); - }); -}); - -async function connectEnterpriseQuickstart() { - // --8<-- [start:connect_enterprise_quickstart] - const uri = "db://your-database-uri"; - const apiKey = "your-api-key"; - const region = "us-east-1"; - const hostOverride = "https://your-enterprise-endpoint.com"; - - const db = await lancedb.connect(uri, { - apiKey, - region, - hostOverride, - }); - // --8<-- [end:connect_enterprise_quickstart] - return db; -} - -test("enterprise quickstart connect uses placeholder config", async () => { - const mockDb = { __mock: true } as unknown as Awaited< - ReturnType - >; - const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb); - - const db = await connectEnterpriseQuickstart(); - expect(db).toBe(mockDb); - expect(spy).toHaveBeenCalledWith("db://your-database-uri", { - apiKey: "your-api-key", - region: "us-east-1", - hostOverride: "https://your-enterprise-endpoint.com", - }); - spy.mockRestore(); -}); - -// --8<-- [start:connect_object_storage] -async function connectObjectStorageExample() { - const uri = "s3://your-bucket/path"; - // You can also use "gs://your-bucket/path" or "az://your-container/path". - const db = await lancedb.connect(uri); - return db; -} -// --8<-- [end:connect_object_storage] - -async function namespaceTableOpsExample() { - // --8<-- [start:namespace_table_ops] - const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" }); - - // Create namespace tree: prod/search and prod/recommendations - await db.createNamespace(["prod"], { mode: "exist_ok" }); - await db.createNamespace(["prod", "search"], { mode: "exist_ok" }); - await db.createNamespace(["prod", "recommendations"], { mode: "exist_ok" }); - - await db.createTable( - "user", - [{ id: 1, vector: [0.1, 0.2], name: "alice" }], - ["prod", "search"], - { mode: "create" }, // use "overwrite" only if you want to replace existing table - ); - - await db.createTable( - "user", - [{ id: 2, vector: [0.3, 0.4], name: "bob" }], - ["prod", "recommendations"], - { mode: "create" }, - ); - - // Verify - console.log((await db.listNamespaces()).namespaces); // ["prod"] - console.log((await db.listNamespaces(["prod"])).namespaces); // ["recommendations", "search"] - console.log(await db.tableNames(["prod", "search"])); // ["user"] - console.log(await db.tableNames(["prod", "recommendations"])); // ["user"] - // --8<-- [end:namespace_table_ops] -} - -async function namespaceAdminOpsExample() { - // --8<-- [start:namespace_admin_ops] - const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" }); - const namespace = ["prod", "search"]; - - await db.createNamespace(["prod"]); - await db.createNamespace(["prod", "search"]); - - const childNamespaces = (await db.listNamespaces(["prod"])).namespaces; - console.log(`Child namespaces under ${JSON.stringify(namespace)}:`, childNamespaces); - // Child namespaces under ["prod","search"]: [ 'search' ] - - const metadata = await db.describeNamespace(["prod", "search"]); - console.log(`Metadata for namespace ${JSON.stringify(namespace)}:`, metadata); - - await db.dropNamespace(["prod", "search"], { mode: "skip" }); - await db.dropNamespace(["prod"], { mode: "skip" }); - // --8<-- [end:namespace_admin_ops] - return { childNamespaces, metadata }; -} - -void [ - connectObjectStorageExample, - connectEnterpriseQuickstart, - namespaceTableOpsExample, - namespaceAdminOpsExample, -]; diff --git a/tests/ts/custom_embedding_function.test.ts b/tests/ts/custom_embedding_function.test.ts deleted file mode 100644 index 528928a..0000000 --- a/tests/ts/custom_embedding_function.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { - type FeatureExtractionPipeline, - pipeline, -} from "@huggingface/transformers"; -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -// --8<-- [start:imports] -import * as lancedb from "@lancedb/lancedb"; -import { - LanceSchema, - TextEmbeddingFunction, - getRegistry, - register, -} from "@lancedb/lancedb/embedding"; -// --8<-- [end:imports] -import { withTempDirectory } from "./util.ts"; - -// --8<-- [start:embedding_impl] -@register("sentence-transformers") -class SentenceTransformersEmbeddings extends TextEmbeddingFunction { - name = "Xenova/all-miniLM-L6-v2"; - #ndims!: number; - extractor!: FeatureExtractionPipeline; - - async init() { - this.extractor = await pipeline("feature-extraction", this.name, { - dtype: "fp32", - }); - this.#ndims = await this.generateEmbeddings(["hello"]).then( - (e) => e[0].length, - ); - } - - ndims() { - return this.#ndims; - } - - toJSON() { - return { - name: this.name, - }; - } - async generateEmbeddings(texts: string[]) { - const output = await this.extractor(texts, { - pooling: "mean", - normalize: true, - }); - return output.tolist(); - } -} -// --8<-- [end:embedding_impl] - -test("Registry examples", async () => { - await withTempDirectory(async (databaseDir) => { - // --8<-- [start:call_custom_function] - const registry = getRegistry(); - - const sentenceTransformer = await registry - .get("sentence-transformers")! - .create(); - - const schema = LanceSchema({ - vector: sentenceTransformer.vectorField(), - text: sentenceTransformer.sourceField(), - }); - - const db = await lancedb.connect(databaseDir); - const table = await db.createEmptyTable("table", schema, { - mode: "overwrite", - }); - - await table.add([{ text: "hello" }, { text: "world" }]); - - const results = await table.search("greeting").limit(1).toArray(); - // --8<-- [end:call_custom_function] - expect(results.length).toBe(1); - }); -}, 100_000); diff --git a/tests/ts/embedding.test.ts b/tests/ts/embedding.test.ts deleted file mode 100644 index a73b808..0000000 --- a/tests/ts/embedding.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -// --8<-- [start:imports] -import * as lancedb from "@lancedb/lancedb"; -import "@lancedb/lancedb/embedding/openai"; -import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding"; -import { EmbeddingFunction } from "@lancedb/lancedb/embedding"; -import { type Float, Float32, Utf8 } from "apache-arrow"; -// --8<-- [end:imports] -import { withTempDirectory } from "./util.ts"; - -const openAiTest = process.env.OPENAI_API_KEY == null ? test.skip : test; - -openAiTest("create embedding function", () => { - // --8<-- [start:create_embedding_function] - const func = getRegistry().get("openai")!.create({ - model: "text-embedding-3-small", - }); - // --8<-- [end:create_embedding_function] - expect(func).toBeDefined(); -}); - -openAiTest("openai embeddings", async () => { - await withTempDirectory(async (databaseDir) => { - // --8<-- [start:openai_embeddings] - const db = await lancedb.connect(databaseDir); - const func = getRegistry() - .get("openai") - ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction; - - const wordsSchema = LanceSchema({ - text: func.sourceField(new Utf8()), - vector: func.vectorField(), - }); - const tbl = await db.createEmptyTable("words", wordsSchema, { - mode: "overwrite", - }); - await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]); - - const query = "greetings"; - const actual = (await tbl.search(query).limit(1).toArray())[0]; - // --8<-- [end:openai_embeddings] - expect(actual).toHaveProperty("text"); - }); -}); - -openAiTest("manual query embeddings", async () => { - await withTempDirectory(async (databaseDir) => { - // --8<-- [start:manual_query_embeddings] - const db = await lancedb.connect(databaseDir); - const func = getRegistry() - .get("openai") - ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction; - - const wordsSchema = LanceSchema({ - text: func.sourceField(new Utf8()), - vector: func.vectorField(), - }); - const tbl = await db.createEmptyTable("words", wordsSchema, { - mode: "overwrite", - }); - await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]); - - const queryVector = await func.computeQueryEmbeddings("greetings"); - // --8<-- [start:manual_query_search] - // queryVector is assumed to already be generated by your embedding function - const actual = (await tbl.search(queryVector).limit(1).toArray())[0]; - // --8<-- [end:manual_query_search] - // --8<-- [end:manual_query_embeddings] - expect(actual).toHaveProperty("text"); - }); -}); - -test("custom embedding function", async () => { - await withTempDirectory(async (databaseDir) => { - // --8<-- [start:embedding_function] - const db = await lancedb.connect(databaseDir); - - @register("my_embedding") - class MyEmbeddingFunction extends EmbeddingFunction { - constructor(optionsRaw = {}) { - super(); - const options = this.resolveVariables(optionsRaw); - // Initialize using options - } - ndims() { - return 3; - } - protected getSensitiveKeys(): string[] { - return []; - } - embeddingDataType(): Float { - return new Float32(); - } - async computeQueryEmbeddings(_data: string) { - // This is a placeholder for a real embedding function - return [1, 2, 3]; - } - async computeSourceEmbeddings(data: string[]) { - // This is a placeholder for a real embedding function - return Array.from({ length: data.length }).fill([ - 1, 2, 3, - ]) as number[][]; - } - } - - const func = new MyEmbeddingFunction(); - - const data = [{ text: "pepperoni" }, { text: "pineapple" }]; - - // Option 1: manually specify the embedding function - const table = await db.createTable("vectors", data, { - embeddingFunction: { - function: func, - sourceColumn: "text", - vectorColumn: "vector", - }, - mode: "overwrite", - }); - - // Option 2: provide the embedding function through a schema - - const schema = LanceSchema({ - text: func.sourceField(new Utf8()), - vector: func.vectorField(), - }); - - const table2 = await db.createTable("vectors2", data, { - schema, - mode: "overwrite", - }); - // --8<-- [end:embedding_function] - expect(await table.countRows()).toBe(2); - expect(await table2.countRows()).toBe(2); - }); -}); - -test("embedding function api_key", async () => { - // --8<-- [start:register_secret] - const registry = getRegistry(); - registry.setVar("api_key", "sk-..."); - - const func = registry.get("openai")!.create({ - apiKey: "$var:api_key", - }); - // --8<-- [end:register_secret] -}); - -openAiTest("embedding function variable fallback", () => { - // --8<-- [start:register_model_fallback] - const registry = getRegistry(); - registry.setVar("openai_model", "text-embedding-3-large"); - - const func = registry.get("openai")!.create({ - model: "$var:openai_model:text-embedding-3-small", - }); - // --8<-- [end:register_model_fallback] - expect(func).toBeDefined(); -}); diff --git a/tests/ts/filtering.test.ts b/tests/ts/filtering.test.ts deleted file mode 100644 index 1c35d51..0000000 --- a/tests/ts/filtering.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -import * as lancedb from "@lancedb/lancedb"; -import { withTempDirectory } from "./util.ts"; - -test("filtering examples", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - const data = Array.from({ length: 10_000 }, (_, i) => ({ - vector: Array(1536).fill(i), - id: i, - item: `item ${i}`, - strId: `${i}`, - })); - - const tbl = await db.createTable("myVectors", data, { mode: "overwrite" }); - - // --8<-- [start:search] - const _result = await tbl - .search(Array(1536).fill(0.5)) - .limit(1) - .where("id = 10") - .toArray(); - // --8<-- [end:search] - - // --8<-- [start:vec_search] - const result = await ( - tbl.search(Array(1536).fill(0)) as lancedb.VectorQuery - ) - .where("(item IN ('item 0', 'item 2')) AND (id > 10)") - .postfilter() - .toArray(); - // --8<-- [end:vec_search] - expect(result.length).toBe(0); - - // --8<-- [start:sql_search] - await tbl.query().where("id = 10").limit(10).toArray(); - // --8<-- [end:sql_search] - }); -}); diff --git a/tests/ts/full_text_search.test.ts b/tests/ts/full_text_search.test.ts deleted file mode 100644 index 9777bda..0000000 --- a/tests/ts/full_text_search.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -import * as lancedb from "@lancedb/lancedb"; -import { withTempDirectory } from "./util.ts"; - -test("full text search", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - const words = [ - "apple", - "banana", - "cherry", - "date", - "elderberry", - "fig", - "grape", - ]; - - const data = Array.from({ length: 10_000 }, (_, i) => ({ - vector: Array(1536).fill(i), - id: i, - item: `item ${i}`, - strId: `${i}`, - doc: words[i % words.length], - })); - - const tbl = await db.createTable("myVectors", data, { mode: "overwrite" }); - - await tbl.createIndex("doc", { - config: lancedb.Index.fts(), - }); - - // --8<-- [start:full_text_search] - const result = await tbl - .query() - .nearestToText("apple") - .select(["id", "doc"]) - .limit(10) - .toArray(); - expect(result.length).toBe(10); - // --8<-- [end:full_text_search] - }); -}, 10_000); diff --git a/tests/ts/integrations.ts b/tests/ts/integrations.ts deleted file mode 100644 index 3ae7967..0000000 --- a/tests/ts/integrations.ts +++ /dev/null @@ -1,134 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -// --8<-- [start:frameworks_genkit_usage] -import { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from "genkitx-lancedb"; -import { textEmbedding004, vertexAI } from "@genkit-ai/vertexai"; -import { gemini } from "@genkit-ai/vertexai"; -import { z, genkit } from "genkit"; -import { Document } from "genkit/retriever"; -import { chunk } from "llm-chunk"; -import { readFile } from "fs/promises"; -import path from "path"; -import pdf from "pdf-parse/lib/pdf-parse"; - -const ai = genkit({ - plugins: [ - // vertexAI provides the textEmbedding004 embedder - vertexAI(), - - // the local vector store requires an embedder to translate from text to vector - lancedb([ - { - dbUri: ".db", // optional lancedb uri, default to .db - tableName: "table", // optional table name, default to table - embedder: textEmbedding004, - }, - ]), - ], -}); -// --8<-- [end:frameworks_genkit_usage] - -// --8<-- [start:frameworks_genkit_custom_indexer] -export const menuPdfIndexer = lancedbIndexerRef({ - // Using all defaults, for dbUri, tableName, and embedder, etc -}); - -const chunkingConfig = { - minLength: 1000, - maxLength: 2000, - splitter: "sentence", - overlap: 100, - delimiters: "", -} as any; - -async function extractTextFromPdf(filePath: string) { - const pdfFile = path.resolve(filePath); - const dataBuffer = await readFile(pdfFile); - const data = await pdf(dataBuffer); - return data.text; -} - -export const indexMenu = ai.defineFlow( - { - name: "indexMenu", - inputSchema: z.string().describe("PDF file path"), - outputSchema: z.void(), - }, - async (filePath: string) => { - filePath = path.resolve(filePath); - - // Read the pdf. - const pdfTxt = await ai.run("extract-text", () => extractTextFromPdf(filePath)); - - // Divide the pdf text into segments. - const chunks = await ai.run("chunk-it", async () => chunk(pdfTxt, chunkingConfig)); - - // Convert chunks of text into documents to store in the index. - const documents = chunks.map((text) => { - return Document.fromText(text, { filePath }); - }); - - // Add documents to the index. - await ai.index({ - indexer: menuPdfIndexer, - documents, - options: { - writeMode: WriteMode.Overwrite, - } as any, - }); - }, -); -// --8<-- [end:frameworks_genkit_custom_indexer] - -// --8<-- [start:frameworks_genkit_custom_retriever] -export const menuRetriever = lancedbRetrieverRef({ - tableName: "table", // Use the same table name as the indexer. - displayName: "Menu", // Use a custom display name. -}); - -export const menuQAFlow = ai.defineFlow( - { name: "Menu", inputSchema: z.string(), outputSchema: z.string() }, - async (input: string) => { - // retrieve relevant documents - const docs = await ai.retrieve({ - retriever: menuRetriever, - query: input, - options: { - k: 3, - }, - }); - - const extractedContent = docs.map((doc) => { - if (doc.content && Array.isArray(doc.content) && doc.content.length > 0) { - if (doc.content[0].media && doc.content[0].media.url) { - return doc.content[0].media.url; - } - } - return "No content found"; - }); - - console.log("Extracted content:", extractedContent); - - const { text } = await ai.generate({ - model: gemini("gemini-2.0-flash"), - prompt: ` -You are acting as a helpful AI assistant that can answer -questions about the food available on the menu at Genkit Grub Pub. - -Use only the context provided to answer the question. -If you don't know, do not make up an answer. -Do not add or change items on the menu. - -Context: -${extractedContent.join("\n\n")} - -Question: ${input}`, - docs, - }); - - return text; - }, -); -// --8<-- [end:frameworks_genkit_custom_retriever] - diff --git a/tests/ts/jest.config.cjs b/tests/ts/jest.config.cjs deleted file mode 100644 index db83d63..0000000 --- a/tests/ts/jest.config.cjs +++ /dev/null @@ -1,6 +0,0 @@ -/** @type {import('ts-jest').JestConfigWithTsJest} */ -module.exports = { - preset: "ts-jest", - testEnvironment: "node", - testPathIgnorePatterns: ["./dist"], -}; diff --git a/tests/ts/merge_insert.test.ts b/tests/ts/merge_insert.test.ts deleted file mode 100644 index 1b33dab..0000000 --- a/tests/ts/merge_insert.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors - -import { expect, test } from "@jest/globals"; -import * as lancedb from "@lancedb/lancedb"; - -test("basic upsert", async () => { - const db = await lancedb.connect("memory://"); - - // --8<-- [start:upsert_basic] - const table = await db.createTable("users", [ - { id: 0, name: "Alice" }, - { id: 1, name: "Bob" }, - ]); - - const newUsers = [ - { id: 1, name: "Bobby" }, - { id: 2, name: "Charlie" }, - ]; - await table - .mergeInsert("id") - .whenMatchedUpdateAll() - .whenNotMatchedInsertAll() - .execute(newUsers); - - await table.countRows(); // 3 - // --8<-- [end:upsert_basic] - expect(await table.countRows()).toBe(3); - - // --8<-- [start:insert_if_not_exists] - const table2 = await db.createTable("domains", [ - { domain: "google.com", name: "Google" }, - { domain: "github.com", name: "GitHub" }, - ]); - - const newDomains = [ - { domain: "google.com", name: "Google" }, - { domain: "facebook.com", name: "Facebook" }, - ]; - await table2 - .mergeInsert("domain") - .whenNotMatchedInsertAll() - .execute(newDomains); - await table2.countRows(); // 3 - // --8<-- [end:insert_if_not_exists] - expect(await table2.countRows()).toBe(3); - - // --8<-- [start:replace_range] - const table3 = await db.createTable("chunks", [ - { doc_id: 0, chunk_id: 0, text: "Hello" }, - { doc_id: 0, chunk_id: 1, text: "World" }, - { doc_id: 1, chunk_id: 0, text: "Foo" }, - { doc_id: 1, chunk_id: 1, text: "Bar" }, - ]); - - const newChunks = [{ doc_id: 1, chunk_id: 0, text: "Baz" }]; - - await table3 - .mergeInsert(["doc_id", "chunk_id"]) - .whenMatchedUpdateAll() - .whenNotMatchedInsertAll() - .whenNotMatchedBySourceDelete({ where: "doc_id = 1" }) - .execute(newChunks); - - await table3.countRows("doc_id = 1"); // 1 - // --8<-- [end:replace_range] - expect(await table3.countRows("doc_id = 1")).toBe(1); -}); diff --git a/tests/ts/multimodal.test.ts b/tests/ts/multimodal.test.ts deleted file mode 100644 index ac9a820..0000000 --- a/tests/ts/multimodal.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -// --8<-- [start:multimodal_imports] -import * as arrow from "apache-arrow"; -import { Buffer } from "node:buffer"; -import * as lancedb from "@lancedb/lancedb"; -// --8<-- [end:multimodal_imports] -import { withTempDirectory } from "./util.ts"; - -test("multimodal snippets (async)", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - // --8<-- [start:create_dummy_data] - const createDummyImage = (color: string): Uint8Array => { - const pngHeader = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]); - return Buffer.concat([Buffer.from(pngHeader), Buffer.from(color, "utf8")]); - }; - - const data = [ - { - id: 1, - filename: "red_square.png", - vector: Array.from({ length: 128 }, (_, i) => (i % 16) / 16), - image_blob: createDummyImage("red"), - label: "red", - }, - { - id: 2, - filename: "blue_square.png", - vector: Array.from({ length: 128 }, (_, i) => ((i + 8) % 16) / 16), - image_blob: createDummyImage("blue"), - label: "blue", - }, - ]; - // --8<-- [end:create_dummy_data] - - // --8<-- [start:define_schema] - const schema = new arrow.Schema([ - new arrow.Field("id", new arrow.Int32()), - new arrow.Field("filename", new arrow.Utf8()), - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 128, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - new arrow.Field("image_blob", new arrow.Binary()), - new arrow.Field("label", new arrow.Utf8()), - ]); - // --8<-- [end:define_schema] - - // --8<-- [start:ingest_data] - const multimodalData = lancedb.makeArrowTable(data, { schema }); - const tbl = await db.createTable("images", multimodalData, { - mode: "overwrite", - }); - // --8<-- [end:ingest_data] - expect(await tbl.countRows()).toBe(2); - - // --8<-- [start:search_data] - const queryVector = Array.from({ length: 128 }, (_, i) => (i % 16) / 16); - const results = await tbl.search(queryVector).limit(1).toArray(); - // --8<-- [end:search_data] - - // --8<-- [start:process_results] - for (const row of results) { - const imageBytes = row.image_blob as Uint8Array; - console.log( - `Retrieved image: ${row.filename}, Byte length: ${imageBytes.length}`, - ); - } - // --8<-- [end:process_results] - expect(results).toHaveLength(1); - - // --8<-- [start:blob_api_schema] - const blobSchema = new arrow.Schema([ - new arrow.Field("id", new arrow.Int64()), - new arrow.Field( - "video", - new arrow.LargeBinary(), - true, - new Map([["lance-encoding:blob", "true"]]), - ), - ]); - // --8<-- [end:blob_api_schema] - - // --8<-- [start:blob_api_ingest] - const blobData = lancedb.makeArrowTable( - [ - { id: 1, video: Buffer.from("fake_video_bytes_1") }, - { id: 2, video: Buffer.from("fake_video_bytes_2") }, - ], - { schema: blobSchema }, - ); - const blobTable = await db.createTable("videos", blobData, { - mode: "overwrite", - }); - // --8<-- [end:blob_api_ingest] - expect(await blobTable.countRows()).toBe(2); - }); -}); diff --git a/tests/ts/package-lock.json b/tests/ts/package-lock.json deleted file mode 100644 index 8713b65..0000000 --- a/tests/ts/package-lock.json +++ /dev/null @@ -1,5501 +0,0 @@ -{ - "name": "examples", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "examples", - "version": "1.0.0", - "license": "Apache-2.0", - "dependencies": { - "@huggingface/transformers": "^3.0.2", - "@lancedb/lancedb": "^0.31.0", - "openai": "^4.29.2", - "sharp": "^0.33.5" - }, - "devDependencies": { - "@biomejs/biome": "^1.7.3", - "@jest/globals": "^29.7.0", - "jest": "^29.7.0", - "jest-environment-node-single-context": "^29.4.0", - "ts-jest": "^29.2.5", - "typescript": "^5.5.4" - } - }, - "..": { - "name": "@lancedb/lancedb", - "version": "0.12.0", - "cpu": [ - "x64", - "arm64" - ], - "extraneous": true, - "license": "Apache 2.0", - "os": [ - "darwin", - "linux", - "win32" - ], - "dependencies": { - "reflect-metadata": "^0.2.2" - }, - "devDependencies": { - "@aws-sdk/client-dynamodb": "^3.33.0", - "@aws-sdk/client-kms": "^3.33.0", - "@aws-sdk/client-s3": "^3.33.0", - "@biomejs/biome": "^1.7.3", - "@jest/globals": "^29.7.0", - "@napi-rs/cli": "^2.18.3", - "@types/axios": "^0.14.0", - "@types/jest": "^29.1.2", - "@types/node": "^22.7.4", - "@types/tmp": "^0.2.6", - "apache-arrow-13": "npm:apache-arrow@13.0.0", - "apache-arrow-14": "npm:apache-arrow@14.0.0", - "apache-arrow-15": "npm:apache-arrow@15.0.0", - "apache-arrow-16": "npm:apache-arrow@16.0.0", - "apache-arrow-17": "npm:apache-arrow@17.0.0", - "eslint": "^8.57.0", - "jest": "^29.7.0", - "shx": "^0.3.4", - "tmp": "^0.2.3", - "ts-jest": "^29.1.2", - "typedoc": "^0.26.4", - "typedoc-plugin-markdown": "^4.2.1", - "typescript": "^5.5.4", - "typescript-eslint": "^7.1.0" - }, - "engines": { - "node": ">= 18" - }, - "optionalDependencies": { - "@huggingface/transformers": "^3.0.2", - "openai": "^4.29.2" - }, - "peerDependencies": { - "apache-arrow": ">=13.0.0 <=17.0.0" - } - }, - "node_modules/@ampproject/remapping": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "dev": true, - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", - "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.27.1", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.26.2", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.2.tgz", - "integrity": "sha512-Z0WgzSEa+aUcdiJuCIqgujCshpMWgUpgOxXotrYPSA53hA3qopNaqcJpyr0hVb1FeWdnqFA35/fUtXgBK8srQg==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", - "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", - "dev": true, - "dependencies": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.26.0", - "@babel/generator": "^7.26.0", - "@babel/helper-compilation-targets": "^7.25.9", - "@babel/helper-module-transforms": "^7.26.0", - "@babel/helpers": "^7.26.0", - "@babel/parser": "^7.26.0", - "@babel/template": "^7.25.9", - "@babel/traverse": "^7.25.9", - "@babel/types": "^7.26.0", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/core/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@babel/generator": { - "version": "7.26.2", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.2.tgz", - "integrity": "sha512-zevQbhbau95nkoxSq3f/DC/SC+EEOUZd3DYqfSkMhY2/wfSeaHV1Ew4vk8e+x8lja31IbyuUa2uQ3JONqKbysw==", - "dev": true, - "dependencies": { - "@babel/parser": "^7.26.2", - "@babel/types": "^7.26.0", - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.25", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz", - "integrity": "sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==", - "dev": true, - "dependencies": { - "@babel/compat-data": "^7.25.9", - "@babel/helper-validator-option": "^7.25.9", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", - "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", - "dev": true, - "dependencies": { - "@babel/traverse": "^7.25.9", - "@babel/types": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", - "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", - "dev": true, - "dependencies": { - "@babel/helper-module-imports": "^7.25.9", - "@babel/helper-validator-identifier": "^7.25.9", - "@babel/traverse": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-plugin-utils": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.25.9.tgz", - "integrity": "sha512-kSMlyUVdWe25rEsRGviIgOWnoT/nfABVWlqt9N19/dIPWViAOW2s9wznP5tURbs/IDuNk4gPy3YdYRgH3uxhBw==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", - "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", - "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.27.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.6.tgz", - "integrity": "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.27.2", - "@babel/types": "^7.27.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.27.5", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.27.5.tgz", - "integrity": "sha512-OsQd175SxWkGlzbny8J3K8TnnDD0N3lrIUtB92xwyRpzaenGZhxDvxN/JgU00U3CDZNj9tPuDJ5H0WS4Nt3vKg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.27.3" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/plugin-syntax-async-generators": { - "version": "7.8.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", - "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-bigint": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", - "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-properties": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", - "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.12.13" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-static-block": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", - "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-attributes": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.26.0.tgz", - "integrity": "sha512-e2dttdsJ1ZTpi3B9UYGLw41hifAubg19AtCu/2I/F1QNVclOBr1dYpTdmdyZ84Xiz43BS/tCUkMAZNLv12Pi+A==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-meta": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", - "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-json-strings": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", - "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.25.9.tgz", - "integrity": "sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-logical-assignment-operators": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", - "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", - "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-numeric-separator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", - "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-object-rest-spread": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", - "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-catch-binding": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", - "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-chaining": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", - "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-private-property-in-object": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", - "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-top-level-await": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", - "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.25.9.tgz", - "integrity": "sha512-hjMgRy5hb8uJJjUcdWunWVcoi9bGpJp8p5Ol1229PoN6aytsLwNMgmdftO23wnCLMfVmTwZDWMPNq/D1SY60JQ==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/template": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", - "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/parser": "^7.27.2", - "@babel/types": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.9.tgz", - "integrity": "sha512-ZCuvfwOwlz/bawvAuvcj8rrithP2/N55Tzz342AkTvq4qaWbGfmCk/tKhNaV2cthijKrPAA8SRJV5WWe7IBMJw==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.25.9", - "@babel/generator": "^7.25.9", - "@babel/parser": "^7.25.9", - "@babel/template": "^7.25.9", - "@babel/types": "^7.25.9", - "debug": "^4.3.1", - "globals": "^11.1.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/types": { - "version": "7.27.6", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.27.6.tgz", - "integrity": "sha512-ETyHEk2VHHvl9b9jZP5IHPavHYk57EhanlRRuae9XCpb/j5bDCbPPMOBfCWhnl/7EDJz0jEMCi/RhccCE8r1+Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true - }, - "node_modules/@biomejs/biome": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.9.4.tgz", - "integrity": "sha512-1rkd7G70+o9KkTn5KLmDYXihGoTaIGO9PIIN2ZB7UJxFrWw04CZHPYiMRjYsaDvVV7hP1dYNRLxSANLaBFGpog==", - "dev": true, - "hasInstallScript": true, - "bin": { - "biome": "bin/biome" - }, - "engines": { - "node": ">=14.21.3" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/biome" - }, - "optionalDependencies": { - "@biomejs/cli-darwin-arm64": "1.9.4", - "@biomejs/cli-darwin-x64": "1.9.4", - "@biomejs/cli-linux-arm64": "1.9.4", - "@biomejs/cli-linux-arm64-musl": "1.9.4", - "@biomejs/cli-linux-x64": "1.9.4", - "@biomejs/cli-linux-x64-musl": "1.9.4", - "@biomejs/cli-win32-arm64": "1.9.4", - "@biomejs/cli-win32-x64": "1.9.4" - } - }, - "node_modules/@biomejs/cli-darwin-arm64": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.9.4.tgz", - "integrity": "sha512-bFBsPWrNvkdKrNCYeAp+xo2HecOGPAy9WyNyB/jKnnedgzl4W4Hb9ZMzYNbf8dMCGmUdSavlYHiR01QaYR58cw==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-darwin-x64": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.9.4.tgz", - "integrity": "sha512-ngYBh/+bEedqkSevPVhLP4QfVPCpb+4BBe2p7Xs32dBgs7rh9nY2AIYUL6BgLw1JVXV8GlpKmb/hNiuIxfPfZg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-linux-arm64": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.9.4.tgz", - "integrity": "sha512-fJIW0+LYujdjUgJJuwesP4EjIBl/N/TcOX3IvIHJQNsAqvV2CHIogsmA94BPG6jZATS4Hi+xv4SkBBQSt1N4/g==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-linux-arm64-musl": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.9.4.tgz", - "integrity": "sha512-v665Ct9WCRjGa8+kTr0CzApU0+XXtRgwmzIf1SeKSGAv+2scAlW6JR5PMFo6FzqqZ64Po79cKODKf3/AAmECqA==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-linux-x64": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.9.4.tgz", - "integrity": "sha512-lRCJv/Vi3Vlwmbd6K+oQ0KhLHMAysN8lXoCI7XeHlxaajk06u7G+UsFSO01NAs5iYuWKmVZjmiOzJ0OJmGsMwg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-linux-x64-musl": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.9.4.tgz", - "integrity": "sha512-gEhi/jSBhZ2m6wjV530Yy8+fNqG8PAinM3oV7CyO+6c3CEh16Eizm21uHVsyVBEB6RIM8JHIl6AGYCv6Q6Q9Tg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-win32-arm64": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.9.4.tgz", - "integrity": "sha512-tlbhLk+WXZmgwoIKwHIHEBZUwxml7bRJgk0X2sPyNR3S93cdRq6XulAZRQJ17FYGGzWne0fgrXBKpl7l4M87Hg==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-win32-x64": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.9.4.tgz", - "integrity": "sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz", - "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@huggingface/transformers": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.0.2.tgz", - "integrity": "sha512-lTyS81eQazMea5UCehDGFMfdcNRZyei7XQLH5X6j4AhA/18Ka0+5qPgMxUxuZLU4xkv60aY2KNz9Yzthv6WVJg==", - "dependencies": { - "@huggingface/jinja": "^0.3.0", - "onnxruntime-node": "1.19.2", - "onnxruntime-web": "1.21.0-dev.20241024-d9ca84ef96", - "sharp": "^0.33.5" - } - }, - "node_modules/@huggingface/transformers/node_modules/@huggingface/jinja": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.2.tgz", - "integrity": "sha512-F2FvuIc+w1blGsaqJI/OErRbWH6bVJDCBI8Rm5D86yZ2wlwrGERsfIaru7XUv9eYC3DMP3ixDRRtF0h6d8AZcQ==", - "engines": { - "node": ">=18" - } - }, - "node_modules/@huggingface/transformers/node_modules/long": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", - "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" - }, - "node_modules/@huggingface/transformers/node_modules/onnxruntime-common": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.19.2.tgz", - "integrity": "sha512-a4R7wYEVFbZBlp0BfhpbFWqe4opCor3KM+5Wm22Az3NGDcQMiU2hfG/0MfnBs+1ZrlSGmlgWeMcXQkDk1UFb8Q==" - }, - "node_modules/@huggingface/transformers/node_modules/onnxruntime-node": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.19.2.tgz", - "integrity": "sha512-9eHMP/HKbbeUcqte1JYzaaRC8JPn7ojWeCeoyShO86TOR97OCyIyAIOGX3V95ErjslVhJRXY8Em/caIUc0hm1Q==", - "hasInstallScript": true, - "os": [ - "win32", - "darwin", - "linux" - ], - "dependencies": { - "onnxruntime-common": "1.19.2", - "tar": "^7.0.1" - } - }, - "node_modules/@huggingface/transformers/node_modules/onnxruntime-web": { - "version": "1.21.0-dev.20241024-d9ca84ef96", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.21.0-dev.20241024-d9ca84ef96.tgz", - "integrity": "sha512-ANSQfMALvCviN3Y4tvTViKofKToV1WUb2r2VjZVCi3uUBPaK15oNJyIxhsNyEckBr/Num3JmSXlkHOD8HfVzSQ==", - "dependencies": { - "flatbuffers": "^1.12.0", - "guid-typescript": "^1.0.9", - "long": "^5.2.3", - "onnxruntime-common": "1.20.0-dev.20241016-2b8fc5529b", - "platform": "^1.3.6", - "protobufjs": "^7.2.4" - } - }, - "node_modules/@huggingface/transformers/node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.20.0-dev.20241016-2b8fc5529b", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.0-dev.20241016-2b8fc5529b.tgz", - "integrity": "sha512-KZK8b6zCYGZFjd4ANze0pqBnqnFTS3GIVeclQpa2qseDpXrCQJfkWBixRcrZShNhm3LpFOZ8qJYFC5/qsJK9WQ==" - }, - "node_modules/@huggingface/transformers/node_modules/protobufjs": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", - "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", - "hasInstallScript": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", - "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", - "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.0.4" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", - "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", - "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", - "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", - "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", - "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", - "cpu": [ - "s390x" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", - "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", - "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", - "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", - "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.0.5" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", - "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.0.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", - "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", - "cpu": [ - "s390x" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.0.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", - "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.0.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", - "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", - "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.0.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", - "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", - "cpu": [ - "wasm32" - ], - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.2.0" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", - "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", - "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", - "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/fs-minipass": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", - "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", - "dependencies": { - "minipass": "^7.0.4" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", - "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", - "dev": true, - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/console": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", - "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/core": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", - "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", - "dev": true, - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/reporters": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-changed-files": "^29.7.0", - "jest-config": "^29.7.0", - "jest-haste-map": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-resolve-dependencies": "^29.7.0", - "jest-runner": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "jest-watcher": "^29.7.0", - "micromatch": "^4.0.4", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/environment": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", - "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", - "dev": true, - "dependencies": { - "@jest/fake-timers": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-mock": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", - "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", - "dev": true, - "dependencies": { - "expect": "^29.7.0", - "jest-snapshot": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect-utils": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", - "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", - "dev": true, - "dependencies": { - "jest-get-type": "^29.6.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/fake-timers": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", - "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.3", - "@sinonjs/fake-timers": "^10.0.2", - "@types/node": "*", - "jest-message-util": "^29.7.0", - "jest-mock": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/globals": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", - "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", - "dev": true, - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/expect": "^29.7.0", - "@jest/types": "^29.6.3", - "jest-mock": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/reporters": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", - "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", - "dev": true, - "dependencies": { - "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@jridgewell/trace-mapping": "^0.3.18", - "@types/node": "*", - "chalk": "^4.0.0", - "collect-v8-coverage": "^1.0.0", - "exit": "^0.1.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-instrument": "^6.0.0", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.1.3", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "jest-worker": "^29.7.0", - "slash": "^3.0.0", - "string-length": "^4.0.1", - "strip-ansi": "^6.0.0", - "v8-to-istanbul": "^9.0.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/schemas": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", - "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", - "dev": true, - "dependencies": { - "@sinclair/typebox": "^0.27.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/source-map": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", - "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.18", - "callsites": "^3.0.0", - "graceful-fs": "^4.2.9" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/test-result": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", - "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", - "dev": true, - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "collect-v8-coverage": "^1.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/test-sequencer": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", - "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", - "dev": true, - "dependencies": { - "@jest/test-result": "^29.7.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/transform": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", - "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", - "dev": true, - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/types": "^29.6.3", - "@jridgewell/trace-mapping": "^0.3.18", - "babel-plugin-istanbul": "^6.1.1", - "chalk": "^4.0.0", - "convert-source-map": "^2.0.0", - "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-util": "^29.7.0", - "micromatch": "^4.0.4", - "pirates": "^4.0.4", - "slash": "^3.0.0", - "write-file-atomic": "^4.0.2" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/types": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", - "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", - "dev": true, - "dependencies": { - "@jest/schemas": "^29.6.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", - "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", - "dev": true, - "dependencies": { - "@jridgewell/set-array": "^1.2.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/set-array": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", - "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", - "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", - "dev": true - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.25", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", - "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", - "dev": true, - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@lancedb/lancedb": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb/-/lancedb-0.31.0.tgz", - "integrity": "sha512-EUEVpheKhaCNE6ybcW760OUyfeei2dKR2ZwgLWeC/ntHL4BBiBLIErh9fuEuUP3/mAx4B5UFraB2m5nDUx5XEA==", - "cpu": [ - "x64", - "arm64" - ], - "license": "Apache-2.0", - "os": [ - "darwin", - "linux", - "win32" - ], - "dependencies": { - "reflect-metadata": "^0.2.2" - }, - "engines": { - "node": ">= 18" - }, - "optionalDependencies": { - "@huggingface/transformers": "3.0.2", - "@lancedb/lancedb-darwin-arm64": "0.31.0", - "@lancedb/lancedb-linux-arm64-gnu": "0.31.0", - "@lancedb/lancedb-linux-arm64-musl": "0.31.0", - "@lancedb/lancedb-linux-x64-gnu": "0.31.0", - "@lancedb/lancedb-linux-x64-musl": "0.31.0", - "@lancedb/lancedb-win32-arm64-msvc": "0.31.0", - "@lancedb/lancedb-win32-x64-msvc": "0.31.0", - "openai": "4.29.2" - }, - "peerDependencies": { - "apache-arrow": ">=15.0.0 <=18.1.0" - } - }, - "node_modules/@lancedb/lancedb-darwin-arm64": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.31.0.tgz", - "integrity": "sha512-6n3VxAenwcNWQpk9Ta4NL/KhpSywskafarBakzFPGe/OXzdKXbmbXdrhGdl8oMacbfgql6wmW3DcAJAbsiThvw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-linux-arm64-gnu": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-gnu/-/lancedb-linux-arm64-gnu-0.31.0.tgz", - "integrity": "sha512-dWVHk5xhTpXQt08y3pxQFf5DK/O2saiWT8AZIIlonFRbGVhLm1KzBCfCgS1WUVbdleSkqAqY3NnMxZzu/dhx9Q==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-linux-arm64-musl": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-musl/-/lancedb-linux-arm64-musl-0.31.0.tgz", - "integrity": "sha512-4y49+83R34IR75H2NPeDtHgSnUIKE9gJoSiELsGmCgICoFTZRVxq88atBEl0BfpsuuUc8S2bSx45mf3H4lZmWA==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-linux-x64-gnu": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.31.0.tgz", - "integrity": "sha512-AFc0qTNdjoPor5bEHRuijft0BGNSiyHlF0cdUdsApLvjrl5jTBzw59+EqzjjCevWz5w/44BBjaTt6ZChWVTc/Q==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-linux-x64-musl": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-musl/-/lancedb-linux-x64-musl-0.31.0.tgz", - "integrity": "sha512-VXtu/xTJXtkfTyUH+MXbMYCGVCAvHJvGDoPNIqDBtrnX7uHlgmxPbw7V97k18/5UVLU2v8NhsM5sCiMD4si+Tw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-win32-arm64-msvc": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-arm64-msvc/-/lancedb-win32-arm64-msvc-0.31.0.tgz", - "integrity": "sha512-AvbnX/24uPh6UP9IRdYZNLc+2iE4uLKIQcEbGXUMLsVIMWQvuHfRamZpIlWb50tz/C7Wo5aPiQIkolPJN5fiuw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb-win32-x64-msvc": { - "version": "0.31.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.31.0.tgz", - "integrity": "sha512-reFE43TCr5Lifni2fC5YiyOnjDTAXOY37+rAY01FFKZyhjZkhRy9NZN1WvWJ35XHC4eeUVOW4+UXOvSV0djyCA==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 18" - } - }, - "node_modules/@lancedb/lancedb/node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "optional": true, - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@lancedb/lancedb/node_modules/openai": { - "version": "4.29.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", - "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7", - "web-streams-polyfill": "^3.2.1" - }, - "bin": { - "openai": "bin/cli" - } - }, - "node_modules/@lancedb/lancedb/node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true, - "engines": { - "node": ">=14" - } - }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, - "node_modules/@sinclair/typebox": { - "version": "0.27.8", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", - "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", - "dev": true - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/fake-timers": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", - "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^3.0.0" - } - }, - "node_modules/@swc/helpers": { - "version": "0.5.17", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", - "integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==", - "peer": true, - "dependencies": { - "tslib": "^2.8.0" - } - }, - "node_modules/@types/babel__core": { - "version": "7.20.5", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", - "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", - "dev": true, - "dependencies": { - "@babel/parser": "^7.20.7", - "@babel/types": "^7.20.7", - "@types/babel__generator": "*", - "@types/babel__template": "*", - "@types/babel__traverse": "*" - } - }, - "node_modules/@types/babel__generator": { - "version": "7.6.8", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", - "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", - "dev": true, - "dependencies": { - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__template": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", - "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", - "dev": true, - "dependencies": { - "@babel/parser": "^7.1.0", - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__traverse": { - "version": "7.20.6", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz", - "integrity": "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==", - "dev": true, - "dependencies": { - "@babel/types": "^7.20.7" - } - }, - "node_modules/@types/command-line-args": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", - "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==", - "peer": true - }, - "node_modules/@types/command-line-usage": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", - "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==", - "peer": true - }, - "node_modules/@types/graceful-fs": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", - "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", - "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", - "dev": true - }, - "node_modules/@types/istanbul-lib-report": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", - "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", - "dev": true, - "dependencies": { - "@types/istanbul-lib-coverage": "*" - } - }, - "node_modules/@types/istanbul-reports": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", - "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", - "dev": true, - "dependencies": { - "@types/istanbul-lib-report": "*" - } - }, - "node_modules/@types/node": { - "version": "20.14.11", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz", - "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@types/node-fetch": { - "version": "2.6.11", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", - "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.0" - } - }, - "node_modules/@types/stack-utils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", - "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", - "dev": true - }, - "node_modules/@types/yargs": { - "version": "17.0.33", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", - "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==", - "dev": true, - "dependencies": { - "@types/yargs-parser": "*" - } - }, - "node_modules/@types/yargs-parser": { - "version": "21.0.3", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", - "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", - "dev": true - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/agentkeepalive": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", - "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, - "node_modules/ansi-escapes": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", - "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", - "dev": true, - "dependencies": { - "type-fest": "^0.21.3" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/apache-arrow": { - "version": "18.1.0", - "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-18.1.0.tgz", - "integrity": "sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==", - "peer": true, - "dependencies": { - "@swc/helpers": "^0.5.11", - "@types/command-line-args": "^5.2.3", - "@types/command-line-usage": "^5.0.4", - "@types/node": "^20.13.0", - "command-line-args": "^5.2.1", - "command-line-usage": "^7.0.1", - "flatbuffers": "^24.3.25", - "json-bignum": "^0.0.3", - "tslib": "^2.6.2" - }, - "bin": { - "arrow2csv": "bin/arrow2csv.js" - } - }, - "node_modules/apache-arrow/node_modules/flatbuffers": { - "version": "24.12.23", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.12.23.tgz", - "integrity": "sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==", - "peer": true - }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/array-back": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", - "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", - "peer": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/async": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", - "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", - "dev": true - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" - }, - "node_modules/babel-jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", - "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", - "dev": true, - "dependencies": { - "@jest/transform": "^29.7.0", - "@types/babel__core": "^7.1.14", - "babel-plugin-istanbul": "^6.1.1", - "babel-preset-jest": "^29.6.3", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.8.0" - } - }, - "node_modules/babel-plugin-istanbul": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", - "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", - "dev": true, - "dependencies": { - "@babel/helper-plugin-utils": "^7.0.0", - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-instrument": "^5.0.4", - "test-exclude": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", - "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", - "dev": true, - "dependencies": { - "@babel/core": "^7.12.3", - "@babel/parser": "^7.14.7", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/babel-plugin-istanbul/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/babel-plugin-jest-hoist": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", - "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", - "dev": true, - "dependencies": { - "@babel/template": "^7.3.3", - "@babel/types": "^7.3.3", - "@types/babel__core": "^7.1.14", - "@types/babel__traverse": "^7.0.6" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/babel-preset-current-node-syntax": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.1.0.tgz", - "integrity": "sha512-ldYss8SbBlWva1bs28q78Ju5Zq1F+8BrqBZZ0VFhLBvhh6lCpC2o3gDJi/5DRLs9FgYZCnmPYIVFU4lRXCkyUw==", - "dev": true, - "dependencies": { - "@babel/plugin-syntax-async-generators": "^7.8.4", - "@babel/plugin-syntax-bigint": "^7.8.3", - "@babel/plugin-syntax-class-properties": "^7.12.13", - "@babel/plugin-syntax-class-static-block": "^7.14.5", - "@babel/plugin-syntax-import-attributes": "^7.24.7", - "@babel/plugin-syntax-import-meta": "^7.10.4", - "@babel/plugin-syntax-json-strings": "^7.8.3", - "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", - "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", - "@babel/plugin-syntax-numeric-separator": "^7.10.4", - "@babel/plugin-syntax-object-rest-spread": "^7.8.3", - "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", - "@babel/plugin-syntax-optional-chaining": "^7.8.3", - "@babel/plugin-syntax-private-property-in-object": "^7.14.5", - "@babel/plugin-syntax-top-level-await": "^7.14.5" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/babel-preset-jest": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", - "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", - "dev": true, - "dependencies": { - "babel-plugin-jest-hoist": "^29.6.3", - "babel-preset-current-node-syntax": "^1.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, - "node_modules/base-64": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==", - "optional": true - }, - "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/browserslist": { - "version": "4.24.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz", - "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "dependencies": { - "caniuse-lite": "^1.0.30001669", - "electron-to-chromium": "^1.5.41", - "node-releases": "^2.0.18", - "update-browserslist-db": "^1.1.1" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/bs-logger": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", - "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", - "dev": true, - "dependencies": { - "fast-json-stable-stringify": "2.x" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/bser": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", - "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", - "dev": true, - "dependencies": { - "node-int64": "^0.4.0" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001677", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001677.tgz", - "integrity": "sha512-fmfjsOlJUpMWu+mAAtZZZHz7UEwsUxIIvu1TJfO1HqFQvB/B+ii0xr9B5HpbZY/mC4XZ8SvjHJqtAY6pDPQEog==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ] - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chalk-template": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", - "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", - "peer": true, - "dependencies": { - "chalk": "^4.1.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/chalk-template?sponsor=1" - } - }, - "node_modules/char-regex": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", - "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "license": "BSD-3-Clause", - "optional": true, - "engines": { - "node": "*" - } - }, - "node_modules/ci-info": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", - "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "engines": { - "node": ">=8" - } - }, - "node_modules/cjs-module-lexer": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz", - "integrity": "sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA==", - "dev": true - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", - "dev": true, - "engines": { - "iojs": ">= 1.0.0", - "node": ">= 0.12.0" - } - }, - "node_modules/collect-v8-coverage": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", - "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", - "dev": true - }, - "node_modules/color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "dependencies": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "engines": { - "node": ">=12.5.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/command-line-args": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", - "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", - "peer": true, - "dependencies": { - "array-back": "^3.1.0", - "find-replace": "^3.0.0", - "lodash.camelcase": "^4.3.0", - "typical": "^4.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/command-line-usage": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.3.tgz", - "integrity": "sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q==", - "peer": true, - "dependencies": { - "array-back": "^6.2.2", - "chalk-template": "^0.4.0", - "table-layout": "^4.1.0", - "typical": "^7.1.1" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/command-line-usage/node_modules/array-back": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", - "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", - "peer": true, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/command-line-usage/node_modules/typical": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", - "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", - "peer": true, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true - }, - "node_modules/create-jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", - "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-config": "^29.7.0", - "jest-util": "^29.7.0", - "prompts": "^2.0.1" - }, - "bin": { - "create-jest": "bin/create-jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "license": "BSD-3-Clause", - "optional": true, - "engines": { - "node": "*" - } - }, - "node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", - "dev": true, - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dedent": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", - "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", - "dev": true, - "peerDependencies": { - "babel-plugin-macros": "^3.1.0" - }, - "peerDependenciesMeta": { - "babel-plugin-macros": { - "optional": true - } - } - }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/detect-libc": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", - "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", - "engines": { - "node": ">=8" - } - }, - "node_modules/detect-newline": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", - "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/diff-sequences": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", - "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", - "dev": true, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/digest-fetch": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", - "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "license": "ISC", - "optional": true, - "dependencies": { - "base-64": "^0.1.0", - "md5": "^2.3.0" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, - "node_modules/ejs": { - "version": "3.1.10", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", - "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", - "dev": true, - "dependencies": { - "jake": "^10.8.5" - }, - "bin": { - "ejs": "bin/cli.js" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/electron-to-chromium": { - "version": "1.5.51", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.51.tgz", - "integrity": "sha512-kKeWV57KSS8jH4alKt/jKnvHPmJgBxXzGUSbMd4eQF+iOsVPl7bz2KUmu6eo80eMP8wVioTfTyTzdMgM15WXNg==", - "dev": true - }, - "node_modules/emittery": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", - "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sindresorhus/emittery?sponsor=1" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "node_modules/error-ex": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", - "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", - "dev": true, - "dependencies": { - "is-arrayish": "^0.2.1" - } - }, - "node_modules/error-ex/node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-string-regexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", - "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "engines": { - "node": ">=6" - } - }, - "node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dev": true, - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/exit": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/expect": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", - "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", - "dev": true, - "dependencies": { - "@jest/expect-utils": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true - }, - "node_modules/fb-watchman": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", - "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", - "dev": true, - "dependencies": { - "bser": "2.1.1" - } - }, - "node_modules/filelist": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", - "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", - "dev": true, - "dependencies": { - "minimatch": "^5.0.1" - } - }, - "node_modules/filelist/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/filelist/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/find-replace": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", - "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", - "peer": true, - "dependencies": { - "array-back": "^3.0.1" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/flatbuffers": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", - "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==" - }, - "node_modules/foreground-child": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", - "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/foreground-child/node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", - "dev": true, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true - }, - "node_modules/guid-typescript": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", - "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==" - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true - }, - "node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "dev": true, - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "dependencies": { - "ms": "^2.0.0" - } - }, - "node_modules/import-local": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", - "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", - "dev": true, - "dependencies": { - "pkg-dir": "^4.2.0", - "resolve-cwd": "^3.0.0" - }, - "bin": { - "import-local-fixture": "fixtures/cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "license": "MIT", - "optional": true - }, - "node_modules/is-core-module": { - "version": "2.15.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", - "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==", - "dev": true, - "dependencies": { - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", - "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", - "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", - "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", - "dev": true, - "dependencies": { - "@babel/core": "^7.23.9", - "@babel/parser": "^7.23.9", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", - "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", - "dev": true, - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^4.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", - "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", - "dev": true, - "dependencies": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-reports": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", - "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", - "dev": true, - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jackspeak": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", - "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, - "node_modules/jake": { - "version": "10.9.2", - "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz", - "integrity": "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==", - "dev": true, - "dependencies": { - "async": "^3.2.3", - "chalk": "^4.0.2", - "filelist": "^1.0.4", - "minimatch": "^3.1.2" - }, - "bin": { - "jake": "bin/cli.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", - "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", - "dev": true, - "dependencies": { - "@jest/core": "^29.7.0", - "@jest/types": "^29.6.3", - "import-local": "^3.0.2", - "jest-cli": "^29.7.0" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-changed-files": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", - "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", - "dev": true, - "dependencies": { - "execa": "^5.0.0", - "jest-util": "^29.7.0", - "p-limit": "^3.1.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-circus": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", - "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", - "dev": true, - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/expect": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "co": "^4.6.0", - "dedent": "^1.0.0", - "is-generator-fn": "^2.0.0", - "jest-each": "^29.7.0", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "p-limit": "^3.1.0", - "pretty-format": "^29.7.0", - "pure-rand": "^6.0.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-cli": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", - "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", - "dev": true, - "dependencies": { - "@jest/core": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "create-jest": "^29.7.0", - "exit": "^0.1.2", - "import-local": "^3.0.2", - "jest-config": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "yargs": "^17.3.1" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-config": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", - "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", - "dev": true, - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/test-sequencer": "^29.7.0", - "@jest/types": "^29.6.3", - "babel-jest": "^29.7.0", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "deepmerge": "^4.2.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-circus": "^29.7.0", - "jest-environment-node": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-runner": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "micromatch": "^4.0.4", - "parse-json": "^5.2.0", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@types/node": "*", - "ts-node": ">=9.0.0" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "ts-node": { - "optional": true - } - } - }, - "node_modules/jest-config/node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-diff": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", - "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^29.6.3", - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-docblock": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", - "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", - "dev": true, - "dependencies": { - "detect-newline": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-each": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", - "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "jest-get-type": "^29.6.3", - "jest-util": "^29.7.0", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-environment-node": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", - "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", - "dev": true, - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/fake-timers": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-mock": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-environment-node-single-context": { - "version": "29.4.0", - "resolved": "https://registry.npmjs.org/jest-environment-node-single-context/-/jest-environment-node-single-context-29.4.0.tgz", - "integrity": "sha512-VOuB0Pf3/+Tu0eImZ888SeHpFIiujRiW/3b6NTST1/zdv6ZdRAblCV2q5SisF0PlDA8y9SHJWjKFtFXNJ7U6CQ==", - "dev": true, - "dependencies": { - "jest-environment-node": "^29.7.0" - }, - "funding": { - "url": "https://github.com/kayahr/jest-environment-node-single-context?sponsor=1" - } - }, - "node_modules/jest-get-type": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", - "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", - "dev": true, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-haste-map": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", - "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.3", - "@types/graceful-fs": "^4.1.3", - "@types/node": "*", - "anymatch": "^3.0.3", - "fb-watchman": "^2.0.0", - "graceful-fs": "^4.2.9", - "jest-regex-util": "^29.6.3", - "jest-util": "^29.7.0", - "jest-worker": "^29.7.0", - "micromatch": "^4.0.4", - "walker": "^1.0.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "optionalDependencies": { - "fsevents": "^2.3.2" - } - }, - "node_modules/jest-leak-detector": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", - "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", - "dev": true, - "dependencies": { - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-matcher-utils": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", - "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "^29.7.0", - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-message-util": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", - "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "^29.6.3", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.4", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-mock": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", - "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-pnp-resolver": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", - "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true, - "engines": { - "node": ">=6" - }, - "peerDependencies": { - "jest-resolve": "*" - }, - "peerDependenciesMeta": { - "jest-resolve": { - "optional": true - } - } - }, - "node_modules/jest-regex-util": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", - "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", - "dev": true, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-resolve": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", - "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-pnp-resolver": "^1.2.2", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "resolve": "^1.20.0", - "resolve.exports": "^2.0.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-resolve-dependencies": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", - "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", - "dev": true, - "dependencies": { - "jest-regex-util": "^29.6.3", - "jest-snapshot": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-runner": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", - "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", - "dev": true, - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/environment": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "graceful-fs": "^4.2.9", - "jest-docblock": "^29.7.0", - "jest-environment-node": "^29.7.0", - "jest-haste-map": "^29.7.0", - "jest-leak-detector": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-resolve": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-util": "^29.7.0", - "jest-watcher": "^29.7.0", - "jest-worker": "^29.7.0", - "p-limit": "^3.1.0", - "source-map-support": "0.5.13" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-runtime": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", - "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", - "dev": true, - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/fake-timers": "^29.7.0", - "@jest/globals": "^29.7.0", - "@jest/source-map": "^29.6.3", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "cjs-module-lexer": "^1.0.0", - "collect-v8-coverage": "^1.0.0", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-mock": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "slash": "^3.0.0", - "strip-bom": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-snapshot": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", - "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", - "dev": true, - "dependencies": { - "@babel/core": "^7.11.6", - "@babel/generator": "^7.7.2", - "@babel/plugin-syntax-jsx": "^7.7.2", - "@babel/plugin-syntax-typescript": "^7.7.2", - "@babel/types": "^7.3.3", - "@jest/expect-utils": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "babel-preset-current-node-syntax": "^1.0.0", - "chalk": "^4.0.0", - "expect": "^29.7.0", - "graceful-fs": "^4.2.9", - "jest-diff": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "natural-compare": "^1.4.0", - "pretty-format": "^29.7.0", - "semver": "^7.5.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-util": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", - "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "graceful-fs": "^4.2.9", - "picomatch": "^2.2.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", - "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.3", - "camelcase": "^6.2.0", - "chalk": "^4.0.0", - "jest-get-type": "^29.6.3", - "leven": "^3.1.0", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-watcher": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", - "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", - "dev": true, - "dependencies": { - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "jest-util": "^29.7.0", - "string-length": "^4.0.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-worker": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", - "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", - "dev": true, - "dependencies": { - "@types/node": "*", - "jest-util": "^29.7.0", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true - }, - "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsesc": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", - "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", - "dev": true, - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-bignum": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", - "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", - "peer": true, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/kleur": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", - "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true - }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "peer": true - }, - "node_modules/lodash.memoize": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", - "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", - "dev": true - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/make-dir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", - "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true - }, - "node_modules/makeerror": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", - "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", - "dev": true, - "dependencies": { - "tmpl": "1.0.5" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", - "dependencies": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "bin": { - "mkdirp": "dist/cjs/src/bin.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/node-int64": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true - }, - "node_modules/node-releases": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", - "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", - "dev": true - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true, - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/openai": { - "version": "4.71.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.71.0.tgz", - "integrity": "sha512-jeJ7+6cZvj+ZbIsbX/Ag8+pug2+vjKbrD/v3Hwp6uv3KZyWjSkZa5MdUshzpNC3jsFzakfbUhEEFQXsKWNgm/g==", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - }, - "bin": { - "openai": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.23.8" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } - } - }, - "node_modules/openai/node_modules/@types/node": { - "version": "18.19.64", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.64.tgz", - "integrity": "sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-locate/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" - }, - "node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true - }, - "node_modules/path-scurry": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", - "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/path-scurry/node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true - }, - "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pirates": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", - "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", - "dev": true, - "engines": { - "node": ">= 6" - } - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true, - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/platform": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", - "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" - }, - "node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/prompts": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", - "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", - "dev": true, - "dependencies": { - "kleur": "^3.0.3", - "sisteransi": "^1.0.5" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/pure-rand": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", - "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/dubzzz" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fast-check" - } - ] - }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true - }, - "node_modules/reflect-metadata": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", - "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==" - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/resolve": { - "version": "1.22.8", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", - "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", - "dev": true, - "dependencies": { - "is-core-module": "^2.13.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/resolve-cwd": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", - "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true, - "dependencies": { - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve.exports": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", - "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/rimraf": { - "version": "5.0.10", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", - "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", - "dependencies": { - "glob": "^10.3.7" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rimraf/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/rimraf/node_modules/glob": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", - "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rimraf/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/semver": { - "version": "7.6.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", - "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", - "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", - "hasInstallScript": true, - "dependencies": { - "color": "^4.2.3", - "detect-libc": "^2.0.3", - "semver": "^7.6.3" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.5", - "@img/sharp-darwin-x64": "0.33.5", - "@img/sharp-libvips-darwin-arm64": "1.0.4", - "@img/sharp-libvips-darwin-x64": "1.0.4", - "@img/sharp-libvips-linux-arm": "1.0.5", - "@img/sharp-libvips-linux-arm64": "1.0.4", - "@img/sharp-libvips-linux-s390x": "1.0.4", - "@img/sharp-libvips-linux-x64": "1.0.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", - "@img/sharp-libvips-linuxmusl-x64": "1.0.4", - "@img/sharp-linux-arm": "0.33.5", - "@img/sharp-linux-arm64": "0.33.5", - "@img/sharp-linux-s390x": "0.33.5", - "@img/sharp-linux-x64": "0.33.5", - "@img/sharp-linuxmusl-arm64": "0.33.5", - "@img/sharp-linuxmusl-x64": "0.33.5", - "@img/sharp-wasm32": "0.33.5", - "@img/sharp-win32-ia32": "0.33.5", - "@img/sharp-win32-x64": "0.33.5" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true - }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/sisteransi": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", - "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", - "dev": true - }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", - "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", - "dev": true, - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true - }, - "node_modules/stack-utils": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", - "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", - "dev": true, - "dependencies": { - "escape-string-regexp": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-length": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", - "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", - "dev": true, - "dependencies": { - "char-regex": "^1.0.2", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", - "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/table-layout": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz", - "integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==", - "peer": true, - "dependencies": { - "array-back": "^6.2.2", - "wordwrapjs": "^5.1.0" - }, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/table-layout/node_modules/array-back": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", - "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", - "peer": true, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", - "dependencies": { - "@isaacs/fs-minipass": "^4.0.0", - "chownr": "^3.0.0", - "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", - "yallist": "^5.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/tar/node_modules/chownr": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", - "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "engines": { - "node": ">=18" - } - }, - "node_modules/tar/node_modules/yallist": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", - "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "engines": { - "node": ">=18" - } - }, - "node_modules/test-exclude": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", - "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", - "dev": true, - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tmpl": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", - "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", - "dev": true - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/ts-jest": { - "version": "29.2.5", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.5.tgz", - "integrity": "sha512-KD8zB2aAZrcKIdGk4OwpJggeLcH1FgrICqDSROWqlnJXGCXK4Mn6FcdK2B6670Xr73lHMG1kHw8R87A0ecZ+vA==", - "dev": true, - "dependencies": { - "bs-logger": "^0.2.6", - "ejs": "^3.1.10", - "fast-json-stable-stringify": "^2.1.0", - "jest-util": "^29.0.0", - "json5": "^2.2.3", - "lodash.memoize": "^4.1.2", - "make-error": "^1.3.6", - "semver": "^7.6.3", - "yargs-parser": "^21.1.1" - }, - "bin": { - "ts-jest": "cli.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" - }, - "peerDependencies": { - "@babel/core": ">=7.0.0-beta.0 <8", - "@jest/transform": "^29.0.0", - "@jest/types": "^29.0.0", - "babel-jest": "^29.0.0", - "jest": "^29.0.0", - "typescript": ">=4.3 <6" - }, - "peerDependenciesMeta": { - "@babel/core": { - "optional": true - }, - "@jest/transform": { - "optional": true - }, - "@jest/types": { - "optional": true - }, - "babel-jest": { - "optional": true - }, - "esbuild": { - "optional": true - } - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" - }, - "node_modules/type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/typescript": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", - "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/typical": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", - "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", - "peer": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" - }, - "node_modules/update-browserslist-db": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", - "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.0" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/v8-to-istanbul": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", - "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.12", - "@types/istanbul-lib-coverage": "^2.0.1", - "convert-source-map": "^2.0.0" - }, - "engines": { - "node": ">=10.12.0" - } - }, - "node_modules/walker": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", - "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", - "dev": true, - "dependencies": { - "makeerror": "1.0.12" - } - }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "engines": { - "node": ">= 14" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wordwrapjs": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz", - "integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==", - "peer": true, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true - }, - "node_modules/write-file-atomic": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", - "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", - "dev": true, - "dependencies": { - "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" - }, - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, - "engines": { - "node": ">=12" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - } -} diff --git a/tests/ts/package.json b/tests/ts/package.json deleted file mode 100644 index 1e1966c..0000000 --- a/tests/ts/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "examples", - "version": "1.0.0", - "description": "Examples for LanceDB", - "main": "index.js", - "type": "module", - "scripts": { - "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers", - "//2": "--testEnvironment is needed to run jest with sentence-transformers", - "//3": "See: https://github.com/huggingface/transformers.js/issues/57", - "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose", - "lint": "biome check *.ts && biome format *.ts", - "lint-ci": "biome ci .", - "lint-fix": "biome check --write *.ts && npm run format", - "format": "biome format --write *.ts" - }, - "author": "Lance Devs", - "license": "Apache-2.0", - "dependencies": { - "@huggingface/transformers": "^3.0.2", - "@lancedb/lancedb": "^0.31.0", - "openai": "^4.29.2", - "sharp": "^0.33.5" - }, - "devDependencies": { - "@biomejs/biome": "^1.7.3", - "@jest/globals": "^29.7.0", - "jest": "^29.7.0", - "jest-environment-node-single-context": "^29.4.0", - "ts-jest": "^29.2.5", - "typescript": "^5.5.4" - } -} diff --git a/tests/ts/quickstart.test.ts b/tests/ts/quickstart.test.ts deleted file mode 100644 index ecd5c09..0000000 --- a/tests/ts/quickstart.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -import * as lancedb from "@lancedb/lancedb"; -import { withTempDirectory } from "./util.ts"; - -test("quickstart example (async)", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - // --8<-- [start:quickstart_data] - const data = [ - { - id: "1", - name: "King Arthur", - role: "King", - description: "Leader of Camelot and wielder of Excalibur.", - stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 }, - vector: [0.7, 0.1, 0.9, 0.7], - }, - { - id: "2", - name: "Merlin", - role: "Wizard", - description: "Advisor and prophet with deep magical knowledge.", - stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 }, - vector: [0.2, 0.9, 0.4, 0.9], - }, - { - id: "3", - name: "Sir Lancelot", - role: "Knight", - description: "Legendary knight known for courage and combat skill.", - stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 }, - vector: [0.9, 0.1, 0.5, 0.4], - }, - ]; - // --8<-- [end:quickstart_data] - - // --8<-- [start:quickstart_create_table] - let table = await db.createTable("characters", data, { mode: "overwrite" }); - // --8<-- [end:quickstart_create_table] - expect(await table.countRows()).toBe(3); - await db.dropTable("characters"); - - // --8<-- [start:quickstart_create_table_no_overwrite] - table = await db.createTable("characters", data); - // --8<-- [end:quickstart_create_table_no_overwrite] - expect(await table.countRows()).toBe(3); - - // --8<-- [start:quickstart_vector_search_1] - // Search for examples similar to a "wise magical advisor" - let queryVector = [0.2, 0.8, 0.4, 0.9]; - - let result = await table - .search(queryVector) - .select(["name", "role", "description", "_distance"]) - .limit(2) - .toArray(); - console.table(result); - // --8<-- [end:quickstart_vector_search_1] - expect(result[0].name).toBe("Merlin"); - - // --8<-- [start:quickstart_curate_with_metadata] - const curated = await table - .search(queryVector) - .where("stats.magic >= 4") - .select(["name", "role", "description", "_distance"]) - .limit(2) - .toArray(); - console.table(curated); - // --8<-- [end:quickstart_curate_with_metadata] - expect(curated[0].name).toBe("Merlin"); - - // --8<-- [start:quickstart_output_array] - result = await table.search(queryVector).limit(2).toArray(); - console.table(result); - // --8<-- [end:quickstart_output_array] - expect(result[0].name).toBe("Merlin"); - - // --8<-- [start:quickstart_add_feature] - await table.addColumns([ - { - name: "power_score", - valueSql: - "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)", - }, - ]); - // --8<-- [end:quickstart_add_feature] - const schemaWithFeature = await table.schema(); - expect(schemaWithFeature.fields.some((f) => f.name === "power_score")).toBe( - true, - ); - - // --8<-- [start:quickstart_query_feature] - const features = await table - .query() - .select(["name", "role", "power_score"]) - .toArray(); - console.table(features); - // --8<-- [end:quickstart_query_feature] - expect(features[0]).toHaveProperty("power_score"); - - // --8<-- [start:quickstart_multimodal_bytes] - const arrow = await import("apache-arrow"); - const path = await import("node:path"); - const { readFile } = await import("node:fs/promises"); - - const imagePath = path.resolve( - "../../docs/static/assets/images/quickstart/sir-lancelot.jpg", - ); - const imageBytes = await readFile(imagePath); - const imageSchema = new arrow.Schema([ - new arrow.Field("id", new arrow.Utf8()), - new arrow.Field("description", new arrow.Utf8()), - new arrow.Field("image", new arrow.Binary()), - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 4, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - ]); - const imageData = lancedb.makeArrowTable( - [ - { - id: "lancelot", - description: "Portrait of Sir Lancelot", - image: imageBytes, - vector: [0.9, 0.1, 0.5, 0.4], - }, - ], - { schema: imageSchema }, - ); - const multimodalTable = await db.createTable( - "character_images", - imageData, - { mode: "overwrite" }, - ); - // --8<-- [end:quickstart_multimodal_bytes] - expect(await multimodalTable.countRows()).toBe(1); - - // --8<-- [start:quickstart_open_table] - table = await db.openTable("characters"); - // --8<-- [end:quickstart_open_table] - - // --8<-- [start:quickstart_add_data] - const moreData = [ - { - id: "4", - name: "Morgana", - role: "Sorceress", - description: "Powerful sorceress of Avalon.", - stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 }, - vector: [0.3, 0.9, 0.6, 0.8], - power_score: 3.75, - }, - ]; - - // Add data to table - await table.add(moreData); - // --8<-- [end:quickstart_add_data] - expect(await table.countRows()).toBe(4); - - // --8<-- [start:quickstart_vector_search_2] - // Search for examples similar to a "powerful sorceress" - queryVector = [0.3, 0.9, 0.6, 0.8]; - - const results = await table.search(queryVector).limit(2).toArray(); - console.table(results); - // --8<-- [end:quickstart_vector_search_2] - expect(results[0].name).toBe("Morgana"); - }); -}); diff --git a/tests/ts/search.test.ts b/tests/ts/search.test.ts deleted file mode 100644 index bc50968..0000000 --- a/tests/ts/search.test.ts +++ /dev/null @@ -1,355 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -// --8<-- [start:import] -import * as lancedb from "@lancedb/lancedb"; -// --8<-- [end:import] -// --8<-- [start:import_bin_util] -import { Field, FixedSizeList, Int32, Schema, Uint8 } from "apache-arrow"; -// --8<-- [end:import_bin_util] -import { Float32, Struct } from "apache-arrow"; -import { withTempDirectory } from "./util.ts"; - -async function buildIndexedTable(db: lancedb.Connection, name: string) { - const data = Array.from({ length: 512 }, (_, i) => ({ - vector: Array.from({ length: 128 }, () => Math.random()), - id: i, - })); - const tbl = await db.createTable(name, data, { mode: "overwrite" }); - await tbl.createIndex("vector", { - config: lancedb.Index.ivfPq({ numPartitions: 4, numSubVectors: 8 }), - }); - return tbl; -} - -test("vector search", async () => { - await withTempDirectory(async (databaseDir) => { - { - const db = await lancedb.connect(databaseDir); - - const data = Array.from({ length: 10_000 }, (_, i) => ({ - vector: Array(128).fill(i), - id: `${i}`, - })); - - await db.createTable("my_vectors", data); - } - - // --8<-- [start:search1] - const db = await lancedb.connect(databaseDir); - const tbl = await db.openTable("my_vectors"); - - const results1 = await tbl.search(Array(128).fill(1.2)).limit(10).toArray(); - // --8<-- [end:search1] - expect(results1.length).toBe(10); - - // --8<-- [start:search2] - const results2 = await ( - tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery - ) - .distanceType("cosine") - .limit(10) - .toArray(); - // --8<-- [end:search2] - expect(results2.length).toBe(10); - - // --8<-- [start:distance_range] - const results3 = await ( - tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery - ) - .distanceType("cosine") - .distanceRange(0.1, 0.2) - .limit(10) - .toArray(); - // --8<-- [end:distance_range] - for (const r of results3) { - expect(r.distance).toBeGreaterThanOrEqual(0.1); - expect(r.distance).toBeLessThan(0.2); - } - - { - // --8<-- [start:ingest_binary_data] - const schema = new Schema([ - new Field("id", new Int32(), true), - new Field("vec", new FixedSizeList(32, new Field("item", new Uint8()))), - ]); - const data = lancedb.makeArrowTable( - Array(1_000) - .fill(0) - .map((_, i) => ({ - // the 256 bits would be store in 32 bytes, - // if your data is already in this format, you can skip the packBits step - id: i, - vec: lancedb.packBits(Array(256).fill(i % 2)), - })), - { schema: schema }, - ); - - const tbl = await db.createTable("binary_table", data); - await tbl.createIndex("vec", { - config: lancedb.Index.ivfFlat({ - numPartitions: 10, - distanceType: "hamming", - }), - }); - // --8<-- [end:ingest_binary_data] - - // --8<-- [start:search_binary_data] - const query = Array(32) - .fill(1) - .map(() => Math.floor(Math.random() * 255)); - const results = await tbl.query().nearestTo(query).limit(10).toArrow(); - // --8<-- [end:search_binary_data] - expect(results.numRows).toBe(10); - } - }); -}); - -test("vector search docs snippets", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - // ---- Nested vector column: inference and explicit selection ---- - { - const nestedSchema = new Schema([ - new Field("id", new Int32()), - new Field( - "image", - new Struct([ - new Field( - "embedding", - new FixedSizeList(2, new Field("item", new Float32(), true)), - ), - ]), - ), - ]); - const arrowTbl = lancedb.makeArrowTable( - [{ id: 0, image: { embedding: [0.0, 1.0] } }], - { schema: nestedSchema }, - ); - await db.createTable("nested", arrowTbl, { mode: "overwrite" }); - - // --8<-- [start:select_vector_column] - const table = await db.openTable("nested"); - - // Inferred: LanceDB finds the single nested vector leaf automatically. - await table.query().nearestTo([0.0, 1.0]).limit(1).toArray(); - - // Explicit: required when more than one vector column matches. - await table - .query() - .nearestTo([0.0, 1.0]) - .column("image.embedding") - .limit(1) - .toArray(); - // --8<-- [end:select_vector_column] - } - - // ---- Index a nested vector column ---- - { - const dim = 16; - const nestedSchema = new Schema([ - new Field("id", new Int32()), - new Field( - "image", - new Struct([ - new Field( - "embedding", - new FixedSizeList(dim, new Field("item", new Float32(), true)), - ), - ]), - ), - ]); - const rows = Array.from({ length: 512 }, (_, i) => ({ - id: i, - image: { embedding: Array.from({ length: dim }, () => Math.random()) }, - })); - const arrowTbl = lancedb.makeArrowTable(rows, { schema: nestedSchema }); - const table = await db.createTable("nested_index", arrowTbl, { - mode: "overwrite", - }); - - // --8<-- [start:index_nested_column] - await table.createIndex("image.embedding"); - // --8<-- [end:index_nested_column] - } - - // ---- Indexed queries: refine, fast search, bypass index ---- - { - const table = await buildIndexedTable(db, "ann_table"); - const embedding = Array.from({ length: 128 }, () => Math.random()); - - // --8<-- [start:exact_vs_approximate] - // Indexed ANN search without refinement (fast, approximate `_distance`) - const fastResults = await (table.search(embedding) as lancedb.VectorQuery) - .limit(10) - .toArray(); - - // Recompute distances on full vectors for reranked candidates - const exactDistanceResults = await ( - table.search(embedding) as lancedb.VectorQuery - ) - .limit(10) - .refineFactor(1) - .toArray(); - - // Rerank a larger candidate set for better recall (higher latency) - const higherRecallResults = await ( - table.search(embedding) as lancedb.VectorQuery - ) - .limit(10) - .refineFactor(20) - .toArray(); - // --8<-- [end:exact_vs_approximate] - expect(fastResults.length).toBe(10); - expect(exactDistanceResults.length).toBe(10); - expect(higherRecallResults.length).toBe(10); - - // --8<-- [start:fast_search] - await table - .query() - .nearestTo(embedding) - .fastSearch() - .limit(5) - .toArray(); - // --8<-- [end:fast_search] - - // --8<-- [start:bypass_vector_index] - await table - .query() - .nearestTo(embedding) - .bypassVectorIndex() - .limit(5) - .toArray(); - // --8<-- [end:bypass_vector_index] - } - - // ---- Brute force search (no index) ---- - { - const data = Array.from({ length: 64 }, (_, i) => ({ - vector: Array(128).fill(i), - id: `${i}`, - })); - await db.createTable("my_vectors", data, { mode: "overwrite" }); - - // --8<-- [start:brute_force_search] - const tbl = await db.openTable("my_vectors"); - - const results1 = await tbl.search(Array(128).fill(1.2)).limit(3).toArray(); - // --8<-- [end:brute_force_search] - expect(results1.length).toBe(3); - } - - // ---- Binary (hamming) vector search ---- - { - const schema = new Schema([ - new Field("id", new Int32(), true), - new Field("vector", new FixedSizeList(32, new Field("item", new Uint8()))), - ]); - const data = lancedb.makeArrowTable( - Array(1000) - .fill(0) - .map((_, i) => ({ - // the 256 bits are stored in 32 bytes; if your data is already in - // this format, you can skip the packBits step - id: i, - vector: lancedb.packBits(Array(256).fill(i % 2)), - })), - { schema }, - ); - - // --8<-- [start:binary_search] - const tbl = await db.createTable("binary_vectors", data, { - mode: "overwrite", - }); - await tbl.createIndex("vector", { - config: lancedb.Index.ivfFlat({ - numPartitions: 10, - distanceType: "hamming", - }), - }); - - const query = Array(32) - .fill(1) - .map(() => Math.floor(Math.random() * 255)); - const results = await tbl.query().nearestTo(query).limit(10).toArray(); - // --8<-- [end:binary_search] - expect(results.length).toBeLessThanOrEqual(10); - } - - // ---- Enterprise-style prefilter / postfilter / batch search ---- - { - const dimensions = 768; - const rows = Array.from({ length: 50 }, (_, i) => ({ - vector: Array.from({ length: dimensions }, () => Math.random() * 2 - 1), - text: `story ${i}`, - keywords: `kw${i}`, - label: i % 4, - })); - await db.createTable("lancedb-enterprise-quickstart", rows, { - mode: "overwrite", - }); - - { - // --8<-- [start:vector_search_prefilter] - // Generate a sample 768-dimension embedding vector (typical for BERT-based models) - // In real applications, you would get this from an embedding model - const dimensions = 768; - const queryEmbed = Array.from( - { length: dimensions }, - () => Math.random() * 2 - 1, - ); - - // Open table and perform search - const tableName = "lancedb-enterprise-quickstart"; - const table = await db.openTable(tableName); - - // Vector search with filters (pre-filtering is the default) - const vectorResults = await table - .search(queryEmbed) - .where("label > 2") - .select(["text", "keywords", "label"]) - .limit(5) - .toArray(); - - console.log("Search results (with pre-filtering):"); - console.log(vectorResults); - // --8<-- [end:vector_search_prefilter] - - // --8<-- [start:vector_search_postfilter] - const vectorResultsWithPostFilter = await ( - table.search(queryEmbed) as lancedb.VectorQuery - ) - .where("label > 2") - .postfilter() - .select(["text", "keywords", "label"]) - .limit(5) - .toArray(); - - console.log("Vector search results with post-filter:"); - console.log(vectorResultsWithPostFilter); - // --8<-- [end:vector_search_postfilter] - - // --8<-- [start:batch_search] - // Batch query - console.log("Performing batch vector search..."); - const batchSize = 5; - const queryVectors = Array.from({ length: batchSize }, () => - Array.from({ length: dimensions }, () => Math.random() * 2 - 1), - ); - let batchQuery = table.search(queryVectors[0]) as lancedb.VectorQuery; - for (let i = 1; i < batchSize; i++) { - batchQuery = batchQuery.addQueryVector(queryVectors[i]); - } - const batchResults = await batchQuery - .select(["text", "keywords", "label"]) - .limit(5) - .toArray(); - console.log("Batch vector search results:"); - console.log(batchResults); - // --8<-- [end:batch_search] - expect(batchResults.length).toBeGreaterThan(0); - } - } - }); -}); diff --git a/tests/ts/sentence-transformers.test.ts b/tests/ts/sentence-transformers.test.ts deleted file mode 100644 index a407e19..0000000 --- a/tests/ts/sentence-transformers.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -import { withTempDirectory } from "./util.ts"; - -// --8<-- [start:quickstart_imports] -import * as lancedb from "@lancedb/lancedb"; -import "@lancedb/lancedb/embedding/transformers"; -import { Utf8 } from "apache-arrow"; -// --8<-- [end:quickstart_imports] - -test("full text search", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - const func = (await lancedb.embedding - .getRegistry() - .get("huggingface") - ?.create()) as lancedb.embedding.EmbeddingFunction; - - const facts = [ - "Albert Einstein was a theoretical physicist.", - "The capital of France is Paris.", - "The Great Wall of China is one of the Seven Wonders of the World.", - "Python is a popular programming language.", - "Mount Everest is the highest mountain in the world.", - "Leonardo da Vinci painted the Mona Lisa.", - "Shakespeare wrote Hamlet.", - "The human body has 206 bones.", - "The speed of light is approximately 299,792 kilometers per second.", - "Water boils at 100 degrees Celsius.", - "The Earth orbits the Sun.", - "The Pyramids of Giza are located in Egypt.", - "Coffee is one of the most popular beverages in the world.", - "Tokyo is the capital city of Japan.", - "Photosynthesis is the process by which plants make their food.", - "The Pacific Ocean is the largest ocean on Earth.", - "Mozart was a prolific composer of classical music.", - "The Internet is a global network of computers.", - "Basketball is a sport played with a ball and a hoop.", - "The first computer virus was created in 1983.", - "Artificial neural networks are inspired by the human brain.", - "Deep learning is a subset of machine learning.", - "IBM's Watson won Jeopardy! in 2011.", - "The first computer programmer was Ada Lovelace.", - "The first chatbot was ELIZA, created in the 1960s.", - ].map((text) => ({ text })); - - const factsSchema = lancedb.embedding.LanceSchema({ - text: func.sourceField(new Utf8()), - vector: func.vectorField(), - }); - - const tbl = await db.createTable("facts", facts, { - mode: "overwrite", - schema: factsSchema, - }); - - const query = "How many bones are in the human body?"; - const actual = await tbl.search(query).limit(1).toArray(); - - expect(actual[0].text).toBe("The human body has 206 bones."); - }); -}, 100_000); - -test.skip("embedding quickstart snippets", async () => { - // --8<-- [start:quickstart_connect] - const db = await lancedb.connect("data/sample-lancedb"); - // --8<-- [end:quickstart_connect] - - // --8<-- [start:quickstart_init_model] - const model = (await lancedb.embedding - .getRegistry() - .get("huggingface") - ?.create()) as lancedb.embedding.EmbeddingFunction; - // --8<-- [end:quickstart_init_model] - - // --8<-- [start:quickstart_schema] - const wordsSchema = lancedb.embedding.LanceSchema({ - text: model.sourceField(new Utf8()), - vector: model.vectorField(), - }); - // --8<-- [end:quickstart_schema] - - // --8<-- [start:quickstart_create_table] - const table = await db.createEmptyTable("words", wordsSchema, { - mode: "overwrite", - }); - await table.add([{ text: "hello world" }, { text: "goodbye world" }]); - // --8<-- [end:quickstart_create_table] - - // --8<-- [start:quickstart_query] - const query = "greetings"; - const actual = (await table.search(query).limit(1).toArray())[0]; - console.log(actual.text); - // --8<-- [end:quickstart_query] -}); diff --git a/tests/ts/storage.test.ts b/tests/ts/storage.test.ts deleted file mode 100644 index 38c27cf..0000000 --- a/tests/ts/storage.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -import * as lancedb from "@lancedb/lancedb"; - -// --8<-- [start:storage_connect_s3] -async function storageConnectS3() { - const db = await lancedb.connect("s3://bucket/path"); - return db; -} -// --8<-- [end:storage_connect_s3] - -// --8<-- [start:storage_connect_gcs] -async function storageConnectGcs() { - const db = await lancedb.connect("gs://bucket/path"); - return db; -} -// --8<-- [end:storage_connect_gcs] - -// --8<-- [start:storage_connect_azure] -async function storageConnectAzure() { - const db = await lancedb.connect("az://bucket/path"); - return db; -} -// --8<-- [end:storage_connect_azure] - -// --8<-- [start:storage_connect_timeout] -async function storageConnectTimeout() { - const db = await lancedb.connect("s3://bucket/path", { - storageOptions: { timeout: "60s" }, - }); - return db; -} -// --8<-- [end:storage_connect_timeout] - -// --8<-- [start:storage_table_timeout] -async function storageTableTimeout() { - const db = await lancedb.connect("s3://bucket/path"); - const table = await db.createTable( - "table", - [{ a: 1, b: 2 }], - { storageOptions: { timeout: "60s" } }, - ); - return table; -} -// --8<-- [end:storage_table_timeout] - -// --8<-- [start:storage_s3_ddb] -async function storageS3Ddb() { - const db = await lancedb.connect( - "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table", - ); - return db; -} -// --8<-- [end:storage_s3_ddb] - -// --8<-- [start:storage_s3_ddb_local] -async function storageS3DdbLocal() { - const db = await lancedb.connect( - "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table", - { - storageOptions: { - endpoint: "http://localhost:4566", - dynamodbEndpoint: "http://localhost:4566", - allowHttp: "true", - }, - }, - ); - return db; -} -// --8<-- [end:storage_s3_ddb_local] - -// --8<-- [start:storage_s3_sse_kms] -async function storageS3SseKms() { - const db = await lancedb.connect("s3://bucket/path", { - storageOptions: { - awsServerSideEncryption: "aws:kms", - awsSseKmsKeyId: "", - }, - }); - return db; -} -// --8<-- [end:storage_s3_sse_kms] - -// --8<-- [start:storage_azure_sas] -async function storageAzureSas() { - const db = await lancedb.connect( - "az://my-container/my-database", - { - storageOptions: { - azureStorageAccountName: "some-account", - azureStorageSasToken: "", - }, - }, - ); - return db; -} -// --8<-- [end:storage_azure_sas] - -// --8<-- [start:storage_s3_minio] -async function storageS3Minio() { - const db = await lancedb.connect("s3://bucket/path", { - storageOptions: { - region: "us-east-1", - endpoint: "http://minio:9000", - }, - }); - return db; -} -// --8<-- [end:storage_s3_minio] - -// --8<-- [start:storage_s3_express] -async function storageS3Express() { - const db = await lancedb.connect( - "s3://my-bucket--use1-az4--x-s3/path", - { - storageOptions: { - region: "us-east-1", - s3Express: "true", - }, - }, - ); - return db; -} -// --8<-- [end:storage_s3_express] - -// --8<-- [start:storage_gcs_service_account] -async function storageGcsServiceAccount() { - const db = await lancedb.connect( - "gs://my-bucket/my-database", - { - storageOptions: { - serviceAccount: "path/to/service-account.json", - }, - }, - ); - return db; -} -// --8<-- [end:storage_gcs_service_account] - -// --8<-- [start:storage_azure_account] -async function storageAzureAccount() { - const db = await lancedb.connect( - "az://my-container/my-database", - { - storageOptions: { - accountName: "some-account", - accountKey: "some-key", - }, - }, - ); - return db; -} -// --8<-- [end:storage_azure_account] - -// --8<-- [start:storage_tigris_connect] -async function storageTigrisConnect() { - const db = await lancedb.connect( - "s3://your-bucket/path", - { - storageOptions: { - endpoint: "https://t3.storage.dev", - region: "auto", - }, - }, - ); - return db; -} -// --8<-- [end:storage_tigris_connect] - -test("storage ts snippets compile", async () => { - expect(storageConnectS3).toBeDefined(); - expect(storageConnectGcs).toBeDefined(); - expect(storageConnectAzure).toBeDefined(); - expect(storageConnectTimeout).toBeDefined(); - expect(storageTableTimeout).toBeDefined(); - expect(storageS3Ddb).toBeDefined(); - expect(storageS3DdbLocal).toBeDefined(); - expect(storageS3Minio).toBeDefined(); - expect(storageS3Express).toBeDefined(); - expect(storageS3SseKms).toBeDefined(); - expect(storageGcsServiceAccount).toBeDefined(); - expect(storageAzureAccount).toBeDefined(); - expect(storageAzureSas).toBeDefined(); - expect(storageTigrisConnect).toBeDefined(); -}); - diff --git a/tests/ts/tables.test.ts b/tests/ts/tables.test.ts deleted file mode 100644 index b2a49b6..0000000 --- a/tests/ts/tables.test.ts +++ /dev/null @@ -1,1029 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; -import * as arrow from "apache-arrow"; -import * as lancedb from "@lancedb/lancedb"; -import { withTempDirectory } from "./util.ts"; - -async function updateConnectEnterpriseExample() { - // --8<-- [start:update_connect_enterprise] - const db = await lancedb.connect("db://your-project-slug", { - apiKey: "your-api-key", - region: "us-east-1", - }); - // --8<-- [end:update_connect_enterprise] - return db; -} - -async function updateConnectLocalExample() { - // --8<-- [start:update_connect_local] - const db = await lancedb.connect("./data"); - // --8<-- [end:update_connect_local] - return db; -} - -test("table creation snippets (async)", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - // --8<-- [start:create_table_from_dicts] - type Location = { - vector: number[]; - lat: number; - long: number; - }; - - const data: Location[] = [ - { vector: [1.1, 1.2], lat: 45.5, long: -122.7 }, - { vector: [0.2, 1.8], lat: 40.1, long: -74.1 }, - ]; - const table = await db.createTable("test_table", data, { - mode: "overwrite", - }); - // --8<-- [end:create_table_from_dicts] - expect(await table.countRows()).toBe(2); - - await db.createTable("conflict_table", data); - // --8<-- [start:create_table_conflict_handling] - // Idempotent open: reuse the existing table if it exists. - // The provided data is ignored; the schema is validated against the - // existing table and a mismatch raises an error. - let conflictTable = await db.createTable("conflict_table", data, { - existOk: true, - }); - - // Overwrite: drop the existing table and create a new one with the - // provided data. This permanently discards the old table's data. - conflictTable = await db.createTable("conflict_table", data, { - mode: "overwrite", - }); - // --8<-- [end:create_table_conflict_handling] - expect(await conflictTable.countRows()).toBe(2); - - // --8<-- [start:create_table_custom_schema] - const customSchema = new arrow.Schema([ - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 4, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - new arrow.Field("lat", new arrow.Float32()), - new arrow.Field("long", new arrow.Float32()), - ]); - - const customSchemaData = lancedb.makeArrowTable( - [ - { vector: [1.1, 1.2, 1.3, 1.4], lat: 45.5, long: -122.7 }, - { vector: [0.2, 1.8, 0.4, 3.6], lat: 40.1, long: -74.1 }, - ], - { schema: customSchema }, - ); - const customSchemaTable = await db.createTable( - "my_table_custom_schema", - customSchemaData, - { mode: "overwrite" }, - ); - // --8<-- [end:create_table_custom_schema] - expect(await customSchemaTable.countRows()).toBe(2); - - // --8<-- [start:create_table_from_arrow] - const arrowSchema = new arrow.Schema([ - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 16, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - new arrow.Field("text", new arrow.Utf8()), - ]); - const arrowData = lancedb.makeArrowTable( - [ - { vector: Array(16).fill(0.1), text: "foo" }, - { vector: Array(16).fill(0.2), text: "bar" }, - ], - { schema: arrowSchema }, - ); - const arrowTable = await db.createTable("f32_tbl", arrowData, { - mode: "overwrite", - }); - // --8<-- [end:create_table_from_arrow] - expect(await arrowTable.countRows()).toBe(2); - - // --8<-- [start:create_table_from_iterator] - const batchSchema = new arrow.Schema([ - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 4, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - new arrow.Field("item", new arrow.Utf8()), - new arrow.Field("price", new arrow.Float32()), - ]); - - const tableForBatches = await db.createEmptyTable( - "batched_table", - batchSchema, - { - mode: "overwrite", - }, - ); - - const rows = Array.from({ length: 10 }, (_, i) => ({ - vector: [i + 0.1, i + 0.2, i + 0.3, i + 0.4], - item: `item-${i + 1}`, - price: (i + 1) * 10, - })); - - const chunkSize = 2; - for (let i = 0; i < rows.length; i += chunkSize) { - const batch = lancedb.makeArrowTable(rows.slice(i, i + chunkSize), { - schema: batchSchema, - }); - await tableForBatches.add(batch); - } - // --8<-- [end:create_table_from_iterator] - expect(await tableForBatches.countRows()).toBe(10); - - // --8<-- [start:open_existing_table] - const openTableData = [{ vector: [1.1, 1.2], lat: 45.5, long: -122.7 }]; - await db.createTable("test_table_open", openTableData, { - mode: "overwrite", - }); - - console.log(await db.tableNames()); - - const openedTable = await db.openTable("test_table_open"); - // --8<-- [end:open_existing_table] - expect(await openedTable.countRows()).toBe(1); - - // --8<-- [start:create_empty_table] - const emptySchema = new arrow.Schema([ - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 2, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - new arrow.Field("item", new arrow.Utf8()), - new arrow.Field("price", new arrow.Float32()), - ]); - const emptyTable = await db.createEmptyTable( - "test_empty_table", - emptySchema, - { - mode: "overwrite", - }, - ); - // --8<-- [end:create_empty_table] - expect(await emptyTable.countRows()).toBe(0); - - // --8<-- [start:drop_table] - await db.createTable("my_table", [{ vector: [1.1, 1.2], lat: 45.5 }], { - mode: "overwrite", - }); - - await db.dropTable("my_table"); - // --8<-- [end:drop_table] - expect(await db.tableNames()).not.toContain("my_table"); - }); -}); - -test("schema evolution snippets (async)", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - // --8<-- [start:schema_add_setup] - const schemaAddData = [ - { - id: 1, - name: "Laptop", - price: 1200.0, - vector: Array.from({ length: 128 }, () => Math.random()), - }, - { - id: 2, - name: "Smartphone", - price: 800.0, - vector: Array.from({ length: 128 }, () => Math.random()), - }, - { - id: 3, - name: "Headphones", - price: 150.0, - vector: Array.from({ length: 128 }, () => Math.random()), - }, - ]; - const schemaAddTable = await db.createTable( - "schema_evolution_add_example", - schemaAddData, - { mode: "overwrite" }, - ); - // --8<-- [end:schema_add_setup] - expect(await schemaAddTable.countRows()).toBe(3); - - // --8<-- [start:add_columns_calculated] - // Add a discounted price column (10% discount) - await schemaAddTable.addColumns([ - { - name: "discounted_price", - valueSql: "cast((price * 0.9) as float)", - }, - ]); - // --8<-- [end:add_columns_calculated] - - // --8<-- [start:add_columns_default_values] - // Add a stock status column with default value - await schemaAddTable.addColumns([ - { - name: "in_stock", - valueSql: "cast(true as boolean)", - }, - ]); - // --8<-- [end:add_columns_default_values] - - // --8<-- [start:add_columns_nullable] - // Add a nullable timestamp column - await schemaAddTable.addColumns([ - { - name: "last_ordered", - valueSql: "cast(NULL as timestamp)", - }, - ]); - // --8<-- [end:add_columns_nullable] - - // --8<-- [start:add_feature_columns_sql] - await schemaAddTable.addColumns([ - { - name: "price_per_id", - valueSql: "cast(price / id as float)", - }, - { - name: "price_log", - valueSql: "ln(price)", - }, - { - name: "price_score", - valueSql: "cast(price / (price + 100.0) as float)", - }, - ]); - // --8<-- [end:add_feature_columns_sql] - expect((await schemaAddTable.schema()).fields.map((field) => field.name)).toEqual( - expect.arrayContaining(["price_per_id", "price_log", "price_score"]), - ); - - // --8<-- [start:schema_alter_setup] - const schemaAlter = new arrow.Schema([ - new arrow.Field("id", new arrow.Int64()), - new arrow.Field("name", new arrow.Utf8()), - new arrow.Field("price", new arrow.Int32()), - new arrow.Field("discount_price", new arrow.Float64()), - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 128, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - ]); - const schemaAlterData = lancedb.makeArrowTable( - [ - { - id: 1, - name: "Laptop", - price: 1200, - discount_price: 1080.0, - vector: Array.from({ length: 128 }, () => Math.random()), - }, - { - id: 2, - name: "Smartphone", - price: 800, - discount_price: 720.0, - vector: Array.from({ length: 128 }, () => Math.random()), - }, - ], - { schema: schemaAlter }, - ); - const schemaAlterTable = await db.createTable( - "schema_evolution_alter_example", - schemaAlterData, - { mode: "overwrite" }, - ); - // --8<-- [end:schema_alter_setup] - expect(await schemaAlterTable.countRows()).toBe(2); - - // --8<-- [start:alter_columns_rename] - // Rename discount_price to sale_price - await schemaAlterTable.alterColumns([ - { path: "discount_price", rename: "sale_price" }, - ]); - // --8<-- [end:alter_columns_rename] - - // --8<-- [start:alter_columns_data_type] - // Change price from int32 to int64 for larger numbers - await schemaAlterTable.alterColumns([ - { path: "price", dataType: new arrow.Int64() }, - ]); - // --8<-- [end:alter_columns_data_type] - - // --8<-- [start:alter_columns_nullable] - // Make the name column nullable - await schemaAlterTable.alterColumns([{ path: "name", nullable: true }]); - // --8<-- [end:alter_columns_nullable] - - // --8<-- [start:alter_columns_multiple] - // Rename, change type, and make nullable in one operation - await schemaAlterTable.alterColumns([ - { - path: "sale_price", - rename: "final_price", - dataType: new arrow.Float64(), - nullable: true, - }, - ]); - // --8<-- [end:alter_columns_multiple] - - // --8<-- [start:alter_columns_with_expression] - // For custom transforms, create a new column from a SQL expression. - const expressionTable = await db.createTable( - "schema_evolution_expression_example", - [{ id: 1, price_text: "$100" }], - { mode: "overwrite" }, - ); - - await expressionTable.addColumns([ - { - name: "price_numeric", - valueSql: "cast(replace(price_text, '$', '') as int)", - }, - ]); - await expressionTable.dropColumns(["price_text"]); - await expressionTable.alterColumns([ - { path: "price_numeric", rename: "price" }, - ]); - // --8<-- [end:alter_columns_with_expression] - expect(await expressionTable.countRows()).toBe(1); - - // --8<-- [start:schema_drop_setup] - const schemaDropData = [ - { - id: 1, - name: "Laptop", - price: 1200.0, - temp_col1: "X", - temp_col2: 100, - vector: Array.from({ length: 128 }, () => Math.random()), - }, - { - id: 2, - name: "Smartphone", - price: 800.0, - temp_col1: "Y", - temp_col2: 200, - vector: Array.from({ length: 128 }, () => Math.random()), - }, - { - id: 3, - name: "Headphones", - price: 150.0, - temp_col1: "Z", - temp_col2: 300, - vector: Array.from({ length: 128 }, () => Math.random()), - }, - ]; - const schemaDropTable = await db.createTable( - "schema_evolution_drop_example", - schemaDropData, - { mode: "overwrite" }, - ); - // --8<-- [end:schema_drop_setup] - expect(await schemaDropTable.countRows()).toBe(3); - - // --8<-- [start:drop_columns_single] - // Remove the first temporary column - await schemaDropTable.dropColumns(["temp_col1"]); - // --8<-- [end:drop_columns_single] - - // --8<-- [start:drop_columns_multiple] - // Remove the second temporary column - await schemaDropTable.dropColumns(["temp_col2"]); - // --8<-- [end:drop_columns_multiple] - - // --8<-- [start:alter_vector_column] - const oldDim = 384; - const newDim = 1024; - const vectorSchema = new arrow.Schema([ - new arrow.Field("id", new arrow.Int64()), - new arrow.Field( - "embedding", - new arrow.FixedSizeList( - oldDim, - new arrow.Field("item", new arrow.Float16(), true), - ), - true, - ), - ]); - const vectorData = lancedb.makeArrowTable( - [{ id: 1, embedding: Array.from({ length: oldDim }, () => Math.random()) }], - { schema: vectorSchema }, - ); - const vectorTable = await db.createTable("vector_alter_example", vectorData, { - mode: "overwrite", - }); - - // Changing FixedSizeList dimensions (384 -> 1024) is not supported via alterColumns. - // Use addColumns + dropColumns + alterColumns(rename) to replace the column. - await vectorTable.addColumns([ - { - name: "embedding_v2", - valueSql: `arrow_cast(NULL, 'FixedSizeList(${newDim}, Float16)')`, - }, - ]); - await vectorTable.dropColumns(["embedding"]); - await vectorTable.alterColumns([{ path: "embedding_v2", rename: "embedding" }]); - // --8<-- [end:alter_vector_column] - expect(await vectorTable.countRows()).toBe(1); - - const fieldMetadataTable = await db.createTable( - "schema_field_metadata_example", - [ - { id: 0, category: "a" }, - { id: 1, category: "b" }, - ], - { mode: "overwrite" }, - ); - - // --8<-- [start:schema_field_metadata_merge] - // Set two metadata keys on the `category` field. - const res = await fieldMetadataTable.updateFieldMetadata([ - { path: "category", metadata: { unit: "label", pii: "false" } }, - ]); - console.log(res.version); - - // Merge: add a new key, delete one via null, keep the rest. - await fieldMetadataTable.updateFieldMetadata([ - { path: "category", metadata: { source: "import", pii: null } }, - ]); - // --8<-- [end:schema_field_metadata_merge] - - // --8<-- [start:schema_field_metadata_replace] - await fieldMetadataTable.updateFieldMetadata([ - { - path: "category", - metadata: { owner: "search-team" }, - replace: true, - }, - ]); - // --8<-- [end:schema_field_metadata_replace] - }); -}); - -test("update snippets (async)", async () => { - // Keep connection snippets in this file, but do not run enterprise/local examples in CI. - void updateConnectEnterpriseExample; - void updateConnectLocalExample; - - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - { - // --8<-- [start:update_example_table_setup] - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - ], - { mode: "overwrite" }, - ); - // --8<-- [end:update_example_table_setup] - await table.countRows(); - } - - { - // --8<-- [start:update_operation] - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - ], - { mode: "overwrite" }, - ); - await table.update({ where: "id = 2", values: { name: "Bobby" } }); - // --8<-- [end:update_operation] - await table.countRows(); - } - - { - // --8<-- [start:update_using_sql] - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - ], - { mode: "overwrite" }, - ); - await table.update({ - where: "id = 2", - valuesSql: { login_count: "login_count + 1" }, - }); - // --8<-- [end:update_using_sql] - await table.countRows(); - } - - { - // --8<-- [start:merge_matched_update_only] - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - ], - { mode: "overwrite" }, - ); - - const incomingUsers = [ - { id: 2, name: "Bobby", login_count: 21 }, - { id: 3, name: "Charlie", login_count: 5 }, - ]; - - await table - .mergeInsert("id") - .whenMatchedUpdateAll() - .execute(incomingUsers); - // --8<-- [end:merge_matched_update_only] - await table.countRows(); - } - - { - // --8<-- [start:insert_if_not_exists] - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - ], - { mode: "overwrite" }, - ); - - const incomingUsers = [ - { id: 2, name: "Bobby", login_count: 21 }, - { id: 3, name: "Charlie", login_count: 5 }, - ]; - - await table - .mergeInsert("id") - .whenNotMatchedInsertAll() - .execute(incomingUsers); - // --8<-- [end:insert_if_not_exists] - await table.countRows(); - } - - { - // --8<-- [start:merge_update_insert] - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - ], - { mode: "overwrite" }, - ); - - const incomingUsers = [ - { id: 2, name: "Bobby", login_count: 21 }, - { id: 3, name: "Charlie", login_count: 5 }, - ]; - - await table - .mergeInsert("id") - .whenMatchedUpdateAll() - .whenNotMatchedInsertAll() - .execute(incomingUsers); - // --8<-- [end:merge_update_insert] - await table.countRows(); - } - - { - // --8<-- [start:merge_delete_missing_by_source] - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - { id: 3, name: "Charlie", login_count: 5 }, - ], - { mode: "overwrite" }, - ); - - const incomingUsers = [ - { id: 2, name: "Bobby", login_count: 21 }, - { id: 3, name: "Charlie", login_count: 5 }, - ]; - - await table - .mergeInsert("id") - .whenMatchedUpdateAll() - .whenNotMatchedInsertAll() - .whenNotMatchedBySourceDelete() - .execute(incomingUsers); - // --8<-- [end:merge_delete_missing_by_source] - await table.countRows(); - } - - { - // --8<-- [start:merge_partial_columns] - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - ], - { mode: "overwrite" }, - ); - - const incomingUsers = [ - { id: 2, name: "Bobby" }, - { id: 3, name: "Charlie" }, - ]; - - await table - .mergeInsert("id") - .whenMatchedUpdateAll() - .whenNotMatchedInsertAll() - .execute(incomingUsers); - // --8<-- [end:merge_partial_columns] - await table.countRows(); - } - - { - const table = await db.createTable( - "users_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - { id: 3, name: "Charlie", login_count: 5 }, - ], - { mode: "overwrite" }, - ); - - // --8<-- [start:delete_operation] - // delete data - const predicate = "id = 3"; - await table.delete(predicate); - // --8<-- [end:delete_operation] - await table.countRows(); - } - - { - const table = await db.createTable( - "users_cleanup_example", - [ - { id: 1, name: "Alice", login_count: 10 }, - { id: 2, name: "Bob", login_count: 20 }, - { id: 3, name: "Charlie", login_count: 5 }, - ], - { mode: "overwrite" }, - ); - - // --8<-- [start:update_optimize_cleanup] - const olderThan = new Date(); - olderThan.setDate(olderThan.getDate() - 1); - await table.optimize({ cleanupOlderThan: olderThan }); - // --8<-- [end:update_optimize_cleanup] - } - }); -}); - -test("versioning snippets (async)", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - - // --8<-- [start:versioning_basic_setup] - const tableName = "quotes_versioning_example"; - const data = [ - { id: 1, author: "Richard", quote: "Wubba Lubba Dub Dub!" }, - { id: 2, author: "Morty", quote: "Rick, what's going on?" }, - { - id: 3, - author: "Richard", - quote: "I turned myself into a pickle, Morty!", - }, - ]; - const table = await db.createTable(tableName, data, { mode: "overwrite" }); - // --8<-- [end:versioning_basic_setup] - expect(await table.countRows()).toBe(3); - - // --8<-- [start:versioning_check_initial_version] - const versions = await table.listVersions(); - const currentVersion = await table.version(); - console.log(`Number of versions after creation: ${versions.length}`); - console.log(`Current version: ${currentVersion}`); - // --8<-- [end:versioning_check_initial_version] - expect(versions.length).toBe(1); - expect(currentVersion).toBe(versions[versions.length - 1].version); - - // --8<-- [start:versioning_update_data] - await table.update({ - where: "author = 'Richard'", - values: { author: "Richard Daniel Sanchez" }, - }); - const rowsAfterUpdate = await table.countRows( - "author = 'Richard Daniel Sanchez'", - ); - console.log(`Rows updated to Richard Daniel Sanchez: ${rowsAfterUpdate}`); - // --8<-- [end:versioning_update_data] - expect(rowsAfterUpdate).toBe(2); - - // --8<-- [start:versioning_add_data] - const moreData = [ - { - id: 4, - author: "Richard Daniel Sanchez", - quote: "That's the way the news goes!", - }, - { id: 5, author: "Morty", quote: "Aww geez, Rick!" }, - ]; - await table.add(moreData); - // --8<-- [end:versioning_add_data] - expect(await table.countRows()).toBe(5); - - // --8<-- [start:versioning_check_versions_after_mod] - const versionsAfterMod = await table.listVersions(); - const versionCountAfterMod = versionsAfterMod.length; - const versionAfterMod = await table.version(); - console.log( - `Number of versions after modifications: ${versionCountAfterMod}`, - ); - console.log(`Current version: ${versionAfterMod}`); - // --8<-- [end:versioning_check_versions_after_mod] - expect(versionCountAfterMod).toBeGreaterThanOrEqual(2); - expect(versionAfterMod).toBe(versionsAfterMod[versionsAfterMod.length - 1].version); - - // --8<-- [start:versioning_list_all_versions] - const allVersions = await table.listVersions(); - for (const v of allVersions) { - console.log(`Version ${v.version}, created at ${v.timestamp}`); - } - // --8<-- [end:versioning_list_all_versions] - expect(allVersions.length).toBeGreaterThanOrEqual(1); - - // --8<-- [start:versioning_rollback] - await table.checkout(versionAfterMod); - await table.restore(); - const versionsAfterRollback = await table.listVersions(); - const versionCountAfterRollback = versionsAfterRollback.length; - console.log( - `Total number of versions after rollback: ${versionCountAfterRollback}`, - ); - // --8<-- [end:versioning_rollback] - expect(versionCountAfterRollback).toBe(versionCountAfterMod + 1); - expect(await table.countRows()).toBe(5); - - // --8<-- [start:versioning_checkout_latest] - await table.checkoutLatest(); - // --8<-- [end:versioning_checkout_latest] - const latestVersion = await table.version(); - const versionsAfterCheckout = await table.listVersions(); - expect(latestVersion).toBe( - versionsAfterCheckout[versionsAfterCheckout.length - 1].version, - ); - - // --8<-- [start:versioning_delete_data] - await table.delete("author = 'Morty'"); - const rowsAfterDeletion = await table.countRows(); - console.log(`Number of rows after deletion: ${rowsAfterDeletion}`); - // --8<-- [end:versioning_delete_data] - expect(rowsAfterDeletion).toBe(3); - - const tagsTable = await db.createTable( - "quotes_tags_example", - [{ id: 1, author: "Richard", quote: "Wubba Lubba Dub Dub!" }], - { mode: "overwrite" }, - ); // v1 - await tagsTable.add([ - { id: 2, author: "Morty", quote: "Aww geez, Rick!" }, - ]); // v2 - await tagsTable.add([ - { id: 3, author: "Summer", quote: "Whatever, Grandpa" }, - ]); // v3 - - // --8<-- [start:versioning_tags] - const tags = await tagsTable.tags(); - - // Create a tag pointing at a specific version - await tags.create("baseline", 1); - await tags.create("with-edits", await tagsTable.version()); - - // List all tags on this table - console.log(await tags.list()); - - // Look up the version a tag points at - console.log(await tags.getVersion("baseline")); - - // Move an existing tag to a different version - await tags.update("baseline", 2); - - // Check out a version by tag name - await tagsTable.checkout("baseline"); - console.log(await tagsTable.version()); - - // Delete a tag (does not delete the underlying version) - await tags.delete("with-edits"); - - // Return to the latest version - await tagsTable.checkoutLatest(); - // --8<-- [end:versioning_tags] - expect(await tagsTable.version()).toBe(3); - const remainingTags = await tags.list(); - expect(remainingTags).toHaveProperty("baseline"); - expect(remainingTags).not.toHaveProperty("with-edits"); - }); -}); - -test("branch snippets (async)", async () => { - await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); - const table = await db.createTable( - "quotes_branches_example", - [ - { id: 1, author: "Lancelot", quote: "My lance never fails." }, - { id: 2, author: "Arthur", quote: "Long live Camelot!" }, - { id: 3, author: "Merlin", quote: "Magic always has a price." }, - ], - { mode: "overwrite" }, - ); - - const branches = await table.branches(); - - // --8<-- [start:branch_create] - // Fork an isolated, writable branch from main's latest version. - // `create` returns a table handle scoped to the new branch. - const branch = await branches.create("exp"); - // --8<-- [end:branch_create] - - // --8<-- [start:branch_write] - // Writes land on the branch handle only; main is left untouched. - await branch.add([{ id: 4, author: "Lancelot", quote: "For the realm!" }]); - console.log(await branch.countRows()); // 4 rows on the branch - console.log(await table.countRows()); // 3 rows; main is unaffected - - // List every branch, each mapped to its metadata (including its fork point). - console.log(await branches.list()); - // --8<-- [end:branch_write] - - // --8<-- [start:branch_reopen] - // Reopen an existing branch by name from the table handle... - const checkedOut = await branches.checkout("exp"); - // ...or open it directly from the database connection. - const branchHandle = await db.openTable( - "quotes_branches_example", - undefined, - { branch: "exp" }, - ); - console.log(await checkedOut.countRows(), await branchHandle.countRows()); // both 4 - // --8<-- [end:branch_reopen] - - // --8<-- [start:branch_delete] - // Delete the branch and its branch-local history. Data on main is safe. - await branches.delete("exp"); - // --8<-- [end:branch_delete] - - expect(await table.countRows()).toBe(3); - expect(await branches.list()).not.toHaveProperty("exp"); - - // Setup: a branch with row results that we want to apply to main. - const candidate = await branches.create("candidate"); - await candidate.update({ - where: "id = 1", - values: { quote: "Revised on the branch" }, - }); - await candidate.add([ - { id: 4, author: "Galahad", quote: "The grail awaits." }, - ]); - - // --8<-- [start:branch_upsert_to_main] - // This is a row-level upsert, not a merge of branch histories. - // `mergeInsert` updates matching rows and inserts new rows using a stable - // unique key. Filter the branch read if you only want to apply some results. - const rowsToApply = await candidate.toArrow(); - await table - .mergeInsert("id") - .whenMatchedUpdateAll() // update rows that already exist on main - .whenNotMatchedInsertAll() // insert rows that are new on the branch - .execute(rowsToApply); - // --8<-- [end:branch_upsert_to_main] - - expect(await table.countRows()).toBe(4); - await branches.delete("candidate"); - - // Setup: a larger table with a vector and a text column to index. - const products = await db.createTable( - "products_branch_index", - Array.from({ length: 512 }, (_, i) => ({ - id: i, - vector: Array.from({ length: 4 }, () => Math.random()), - text: `product number ${i}`, - })), - { mode: "overwrite" }, - ); - const productBranches = await products.branches(); - - // --8<-- [start:branch_index] - // Build and validate indexes on a branch before using the configuration on - // main. - const dev = await productBranches.create("index-dev"); - - // A vector (ANN) index and a full-text search index, both branch-scoped. - await dev.createIndex("vector", { - config: lancedb.Index.ivfPq({ - distanceType: "cosine", - numPartitions: 1, - numSubVectors: 2, - }), - }); - await dev.createIndex("text", { config: lancedb.Index.fts() }); - - // Both indexes live only on the branch; main still has none. - console.log((await dev.listIndices()).map((ix) => ix.name)); // branch: two indexes - console.log((await products.listIndices()).map((ix) => ix.name)); // main: [] (untouched) - // --8<-- [end:branch_index] - - expect(await dev.listIndices()).toHaveLength(2); - expect(await products.listIndices()).toHaveLength(0); - await productBranches.delete("index-dev"); - }); -}); - -test("consistency snippets (async)", async () => { - await withTempDirectory(async (databaseDir) => { - // --8<-- [start:consistency_strong] - const strongWriterDb = await lancedb.connect(databaseDir); - const strongReaderDb = await lancedb.connect(databaseDir, { - readConsistencyInterval: 0, - }); - const strongWriterTable = await strongWriterDb.createTable( - "consistency_strong_table", - [{ id: 1 }], - { mode: "overwrite" }, - ); - const strongReaderTable = await strongReaderDb.openTable( - "consistency_strong_table", - ); - await strongWriterTable.add([{ id: 2 }]); - const strongRowsAfterWrite = await strongReaderTable.countRows(); - console.log(`Rows visible with strong consistency: ${strongRowsAfterWrite}`); - // --8<-- [end:consistency_strong] - expect(strongRowsAfterWrite).toBe(2); - - // --8<-- [start:consistency_eventual] - const eventualWriterDb = await lancedb.connect(databaseDir); - const eventualReaderDb = await lancedb.connect(databaseDir, { - readConsistencyInterval: 3600, - }); - const eventualWriterTable = await eventualWriterDb.createTable( - "consistency_eventual_table", - [{ id: 1 }], - { mode: "overwrite" }, - ); - const eventualReaderTable = await eventualReaderDb.openTable( - "consistency_eventual_table", - ); - await eventualWriterTable.add([{ id: 2 }]); - const eventualRowsAfterWrite = await eventualReaderTable.countRows(); - console.log( - `Rows visible before eventual refresh interval: ${eventualRowsAfterWrite}`, - ); - // --8<-- [end:consistency_eventual] - expect(eventualRowsAfterWrite).toBe(1); - - // --8<-- [start:consistency_checkout_latest] - const checkoutWriterDb = await lancedb.connect(databaseDir); - const checkoutReaderDb = await lancedb.connect(databaseDir); - const checkoutWriterTable = await checkoutWriterDb.createTable( - "consistency_checkout_latest_table", - [{ id: 1 }], - { mode: "overwrite" }, - ); - const checkoutReaderTable = await checkoutReaderDb.openTable( - "consistency_checkout_latest_table", - ); - await checkoutWriterTable.add([{ id: 2 }]); - const rowsBeforeRefresh = await checkoutReaderTable.countRows(); - console.log(`Rows before checkoutLatest: ${rowsBeforeRefresh}`); - await checkoutReaderTable.checkoutLatest(); - const rowsAfterRefresh = await checkoutReaderTable.countRows(); - console.log(`Rows after checkoutLatest: ${rowsAfterRefresh}`); - // --8<-- [end:consistency_checkout_latest] - expect(rowsBeforeRefresh).toBe(1); - expect(rowsAfterRefresh).toBe(2); - }); -}); diff --git a/tests/ts/tsconfig.json b/tests/ts/tsconfig.json deleted file mode 100644 index 49a04ac..0000000 --- a/tests/ts/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "include": ["*.test.ts"], - "compilerOptions": { - "target": "es2022", - "module": "NodeNext", - "declaration": true, - "outDir": "./dist", - "strict": true, - "allowJs": true, - "resolveJsonModule": true, - "emitDecoratorMetadata": true, - "experimentalDecorators": true, - "moduleResolution": "NodeNext", - "allowImportingTsExtensions": true, - "emitDeclarationOnly": true - } -} diff --git a/tests/ts/util.ts b/tests/ts/util.ts deleted file mode 100644 index 404abed..0000000 --- a/tests/ts/util.ts +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The LanceDB Authors -import * as fs from "node:fs"; -import { tmpdir } from "node:os"; -import * as path from "node:path"; - -export async function withTempDirectory( - fn: (tempDir: string) => Promise, -) { - const tmpDirPath = fs.mkdtempSync(path.join(tmpdir(), "temp-dir-")); - try { - await fn(tmpDirPath); - } finally { - fs.rmSync(tmpDirPath, { recursive: true }); - } -}