diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3462cb1c5..50b5cbe8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,6 +195,117 @@ jobs: if: always() run: docker compose -f /tmp/milvus-compose.yml down -v + integration-seekdb: + name: integration tests (SeekDB embedded 1.4) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.2.0 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Set up Python + run: uv python install 3.12 + + # pyseekdb pulls the matching pylibseekdb wheel on Linux. The version is + # locked with the rest of the development environment in uv.lock. + - name: Install dependencies (frozen) + run: uv sync --frozen --extra seekdb-embedded + + - name: SeekDB repository and derived-index contracts + env: + EVEROS_TEST_SEEKDB_PATH: /tmp/everos-seekdb + run: >- + uv run --frozen pytest + tests/integration/test_seekdb_backend.py + tests/unit/test_infra/test_index_contract.py + -v + + - name: SeekDB end-to-end (tiered API suites) + env: + EVEROS_TEST_SEEKDB_PATH: /tmp/everos-seekdb + run: uv run --frozen pytest tests/integration/test_tiers -v -k seekdb + + integration-seekdb-remote: + name: integration tests (SeekDB remote latest) + runs-on: ubuntu-latest + services: + seekdb: + image: oceanbase/seekdb:latest + env: + ROOT_PASSWORD: everos-test + CPU_COUNT: "2" + MEMORY_LIMIT: 2G + LOG_DISK_SIZE: 2G + DATAFILE_SIZE: 2G + ports: + - 2881:2881 + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.2.0 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Set up Python + run: uv python install 3.12 + + - name: Install dependencies (frozen) + run: uv sync --frozen --extra seekdb + + - name: Wait for the MySQL endpoint + run: | + uv run python - <<'PY' + import time + import pymysql + + deadline = time.monotonic() + 180 + while True: + try: + connection = pymysql.connect( + host="127.0.0.1", + port=2881, + user="root", + password="everos-test", + connect_timeout=3, + ) + connection.close() + break + except pymysql.MySQLError: + if time.monotonic() >= deadline: + raise + time.sleep(3) + PY + + - name: SeekDB remote repository and derived-index contracts + env: + EVEROS_TEST_SEEKDB_HOST: 127.0.0.1 + EVEROS_TEST_SEEKDB_PORT: "2881" + EVEROS_TEST_SEEKDB_TENANT: "" + EVEROS_TEST_SEEKDB_USER: root + EVEROS_TEST_SEEKDB_PASSWORD: everos-test + EVEROS_TEST_SEEKDB_DATABASE: everos_test + run: >- + uv run --frozen pytest + tests/integration/test_seekdb_backend.py + tests/unit/test_infra/test_index_contract.py + -v + + - name: SeekDB remote end-to-end (tiered API suites) + env: + EVEROS_TEST_SEEKDB_HOST: 127.0.0.1 + EVEROS_TEST_SEEKDB_PORT: "2881" + EVEROS_TEST_SEEKDB_TENANT: "" + EVEROS_TEST_SEEKDB_USER: root + EVEROS_TEST_SEEKDB_PASSWORD: everos-test + EVEROS_TEST_SEEKDB_DATABASE: everos_test + run: uv run --frozen pytest tests/integration/test_tiers -v -k seekdb + package: name: package build runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 9afde6049..00605d132 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Optional SeekDB derived-index backend.** Set `[index] backend = "seekdb"` + to store the seven rebuildable business indexes in embedded SeekDB or a + remote seekdb Server/OceanBase database. The adapter implements the same + repository and lifecycle ports as LanceDB and Milvus, including typed scalar + filters, JSON array membership, native pagination, BM25 token columns, and + cosine search across multiple vector columns. Install `everos[seekdb]` for + remote mode or `everos[seekdb-embedded]` on Linux/macOS. Markdown remains the + source of truth and the default LanceDB installation is unchanged. + +### Changed + +- **Derived-index routing now supports registered backends uniformly.** Stable + repository objects resolve LanceDB, Milvus, or SeekDB at call time without + leaking a concrete adapter into the port definitions. + ## [1.3.1] - 2026-09-08 **One reproducible runner for four long-term-memory benchmarks, plus an diff --git a/README.md b/README.md index dae432f19..d37dccd15 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,9 @@ EverOS is a Python library and local-first memory runtime for agents and makers. It gives one portable memory layer across coding assistants, apps, devices, and workflows from day one. It stores conversations, files, and agent -trajectories as readable Markdown, then syncs local SQLite and LanceDB indexes -for fast retrieval and self-evolving reuse. +trajectories as readable Markdown, then syncs SQLite and a rebuildable derived +index (LanceDB by default, with optional Milvus or SeekDB) for fast retrieval +and self-evolving reuse. @@ -60,7 +61,7 @@ for fast retrieval and self-evolving reuse. - + @@ -664,7 +665,7 @@ Explore stored entities and relationships in a graph interface. Frontend demo; b ## Documentation - [docs/everos-demo.md](docs/everos-demo.md) — Demo scope and TUI source layout -- [docs/how-memory-works.md](docs/how-memory-works.md) — Markdown, SQLite, LanceDB, and recall flow +- [docs/how-memory-works.md](docs/how-memory-works.md) — Markdown, SQLite, derived indexes, and recall flow - [docs/use-cases.md](docs/use-cases.md) — Full use-case gallery and integration examples - [docs/engineering.md](docs/engineering.md) — Contributor engineering reference: build, test, CI, conventions - [docs/migration-to-1.0.0.md](docs/migration-to-1.0.0.md) — Legacy API migration notes diff --git a/README.zh-CN.md b/README.zh-CN.md index 53df0c24b..6b678c004 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -39,8 +39,8 @@ EverOS 是面向 agents 和 makers 的 Python library 与 local-first memory runtime。它从 day one 开始就提供一层可携带的记忆层，让记忆穿过 coding assistants、apps、devices 和 workflows。它会把 conversations、files 和 -agent trajectories 保存为可读 Markdown，并同步本地 SQLite 与 LanceDB -索引，用于快速检索和自进化复用。 +agent trajectories 保存为可读 Markdown，并同步 SQLite 与可重建派生索引 +（默认 LanceDB，也可选 Milvus 或 SeekDB），用于快速检索和自进化复用。
Local three-part stack✅ Markdown + SQLite + LanceDB; no MongoDB, Elasticsearch, or Redis required✅ Markdown + SQLite + LanceDB by default; optional Milvus or SeekDB backends ❌ Often depends on managed services, vector DBs, graph DBs, or server stacks
@@ -60,7 +60,7 @@ agent trajectories 保存为可读 Markdown，并同步本地 SQLite 与 LanceDB - + @@ -663,7 +663,7 @@ Claude Code 的持久记忆插件。自动保存并回忆过去 coding sessions ## 文档 - [docs/everos-demo.md](docs/everos-demo.md) - Demo 范围与 TUI 源码布局 -- [docs/how-memory-works.md](docs/how-memory-works.md) - Markdown、SQLite、LanceDB 与 recall flow +- [docs/how-memory-works.md](docs/how-memory-works.md) - Markdown、SQLite、派生索引与 recall flow - [docs/use-cases.md](docs/use-cases.md) - 完整使用场景 gallery 和集成示例 - [docs/engineering.md](docs/engineering.md) - 贡献者工程参考:构建、测试、CI 与规范 - [docs/migration-to-1.0.0.md](docs/migration-to-1.0.0.md) - Legacy API 迁移说明 diff --git a/config.example.toml b/config.example.toml index 48cff25fd..8b8fdd33b 100644 --- a/config.example.toml +++ b/config.example.toml @@ -58,7 +58,7 @@ max_concurrent = 5 # read_consistency_seconds = 5.0 # # [index] -# backend = "lancedb" # or "milvus" +# backend = "lancedb" # or "milvus" / "seekdb" # # # Required only when index.backend = "milvus". Install with: # # pip install "everos[milvus]" @@ -68,3 +68,22 @@ max_concurrent = 5 # db_name = "" # consistency_level = "Session" # collection_prefix = "everos" +# +# # SeekDB supports an in-process engine on Linux/macOS and a remote seekdb +# # Server or OceanBase endpoint on every platform. Install with: +# # pip install "everos[seekdb]" +# # Add the embedded extra for local in-process storage: +# # pip install "everos[seekdb-embedded]" +# [seekdb] +# mode = "embedded" # "embedded" or "remote" +# path = "" # empty -> /.index/seekdb +# host = "" # required in remote mode +# port = 2881 +# tenant = "" # set for OceanBase, e.g. "test" +# user = "root" +# password = "" # or export SEEKDB_PASSWORD +# database = "everos" +# table_prefix = "everos" +# vector_sync_mode = "immediate" # "immediate" or "async" +# connect_timeout_seconds = 10.0 +# read_timeout_seconds = 60.0 diff --git a/docs/architecture.md b/docs/architecture.md index 4aae7826a..fffaf0bff 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,7 +69,8 @@ layers = [ ┌──────────────┐ ┌──────────────┐ ┌─────────────────┐ │ Markdown │ │ SQLite │ │ Derived index │ - │ (truth) │ │ (state) │ │ LanceDB/Milvus │ + │ (truth) │ │ (state) │ │ Lance/Milvus/ │ + │ │ │ │ │ SeekDB │ ├──────────────┤ ├──────────────┤ ├─────────────────┤ │ entries + │ │ change queue │ │ vector ANN │ │ frontmatter │ │ + state/LSN │ │ BM25 (Tantivy) │ diff --git a/docs/configuration.md b/docs/configuration.md index 5cf85a198..bb3be27ef 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -106,7 +106,7 @@ truth regardless of backend. | Field | Type | Default | Description | |---|---|---|---| -| `backend` | string | `"lancedb"` | Index implementation: `lancedb` or `milvus`. | +| `backend` | string | `"lancedb"` | Index implementation: `lancedb`, `milvus`, or `seekdb`. | ### `[milvus]` @@ -122,6 +122,41 @@ Zilliz Cloud endpoint; a Milvus Lite filesystem path is rejected. | `consistency_level` | string | `"Session"` | Milvus consistency level used by collections. | | `collection_prefix` | string | `"everos"` | Prefix for the seven derived-index collections. | +### `[seekdb]` + +Used only when `index.backend = "seekdb"`. Install `everos[seekdb]` for a +remote seekdb Server or OceanBase endpoint. Install `everos[seekdb-embedded]` +for in-process storage on Linux or macOS; pylibseekdb does not currently ship a +Windows wheel. Embedded mode owns its database directory exclusively, so do +not run a server and a cascade CLI process against the same path concurrently. + +| Field | Type | Default | Description | +|---|---|---|---| +| `mode` | string | `"embedded"` | `embedded` for an in-process directory, or `remote` for a server endpoint. | +| `path` | string | `""` | Embedded directory; empty resolves to `/.index/seekdb`. | +| `host` | string | `""` | Required hostname in remote mode. | +| `port` | int | `2881` | MySQL-compatible server port. | +| `tenant` | string | `""` | Leave empty for seekdb Server; set the OceanBase tenant name (for example, `test`) when connecting to OceanBase. | +| `user` | string | `"root"` | User name without the tenant suffix. | +| `password` | string | `""` | Password; an empty value falls back to `SEEKDB_PASSWORD`. | +| `database` | string | `"everos"` | Database created on first use when permitted; SQL identifier, at most 64 characters. | +| `table_prefix` | string | `"everos"` | Prefix for the seven derived-index tables; SQL identifier, at most 48 characters. | +| `vector_sync_mode` | string | `"immediate"` | Vector-index synchronization: `immediate` for write-then-search consistency, or `async` for higher ingest throughput and eventual index visibility. | +| `connect_timeout_seconds` | float | `10.0` | Remote connection timeout. | +| `read_timeout_seconds` | float | `60.0` | Remote read and write timeout. | + +SeekDB tables use `utf8mb4_bin`, typed scalar/JSON columns, +whitespace-tokenized full-text indexes, and cosine HNSW indexes. The default +`immediate` vector synchronization makes a successful upsert visible to the +following search, at a significant write-throughput cost. Choose `async` when +bulk-ingest throughput matters more than immediate visibility, or when a +compatible OceanBase deployment does not support seekdb's `immediate` option; +new vectors may then be absent from search results until background index +synchronization catches up. + +Markdown remains the source of truth; changing the backend requires +`everos cascade rebuild`, not a data migration. + ### `[llm]` | Field | Type | Default | Required | Description | diff --git a/docs/how-memory-works.md b/docs/how-memory-works.md index be6fda712..9f6726f62 100644 --- a/docs/how-memory-works.md +++ b/docs/how-memory-works.md @@ -23,14 +23,14 @@ This is the narrative companion to the reference docs: see ## The storage stack -Three embedded pieces, each owning what it is best at. Markdown is the +Three storage layers, each owning what it is best at. Markdown is the **source of truth**; the other two are **derived and rebuildable**. | Layer | Backed by | Holds | Rebuildable? | |---|---|---|---| | **Markdown + YAML frontmatter** | plain `.md` files | the memory content itself — the only portable, human-editable asset | — (it *is* the truth) | | **SQLite** (`aiosqlite`) | `.index/sqlite/*.db` | system state, audit log, the cascade queue, the boundary buffer, OME engine state | ✅ from markdown | -| **LanceDB** (Arrow) | `.index/lancedb/*.lance` | vector + BM25 + scalar columns for retrieval | ✅ from markdown | +| **Derived index** (LanceDB by default; Milvus or SeekDB optional) | `.index/lancedb/*.lance`, a remote service, or `.index/seekdb/` | vector + BM25 + scalar columns for retrieval | ✅ from markdown | !!! note "The one rule that follows from this" Delete the entire `.index/` directory and **no memory is lost** — it @@ -77,8 +77,10 @@ visually distinct from a user-named one). │ │ ├── ome.db Offline Memory Engine state │ │ ├── ome.aps.db APScheduler jobstore (split to avoid lock contention) │ │ └── ome.db.lock OME single-engine guard (portalocker) -│ └── lancedb/ -│ └── .lance/ one Arrow table per kind +│ ├── lancedb/ +│ │ └── .lance/ default: one Arrow table per kind +│ └── seekdb/ +│ └── ... optional embedded SeekDB files │ ├── ome.toml ← user-editable OME strategy overrides (hot-reloaded) └── .tmp/ atomic-write staging @@ -97,8 +99,9 @@ visually distinct from a user-named one). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py); every path above is a property on it. `MemoryRoot.ensure()` creates the runtime dirs -(`.index/{sqlite,lancedb}/`, `.tmp/`); user-visible dirs appear on first -write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. +(`.index/{sqlite,lancedb}/`, `.tmp/`); the optional SeekDB directory is +created lazily. User-visible dirs appear on first write. Config files +(`everos.toml`, `ome.toml`) are created by `everos init`. ## How a memory is born @@ -130,7 +133,7 @@ index catches up asynchronously. ▼ md_change_state queue (SQLite, durable) ▼ - rebuild LanceDB rows ──▶ searchable + rebuild index rows ──▶ searchable ``` - **`/add`** appends messages to a per-`(session_id, app_id, project_id)` @@ -143,7 +146,7 @@ index catches up asynchronously. - Everything else (atomic facts, foresight, profile, agent cases/skills) is produced **asynchronously** by the OME — see [the OME section](#the-offline-memory-engine-ome). -- The **cascade daemon** turns every `.md` write into LanceDB rows so the +- The **cascade daemon** turns every `.md` write into derived-index rows so the content becomes searchable. ## Memory types & storage strategies @@ -178,7 +181,8 @@ The three strategies: ## The cascade daemon -The cascade subsystem keeps LanceDB in sync with the markdown tree. It runs +The cascade subsystem keeps the configured derived index in sync with the +markdown tree. It runs **in-process** with the server (a coroutine started by the app lifespan), not as a separate OS daemon. @@ -188,7 +192,7 @@ not as a separate OS daemon. durable, so a crash mid-sync replays on restart. 3. A worker drains the queue at **entry-level** granularity: it diffs the file, re-embeds only changed entries (keyed by `content_sha256`), and - upserts the LanceDB rows. + upserts the configured backend's rows. Because markdown is the source of truth, **editing a file directly is fully supported** — open an episode in VSCode / Obsidian / Vim, change an @@ -258,8 +262,8 @@ Two paths, two guarantees: | Path | Guarantee | Detail | |---|---|---| -| **Write** (`/add`, `/flush`) | **strong** | the episode `.md` is on disk before the call returns `extracted`; never blocks on LanceDB | -| **Read** (`/search`, `/get`) | **eventual** | reads LanceDB, which lags md by the cascade processing time — sub-second typically, up to ~10–15 s under load | +| **Write** (`/add`, `/flush`) | **strong** | the episode `.md` is on disk before the call returns `extracted`; never blocks on derived-index work | +| **Read** (`/search`, `/get`) | **eventual** | reads the configured derived index, which lags md by the cascade processing time — sub-second typically, up to ~10–15 s under load | So a `/search` immediately after the `/flush` that produced a record may miss it. The markdown is durable regardless; index lag never loses data. If @@ -275,15 +279,15 @@ trail. ## Zero external services -No database server, message broker, or vector service to run. Vector ANN, -full-text BM25, and scalar filtering all execute inside the **embedded -LanceDB** engine in one query; SQLite is a local file. The whole stack is a -single directory you can copy, back up, or check the user-visible parts of -into git. +With the default LanceDB backend there is no database server, message broker, +or vector service to run. Vector ANN, full-text BM25, and scalar filtering all +execute inside the embedded engine; SQLite is a local file. Embedded SeekDB +also needs no service on supported platforms. Milvus and remote SeekDB are +explicit opt-in service-backed modes. !!! note There is no automatic "grep over markdown" search fallback today — if - the LanceDB index is unavailable, rebuild it from markdown (it is + the derived index is unavailable, rebuild it from markdown (it is derived and disposable) rather than relying on a degraded search path. ## Operating it @@ -295,17 +299,17 @@ The CLI ([cli.md](cli.md)) is intentionally small: | `everos init` | generate starter config files (`everos.toml` + `ome.toml`) | | `everos server start` | run the HTTP API (cascade + OME start with it) | | `everos cascade status` | queue / LSN summary | -| `everos cascade sync` | drain the cascade queue now (force md → LanceDB) | +| `everos cascade sync` | drain the cascade queue now (force md → derived index) | | `everos cascade fix` | list failed rows / re-enqueue retryable ones | | `everos cascade rebuild` | rebuild the whole index from markdown (drift / corruption recovery) | !!! warning "There is no `everos reindex` or `everos flush`" - **Reindex** = the index is rebuildable from markdown. To rebuild the whole index, run `everos cascade rebuild` — it drops the - LanceDB tables and re-indexes from md, re-populating even entries + derived-index tables and re-indexes from md, re-populating even entries the queue already marked `done` and preserving un-extracted - buffered messages. (A bare `rm -rf /.index/lancedb` - is **not** enough: the cascade queue still shows those files + buffered messages. (Deleting only the backend's physical data is + **not** enough: the cascade queue still shows those files `done`, so the scanner skips them and the index comes back empty.) For an incremental catch-up, use `everos cascade sync`. - **Flush** is an HTTP endpoint (`POST /api/v2/memory/flush`), not a diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 3d90c775d..5afa108f2 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -52,6 +52,8 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ └── ome.db.lock OME single-engine guard (portalocker) │ ├── lancedb/ │ │ └── .lance/ default derived index backend +│ └── seekdb/ optional embedded SeekDB backend +│ └── ... engine-managed files │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) └── .tmp/ staging dir for batch / multi-step writes @@ -65,7 +67,8 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py), exposing every path as a property. `MemoryRoot.ensure()` creates the -runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); the +runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); SeekDB's optional +embedded directory is created lazily when that backend connects. The user-visible dirs are *not* pre-created — they appear on first write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. @@ -181,6 +184,8 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ unprocessed_buffer, conversation_status, cluster) ├── lancedb/ │ └── .lance/ default derived index backend +├── seekdb/ +│ └── ... optional embedded SeekDB data directory ``` - **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) @@ -192,9 +197,14 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers status). - The **derived index backend** holds the per-kind business rows, keyed `_` (so cross-table joins use `(owner_id, entry_id)`). - LanceDB is the default backend under `.index/lancedb/`; Milvus can be enabled - as the same rebuildable index backend and lives outside the memory root in a - configured Milvus Server or Zilliz Cloud deployment. + LanceDB is the default backend under `.index/lancedb/`. Milvus can be enabled + against a configured Milvus Server or Zilliz Cloud deployment. SeekDB can run + in process under `.index/seekdb/` on Linux/macOS, or connect to a remote + seekdb Server or OceanBase database. All three implement the same repository + contract and keep no authoritative data. `everos cascade rebuild` drops and + recreates SeekDB business tables but deliberately retains the embedded + `.index/seekdb/` directory and its `.everos.lock` anchor; removing the whole + directory is a separate offline purge operation. Episode and AtomicFact index rows carry a `deprecated_by: str | None` column. When an episode is superseded by a Reflection merge, @@ -227,5 +237,5 @@ this primitive is **schema-agnostic** — field-level semantics - Code: - [`core/persistence/memory_root.py`](../src/everos/core/persistence/memory_root.py) - [`core/persistence/markdown/`](../src/everos/core/persistence/markdown/) - - [`infra/persistence/{markdown,sqlite,lancedb,milvus,index}/`](../src/everos/infra/persistence/) + - [`infra/persistence/{markdown,sqlite,lancedb,milvus,seekdb,index}/`](../src/everos/infra/persistence/) - [`memory/cascade/`](../src/everos/memory/cascade/) (md → derived index sync) diff --git a/pyproject.toml b/pyproject.toml index f8b6c22b3..68d45b10d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,11 @@ otel = [ "opentelemetry-exporter-otlp-proto-http>=1.27.0", ] milvus = ["pymilvus>=3.0.0"] +seekdb = ["pyseekdb>=1.4.0.post1,<1.5"] +seekdb-embedded = [ + "pyseekdb>=1.4.0.post1,<1.5", + "pylibseekdb>=1.4.0.post1,<1.5; sys_platform == 'linux' or sys_platform == 'darwin'", +] [project.urls] Homepage = "https://evermind.ai" @@ -256,6 +261,7 @@ forbidden_modules = [ "everos.infra.persistence.sqlite.**", "everos.infra.persistence.index.**", "everos.infra.persistence.milvus.**", + "everos.infra.persistence.seekdb.**", "everos.infra.persistence.backends.**", ] # `forbidden` contracts run a *transitive closure* — any path from a @@ -289,6 +295,7 @@ ignore_imports = [ "everos.infra.persistence.index -> everos.infra.persistence.index.schema", "everos.infra.persistence.index -> everos.infra.persistence.backends.lancedb", "everos.infra.persistence.index -> everos.infra.persistence.backends.milvus", + "everos.infra.persistence.index -> everos.infra.persistence.backends.seekdb", ] [[tool.importlinter.contracts]] @@ -309,6 +316,7 @@ forbidden_modules = [ "everos.infra.persistence.backends", "everos.infra.persistence.lancedb", "everos.infra.persistence.milvus", + "everos.infra.persistence.seekdb", "everos.core.persistence.lancedb", ] diff --git a/src/everos/config/__init__.py b/src/everos/config/__init__.py index fb3108132..5715d7154 100644 --- a/src/everos/config/__init__.py +++ b/src/everos/config/__init__.py @@ -22,6 +22,7 @@ from .settings import MilvusSettings as MilvusSettings from .settings import MultimodalSettings as MultimodalSettings from .settings import RerankSettings as RerankSettings +from .settings import SeekdbSettings as SeekdbSettings from .settings import Settings as Settings from .settings import SqliteSettings as SqliteSettings from .settings import load_settings as load_settings @@ -37,6 +38,7 @@ "MilvusSettings", "MultimodalSettings", "RerankSettings", + "SeekdbSettings", "Settings", "SqliteSettings", "load_settings", diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index 3dceee2f4..cdebf4f10 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -62,6 +62,22 @@ db_name = "" consistency_level = "Session" collection_prefix = "everos" +[seekdb] +# Used only when index.backend = "seekdb". Install everos[seekdb] for remote +# mode, or everos[seekdb-embedded] on Linux/macOS for the in-process engine. +mode = "embedded" +path = "" +host = "" +port = 2881 +tenant = "" +user = "root" +password = "" +database = "everos" +table_prefix = "everos" +vector_sync_mode = "immediate" +connect_timeout_seconds = 10.0 +read_timeout_seconds = 60.0 + [llm] # Provider-agnostic OpenAI-protocol client config. Override via env: # EVEROS_LLM__MODEL, EVEROS_LLM__API_KEY, EVEROS_LLM__BASE_URL diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index fc1be21ad..791b81fc5 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -560,7 +560,7 @@ class CascadeSettings(BaseModel): class IndexSettings(BaseModel): """Rebuildable derived-index backend selection.""" - backend: Literal["lancedb", "milvus"] = "lancedb" + backend: Literal["lancedb", "milvus", "seekdb"] = "lancedb" class MilvusSettings(BaseModel): @@ -583,6 +583,54 @@ def _validate_collection_prefix(cls, value: str) -> str: return value +class SeekdbSettings(BaseModel): + """Embedded or remote SeekDB connection settings. + + Env binding (via parent ``Settings``): + EVEROS_SEEKDB__MODE + EVEROS_SEEKDB__PATH + EVEROS_SEEKDB__HOST + EVEROS_SEEKDB__PORT + EVEROS_SEEKDB__TENANT + EVEROS_SEEKDB__USER + EVEROS_SEEKDB__PASSWORD + EVEROS_SEEKDB__DATABASE + EVEROS_SEEKDB__TABLE_PREFIX + EVEROS_SEEKDB__VECTOR_SYNC_MODE + EVEROS_SEEKDB__CONNECT_TIMEOUT_SECONDS + EVEROS_SEEKDB__READ_TIMEOUT_SECONDS + """ + + mode: Literal["embedded", "remote"] = "embedded" + path: str = "" + host: str = "" + port: int = Field(default=2881, ge=1, le=65535) + tenant: str = "" + user: str = "root" + password: SecretStr = SecretStr("") + database: str = Field( + default="everos", + max_length=64, + pattern=r"^[A-Za-z_][A-Za-z0-9_]*$", + ) + # SeekDB follows MySQL's 64-character table-name limit. The longest + # logical suffix is ``_knowledge_topic`` (16 characters). + table_prefix: str = Field(default="everos", min_length=1, max_length=48) + vector_sync_mode: Literal["immediate", "async"] = "immediate" + connect_timeout_seconds: float = Field(default=10.0, gt=0) + read_timeout_seconds: float = Field(default=60.0, gt=0) + + @field_validator("table_prefix") + @classmethod + def _validate_table_prefix(cls, value: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value): + raise ValueError( + "table_prefix must start with a letter or underscore and " + "contain only letters, digits, and underscores" + ) + return value + + class KnowledgeSearchSettings(BaseModel): """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" @@ -658,6 +706,7 @@ class Settings(BaseSettings): lancedb: LanceDBSettings = LanceDBSettings() index: IndexSettings = IndexSettings() milvus: MilvusSettings = MilvusSettings() + seekdb: SeekdbSettings = SeekdbSettings() llm: LLMSettings = LLMSettings() decider: DeciderSettings = DeciderSettings() embedding: EmbeddingSettings = EmbeddingSettings() diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index d8095f56e..a1b124c3d 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -15,7 +15,7 @@ vectors, build clusters, extract skills. See :func:`everos.entrypoints.cli.commands._backfill_cmd.run_backfill` for the phase orchestration. -- ``cascade rebuild`` — drop every business LanceDB table and re-index +- ``cascade rebuild`` — drop every business derived-index table and re-index all md from scratch. Recovery for a drifted / corrupt index; safe because md is the source of truth and un-extracted buffered messages are preserved. Skips the schema-verify guard (which the drift would @@ -69,7 +69,7 @@ app = typer.Typer( name="cascade", - help="Inspect and operate the md → LanceDB sync queue", + help="Inspect and operate the md → derived-index sync queue", no_args_is_help=True, ) @@ -137,7 +137,7 @@ def _apply_verbose_logging(verbose: bool | None) -> None: @asynccontextmanager async def _runtime(*, verify: bool = True, ensure: bool = True) -> AsyncIterator[None]: - """Stand up sqlite + lancedb the same way the API lifespan would. + """Stand up SQLite and the configured index backend like the API lifespan. The CLI uses the same lazy, process-wide singletons the API lifespan does. They are **per-process**: a running daemon has its own @@ -450,7 +450,7 @@ def rebuild( typer.Option("--yes", "-y", help="Skip the confirmation prompt."), ] = False, ) -> None: - """Rebuild the LanceDB index from markdown (recover from schema drift). + """Rebuild the configured derived index from markdown. **Stop the ``everos server`` first** — this is the one cascade command that is not safe alongside a live daemon. It drops and recreates the @@ -458,15 +458,15 @@ def rebuild( the dropped dataset; the command refuses to start while a server holds the OME lock. - Drops every business LanceDB table and re-indexes all md from - scratch. Markdown is the source of truth, so no memory content is + Drops every business table in the configured index backend and re-indexes + all md from scratch. Markdown is the source of truth, so no memory content is lost, and this is the safe recovery from a drifted / corrupt index (e.g. the ``verify_business_schemas`` startup failure): - unlike ``rm -rf ~/.everos/.index/lancedb``, it re-populates already-indexed entries (that command leaves the cascade queue marked ``done``, so nothing re-indexes and the index comes back - empty); + empty; an embedded SeekDB directory itself is retained); - unlike ``rm -rf ~/.everos/.index``, it preserves SQLite state that is NOT rebuildable from md — notably ``unprocessed_buffer`` (messages received but not yet extracted). @@ -475,7 +475,7 @@ def rebuild( typer.echo( "error: a server (or another exclusive CLI phase) is running on " "this memory root.\n" - " cascade rebuild drops and recreates the LanceDB tables; a live " + " cascade rebuild drops and recreates derived-index tables; a live " "daemon holds cached\n" " table handles and would keep writing to the dropped dataset. " "Stop `everos server`\n" @@ -485,7 +485,7 @@ def rebuild( raise typer.Exit(code=3) if not yes: typer.confirm( - "Drop all LanceDB business tables and re-index from markdown? " + "Drop all derived-index business tables and re-index from markdown? " "(requires the server to be stopped)", abort=True, ) @@ -506,7 +506,7 @@ async def _run() -> None: typer.echo(f"reset {cleared} cascade queue row(s)") dropped = await drop_business_tables() typer.echo( - f"dropped {len(dropped)} LanceDB table(s): " + f"dropped {len(dropped)} derived-index table(s): " f"{', '.join(dropped) or '(none)'}" ) # Recreate the tables (current schema) + FTS indexes. diff --git a/src/everos/infra/persistence/backends/__init__.py b/src/everos/infra/persistence/backends/__init__.py index 7547c8ce3..196833d15 100644 --- a/src/everos/infra/persistence/backends/__init__.py +++ b/src/everos/infra/persistence/backends/__init__.py @@ -3,12 +3,12 @@ Each module here implements the ports in :mod:`everos.infra.persistence.index.protocols` for one storage engine, and owns everything specific to it: predicate rendering, physical schema, and the -connection lifecycle. +connection lifecycle. LanceDB, Milvus, and SeekDB are the registered engines. They live outside :mod:`everos.infra.persistence.index` on purpose. That package is the boundary outer layers depend on, and mixing adapters into it made "which of these is the abstraction?" a question a reader had to answer by -opening files. Adding a third backend means adding a module here, not editing +opening files. Adding another backend means adding a module here, not editing the abstraction. Nothing outside ``persistence`` should import these directly — go through the diff --git a/src/everos/infra/persistence/backends/seekdb.py b/src/everos/infra/persistence/backends/seekdb.py new file mode 100644 index 000000000..12a3fd196 --- /dev/null +++ b/src/everos/infra/persistence/backends/seekdb.py @@ -0,0 +1,50 @@ +"""SeekDB lifecycle adapter for the derived-index backend port.""" + +from __future__ import annotations + +from types import ModuleType +from typing import TYPE_CHECKING, Any, ClassVar + +if TYPE_CHECKING: + from ..index.protocols import IndexRepository + + +class SeekdbIndexBackend: + """Own embedded or remote SeekDB table and connection lifecycle.""" + + name: ClassVar[str] = "seekdb" + + @property + def repositories(self) -> tuple[IndexRepository[Any], ...]: + return tuple(_seekdb().ALL_REPOS) + + async def connect(self) -> Any: + return await _seekdb().get_session() + + async def startup(self) -> Any: + session = await self.connect() + await self.ensure_business_indexes() + return session + + async def shutdown(self) -> None: + await _seekdb().dispose_connection() + + async def ensure_business_indexes(self) -> None: + await _seekdb().ensure_business_indexes() + + async def verify_business_schemas(self) -> None: + await self.ensure_business_indexes() + + async def drop_business_tables(self) -> list[str]: + return await _seekdb().drop_business_tables() + + +def _seekdb() -> ModuleType: + from everos.infra.persistence import seekdb + + return seekdb + + +seekdb_index_backend = SeekdbIndexBackend() + +__all__ = ["SeekdbIndexBackend", "seekdb_index_backend"] diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index abf01382c..7c721bf2e 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -2,13 +2,13 @@ Markdown remains the source of truth and SQLite remains the system-state store. This boundary owns only the derived business indexes used by cascade, -search, and get. LanceDB and Milvus implement the same typed ports, so callers -do not branch on physical storage. +search, and get. LanceDB, Milvus, and SeekDB implement the same typed ports, so +callers do not branch on physical storage. """ from __future__ import annotations -from typing import Any +from typing import Any, Final from everos.config import load_settings from everos.infra.persistence import lancedb as _lancedb @@ -38,6 +38,7 @@ user_profile_repo as _lance_user_profile_repo, ) from ..backends.milvus import milvus_index_backend +from ..backends.seekdb import seekdb_index_backend from .predicate import ( All, AnyOf, @@ -104,6 +105,12 @@ knowledge_topic_repo, ) +_BACKENDS: Final[dict[str, IndexBackend]] = { + "lancedb": lance_index_backend, + "milvus": milvus_index_backend, + "seekdb": seekdb_index_backend, +} + def active_backend() -> str: """Name of the configured derived-index backend.""" @@ -111,9 +118,7 @@ def active_backend() -> str: def _backend() -> IndexBackend: - if active_backend() == "milvus": - return milvus_index_backend - return lance_index_backend + return _BACKENDS[active_backend()] async def connect() -> Any: diff --git a/src/everos/infra/persistence/index/router.py b/src/everos/infra/persistence/index/router.py index 83480c04d..0596bf087 100644 --- a/src/everos/infra/persistence/index/router.py +++ b/src/everos/infra/persistence/index/router.py @@ -3,8 +3,9 @@ from __future__ import annotations import datetime as dt +import importlib from collections.abc import Sequence -from typing import Any, cast +from typing import Any, Final, cast from pydantic import BaseModel @@ -17,6 +18,11 @@ IndexRepository, ) +_BACKEND_MODULES: Final[dict[str, str]] = { + "milvus": "everos.infra.persistence.milvus", + "seekdb": "everos.infra.persistence.seekdb", +} + class RoutedIndexRepository[T: BaseModel]: """Stable repository identity with a backend selected at call time.""" @@ -24,10 +30,10 @@ class RoutedIndexRepository[T: BaseModel]: def __init__( self, lance_repo: IndexRepository[T], - milvus_repo_name: str, + repo_name: str, ) -> None: self._lance_repo = lance_repo - self._milvus_repo_name = milvus_repo_name + self._repo_name = repo_name self.schema = lance_repo.schema @property @@ -35,11 +41,11 @@ def table_name(self) -> str: return self._lance_repo.table_name def _repo(self) -> IndexRepository[T]: - if load_settings().index.backend == "milvus": - from everos.infra.persistence import milvus - - return getattr(milvus, self._milvus_repo_name) - return self._lance_repo + backend = load_settings().index.backend + if backend == "lancedb": + return self._lance_repo + module = importlib.import_module(_BACKEND_MODULES[backend]) + return getattr(module, self._repo_name) async def add(self, records: Sequence[T]) -> None: await self._repo().add(records) diff --git a/src/everos/infra/persistence/index/schema.py b/src/everos/infra/persistence/index/schema.py index 0c6178b2b..9b1f2a7f8 100644 --- a/src/everos/infra/persistence/index/schema.py +++ b/src/everos/infra/persistence/index/schema.py @@ -50,7 +50,9 @@ def field(self, name: str) -> IndexField: for field in self.fields: if field.name == name: return field - raise KeyError(name) + raise ValueError( + f"derived-index schema {self.table_name!r} has no field {name!r}" + ) @property def vector_fields(self) -> tuple[IndexField, ...]: diff --git a/src/everos/infra/persistence/seekdb/__init__.py b/src/everos/infra/persistence/seekdb/__init__.py new file mode 100644 index 000000000..5d1a6f808 --- /dev/null +++ b/src/everos/infra/persistence/seekdb/__init__.py @@ -0,0 +1,88 @@ +"""SeekDB derived-index backend selected through ``Settings.index.backend``. + +The package exposes the same seven repository singletons as LanceDB and +Milvus. pyseekdb itself stays lazily imported by the connection manager, so +default installations can import EverOS without the optional dependency. +""" + +from __future__ import annotations + +from everos.core.observability.logging import get_logger + +from .errors import SeekdbConfigurationError as SeekdbConfigurationError +from .errors import SeekdbIntegrityError as SeekdbIntegrityError +from .errors import SeekdbOperationalError as SeekdbOperationalError +from .errors import SeekdbSchemaMismatchError as SeekdbSchemaMismatchError +from .errors import SeekdbValueLimitError as SeekdbValueLimitError +from .repos import ALL_REPOS as ALL_REPOS +from .repos import agent_case_repo as agent_case_repo +from .repos import agent_skill_repo as agent_skill_repo +from .repos import atomic_fact_repo as atomic_fact_repo +from .repos import episode_repo as episode_repo +from .repos import foresight_repo as foresight_repo +from .repos import knowledge_topic_repo as knowledge_topic_repo +from .repos import user_profile_repo as user_profile_repo +from .repository import SeekdbRepoBase as SeekdbRepoBase +from .seekdb_manager import dispose_connection as _dispose_connection +from .seekdb_manager import get_session as get_session +from .seekdb_manager import run as _run +from .seekdb_manager import table_name as table_name +from .sql import quote_identifier as _quote_identifier + +logger = get_logger(__name__) + + +async def ensure_business_indexes() -> None: + """Create or verify every configured SeekDB business table.""" + for repo in ALL_REPOS: + await repo.ensure_table() + + +async def drop_business_tables() -> list[str]: + """Drop every configured SeekDB business table and return physical names.""" + session = await get_session() + dropped: list[str] = [] + for repo in ALL_REPOS: + if not await repo.table_exists(): + continue + name = repo.physical_table_name + await _run( + session.execute, + f"DROP TABLE IF EXISTS {_quote_identifier(name)}", + table=name, + ) + dropped.append(name) + logger.info("seekdb_table_dropped", table=repo.table_name, physical_table=name) + SeekdbRepoBase._reset_table_cache() + return dropped + + +async def dispose_connection() -> None: + """Close SeekDB and clear per-process table readiness state.""" + try: + await _dispose_connection() + finally: + SeekdbRepoBase._reset_table_cache() + + +__all__ = [ + "ALL_REPOS", + "SeekdbConfigurationError", + "SeekdbIntegrityError", + "SeekdbOperationalError", + "SeekdbRepoBase", + "SeekdbSchemaMismatchError", + "SeekdbValueLimitError", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "dispose_connection", + "drop_business_tables", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "get_session", + "knowledge_topic_repo", + "table_name", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/seekdb/codec.py b/src/everos/infra/persistence/seekdb/codec.py new file mode 100644 index 000000000..d565cbada --- /dev/null +++ b/src/everos/infra/persistence/seekdb/codec.py @@ -0,0 +1,190 @@ +"""Encode logical records for SeekDB and restore typed result rows. + +Datetime values use exact UTC epoch milliseconds, arrays use JSON, and VECTOR +values use textual input on writes while accepting text, native lists, or +little-endian float32 bytes on reads. +""" + +from __future__ import annotations + +import datetime as dt +import json +import math +import struct +from collections.abc import Mapping, Sequence +from typing import Any + +from pydantic import BaseModel + +from everos.component.utils.datetime import ( + ensure_utc, + from_timestamp_ms, + to_timestamp_ms, +) +from everos.infra.persistence.index.schema import ( + IndexField, + IndexFieldKind, + IndexSchema, +) + +from .errors import SeekdbValueLimitError +from .schema import physical_column + + +def to_row(record: BaseModel, schema: IndexSchema) -> dict[str, Any]: + """Encode one validated Pydantic record into physical column values.""" + raw = record.model_dump(mode="python") + out: dict[str, Any] = {} + for field in schema.fields: + name, value = write_field_value(field, raw.get(field.name), schema) + out[name] = value + return out + + +def from_row(row: Mapping[str, Any], schema: IndexSchema) -> dict[str, Any]: + """Restore all logical fields present in a raw SeekDB result row.""" + out: dict[str, Any] = {} + for field in schema.fields: + storage_name = _storage_name(field) + if storage_name not in row: + continue + value = row[storage_name] + if field.kind is IndexFieldKind.DATETIME: + out[field.name] = None if value is None else from_timestamp_ms(int(value)) + elif field.kind is IndexFieldKind.STRING_ARRAY: + out[field.name] = _decode_json(value) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + out[field.name] = _decode_vector(value, field.dimension) + else: + out[field.name] = _decode_scalar(value) + return out + + +def model_from_row( + row: Mapping[str, Any], schema: IndexSchema, model: type[BaseModel] +) -> BaseModel: + """Restore and validate a complete result row as ``model``.""" + return model.model_validate(from_row(row, schema)) + + +def write_field_value( + field: IndexField, value: Any, schema: IndexSchema +) -> tuple[str, Any]: + """Validate and encode one logical field for INSERT or UPDATE.""" + table_name = schema.table_name + if value is None: + if not field.nullable: + raise SeekdbValueLimitError(f"{table_name}.{field.name} cannot be null") + return _storage_name(field), None + if field.kind is IndexFieldKind.STRING: + text = str(value) + column = physical_column(schema, field.name) + if limit := _varchar_length(column.sql_type): + _validate_string_length(table_name, field.name, text, limit) + return field.name, text + if field.kind is IndexFieldKind.STRING_ARRAY: + items = [str(item) for item in value] + if field.max_capacity is not None and len(items) > field.max_capacity: + raise SeekdbValueLimitError( + f"{table_name}.{field.name} has {len(items)} items; " + f"SeekDB limit is {field.max_capacity}" + ) + for position, item in enumerate(items): + if field.max_length is not None: + _validate_string_length( + table_name, + f"{field.name}[{position}]", + item, + field.max_length, + ) + return field.name, json.dumps(items, ensure_ascii=False, separators=(",", ":")) + if field.kind is IndexFieldKind.DATETIME: + return f"{field.name}_ms", _datetime_to_ms(value) + if field.kind is IndexFieldKind.DENSE_VECTOR: + vector = _validate_vector(value, field, table_name) + return field.name, json.dumps(vector, separators=(",", ":")) + return field.name, value + + +def _storage_name(field: IndexField) -> str: + return f"{field.name}_ms" if field.kind is IndexFieldKind.DATETIME else field.name + + +def _varchar_length(sql_type: str) -> int | None: + normalized = sql_type.strip().upper() + if not normalized.startswith("VARCHAR(") or not normalized.endswith(")"): + return None + return int(normalized[8:-1]) + + +def _validate_string_length(table: str, field: str, value: str, limit: int) -> None: + size = len(value) + if size > limit: + raise SeekdbValueLimitError( + f"{table}.{field} is {size} characters; SeekDB limit is {limit}" + ) + + +def _datetime_to_ms(value: Any) -> int: + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + raise TypeError(f"expected datetime or epoch ms, got {type(value).__name__}") + + +def _validate_vector( + value: Sequence[float], field: IndexField, table_name: str +) -> list[float]: + vector = [float(item) for item in value] + if len(vector) != field.dimension: + raise SeekdbValueLimitError( + f"{table_name}.{field.name} has dimension {len(vector)}; " + f"expected {field.dimension}" + ) + if not all(math.isfinite(item) for item in vector): + raise SeekdbValueLimitError( + f"{table_name}.{field.name} contains a non-finite value" + ) + return vector + + +def _decode_scalar(value: Any) -> Any: + return value.decode("utf-8") if isinstance(value, bytes) else value + + +def _decode_json(value: Any) -> Any: + if isinstance(value, bytes): + value = value.decode("utf-8") + return json.loads(value) if isinstance(value, str) else value + + +def _decode_vector(value: Any, dimension: int | None) -> list[float] | None: + if value is None: + return None + if isinstance(value, str): + return [float(item) for item in json.loads(value)] + if isinstance(value, bytes): + try: + decoded = value.decode("utf-8") + except UnicodeDecodeError: + decoded = "" + if decoded: + try: + parsed = json.loads(decoded) + except (json.JSONDecodeError, TypeError): + pass + else: + if isinstance(parsed, list) and ( + dimension is None or len(parsed) == dimension + ): + return [float(item) for item in parsed] + if dimension is not None and len(value) == dimension * 4: + return list(struct.unpack(f"<{dimension}f", value)) + raise ValueError("unexpected binary VECTOR representation") + return [float(item) for item in value] + + +__all__ = ["from_row", "model_from_row", "to_row", "write_field_value"] diff --git a/src/everos/infra/persistence/seekdb/errors.py b/src/everos/infra/persistence/seekdb/errors.py new file mode 100644 index 000000000..6c908e8de --- /dev/null +++ b/src/everos/infra/persistence/seekdb/errors.py @@ -0,0 +1,34 @@ +"""Typed failures raised at the SeekDB adapter boundary.""" + +from __future__ import annotations + +from everos.core.errors import ConfigurationError + + +class SeekdbConfigurationError(ConfigurationError): + """SeekDB settings or optional runtime dependencies are invalid.""" + + +class SeekdbSchemaMismatchError(RuntimeError): + """A physical SeekDB table no longer matches its logical schema.""" + + +class SeekdbValueLimitError(ValueError): + """A record cannot be represented by the declared SeekDB schema.""" + + +class SeekdbIntegrityError(RuntimeError): + """A SeekDB write violates a uniqueness or integrity constraint.""" + + +class SeekdbOperationalError(RuntimeError): + """SeekDB rejected or could not execute an adapter operation.""" + + +__all__ = [ + "SeekdbConfigurationError", + "SeekdbIntegrityError", + "SeekdbOperationalError", + "SeekdbSchemaMismatchError", + "SeekdbValueLimitError", +] diff --git a/src/everos/infra/persistence/seekdb/predicate.py b/src/everos/infra/persistence/seekdb/predicate.py new file mode 100644 index 000000000..7382b8e79 --- /dev/null +++ b/src/everos/infra/persistence/seekdb/predicate.py @@ -0,0 +1,111 @@ +"""Render backend-neutral predicates into the SeekDB MySQL dialect.""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Collection +from typing import Final + +from everos.component.utils.datetime import ensure_utc, to_timestamp_ms +from everos.infra.persistence.predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, +) + +from .sql import json_literal, literal, quote_identifier + +_OPERATORS: Final[dict[str, str]] = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate( + predicate: Predicate | None, + *, + datetime_fields: Collection[str] = (), + vector_fields: Collection[str] = (), +) -> str: + """Render ``predicate`` without a leading ``WHERE`` keyword.""" + if predicate is None: + return "" + if not isinstance(predicate, Predicate): + raise TypeError( + "SeekDB predicates must use the neutral Predicate AST, " + f"got {type(predicate).__name__}" + ) + if isinstance(predicate, Comparison): + return _comparison(predicate, datetime_fields) + if isinstance(predicate, In): + if not predicate.values: + raise ValueError("SeekDB IN predicates require at least one value") + values = ", ".join( + _field_literal(predicate.field, value, datetime_fields) + for value in predicate.values + ) + return f"{_field(predicate.field, datetime_fields)} IN ({values})" + if isinstance(predicate, Contains): + column = _field(predicate.field, datetime_fields) + return f"JSON_CONTAINS({column}, {json_literal(predicate.value)})" + if isinstance(predicate, IsNull): + # SeekDB 1.4 permits nullable VECTOR columns; vector_fields remains an + # explicit argument so a future presence-marker fallback is contained. + _ = vector_fields + return f"{_field(predicate.field, datetime_fields)} IS NULL" + if isinstance(predicate, All): + return _render_group(predicate.children, "AND", datetime_fields, vector_fields) + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "OR", datetime_fields, vector_fields) + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _comparison(node: Comparison, datetime_fields: Collection[str]) -> str: + return ( + f"{_field(node.field, datetime_fields)} {_OPERATORS[node.operator]} " + f"{_field_literal(node.field, node.value, datetime_fields)}" + ) + + +def _field(name: str, datetime_fields: Collection[str]) -> str: + physical = f"{name}_ms" if name in datetime_fields else name + return quote_identifier(physical) + + +def _field_literal(name: str, value: Scalar, datetime_fields: Collection[str]) -> str: + if name in datetime_fields and isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return str(to_timestamp_ms(aware)) + return literal(value) + + +def _render_group( + children: tuple[Predicate, ...], + operator: str, + datetime_fields: Collection[str], + vector_fields: Collection[str], +) -> str: + rendered = [ + render_predicate( + child, + datetime_fields=datetime_fields, + vector_fields=vector_fields, + ) + for child in children + ] + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/seekdb/repos.py b/src/everos/infra/persistence/seekdb/repos.py new file mode 100644 index 000000000..1b71a2435 --- /dev/null +++ b/src/everos/infra/persistence/seekdb/repos.py @@ -0,0 +1,177 @@ +"""SeekDB repository singletons for all seven derived business indexes.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from everos.component.utils.datetime import from_timestamp +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + UserProfile, +) +from everos.infra.persistence.predicate import all_of, eq, gt, is_null + +from .codec import from_row +from .repository import SeekdbRepoBase + + +class _EpisodeRepo(SeekdbRepoBase[Episode]): + schema = Episode + + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await self.count_where( + all_of( + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + eq("parent_type", parent_type) if parent_type is not None else None, + ) + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Episode] | list[dict[str, Any]]: + predicate = all_of( + eq("owner_id", owner_id), + gt("timestamp", from_timestamp(after_ts)), + eq("parent_type", parent_type), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + ) + if columns is None: + raw = await self._query_rows( + predicate, + columns=self._stored_columns(include_vectors=True), + order_by="`timestamp_ms` ASC, `id` ASC", + limit=limit, + ) + return [self._model(row) for row in raw] + projection = list(dict.fromkeys([*columns, "timestamp"])) + fields = [self.index_schema.field(name) for name in projection] + physical = [self._storage_name(field) for field in fields] + raw = await self._query_rows( + predicate, + columns=physical, + order_by="`timestamp_ms` ASC, `id` ASC", + limit=limit, + ) + result: list[dict[str, Any]] = [] + for row in raw: + restored = from_row(row, self.index_schema) + result.append({name: restored.get(name) for name in projection}) + return result + + +class _AtomicFactRepo(SeekdbRepoBase[AtomicFact]): + schema = AtomicFact + + +class _ForesightRepo(SeekdbRepoBase[Foresight]): + schema = Foresight + + +class _AgentCaseRepo(SeekdbRepoBase[AgentCase]): + schema = AgentCase + + +class _AgentSkillRepo(SeekdbRepoBase[AgentSkill]): + schema = AgentSkill + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + return await self.count_where( + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)) + ) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[AgentSkill]: + return await self.find_where( + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), + limit=limit, + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[AgentSkill]: + if not query_vector: + raise ValueError( + "query_vector must be non-empty; " + "call find_in_cluster for the scalar fallback" + ) + rows = await self.dense_search( + query_vector, + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), + limit=top_k, + ) + out: list[AgentSkill] = [] + for row in rows: + rid = row.get("id") + if isinstance(rid, str) and (item := await self.get_by_id(rid)) is not None: + out.append(item) + return out + + +class _UserProfileRepo(SeekdbRepoBase[UserProfile]): + schema = UserProfile + + +class _KnowledgeTopicRepo(SeekdbRepoBase[KnowledgeTopic]): + schema = KnowledgeTopic + + +episode_repo = _EpisodeRepo() +atomic_fact_repo = _AtomicFactRepo() +foresight_repo = _ForesightRepo() +agent_case_repo = _AgentCaseRepo() +agent_skill_repo = _AgentSkillRepo() +user_profile_repo = _UserProfileRepo() +knowledge_topic_repo = _KnowledgeTopicRepo() + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/seekdb/repository.py b/src/everos/infra/persistence/seekdb/repository.py new file mode 100644 index 000000000..d164ab241 --- /dev/null +++ b/src/everos/infra/persistence/seekdb/repository.py @@ -0,0 +1,595 @@ +"""SQL-first SeekDB repository implementing the complete index port. + +Every business table is created from the backend-neutral logical schema. Reads +always name their columns explicitly so tuple rows from embedded pylibseekdb +and dictionary rows from remote PyMySQL have identical behavior. +""" + +from __future__ import annotations + +import asyncio +import datetime as dt +from collections.abc import Mapping, Sequence +from typing import Any, ClassVar, Literal, cast + +from pydantic import BaseModel + +from everos.config import load_settings +from everos.core.observability.logging import get_logger +from everos.infra.persistence.index.schema import ( + IndexField, + IndexFieldKind, + IndexSchema, + schema_for, +) +from everos.infra.persistence.predicate import Predicate, all_of, eq, one_of + +from .codec import from_row, model_from_row, to_row, write_field_value +from .errors import SeekdbSchemaMismatchError +from .predicate import render_predicate +from .schema import ( + build_create_table, + column_drift, + index_drift, + physical_column, + physical_columns, + physical_indexes, +) +from .seekdb_manager import get_session, run, table_name +from .sql import literal, quote_identifier + +logger = get_logger(__name__) + +_SCAN_BATCH_SIZE = 1_000 +_SEARCH_LIMIT_MAX = 16_384 +_WRITE_BATCH_SIZE = 64 + + +class SeekdbRepoBase[T: BaseModel]: + """Generic SeekDB repository backed by one portable record model.""" + + schema: type[T] + _ready_tables: ClassVar[set[str]] = set() + _table_locks: ClassVar[dict[str, asyncio.Lock]] = {} + + @property + def index_schema(self) -> IndexSchema: + return schema_for(self.schema) + + @property + def table_name(self) -> str: + return self.index_schema.table_name + + @property + def physical_table_name(self) -> str: + return table_name(self.table_name) + + @classmethod + def _table_lock(cls, name: str) -> asyncio.Lock: + return cls._table_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _reset_table_cache(cls) -> None: + cls._ready_tables.clear() + + @classmethod + def _reset_locks_for_tests(cls) -> None: + cls._table_locks.clear() + cls._reset_table_cache() + + async def ensure_table(self) -> None: + """Create or drift-check this table once per process.""" + name = self.physical_table_name + if name in self._ready_tables: + return + async with self._table_lock(name): + if name in self._ready_tables: + return + if await self.table_exists(): + await self.verify_table() + else: + await self._execute( + build_create_table( + name, + self.index_schema, + self._vector_sync_mode(), + ) + ) + logger.info( + "seekdb_table_created", table=self.table_name, physical_table=name + ) + self._ready_tables.add(name) + + async def table_exists(self) -> bool: + sql = ( + "SELECT COUNT(*) AS n FROM information_schema.TABLES " + "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = " + f"{literal(self.physical_table_name)}" + ) + return bool(int(await self._fetch_scalar(sql) or 0)) + + async def verify_table(self) -> None: + """Reject column or index drift with a rebuild-oriented error.""" + table = self.physical_table_name + columns = await self._fetch_all( + "SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY, " + "CHARACTER_SET_NAME, COLLATION_NAME " + "FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() " + f"AND TABLE_NAME = {literal(table)} ORDER BY ORDINAL_POSITION", + [ + "COLUMN_NAME", + "COLUMN_TYPE", + "IS_NULLABLE", + "COLUMN_KEY", + "CHARACTER_SET_NAME", + "COLLATION_NAME", + ], + ) + indexes = await self._fetch_all( + "SELECT INDEX_NAME, COLUMN_NAME, INDEX_TYPE, SEQ_IN_INDEX, SUB_PART " + "FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() " + f"AND TABLE_NAME = {literal(table)} ORDER BY INDEX_NAME, SEQ_IN_INDEX", + ["INDEX_NAME", "COLUMN_NAME", "INDEX_TYPE", "SEQ_IN_INDEX", "SUB_PART"], + ) + create_rows = await self._fetch_all( + f"SHOW CREATE TABLE {quote_identifier(table)}", ["Table", "Create Table"] + ) + create_value = create_rows[0].get("Create Table") if create_rows else "" + create_sql = ( + create_value.decode("utf-8") + if isinstance(create_value, bytes) + else str(create_value or "") + ) + missing, stale, incompatible = column_drift( + physical_columns(self.index_schema), columns + ) + missing_indexes, stale_indexes, incompatible_indexes = index_drift( + physical_indexes(self.index_schema, self._vector_sync_mode()), + indexes, + create_sql=create_sql, + ) + if ( + missing + or stale + or incompatible + or missing_indexes + or stale_indexes + or incompatible_indexes + ): + details = ( + f"missing_columns={missing}, stale_columns={stale}, " + f"incompatible_columns={incompatible}, " + f"missing_indexes={missing_indexes}, " + f"stale_indexes={stale_indexes}, " + f"incompatible_indexes={incompatible_indexes}" + ) + logger.error("seekdb_schema_drift", table=table, details=details) + raise SeekdbSchemaMismatchError( + f"SeekDB table {table!r} schema drift: {details}. The index is " + "rebuildable from markdown; run `everos cascade rebuild`." + ) + + async def add(self, records: Sequence[T]) -> None: + if not records: + return + await self.ensure_table() + await self._write_records(records, upsert=False) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + if by != "id": + raise ValueError("SeekdbRepoBase only supports upsert by id") + if not records: + return + await self.ensure_table() + await self._write_records(records, upsert=True) + + async def _write_records(self, records: Sequence[T], *, upsert: bool) -> None: + rows = [to_row(record, self.index_schema) for record in records] + for start in range(0, len(rows), _WRITE_BATCH_SIZE): + sql = self._insert_sql(rows[start : start + _WRITE_BATCH_SIZE], upsert) + await self._execute(sql) + + def _insert_sql(self, rows: Sequence[Mapping[str, Any]], upsert: bool) -> str: + columns = self._stored_columns(include_vectors=True) + column_sql = ", ".join(quote_identifier(name) for name in columns) + values = ", ".join( + "(" + ", ".join(literal(row[name]) for name in columns) + ")" + for row in rows + ) + sql = ( + f"INSERT INTO {quote_identifier(self.physical_table_name)} " + f"({column_sql}) VALUES {values}" + ) + if upsert: + primary = next( + column.name + for column in physical_columns(self.index_schema) + if column.primary + ) + # seekdb and OceanBase's supported MySQL dialect intentionally uses + # VALUES(col). MySQL upstream deprecates it, but adopting MySQL 8's + # row-alias syntax would make current seekdb releases incompatible. + updates = ", ".join( + f"{quote_identifier(name)}=VALUES({quote_identifier(name)})" + for name in columns + if name != primary + ) + sql += f" ON DUPLICATE KEY UPDATE {updates}" + return sql + + async def count(self) -> int: + return await self.count_where() + + async def count_where(self, where: Predicate | None = None) -> int: + await self.ensure_table() + sql = f"SELECT COUNT(*) AS n FROM {quote_identifier(self.physical_table_name)}" + sql += self._where_clause(where) + return int(await self._fetch_scalar(sql) or 0) + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + self.index_schema.field(id_field) + return await self.find_one_where(eq(id_field, id_value)) + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: + rows = await self._query_rows( + where, + columns=self._stored_columns(include_vectors=True), + order_by=f"{quote_identifier('id')} ASC", + limit=max(0, limit), + ) + return [self._model(row) for row in rows] + + async def find_one_where(self, where: Predicate) -> T | None: + rows = await self.find_where(where, limit=1) + return rows[0] if rows else None + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + """Page natively; ``max_fetch`` is retained only for port compatibility.""" + _ = max_fetch + if page < 1 or page_size < 1: + raise ValueError("page and page_size must be positive") + field = self.index_schema.field(sort_by) + column = physical_column(self.index_schema, sort_by) + if column.dimension is not None or column.sql_type in {"JSON", "LONGTEXT"}: + raise ValueError( + f"SeekDB cannot paginate by non-scalar field {sort_by!r} " + f"({column.sql_type})" + ) + physical = self._storage_name(field) + direction = "DESC" if descending else "ASC" + order = f"{quote_identifier(physical)} {direction}" + if physical != "id": + order += f", {quote_identifier('id')} ASC" + total = await self.count_where(where) + rows = await self._query_rows( + where, + columns=self._stored_columns(include_vectors=True), + order_by=order, + limit=page_size, + offset=(page - 1) * page_size, + ) + return [self._model(row) for row in rows], total + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + if vector is not None: + return await self.dense_search(vector, where, limit=limit) + rows = await self._query_rows( + where, + columns=self._stored_columns(include_vectors=False), + order_by=f"{quote_identifier('id')} ASC", + limit=max(0, limit), + ) + return [from_row(row, self.index_schema) for row in rows] + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + if not vector or limit <= 0: + return [] + field = self.index_schema.field(vector_field) + if field.kind is not IndexFieldKind.DENSE_VECTOR: + raise ValueError(f"{vector_field!r} is not a dense-vector field") + _, encoded = write_field_value(field, vector, self.index_schema) + await self.ensure_table() + columns = self._stored_columns(include_vectors=False) + select = self._select_clause(columns) + vector_column = quote_identifier(vector_field) + distance = f"cosine_distance({vector_column}, {literal(encoded)})" + predicate = self._render(where) + conditions = [ + item for item in (predicate, f"{vector_column} IS NOT NULL") if item + ] + sql = ( + f"SELECT {select}, {distance} AS _distance FROM " + f"{quote_identifier(self.physical_table_name)} WHERE " + + " AND ".join(f"({item})" for item in conditions) + + f" ORDER BY _distance APPROXIMATE LIMIT {min(limit, _SEARCH_LIMIT_MAX)}" + ) + rows = await self._fetch_all(sql, [*columns, "_distance"]) + return [self._search_row(row, score_field="_distance") for row in rows] + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + terms = [term for term in query_terms if term] + fields = list(columns or self.index_schema.bm25_fields) + if not terms or not fields or limit <= 0: + return [] + unknown = set(fields) - set(self.index_schema.bm25_fields) + if unknown: + raise ValueError(f"unknown BM25 fields: {sorted(unknown)}") + await self.ensure_table() + query = literal(" ".join(terms)) + output = self._stored_columns(include_vectors=False) + select = self._select_clause(output) + predicate = self._render(where) + best: dict[str, dict[str, Any]] = {} + for field in fields: + match = ( + f"MATCH({quote_identifier(field)}) AGAINST " + f"({query} IN NATURAL LANGUAGE MODE)" + ) + conditions = [match, *([predicate] if predicate else [])] + sql = ( + f"SELECT {select}, {match} AS _score FROM " + f"{quote_identifier(self.physical_table_name)} WHERE " + + " AND ".join(f"({item})" for item in conditions) + + f" ORDER BY _score DESC LIMIT {min(limit, _SEARCH_LIMIT_MAX)}" + ) + rows = await self._fetch_all(sql, [*output, "_score"]) + for row in rows: + shaped = self._search_row(row, score_field="_score") + rid = shaped.get("id") + if not isinstance(rid, str): + continue + prior = best.get(rid) + if prior is None or shaped["_score"] > prior["_score"]: + best[rid] = shaped + return sorted(best.values(), key=lambda item: item["_score"], reverse=True)[ + :limit + ] + + async def scan(self, where: Predicate | None = None) -> list[T]: + await self.ensure_table() + columns = self._stored_columns(include_vectors=True) + predicate = self._render(where) + rows: list[T] = [] + last_id: str | None = None + while True: + conditions = [predicate] if predicate else [] + if last_id is not None: + conditions.append(f"{quote_identifier('id')} > {literal(last_id)}") + sql = ( + f"SELECT {self._select_clause(columns)} FROM " + f"{quote_identifier(self.physical_table_name)}" + ) + if conditions: + sql += " WHERE " + " AND ".join(f"({item})" for item in conditions) + sql += f" ORDER BY {quote_identifier('id')} ASC LIMIT {_SCAN_BATCH_SIZE}" + batch = await self._fetch_all(sql, columns) + if not batch: + break + models = [self._model(row) for row in batch] + rows.extend(models) + last_id = str(models[-1].model_dump(mode="python")["id"]) + if len(batch) < _SCAN_BATCH_SIZE: + break + return rows + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + if not isinstance(where, Predicate): + raise TypeError("SeekDB update requires a neutral Predicate") + if not updates: + return + await self.ensure_table() + assignments: list[str] = [] + for name, value in updates.items(): + field = self.index_schema.field(name) + storage_name, encoded = write_field_value(field, value, self.index_schema) + assignments.append(f"{quote_identifier(storage_name)} = {literal(encoded)}") + sql = ( + f"UPDATE {quote_identifier(self.physical_table_name)} SET " + + ", ".join(assignments) + + f" WHERE {self._render(where)}" + ) + await self._execute(sql) + + async def delete(self, predicate: Predicate) -> None: + if not isinstance(predicate, Predicate): + raise TypeError("SeekDB delete requires a neutral Predicate") + await self.ensure_table() + sql = ( + f"DELETE FROM {quote_identifier(self.physical_table_name)} " + f"WHERE {self._render(predicate)}" + ) + await self._execute(sql) + + async def delete_by_md_path(self, md_path: str) -> int: + predicate = eq("md_path", md_path) + count = await self.count_where(predicate) + if count: + await self.delete(predicate) + return count + + async def optimize(self) -> None: + """Immediate vector indexes require no explicit refresh.""" + + async def prune(self, older_than: dt.timedelta) -> None: + """SeekDB owns physical compaction and retention.""" + _ = older_than + + async def rebuild_indexes(self) -> None: + """SeekDB owns physical index maintenance.""" + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self.find_where(eq("owner_id", owner_id), limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self.find_one_where(eq("md_path", md_path)) + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self.find_one_where( + all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), + ) + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + if not entry_ids: + return [] + return await self.find_where( + all_of( + eq("owner_id", owner_id), + one_of("entry_id", list(entry_ids)), + eq("app_id", app_id), + eq("project_id", project_id), + ), + limit=len(entry_ids), + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("owner_id", owner_id), eq("session_id", session_id)), + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("parent_type", parent_type), eq("parent_id", parent_id)), + limit=limit, + ) + + async def _query_rows( + self, + where: Predicate | None, + *, + columns: Sequence[str], + order_by: str | None = None, + limit: int | None = None, + offset: int | None = None, + ) -> list[dict[str, Any]]: + await self.ensure_table() + sql = ( + f"SELECT {self._select_clause(columns)} FROM " + f"{quote_identifier(self.physical_table_name)}" + self._where_clause(where) + ) + if order_by: + sql += f" ORDER BY {order_by}" + if limit is not None: + sql += f" LIMIT {max(0, limit)}" + if offset is not None: + sql += f" OFFSET {max(0, offset)}" + return await self._fetch_all(sql, columns) + + def _model(self, row: Mapping[str, Any]) -> T: + return cast(T, model_from_row(row, self.index_schema, self.schema)) + + def _search_row( + self, row: Mapping[str, Any], *, score_field: str + ) -> dict[str, Any]: + shaped = from_row(row, self.index_schema) + raw = row.get(score_field) + score = 0.0 if raw is None else float(raw) + shaped[score_field] = max(0.0, score) if score_field == "_score" else score + return shaped + + def _stored_columns(self, *, include_vectors: bool) -> list[str]: + return [ + column.name + for column in physical_columns(self.index_schema) + if include_vectors or column.dimension is None + ] + + def _vector_sync_mode(self) -> Literal["immediate", "async"]: + return load_settings().seekdb.vector_sync_mode + + def _select_clause(self, columns: Sequence[str]) -> str: + return ", ".join(quote_identifier(name) for name in columns) + + def _where_clause(self, where: Predicate | None) -> str: + rendered = self._render(where) + return f" WHERE {rendered}" if rendered else "" + + def _render(self, where: Predicate | None) -> str: + return render_predicate( + where, + datetime_fields=self.index_schema.datetime_fields, + vector_fields={field.name for field in self.index_schema.vector_fields}, + ) + + def _storage_name(self, field: IndexField) -> str: + return ( + f"{field.name}_ms" if field.kind is IndexFieldKind.DATETIME else field.name + ) + + async def _execute(self, sql: str) -> None: + session = await get_session() + await run(session.execute, sql, table=self.physical_table_name) + + async def _fetch_all( + self, sql: str, columns: Sequence[str] + ) -> list[dict[str, Any]]: + session = await get_session() + return await run( + session.fetch_all, + sql, + columns, + table=self.physical_table_name, + ) + + async def _fetch_scalar(self, sql: str) -> Any: + session = await get_session() + return await run( + session.fetch_scalar, + sql, + table=self.physical_table_name, + ) + + +__all__ = ["SeekdbRepoBase"] diff --git a/src/everos/infra/persistence/seekdb/schema.py b/src/everos/infra/persistence/seekdb/schema.py new file mode 100644 index 000000000..a878fa74f --- /dev/null +++ b/src/everos/infra/persistence/seekdb/schema.py @@ -0,0 +1,514 @@ +"""Map portable index schemas to SeekDB columns, indexes, and DDL. + +The same immutable descriptors drive creation and drift verification. This +keeps catalog spelling differences separate from real semantic differences. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from functools import cache +from typing import Any, Literal + +from everos.infra.persistence.index.schema import ( + IndexField, + IndexFieldKind, + IndexSchema, +) + +from .sql import quote_identifier + +_SHORT_STRING_FIELDS = frozenset( + {"md_path", "content_sha256", "deprecated_by", "cluster_id", "entry_id"} +) +_INTEGER_DISPLAY_WIDTH = re.compile( + r"^(tinyint|smallint|mediumint|int|integer|bigint)\(\d+\)" +) +_INDEX_HEADER = re.compile( + r"^\s*(?:(?PVECTOR|FULLTEXT)\s+)?(?:INDEX|KEY)\s+" + r"`?(?P[A-Za-z_][A-Za-z0-9_]*)`?\s*\(", + re.IGNORECASE, +) +_INDEX_COLUMN = re.compile( + r"^\s*`?(?P[A-Za-z_][A-Za-z0-9_]*)`?" + r"(?:\s*\(\s*(?P\d+)\s*\))?\s*$" +) +_VECTOR_OPTION = re.compile( + r"\b(DISTANCE|TYPE|LIB|SYNC_MODE)\s*=\s*([A-Za-z0-9_]+)", + re.IGNORECASE, +) +_TEXT_SQL_TYPES = ("VARCHAR(", "LONGTEXT") +_INDEX_PREFIX_LENGTH = 128 + + +@dataclass(frozen=True) +class PhysicalColumn: + """One physical column as declared in SeekDB.""" + + name: str + sql_type: str + nullable: bool + primary: bool = False + dimension: int | None = None + character_set: str | None = None + collation: str | None = None + + def ddl(self) -> str: + nullability = "NULL" if self.nullable and not self.primary else "NOT NULL" + return f"{quote_identifier(self.name)} {self.sql_type} {nullability}" + + def mismatches(self, reported: Mapping[str, Any]) -> list[str]: + actual_type = normalize_sql_type(_as_text(_get(reported, "COLUMN_TYPE", ""))) + expected_type = normalize_sql_type(self.sql_type) + actual_nullable = _as_text(_get(reported, "IS_NULLABLE", "NO")).upper() == "YES" + actual_primary = _as_text(_get(reported, "COLUMN_KEY", "")).upper() == "PRI" + out: list[str] = [] + if actual_type != expected_type: + out.append(f"{self.name}: type {actual_type!r} != {expected_type!r}") + if actual_nullable != (self.nullable and not self.primary): + out.append( + f"{self.name}: nullable {actual_nullable} != " + f"{self.nullable and not self.primary}" + ) + if actual_primary != self.primary: + out.append(f"{self.name}: primary {actual_primary} != {self.primary}") + _append_text_metadata_mismatches(out, self, reported) + return out + + +@dataclass(frozen=True) +class PhysicalIndex: + """One named secondary, full-text, or vector index.""" + + name: str + columns: tuple[str, ...] + kind: Literal["btree", "fulltext", "vector"] + prefix_lengths: tuple[int | None, ...] = () + parser: str | None = None + distance: str | None = None + vector_type: str | None = None + library: str | None = None + sync_mode: Literal["immediate", "async"] | None = None + + def ddl(self) -> str: + prefixes = self.prefix_lengths or (None,) * len(self.columns) + columns = ", ".join( + quote_identifier(column) + (f"({prefix})" if prefix else "") + for column, prefix in zip(self.columns, prefixes, strict=True) + ) + name = quote_identifier(self.name) + if self.kind == "fulltext": + return f"FULLTEXT INDEX {name} ({columns}) WITH PARSER {self.parser}" + if self.kind == "vector": + return ( + f"VECTOR INDEX {name} ({columns}) WITH " + f"(DISTANCE={self.distance}, TYPE={self.vector_type}, " + f"LIB={self.library}, SYNC_MODE={self.sync_mode})" + ) + return f"INDEX {name} ({columns})" + + +@dataclass(frozen=True) +class _ParsedIndex: + name: str + kind: Literal["btree", "fulltext", "vector"] + columns: tuple[str, ...] + prefix_lengths: tuple[int | None, ...] + parser: str | None = None + options: tuple[tuple[str, str], ...] = () + + +def normalize_sql_type(value: str) -> str: + """Collapse non-semantic MySQL/OceanBase catalog spelling differences.""" + lowered = value.strip().lower().replace(" ", "") + return _INTEGER_DISPLAY_WIDTH.sub(r"\1", lowered) + + +@cache +def physical_columns(schema: IndexSchema) -> tuple[PhysicalColumn, ...]: + """Return the complete physical column set in deterministic order.""" + return tuple(_physical_column(field, schema) for field in schema.fields) + + +def physical_column(schema: IndexSchema, name: str) -> PhysicalColumn: + """Return the physical descriptor for one logical field.""" + field = schema.field(name) + return next( + column + for column in physical_columns(schema) + if column.name == _storage_name(field) + ) + + +@cache +def physical_indexes( + schema: IndexSchema, + vector_sync_mode: Literal["immediate", "async"] = "immediate", +) -> tuple[PhysicalIndex, ...]: + """Return indexes useful to EverOS reads plus every search index.""" + names = {field.name for field in schema.fields} + columns = {column.name: column for column in physical_columns(schema)} + out: list[PhysicalIndex] = [] + _append_index( + out, + names, + columns, + "ix_owner_scope", + "owner_id", + "app_id", + "project_id", + ) + _append_index(out, names, columns, "ix_md_path", "md_path") + _append_index(out, names, columns, "ix_owner_entry", "owner_id", "entry_id") + _append_index(out, names, columns, "ix_owner_cluster", "owner_id", "cluster_id") + _append_index(out, names, columns, "ix_parent", "parent_type", "parent_id") + out.extend( + PhysicalIndex(f"ft_{name}", (name,), "fulltext", parser="space") + for name in schema.bm25_fields + ) + out.extend( + PhysicalIndex( + f"vec_{field.name}", + (field.name,), + "vector", + distance="cosine", + vector_type="hnsw", + library="vsag", + sync_mode=vector_sync_mode, + ) + for field in schema.vector_fields + ) + return tuple(out) + + +def build_create_table( + table: str, + schema: IndexSchema, + vector_sync_mode: Literal["immediate", "async"] = "immediate", +) -> str: + """Build the complete idempotent CREATE TABLE statement.""" + columns = physical_columns(schema) + definitions = [column.ddl() for column in columns] + primary = next(column for column in columns if column.primary) + definitions.append(f"PRIMARY KEY ({quote_identifier(primary.name)})") + definitions.extend( + index.ddl() for index in physical_indexes(schema, vector_sync_mode) + ) + body = ",\n ".join(definitions) + return ( + f"CREATE TABLE IF NOT EXISTS {quote_identifier(table)} (\n" + f" {body}\n" + ") DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin " + "ORGANIZATION = HEAP;" + ) + + +def column_drift( + expected: Sequence[PhysicalColumn], reported: Sequence[Mapping[str, Any]] +) -> tuple[list[str], list[str], list[str]]: + """Return missing, stale, and incompatible column details.""" + wanted = {column.name: column for column in expected} + actual = {_as_text(_get(row, "COLUMN_NAME", "")): row for row in reported} + missing = sorted(set(wanted) - set(actual)) + stale = sorted(set(actual) - set(wanted)) + incompatible: list[str] = [] + for name, column in wanted.items(): + if name in actual: + incompatible.extend(column.mismatches(actual[name])) + return missing, stale, incompatible + + +def index_drift( + expected: Sequence[PhysicalIndex], + reported: Sequence[Mapping[str, Any]], + *, + create_sql: str = "", +) -> tuple[list[str], list[str], list[str]]: + """Return missing, stale, and incompatible index details.""" + by_name: dict[str, list[Mapping[str, Any]]] = {} + for row in reported: + name = _as_text(_get(row, "INDEX_NAME", "")) + if name.upper() != "PRIMARY": + by_name.setdefault(name, []).append(row) + parsed = _parse_indexes(create_sql) + wanted = {index.name: index for index in expected} + actual_names = set(by_name) | set(parsed) + missing: list[str] = [] + incompatible: list[str] = [] + for name, index in wanted.items(): + rows = by_name.get(name, []) + declaration = parsed.get(name) + if not rows and declaration is None: + missing.append(name) + continue + if rows: + incompatible.extend(_statistics_mismatches(index, rows)) + if declaration is not None: + incompatible.extend(_ddl_index_mismatches(index, declaration)) + elif index.kind in {"fulltext", "vector"}: + incompatible.append(f"{name}: SHOW CREATE TABLE declaration is missing") + stale = sorted(actual_names - set(wanted)) + return sorted(missing), stale, incompatible + + +def _physical_column(field: IndexField, schema: IndexSchema) -> PhysicalColumn: + if field.kind is IndexFieldKind.STRING: + short = field.primary or field.name.endswith(("_id", "_type")) + sql_type = ( + "LONGTEXT" + if field.name in schema.bm25_fields + or not (short or field.name in _SHORT_STRING_FIELDS) + else "VARCHAR(512)" + ) + elif field.kind is IndexFieldKind.STRING_ARRAY: + sql_type = "JSON" + elif field.kind is IndexFieldKind.FLOAT: + sql_type = "DOUBLE" + elif field.kind is IndexFieldKind.INTEGER: + sql_type = "BIGINT" + elif field.kind is IndexFieldKind.DATETIME: + return PhysicalColumn(f"{field.name}_ms", "BIGINT", field.nullable) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + sql_type = f"VECTOR({field.dimension})" + else: # pragma: no cover - enum exhaustiveness guard + raise TypeError(f"unsupported index field kind: {field.kind}") + is_text = sql_type.startswith(_TEXT_SQL_TYPES) + return PhysicalColumn( + field.name, + sql_type, + field.nullable, + primary=field.primary, + dimension=field.dimension, + character_set="utf8mb4" if is_text else None, + collation="utf8mb4_bin" if is_text else None, + ) + + +def _storage_name(field: IndexField) -> str: + return f"{field.name}_ms" if field.kind is IndexFieldKind.DATETIME else field.name + + +def _append_index( + indexes: list[PhysicalIndex], + available: set[str], + physical: Mapping[str, PhysicalColumn], + name: str, + *columns: str, +) -> None: + if not set(columns) <= available: + return + use_prefixes = len(columns) > 1 + prefixes = tuple( + _INDEX_PREFIX_LENGTH + if use_prefixes and physical[column].sql_type.startswith("VARCHAR(") + else None + for column in columns + ) + indexes.append(PhysicalIndex(name, tuple(columns), "btree", prefixes)) + + +def _append_text_metadata_mismatches( + out: list[str], column: PhysicalColumn, reported: Mapping[str, Any] +) -> None: + checks = ( + ("CHARACTER_SET_NAME", column.character_set, "character set"), + ("COLLATION_NAME", column.collation, "collation"), + ) + for key, expected, label in checks: + if expected is None or not _has_key(reported, key): + continue + actual = _get(reported, key, None) + actual_text = None if actual is None else _as_text(actual).casefold() + if actual_text != expected.casefold(): + out.append(f"{column.name}: {label} {actual_text!r} != {expected!r}") + + +def _statistics_mismatches( + expected: PhysicalIndex, rows: Sequence[Mapping[str, Any]] +) -> list[str]: + ordered = sorted(rows, key=_sequence_in_index) + columns = tuple(_as_text(_get(row, "COLUMN_NAME", "")) for row in ordered) + prefixes = tuple(_optional_int(_get(row, "SUB_PART", None)) for row in ordered) + expected_prefixes = expected.prefix_lengths or (None,) * len(expected.columns) + out: list[str] = [] + if columns != expected.columns: + out.append(f"{expected.name}: columns {columns!r} != {expected.columns!r}") + if prefixes != expected_prefixes: + out.append(f"{expected.name}: prefixes {prefixes!r} != {expected_prefixes!r}") + actual_types = {_as_text(_get(row, "INDEX_TYPE", "")).upper() for row in ordered} + if not _statistics_kind_matches(expected.kind, actual_types): + out.append( + f"{expected.name}: type {sorted(actual_types)!r} != {expected.kind!r}" + ) + return out + + +def _ddl_index_mismatches(expected: PhysicalIndex, actual: _ParsedIndex) -> list[str]: + out: list[str] = [] + if actual.kind != expected.kind: + out.append(f"{expected.name}: DDL type {actual.kind!r} != {expected.kind!r}") + if actual.columns != expected.columns: + out.append( + f"{expected.name}: DDL columns {actual.columns!r} != {expected.columns!r}" + ) + expected_prefixes = expected.prefix_lengths or (None,) * len(expected.columns) + if actual.prefix_lengths != expected_prefixes: + out.append( + f"{expected.name}: DDL prefixes {actual.prefix_lengths!r} " + f"!= {expected_prefixes!r}" + ) + if expected.kind == "fulltext" and actual.parser != expected.parser: + out.append(f"{expected.name}: parser {actual.parser!r} != {expected.parser!r}") + if expected.kind == "vector": + options = dict(actual.options) + expected_options = { + "distance": expected.distance, + "type": expected.vector_type, + "lib": expected.library, + "sync_mode": expected.sync_mode, + } + for key, wanted in expected_options.items(): + if options.get(key) != wanted: + out.append(f"{expected.name}: {key} {options.get(key)!r} != {wanted!r}") + return out + + +def _parse_indexes(create_sql: str) -> dict[str, _ParsedIndex]: + parsed: dict[str, _ParsedIndex] = {} + for definition in _table_definitions(create_sql): + match = _INDEX_HEADER.match(definition) + if match is None: + continue + opening = match.end() - 1 + closing = _matching_parenthesis(definition, opening) + if closing is None: + continue + columns, prefixes = _parse_index_columns(definition[opening + 1 : closing]) + raw_kind = (match.group("kind") or "").casefold() + kind: Literal["btree", "fulltext", "vector"] = ( + "fulltext" + if raw_kind == "fulltext" + else "vector" + if raw_kind == "vector" + else "btree" + ) + tail = definition[closing + 1 :] + parser_match = re.search( + r"\bWITH\s+PARSER\s+([A-Za-z0-9_]+)", tail, re.IGNORECASE + ) + options = tuple( + (key.casefold(), value.casefold()) + for key, value in _VECTOR_OPTION.findall(tail) + ) + name = match.group("name") + parsed[name] = _ParsedIndex( + name, + kind, + columns, + prefixes, + parser_match.group(1).casefold() if parser_match else None, + options, + ) + return parsed + + +def _table_definitions(create_sql: str) -> tuple[str, ...]: + if not create_sql: + return () + opening = create_sql.find("(") + closing = create_sql.rfind(")") + if opening < 0 or closing <= opening: + return () + body = create_sql[opening + 1 : closing] + out: list[str] = [] + start = 0 + depth = 0 + quote: str | None = None + for position, char in enumerate(body): + if char in {"'", '"', "`"}: + quote = None if quote == char else char if quote is None else quote + elif quote is None and char == "(": + depth += 1 + elif quote is None and char == ")": + depth -= 1 + elif quote is None and char == "," and depth == 0: + out.append(body[start:position].strip()) + start = position + 1 + out.append(body[start:].strip()) + return tuple(item for item in out if item) + + +def _parse_index_columns(value: str) -> tuple[tuple[str, ...], tuple[int | None, ...]]: + columns: list[str] = [] + prefixes: list[int | None] = [] + for item in value.split(","): + match = _INDEX_COLUMN.match(item) + if match is None: + return (), () + columns.append(match.group("name")) + prefixes.append(_optional_int(match.group("prefix"))) + return tuple(columns), tuple(prefixes) + + +def _matching_parenthesis(value: str, opening: int) -> int | None: + depth = 0 + for position in range(opening, len(value)): + if value[position] == "(": + depth += 1 + elif value[position] == ")": + depth -= 1 + if depth == 0: + return position + return None + + +def _statistics_kind_matches( + kind: Literal["btree", "fulltext", "vector"], actual_types: set[str] +) -> bool: + if kind == "fulltext": + return actual_types == {"FULLTEXT"} + if kind == "vector": + return any("VECTOR" in value or "HNSW" in value for value in actual_types) + return not actual_types or actual_types == {"BTREE"} + + +def _get(row: Mapping[str, Any], name: str, default: Any) -> Any: + if name in row: + return row[name] + folded = name.casefold() + return next( + (value for key, value in row.items() if _as_text(key).casefold() == folded), + default, + ) + + +def _has_key(row: Mapping[str, Any], name: str) -> bool: + folded = name.casefold() + return any(_as_text(key).casefold() == folded for key in row) + + +def _as_text(value: Any) -> str: + return value.decode("utf-8") if isinstance(value, bytes) else str(value) + + +def _sequence_in_index(row: Mapping[str, Any]) -> int: + return int(_get(row, "SEQ_IN_INDEX", 0) or 0) + + +def _optional_int(value: Any) -> int | None: + return None if value in {None, ""} else int(value) + + +__all__ = [ + "PhysicalColumn", + "PhysicalIndex", + "build_create_table", + "column_drift", + "index_drift", + "normalize_sql_type", + "physical_column", + "physical_columns", + "physical_indexes", +] diff --git a/src/everos/infra/persistence/seekdb/seekdb_manager.py b/src/everos/infra/persistence/seekdb/seekdb_manager.py new file mode 100644 index 000000000..cc8526c2d --- /dev/null +++ b/src/everos/infra/persistence/seekdb/seekdb_manager.py @@ -0,0 +1,488 @@ +"""Own the optional pyseekdb connection and serialize SQL execution. + +Both the embedded DB-API connection and PyMySQL connections are treated as +single-threaded resources. The async boundary therefore holds one process-wide +lock while offloading blocking work, and only this module imports pyseekdb. +""" + +from __future__ import annotations + +import asyncio +import importlib +import os +import re +import sys +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, Protocol, cast + +from everos.config import SeekdbSettings, load_settings, resolve_root +from everos.core.observability.logging import get_logger + +from .errors import ( + SeekdbConfigurationError, + SeekdbIntegrityError, + SeekdbOperationalError, +) +from .sql import literal, quote_identifier + +logger = get_logger(__name__) + + +class _SqlServer(Protocol): + """The pyseekdb 1.4.x private surface isolated by this adapter.""" + + def _execute(self, sql: str) -> Any: ... + + def get_raw_connection(self) -> Any: ... + + def _cleanup(self) -> None: ... + + +class _HeldFileLock(Protocol): + """Lifetime lock retained by an embedded session.""" + + def release(self) -> None: ... + + +@dataclass(frozen=True) +class SeekdbTarget: + """A fully resolved embedded directory or remote endpoint.""" + + mode: Literal["embedded", "remote"] + database: str + path: Path | None = None + host: str | None = None + port: int = 2881 + tenant: str = "" + user: str = "root" + password: str = "" + connect_timeout_seconds: float = 10.0 + read_timeout_seconds: float = 60.0 + + +class SeekdbSession: + """Normalize pyseekdb SQL result rows without relying on cursor metadata.""" + + def __init__( + self, + server: _SqlServer, + *, + mode: Literal["embedded", "remote"], + directory_lock: _HeldFileLock | None = None, + ): + self._server = server + self._directory_lock = directory_lock + self._invalidated = False + self.mode = mode + + def execute(self, sql: str, *, table: str | None = None) -> None: + self._call(sql, table=table) + + def fetch_all( + self, + sql: str, + columns: Sequence[str], + *, + table: str | None = None, + ) -> list[dict[str, Any]]: + rows = self._call(sql, table=table) or [] + return [_normalize_row(row, columns) for row in rows] + + def fetch_scalar(self, sql: str, *, table: str | None = None) -> Any: + rows = self._call(sql, table=table) or [] + if not rows: + return None + row = rows[0] + if isinstance(row, Mapping): + return next(iter(row.values()), None) + if isinstance(row, (tuple, list)): + return row[0] if row else None + return row + + def close(self) -> None: + try: + self._server._cleanup() + finally: + if self._directory_lock is not None: + self._directory_lock.release() + self._directory_lock = None + + def _call(self, sql: str, *, table: str | None) -> Any: + try: + if self.mode == "remote": + raw = self._server.get_raw_connection() + ping = getattr(raw, "ping", None) + if callable(ping): + ping(reconnect=True) + return self._server._execute(sql) + except Exception as exc: + kind = _sql_kind(sql) + code = _error_code(exc) + disconnected = self.mode == "remote" and code in {2006, 2013} + if disconnected: + self._invalidate() + logger.warning( + "seekdb_operation_failed", + sql_kind=kind, + table=table, + error_type=type(exc).__name__, + error_code=code, + session_invalidated=disconnected, + ) + detail = _error_detail(exc, code) + if _is_integrity_error(exc): + raise SeekdbIntegrityError( + f"SeekDB rejected a {kind} operation on {table or 'SeekDB'} " + f"due to an integrity constraint [{detail}]" + ) from exc + raise SeekdbOperationalError( + f"SeekDB failed to execute a {kind} operation on " + f"{table or 'SeekDB'} [{detail}]" + ) from exc + + def _invalidate(self) -> None: + global _session + if self._invalidated: + return + self._invalidated = True + if _session is self: + _session = None + try: + self._server._cleanup() + except Exception: + logger.warning("seekdb_disconnected_session_cleanup_failed") + + +@dataclass +class _EmbeddedDirectoryLock: + """A non-blocking POSIX flock held for an embedded engine's lifetime.""" + + fd: int + path: Path + fcntl: Any + + def release(self) -> None: + if self.fd < 0: + return + fd, self.fd = self.fd, -1 + try: + self.fcntl.flock(fd, self.fcntl.LOCK_UN) + finally: + os.close(fd) + + +_session: SeekdbSession | None = None +_connection_lock = asyncio.Lock() +_operation_lock = asyncio.Lock() + + +def resolve_target(settings: SeekdbSettings | None = None) -> SeekdbTarget: + """Validate settings and resolve the default embedded data directory.""" + cfg = settings or load_settings().seekdb + password = cfg.password.get_secret_value() or os.environ.get("SEEKDB_PASSWORD", "") + if cfg.mode == "embedded": + if sys.platform not in {"linux", "darwin"}: + raise SeekdbConfigurationError( + "Embedded SeekDB requires a pylibseekdb wheel, which is currently " + "available on Linux and macOS only. Use remote mode, Docker, or WSL2." + ) + path = Path(cfg.path).expanduser() if cfg.path else _default_path() + return SeekdbTarget(mode="embedded", database=cfg.database, path=path) + if not cfg.host.strip(): + raise SeekdbConfigurationError("[seekdb] host is required when mode = 'remote'") + return SeekdbTarget( + mode="remote", + database=cfg.database, + host=cfg.host.strip(), + port=cfg.port, + tenant=cfg.tenant, + user=cfg.user, + password=password, + connect_timeout_seconds=cfg.connect_timeout_seconds, + read_timeout_seconds=cfg.read_timeout_seconds, + ) + + +def table_name(logical: str, settings: SeekdbSettings | None = None) -> str: + """Return the configured physical table name after identifier validation.""" + cfg = settings or load_settings().seekdb + name = f"{cfg.table_prefix}_{logical}" + quote_identifier(name) + return name + + +async def get_session() -> SeekdbSession: + """Create and cache the configured SeekDB session.""" + global _session + if _session is not None: + return _session + async with _connection_lock: + if _session is None: + target = resolve_target() + _session = await asyncio.to_thread(_open_session, target) + logger.info( + "seekdb_connection_opened", + mode=target.mode, + database=target.database, + host=target.host, + ) + return _session + + +async def run[**P, R](fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> R: + """Run one blocking client operation under the connection lock.""" + async with _operation_lock: + owner = getattr(fn, "__self__", None) + if isinstance(owner, SeekdbSession) and owner._invalidated: + current = await get_session() + rebound = cast(Callable[P, R], getattr(current, fn.__name__)) + return await asyncio.to_thread(rebound, *args, **kwargs) + return await asyncio.to_thread(fn, *args, **kwargs) + + +async def dispose_connection() -> None: + """Close the process session and permit a fresh target on next use.""" + global _session + async with _connection_lock: + session = _session + _session = None + if session is not None: + async with _operation_lock: + await asyncio.to_thread(session.close) + logger.info("seekdb_connection_closed") + + +def _open_session(target: SeekdbTarget) -> SeekdbSession: + directory_lock = _acquire_embedded_lock(target) + try: + server = _connect_target_database(target) + _configure_session(server, target) + return SeekdbSession( + server, + mode=target.mode, + directory_lock=directory_lock, + ) + except BaseException: + if directory_lock is not None: + directory_lock.release() + raise + + +def _connect_target_database(target: SeekdbTarget) -> _SqlServer: + try: + return _new_connected_server(target, database=target.database) + except SeekdbConfigurationError: + raise + except Exception as exc: + if not _is_missing_database_error(exc): + raise SeekdbOperationalError( + f"Could not connect to SeekDB database {target.database!r} " + f"[{_error_detail(exc, _error_code(exc))}]" + ) from exc + _ensure_database(target) + try: + return _new_connected_server(target, database=target.database) + except Exception as exc: + raise SeekdbOperationalError( + f"Could not connect to newly created SeekDB database " + f"{target.database!r} [{_error_detail(exc, _error_code(exc))}]" + ) from exc + + +def _new_connected_server(target: SeekdbTarget, *, database: str) -> _SqlServer: + try: + server = _new_server(target, database=database) + except RuntimeError as exc: + if target.mode == "embedded" and _is_driver_unavailable(exc): + raise SeekdbConfigurationError( + "SeekDB embedded support is unavailable. Install " + "everos[seekdb-embedded]." + ) from exc + raise + try: + server.get_raw_connection() + return server + except Exception: + server._cleanup() + raise + + +def _ensure_database(target: SeekdbTarget) -> None: + # Embedded seekdb guarantees its default ``test`` database; selecting a + # virtual system schema as the initial embedded database is not part of + # pylibseekdb's public contract. Remote MySQL endpoints can select + # information_schema directly without requiring access to another user DB. + admin_database = "test" if target.mode == "embedded" else "information_schema" + server = _new_connected_server(target, database=admin_database) + try: + rows = server._execute( + "SELECT SCHEMA_NAME FROM information_schema.SCHEMATA " + f"WHERE SCHEMA_NAME = {literal(target.database)}" + ) + if rows: + return + server._execute( + f"CREATE DATABASE IF NOT EXISTS {quote_identifier(target.database)} " + "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" + ) + except Exception as exc: + if _is_permission_error(exc): + raise SeekdbConfigurationError( + f"SeekDB database {target.database!r} does not exist and " + "the configured account cannot create it; pre-create the " + "database with utf8mb4_bin collation or grant CREATE permission" + ) from exc + raise SeekdbOperationalError( + f"Could not ensure SeekDB database {target.database!r} " + f"[{_error_detail(exc, _error_code(exc))}]" + ) from exc + finally: + server._cleanup() + + +def _configure_session(server: _SqlServer, target: SeekdbTarget) -> None: + try: + server._execute("SET NAMES utf8mb4 COLLATE utf8mb4_bin") + server._execute( + "SET SESSION sql_mode = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', " + "@@SESSION.sql_mode, ','), ',NO_BACKSLASH_ESCAPES,', ','))" + ) + except Exception as exc: + server._cleanup() + raise SeekdbConfigurationError( + f"Could not configure the SeekDB session for {target.database!r}; " + "the account must be allowed to set its session charset and sql_mode" + ) from exc + + +def _acquire_embedded_lock(target: SeekdbTarget) -> _EmbeddedDirectoryLock | None: + if target.mode != "embedded": + return None + assert target.path is not None + target.path.mkdir(parents=True, exist_ok=True) + lock_path = target.path / ".everos.lock" + fcntl = importlib.import_module("fcntl") + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o644) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + os.close(fd) + raise SeekdbConfigurationError( + f"embedded SeekDB at {target.path} is already opened by another " + "process; use remote mode or stop the other process" + ) from exc + except BaseException: + os.close(fd) + raise + return _EmbeddedDirectoryLock(fd, lock_path, fcntl) + + +def _new_server(target: SeekdbTarget, *, database: str) -> _SqlServer: + try: + client = importlib.import_module("pyseekdb.client") + if target.mode == "embedded": + cls = client.SeekdbEmbeddedClient + assert target.path is not None + return cls(path=str(target.path), database=database) + cls = client.RemoteServerClient + assert target.host is not None + options: dict[str, Any] = dict( + host=target.host, + port=target.port, + database=database, + user=target.user, + password=target.password, + connect_timeout=target.connect_timeout_seconds, + read_timeout=target.read_timeout_seconds, + write_timeout=target.read_timeout_seconds, + ) + if target.tenant: + options["tenant"] = target.tenant + return cls(**options) + except (ImportError, AttributeError) as exc: + extra = "seekdb-embedded" if target.mode == "embedded" else "seekdb" + raise SeekdbConfigurationError( + f"SeekDB {target.mode} support is unavailable. Install everos[{extra}]." + ) from exc + + +def _default_path() -> Path: + return resolve_root() / ".index" / "seekdb" + + +def _normalize_row(row: Any, columns: Sequence[str]) -> dict[str, Any]: + if isinstance(row, Mapping): + folded = {str(key).casefold(): value for key, value in row.items()} + return {name: folded.get(name.casefold()) for name in columns} + if isinstance(row, (tuple, list)): + if len(row) != len(columns): + raise SeekdbOperationalError( + f"SeekDB returned {len(row)} values for {len(columns)} columns" + ) + return dict(zip(columns, row, strict=True)) + if len(columns) == 1: + return {columns[0]: row} + raise SeekdbOperationalError( + f"SeekDB returned an unsupported row type: {type(row).__name__}" + ) + + +def _sql_kind(sql: str) -> str: + return sql.lstrip().split(maxsplit=1)[0].upper() if sql.strip() else "SQL" + + +def _is_integrity_error(exc: Exception) -> bool: + if "integrity" in type(exc).__name__.casefold(): + return True + return _error_code(exc) in {1062, 1586} + + +def _is_missing_database_error(exc: Exception) -> bool: + message = str(exc).casefold() + return _error_code(exc) == 1049 or any( + marker in message for marker in ("unknown database", "database does not exist") + ) + + +def _is_permission_error(exc: Exception) -> bool: + return _error_code(exc) in {1044, 1045, 1142, 1227} or any( + marker in str(exc).casefold() + for marker in ("access denied", "permission denied") + ) + + +def _is_driver_unavailable(exc: Exception) -> bool: + message = str(exc).casefold() + return "pylibseekdb" in message and any( + marker in message for marker in ("not found", "not installed", "unavailable") + ) + + +def _error_code(exc: Exception) -> int | None: + for name in ("errno", "code"): + value = getattr(exc, name, None) + if isinstance(value, int) and not isinstance(value, bool): + return abs(value) + for value in exc.args: + if isinstance(value, int) and not isinstance(value, bool): + return abs(value) + match = re.search(r"(?:error|code)\s*[=:]?\s*-?(\d+)", str(exc), re.IGNORECASE) + return int(match.group(1)) if match else None + + +def _error_detail(exc: Exception, code: int | None) -> str: + name = type(exc).__name__ + return f"{name} {code}" if code is not None else name + + +__all__ = [ + "SeekdbSession", + "SeekdbTarget", + "dispose_connection", + "get_session", + "resolve_target", + "run", + "table_name", +] diff --git a/src/everos/infra/persistence/seekdb/sql.py b/src/everos/infra/persistence/seekdb/sql.py new file mode 100644 index 000000000..cd0bf2107 --- /dev/null +++ b/src/everos/infra/persistence/seekdb/sql.py @@ -0,0 +1,73 @@ +"""Safe SQL literal and identifier rendering for both SeekDB client modes. + +Embedded pylibseekdb cursors do not support DB-API parameters, so every query +uses these renderers. Identifiers are allow-listed and values are escaped in +one place; repository code must never interpolate caller strings directly. +""" + +from __future__ import annotations + +import datetime as dt +import json +import math +import re +from typing import Any, Final + +from everos.component.utils.datetime import ensure_utc, to_timestamp_ms + +_IDENTIFIER: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def quote_identifier(value: str) -> str: + """Validate and quote one unqualified SQL identifier.""" + if _IDENTIFIER.fullmatch(value) is None: + raise ValueError(f"invalid SQL identifier: {value!r}") + return f"`{value}`" + + +def literal(value: Any) -> str: + """Render a scalar as a SeekDB/MySQL SQL literal.""" + if value is None: + return "NULL" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return str(to_timestamp_ms(aware)) + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("SQL numeric literals must be finite") + return repr(value) + if isinstance(value, bytes): + value = value.decode("utf-8") + if isinstance(value, str): + return f"'{_escape_string(value)}'" + raise TypeError(f"unsupported SQL literal type: {type(value).__name__}") + + +def json_literal(value: Any) -> str: + """Render a JSON value as a quoted UTF-8 JSON document.""" + document = json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + return literal(document) + + +def _escape_string(value: str) -> str: + return ( + value.replace("\\", "\\\\") + .replace("\0", "\\0") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\x1a", "\\Z") + .replace("'", "''") + ) + + +__all__ = ["json_literal", "literal", "quote_identifier"] diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index bdd99af88..5034bfb52 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -127,6 +127,10 @@ def _index_backends() -> list[str]: backends = ["lancedb"] if os.environ.get("EVEROS_TEST_MILVUS_URI"): backends.append("milvus") + if os.environ.get("EVEROS_TEST_SEEKDB_PATH") or os.environ.get( + "EVEROS_TEST_SEEKDB_HOST" + ): + backends.append("seekdb") return backends @@ -150,10 +154,33 @@ async def index_backend( monkeypatch.setenv( "EVEROS_MILVUS__COLLECTION_PREFIX", f"everos_e2e_{uuid.uuid4().hex}" ) + elif backend == "seekdb": + monkeypatch.setenv( + "EVEROS_SEEKDB__TABLE_PREFIX", f"everos_e2e_{uuid.uuid4().hex}" + ) + if path := os.environ.get("EVEROS_TEST_SEEKDB_PATH"): + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "embedded") + monkeypatch.setenv("EVEROS_SEEKDB__PATH", path) + else: + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "remote") + monkeypatch.setenv( + "EVEROS_SEEKDB__HOST", os.environ["EVEROS_TEST_SEEKDB_HOST"] + ) + for name, default in ( + ("PORT", "2881"), + ("TENANT", ""), + ("USER", "root"), + ("PASSWORD", ""), + ("DATABASE", "everos_test"), + ): + monkeypatch.setenv( + f"EVEROS_SEEKDB__{name}", + os.environ.get(f"EVEROS_TEST_SEEKDB_{name}", default), + ) yield backend - if backend == "milvus": + if backend in {"milvus", "seekdb"}: from everos.config import load_settings from everos.infra.persistence.index import drop_business_tables, shutdown diff --git a/tests/e2e/test_get_endpoint_e2e.py b/tests/e2e/test_get_endpoint_e2e.py index 18f1c46ea..44be122d4 100644 --- a/tests/e2e/test_get_endpoint_e2e.py +++ b/tests/e2e/test_get_endpoint_e2e.py @@ -697,6 +697,9 @@ async def test_get_truncates_above_max_fetch( # wired up and ``caplog`` can observe the chassis warning. from everos.core.observability.logging import configure_logging + if index_backend == "seekdb": + pytest.skip("SeekDB paginates natively without a max_fetch window") + if index_backend == "milvus": requested_prefix = os.environ.get("EVEROS_TEST_MILVUS_PREFIX") if requested_prefix: diff --git a/tests/integration/test_seekdb_backend.py b/tests/integration/test_seekdb_backend.py new file mode 100644 index 000000000..4f6877820 --- /dev/null +++ b/tests/integration/test_seekdb_backend.py @@ -0,0 +1,320 @@ +"""Behavioral contract for embedded SeekDB and remote seekdb/OceanBase. + +Set ``EVEROS_TEST_SEEKDB_PATH`` for the embedded engine or +``EVEROS_TEST_SEEKDB_HOST`` plus the optional connection variables for a +server. Each test owns a uniquely prefixed set of disposable tables. +""" + +from __future__ import annotations + +import datetime as dt +import os +import re +import uuid +from collections.abc import AsyncIterator + +import pytest +import pytest_asyncio + +from everos.config import load_settings + +_PATH = os.environ.get("EVEROS_TEST_SEEKDB_PATH", "") +_HOST = os.environ.get("EVEROS_TEST_SEEKDB_HOST", "") + +pytestmark = pytest.mark.skipif( + not (_PATH or _HOST), + reason="EVEROS_TEST_SEEKDB_PATH or EVEROS_TEST_SEEKDB_HOST is not configured", +) + + +@pytest_asyncio.fixture(autouse=True) +async def _seekdb_runtime( + monkeypatch: pytest.MonkeyPatch, +) -> AsyncIterator[None]: + prefix = os.environ.get( + "EVEROS_TEST_SEEKDB_PREFIX", f"everos_e2e_{uuid.uuid4().hex}" + ) + assert re.fullmatch(r"everos_e2e_[0-9a-f]{32}", prefix) + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "seekdb") + monkeypatch.setenv("EVEROS_SEEKDB__TABLE_PREFIX", prefix) + if _PATH: + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "embedded") + monkeypatch.setenv("EVEROS_SEEKDB__PATH", _PATH) + else: + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "remote") + monkeypatch.setenv("EVEROS_SEEKDB__HOST", _HOST) + for name, default in ( + ("PORT", "2881"), + ("TENANT", ""), + ("USER", "root"), + ("PASSWORD", ""), + ("DATABASE", "everos_test"), + ): + monkeypatch.setenv( + f"EVEROS_SEEKDB__{name}", + os.environ.get(f"EVEROS_TEST_SEEKDB_{name}", default), + ) + load_settings.cache_clear() + + from everos.infra.persistence.index import startup + + await startup() + try: + yield + finally: + from everos.infra.persistence.index import drop_business_tables, shutdown + + try: + await drop_business_tables() + finally: + await shutdown() + load_settings.cache_clear() + + +def _episode(number: int, *, vector: bool = True, **overrides: object): # type: ignore[no-untyped-def] + from everos.infra.persistence.index import Episode + + dense = [0.0] * 1024 + dense[number % 2] = 1.0 + values: dict[str, object] = dict( + id=f"u1_ep_{number:04d}", + entry_id=f"ep_{number:04d}", + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + session_id="seekdb", + timestamp=dt.datetime(1999, 1, 1, tzinfo=dt.UTC) + dt.timedelta(seconds=number), + parent_id="mc1", + sender_ids=["u1"], + subject="red apple" if number % 2 == 0 else "blue banana", + episode="red apple memory" if number % 2 == 0 else "blue banana memory", + episode_tokens=( + "red apple memory" if number % 2 == 0 else "blue banana memory" + ), + md_path="test_app/test_project/users/u1/episodes/day.md", + content_sha256=f"{number:064x}", + vector=dense if vector else None, + subject_vector=dense if vector else None, + ) + values.update(overrides) + return Episode(**values) # type: ignore[arg-type] + + +def _foresight(number: int, *, primary_text: str, evidence_text: str): # type: ignore[no-untyped-def] + from everos.infra.persistence.index import Foresight + + dense = [0.0] * 1024 + dense[number] = 1.0 + return Foresight( + id=f"u1_fs_{number}", + entry_id=f"fs_{number}", + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + session_id="seekdb", + timestamp=dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + dt.timedelta(seconds=number), + parent_id="mc1", + sender_ids=["u1"], + foresight=primary_text, + foresight_tokens=primary_text, + evidence=evidence_text, + evidence_tokens=evidence_text, + md_path="test_app/test_project/users/u1/.foresights/day.md", + content_sha256=f"f{number}", + vector=dense, + ) + + +def _agent_skill(number: int): # type: ignore[no-untyped-def] + from everos.infra.persistence.index import AgentSkill + + dense = [0.0] * 1024 + dense[number] = 1.0 + return AgentSkill( + id=f"agent1_skill_{number}", + owner_id="agent1", + owner_type="agent", + app_id="test_app", + project_id="test_project", + name=f"skill_{number}", + description=f"SeekDB skill {number}", + description_tokens=f"seekdb skill {number}", + content=f"Reusable procedure {number}", + content_tokens=f"reusable procedure {number}", + confidence=0.9, + maturity_score=0.8, + source_case_ids=["case1"], + cluster_id="cluster1", + md_path=f"test_app/test_project/agents/agent1/skills/skill_{number}/SKILL.md", + content_sha256=f"s{number}", + vector=dense, + ) + + +async def test_seekdb_matches_the_derived_index_contract() -> None: + from everos.infra.persistence.index import ( + Episode, + episode_repo, + eq, + is_null, + ) + + records = [_episode(number) for number in range(2)] + await episode_repo.upsert(records) + assert await episode_repo.count() == 2 + assert (await episode_repo.get_by_id(records[0].id)).timestamp == records[ + 0 + ].timestamp # type: ignore[union-attr] + + await episode_repo.update({"subject": "updated"}, where=eq("id", records[0].id)) + updated = await episode_repo.get_by_id(records[0].id) + assert updated is not None and updated.subject == "updated" + + sparse = await episode_repo.sparse_search( + ["apple"], None, columns=Episode.BM25_FIELDS, limit=5 + ) + assert sparse[0]["id"] == records[0].id + assert sparse[0]["_score"] > 0 + + query = [1.0] + [0.0] * 1023 + dense = await episode_repo.dense_search(query, None, limit=5) + assert dense[0]["id"] == records[0].id + assert dense[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + by_subject = await episode_repo.dense_search( + query, None, limit=5, vector_field="subject_vector" + ) + assert by_subject[0]["id"] == records[0].id + assert by_subject[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + page, total = await episode_repo.find_where_paginated( + eq("owner_id", "u1"), sort_by="timestamp", page=2, page_size=1 + ) + assert total == 2 + assert len(page) == 1 + + await episode_repo.upsert( + [_episode(number, vector=False) for number in range(2, 103)] + ) + assert await episode_repo.count_where(is_null("vector")) == 101 + assert len(await episode_repo.scan()) == 103 + + assert ( + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) + == 103 + ) + assert await episode_repo.count() == 0 + + +async def test_ids_differing_only_in_case_are_distinct() -> None: + from everos.infra.persistence.index import episode_repo + + lower = _episode(200, id="case-sensitive-id", entry_id="case-lower") + upper = _episode(201, id="CASE-SENSITIVE-ID", entry_id="case-upper") + await episode_repo.add([lower, upper]) + assert await episode_repo.count() == 2 + assert (await episode_repo.get_by_id(lower.id)).id == lower.id # type: ignore[union-attr] + assert (await episode_repo.get_by_id(upper.id)).id == upper.id # type: ignore[union-attr] + + +async def test_seekdb_runs_specialized_repositories_and_real_filters() -> None: + from everos.infra.persistence.index import ( + Episode, + agent_skill_repo, + episode_repo, + foresight_repo, + ) + from everos.memory.search import FilterNode + from everos.memory.search.filters import compile_filters + + await episode_repo.upsert([_episode(0), _episode(1)]) + where = compile_filters( + FilterNode.model_validate({"session_id": "seekdb"}), + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + assert {row.id for row in await episode_repo.find_where(where, limit=10)} == { + "u1_ep_0000", + "u1_ep_0001", + } + + projected = await episode_repo.list_by_owner_after_ts( + owner_id="u1", + after_ts=int(dt.datetime(1998, 1, 1, tzinfo=dt.UTC).timestamp()), + parent_type="memcell", + app_id="test_app", + project_id="test_project", + columns=["id", "subject"], + ) + assert [row["id"] for row in projected] == ["u1_ep_0000", "u1_ep_0001"] + assert all(set(row) == {"id", "subject", "timestamp"} for row in projected) + + await foresight_repo.upsert( + [ + _foresight(0, primary_text="orchard plan", evidence_text="history"), + _foresight(1, primary_text="travel plan", evidence_text="orchard note"), + ] + ) + sparse = await foresight_repo.sparse_search( + ["orchard"], None, columns=["foresight_tokens", "evidence_tokens"], limit=5 + ) + assert {row["id"] for row in sparse} == {"u1_fs_0", "u1_fs_1"} + assert all(row["_score"] > 0 for row in sparse) + + skills = [_agent_skill(0), _agent_skill(1)] + await agent_skill_repo.upsert(skills) + assert ( + await agent_skill_repo.count_in_cluster( + owner_id="agent1", cluster_id="cluster1" + ) + == 2 + ) + assert ( + len( + await agent_skill_repo.find_in_cluster( + owner_id="agent1", cluster_id="cluster1", limit=10 + ) + ) + == 2 + ) + query = [1.0] + [0.0] * 1023 + top = await agent_skill_repo.find_topk_relevant_in_cluster( + owner_id="agent1", + cluster_id="cluster1", + query_vector=query, + top_k=1, + ) + assert [row.id for row in top] == [skills[0].id] + + # Keep the imported model live so the test also confirms its BM25 metadata. + assert Episode.BM25_FIELDS == ["episode_tokens"] + + +async def test_seekdb_schema_drift_is_loud_and_rebuildable() -> None: + from everos.infra.persistence.index import ( + drop_business_tables, + ensure_business_indexes, + verify_business_schemas, + ) + from everos.infra.persistence.seekdb import episode_repo as concrete_episode_repo + from everos.infra.persistence.seekdb.errors import SeekdbSchemaMismatchError + from everos.infra.persistence.seekdb.repository import SeekdbRepoBase + + await concrete_episode_repo._execute( # type: ignore[attr-defined] + f"ALTER TABLE `{concrete_episode_repo.physical_table_name}` " + "DROP COLUMN `subject`" + ) + SeekdbRepoBase._reset_table_cache() + with pytest.raises(SeekdbSchemaMismatchError, match="cascade rebuild"): + await verify_business_schemas() + + await drop_business_tables() + await ensure_business_indexes() + SeekdbRepoBase._reset_table_cache() + await verify_business_schemas() diff --git a/tests/integration/test_tiers/conftest.py b/tests/integration/test_tiers/conftest.py index 27197dfe8..907bbc402 100644 --- a/tests/integration/test_tiers/conftest.py +++ b/tests/integration/test_tiers/conftest.py @@ -188,6 +188,10 @@ def _index_backends() -> list[str]: backends = ["lancedb"] if os.environ.get("EVEROS_TEST_MILVUS_URI"): backends.append("milvus") + if os.environ.get("EVEROS_TEST_SEEKDB_PATH") or os.environ.get( + "EVEROS_TEST_SEEKDB_HOST" + ): + backends.append("seekdb") return backends @@ -211,10 +215,33 @@ async def index_backend( monkeypatch.setenv( "EVEROS_MILVUS__COLLECTION_PREFIX", f"everos_tier_{uuid.uuid4().hex}" ) + elif backend == "seekdb": + monkeypatch.setenv( + "EVEROS_SEEKDB__TABLE_PREFIX", f"everos_tier_{uuid.uuid4().hex}" + ) + if path := os.environ.get("EVEROS_TEST_SEEKDB_PATH"): + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "embedded") + monkeypatch.setenv("EVEROS_SEEKDB__PATH", path) + else: + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "remote") + monkeypatch.setenv( + "EVEROS_SEEKDB__HOST", os.environ["EVEROS_TEST_SEEKDB_HOST"] + ) + for name, default in ( + ("PORT", "2881"), + ("TENANT", ""), + ("USER", "root"), + ("PASSWORD", ""), + ("DATABASE", "everos_test"), + ): + monkeypatch.setenv( + f"EVEROS_SEEKDB__{name}", + os.environ.get(f"EVEROS_TEST_SEEKDB_{name}", default), + ) yield backend - if backend == "milvus": + if backend in {"milvus", "seekdb"}: from everos.config import load_settings from everos.infra.persistence.index import drop_business_tables, shutdown diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index 6d3f2c59d..c57a245e9 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -144,6 +144,44 @@ def test_index_milvus_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: assert s.milvus.consistency_level == "Strong" +def test_index_seekdb_defaults_and_env_overrides( + monkeypatch: pytest.MonkeyPatch, +) -> None: + defaults = Settings() + assert defaults.seekdb.mode == "embedded" + assert defaults.seekdb.database == "everos" + assert defaults.seekdb.port == 2881 + assert defaults.seekdb.tenant == "" + assert defaults.seekdb.vector_sync_mode == "immediate" + + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "seekdb") + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "remote") + monkeypatch.setenv("EVEROS_SEEKDB__HOST", "seekdb.example") + monkeypatch.setenv("EVEROS_SEEKDB__PASSWORD", "secret") + monkeypatch.setenv("EVEROS_SEEKDB__VECTOR_SYNC_MODE", "async") + configured = Settings() + assert configured.index.backend == "seekdb" + assert configured.seekdb.mode == "remote" + assert configured.seekdb.host == "seekdb.example" + assert configured.seekdb.password.get_secret_value() == "secret" + assert configured.seekdb.vector_sync_mode == "async" + + +def test_seekdb_identifier_settings_are_validated() -> None: + from pydantic import ValidationError + + from everos.config import SeekdbSettings + + with pytest.raises(ValidationError, match="table_prefix"): + SeekdbSettings(table_prefix="bad-prefix") + with pytest.raises(ValidationError, match="database"): + SeekdbSettings(database="bad-database") + with pytest.raises(ValidationError, match="database"): + SeekdbSettings(database="d" * 65) + with pytest.raises(ValidationError, match="table_prefix"): + SeekdbSettings(table_prefix="p" * 49) + + def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None: """No --root, no EVEROS_ROOT → ~/.everos.""" monkeypatch.delenv("EVEROS_ROOT", raising=False) diff --git a/tests/unit/test_infra/test_index_contract.py b/tests/unit/test_infra/test_index_contract.py index 87dbc32ae..46deadb9a 100644 --- a/tests/unit/test_infra/test_index_contract.py +++ b/tests/unit/test_infra/test_index_contract.py @@ -18,6 +18,7 @@ render_predicate, ) from everos.infra.persistence.backends.milvus import milvus_index_backend +from everos.infra.persistence.backends.seekdb import seekdb_index_backend from everos.infra.persistence.index import ( ALL_REPOS, All, @@ -49,6 +50,10 @@ def _behavioural_backends() -> list[str]: backends = ["lancedb"] if os.environ.get("EVEROS_TEST_MILVUS_URI"): backends.append("milvus") + if os.environ.get("EVEROS_TEST_SEEKDB_PATH") or os.environ.get( + "EVEROS_TEST_SEEKDB_HOST" + ): + backends.append("seekdb") return backends @@ -63,6 +68,17 @@ def _milvus_collection_prefix() -> str: return external_prefix +def _seekdb_table_prefix() -> str: + external_prefix = os.environ.get("EVEROS_TEST_SEEKDB_PREFIX") + if external_prefix is None: + return f"everos_ct_{uuid.uuid4().hex}" + if _EXTERNAL_PREFIX_PATTERN.fullmatch(external_prefix) is None: + raise ValueError( + "EVEROS_TEST_SEEKDB_PREFIX must match everos_e2e_<32 lowercase hex>" + ) + return external_prefix + + @pytest.fixture(params=_behavioural_backends(), ids=lambda b: f"index={b}") async def index_backend( request: pytest.FixtureRequest, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -85,6 +101,28 @@ async def index_backend( "EVEROS_MILVUS__DB_NAME", os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", "") ) monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", collection_prefix) + elif backend == "seekdb": + collection_prefix = _seekdb_table_prefix() + monkeypatch.setenv("EVEROS_SEEKDB__TABLE_PREFIX", collection_prefix) + if path := os.environ.get("EVEROS_TEST_SEEKDB_PATH"): + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "embedded") + monkeypatch.setenv("EVEROS_SEEKDB__PATH", path) + else: + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "remote") + monkeypatch.setenv( + "EVEROS_SEEKDB__HOST", os.environ["EVEROS_TEST_SEEKDB_HOST"] + ) + for name, default in ( + ("PORT", "2881"), + ("TENANT", ""), + ("USER", "root"), + ("PASSWORD", ""), + ("DATABASE", "everos_test"), + ): + monkeypatch.setenv( + f"EVEROS_SEEKDB__{name}", + os.environ.get(f"EVEROS_TEST_SEEKDB_{name}", default), + ) from everos.config import load_settings @@ -97,12 +135,22 @@ async def index_backend( if repo.table_name == "episode" ) assert milvus_episode_repo.collection_name == f"{collection_prefix}_episode" + elif backend == "seekdb": + assert collection_prefix is not None + seekdb_episode_repo = next( + repo + for repo in seekdb_index_backend.repositories + if repo.table_name == "episode" + ) + assert seekdb_episode_repo.physical_table_name == ( + f"{collection_prefix}_episode" + ) try: yield backend finally: try: - if backend == "milvus": + if backend in {"milvus", "seekdb"}: try: load_settings.cache_clear() finally: @@ -157,6 +205,7 @@ def _episode(number: int, *, owner_id: str = "owner") -> Episode: [ pytest.param(lance_index_backend, id="lancedb"), pytest.param(milvus_index_backend, id="milvus"), + pytest.param(seekdb_index_backend, id="seekdb"), ], ) def test_every_backend_satisfies_the_ports(backend: IndexBackend) -> None: @@ -204,6 +253,7 @@ def test_lance_predicate_renderer_owns_escaping() -> None: [ "everos.infra.persistence.backends.lancedb", "everos.infra.persistence.backends.milvus", + "everos.infra.persistence.backends.seekdb", "everos.infra.persistence.index", ], ) @@ -224,6 +274,25 @@ def test_adapter_and_port_modules_import_in_any_order(module: str) -> None: assert proc.returncode == 0, f"importing {module} first failed:\n{proc.stderr}" +def test_seekdb_adapter_does_not_eagerly_import_optional_client() -> None: + proc = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; " + "from everos.infra.persistence.backends.seekdb import " + "seekdb_index_backend; " + "assert len(seekdb_index_backend.repositories) == 7; " + "assert 'pyseekdb' not in sys.modules" + ), + ], + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + + @pytest.mark.parametrize("group", [All, AnyOf]) def test_empty_predicate_group_is_rejected_at_construction( group: type[Predicate], diff --git a/tests/unit/test_infra/test_seekdb/__init__.py b/tests/unit/test_infra/test_seekdb/__init__.py new file mode 100644 index 000000000..5e71d612c --- /dev/null +++ b/tests/unit/test_infra/test_seekdb/__init__.py @@ -0,0 +1 @@ +"""Unit tests for the optional SeekDB derived-index adapter.""" diff --git a/tests/unit/test_infra/test_seekdb/test_codec.py b/tests/unit/test_infra/test_seekdb/test_codec.py new file mode 100644 index 000000000..98a92b480 --- /dev/null +++ b/tests/unit/test_infra/test_seekdb/test_codec.py @@ -0,0 +1,92 @@ +"""Pin exact datetime, JSON, vector, and limit behavior in SeekDB row codecs.""" + +from __future__ import annotations + +import datetime as dt +import struct + +import pytest + +from everos.infra.persistence.index import Episode +from everos.infra.persistence.index.schema import schema_for +from everos.infra.persistence.seekdb.codec import from_row, to_row, write_field_value +from everos.infra.persistence.seekdb.errors import SeekdbValueLimitError + + +def _episode(**overrides: object) -> Episode: + values: dict[str, object] = { + "id": "u1_ep1", + "entry_id": "ep1", + "owner_id": "u1", + "owner_type": "user", + "session_id": "session", + "timestamp": dt.datetime(1999, 1, 1, tzinfo=dt.UTC), + "parent_id": "mc1", + "sender_ids": ["u1"], + "episode": "red apple memory", + "episode_tokens": "red apple memory", + "md_path": "users/u1/episodes/day.md", + "content_sha256": "a" * 64, + "vector": [1.0] + [0.0] * 1023, + "subject_vector": None, + } + values.update(overrides) + return Episode(**values) # type: ignore[arg-type] + + +def test_record_round_trip_preserves_epoch_ms_json_and_null_vector() -> None: + schema = schema_for(Episode) + stored = to_row(_episode(), schema) + assert stored["timestamp_ms"] == 915148800000 + assert stored["sender_ids"] == '["u1"]' + assert stored["subject_vector"] is None + restored = from_row(stored, schema) + assert restored["timestamp"] == dt.datetime(1999, 1, 1, tzinfo=dt.UTC) + assert restored["sender_ids"] == ["u1"] + assert restored["vector"][:2] == [1.0, 0.0] + + +def test_binary_float32_vectors_are_accepted() -> None: + schema = schema_for(Episode) + raw = {"vector": struct.pack("<1024f", *([0.5] * 1024))} + restored = from_row(raw, schema) + assert restored["vector"][0] == pytest.approx(0.5) + + +def test_binary_vector_starting_with_json_marker_is_not_misclassified() -> None: + schema = schema_for(Episode) + raw = {"vector": bytes([0x5B, 0, 0, 0]) * 1024} + restored = from_row(raw, schema) + assert len(restored["vector"]) == 1024 + + +@pytest.mark.parametrize( + ("moment", "epoch_ms"), + [ + (dt.datetime(1970, 1, 1, tzinfo=dt.UTC), 0), + (dt.datetime(2001, 9, 9, 1, 46, 40, 123000, tzinfo=dt.UTC), 1000000000123), + ], +) +def test_datetime_boundaries_round_trip_exactly( + moment: dt.datetime, epoch_ms: int +) -> None: + schema = schema_for(Episode) + stored = to_row(_episode(timestamp=moment), schema) + assert stored["timestamp_ms"] == epoch_ms + assert from_row(stored, schema)["timestamp"] == moment + + +def test_short_string_array_and_vector_limits_are_loud() -> None: + schema = schema_for(Episode) + with pytest.raises(SeekdbValueLimitError, match="id is 513 characters"): + to_row(_episode(id="x" * 513), schema) + with pytest.raises(SeekdbValueLimitError, match="sender_ids has 257 items"): + to_row(_episode(sender_ids=["u"] * 257), schema) + with pytest.raises(SeekdbValueLimitError, match="dimension 2"): + write_field_value(schema.field("vector"), [1.0, 0.0], schema) + with pytest.raises(SeekdbValueLimitError, match="non-finite"): + write_field_value( + schema.field("vector"), + [float("nan")] + [0.0] * 1023, + schema, + ) diff --git a/tests/unit/test_infra/test_seekdb/test_manager.py b/tests/unit/test_infra/test_seekdb/test_manager.py new file mode 100644 index 000000000..b24a103c0 --- /dev/null +++ b/tests/unit/test_infra/test_seekdb/test_manager.py @@ -0,0 +1,323 @@ +"""Exercise SeekDB lifecycle, diagnostics, locking, and row normalization.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from everos.config import SeekdbSettings +from everos.infra.persistence.seekdb.errors import ( + SeekdbConfigurationError, + SeekdbIntegrityError, + SeekdbOperationalError, +) +from everos.infra.persistence.seekdb.seekdb_manager import ( + SeekdbSession, + SeekdbTarget, + resolve_target, + table_name, +) + + +class OperationalError(Exception): + """Fake DB-API operational error retaining the numeric code.""" + + +class IntegrityError(Exception): + """Fake DB-API integrity error.""" + + +class _RawConnection: + def __init__(self) -> None: + self.pings: list[bool] = [] + + def ping(self, *, reconnect: bool) -> None: + self.pings.append(reconnect) + + +class _FakeServer: + def __init__( + self, + rows: object = None, + *, + connection_error: Exception | None = None, + execute_errors: list[Exception] | None = None, + ) -> None: + self.rows = rows + self.connection_error = connection_error + self.execute_errors = list(execute_errors or []) + self.executed: list[str] = [] + self.raw = _RawConnection() + self.cleanup_count = 0 + + def _execute(self, sql: str) -> object: + self.executed.append(sql) + if self.execute_errors: + raise self.execute_errors.pop(0) + return self.rows + + def get_raw_connection(self) -> object: + if self.connection_error is not None: + raise self.connection_error + return self.raw + + def _cleanup(self) -> None: + self.cleanup_count += 1 + + +def test_remote_target_requires_host_and_resolves_password( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(SeekdbConfigurationError, match="host is required"): + resolve_target(SeekdbSettings(mode="remote")) + monkeypatch.setenv("SEEKDB_PASSWORD", "secret") + target = resolve_target(SeekdbSettings(mode="remote", host="db.example")) + assert target.host == "db.example" + assert target.password == "secret" + + +def test_embedded_target_rejects_windows_and_resolves_default_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + monkeypatch.setattr(manager.sys, "platform", "win32") + with pytest.raises(SeekdbConfigurationError, match="Linux and macOS"): + resolve_target(SeekdbSettings()) + monkeypatch.setattr(manager.sys, "platform", "linux") + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + target = resolve_target(SeekdbSettings()) + assert target.path == tmp_path / ".index" / "seekdb" + + +def test_session_normalizes_rows_and_pings_remote_connection() -> None: + embedded = SeekdbSession(_FakeServer([("one", 2)]), mode="embedded") + remote_server = _FakeServer([{"ID": "one", "COUNT": 2}]) + remote = SeekdbSession(remote_server, mode="remote") + assert embedded.fetch_all("SELECT", ["id", "count"]) == [{"id": "one", "count": 2}] + assert remote.fetch_all("SELECT", ["id", "count"]) == [{"id": "one", "count": 2}] + assert remote_server.raw.pings == [True] + + +def test_session_maps_integrity_and_preserves_operational_context() -> None: + duplicate = SeekdbSession( + _FakeServer(execute_errors=[IntegrityError(1062, "duplicate")]), + mode="embedded", + ) + with pytest.raises(SeekdbIntegrityError, match=r"INSERT.*unit_episode.*1062"): + duplicate.execute("INSERT INTO unit_episode VALUES (1)", table="unit_episode") + + broken = SeekdbSession( + _FakeServer(execute_errors=[OperationalError(1064, "syntax")]), + mode="embedded", + ) + with pytest.raises(SeekdbOperationalError, match=r"SELECT.*unit_episode.*1064"): + broken.fetch_scalar("SELECT broken", table="unit_episode") + + +async def test_disconnect_invalidates_cache_and_next_get_reconnects( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + failed_server = _FakeServer( + execute_errors=[OperationalError(2013, "lost connection")] + ) + failed = SeekdbSession(failed_server, mode="remote") + monkeypatch.setattr(manager, "_session", failed) + with pytest.raises(SeekdbOperationalError, match="2013"): + failed.execute("SELECT 1", table="unit_episode") + assert manager._session is None + assert failed_server.cleanup_count == 1 + + healthy = SeekdbSession(_FakeServer([(1,)]), mode="remote") + monkeypatch.setattr(manager, "_open_session", lambda target: healthy) + monkeypatch.setattr( + manager, + "resolve_target", + lambda: SeekdbTarget(mode="remote", database="everos", host="db.example"), + ) + # A coroutine may have captured the old bound method before the first + # failure. run() rebinds it after taking the operation lock. + assert await manager.run(failed.fetch_scalar, "SELECT 1") == 1 + assert await manager.get_session() is healthy + await manager.dispose_connection() + + +def test_embedded_engine_error_does_not_trigger_remote_reconnect_logic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + embedded = SeekdbSession( + _FakeServer(execute_errors=[OperationalError(2013, "engine error")]), + mode="embedded", + ) + monkeypatch.setattr(manager, "_session", embedded) + with pytest.raises(SeekdbOperationalError, match="2013"): + embedded.execute("SELECT 1") + assert manager._session is embedded + + +def test_open_existing_database_never_attempts_create( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + server = _FakeServer() + databases: list[str] = [] + + def make_server(target: SeekdbTarget, *, database: str) -> _FakeServer: + databases.append(database) + return server + + monkeypatch.setattr(manager, "_new_server", make_server) + session = manager._open_session( + SeekdbTarget(mode="remote", database="everos", host="db.example") + ) + assert databases == ["everos"] + assert server.executed == [ + "SET NAMES utf8mb4 COLLATE utf8mb4_bin", + "SET SESSION sql_mode = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', " + "@@SESSION.sql_mode, ','), ',NO_BACKSLASH_ESCAPES,', ','))", + ] + session.close() + + +def test_missing_database_is_checked_then_created_with_binary_collation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + missing = _FakeServer(connection_error=OperationalError(1049, "unknown database")) + admin = _FakeServer(rows=[]) + connected = _FakeServer() + servers = iter([missing, admin, connected]) + databases: list[str] = [] + + def make_server(target: SeekdbTarget, *, database: str) -> _FakeServer: + databases.append(database) + return next(servers) + + monkeypatch.setattr(manager, "_new_server", make_server) + session = manager._open_session( + SeekdbTarget(mode="remote", database="everos", host="db.example") + ) + assert databases == ["everos", "information_schema", "everos"] + assert any( + sql == "CREATE DATABASE IF NOT EXISTS `everos` DEFAULT CHARACTER SET " + "utf8mb4 COLLATE utf8mb4_bin" + for sql in admin.executed + ) + session.close() + + +def test_embedded_database_creation_uses_real_default_database( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + missing = _FakeServer(connection_error=OperationalError(1049, "unknown database")) + admin = _FakeServer(rows=[]) + connected = _FakeServer() + servers = iter([missing, admin, connected]) + databases: list[str] = [] + + def make_server(target: SeekdbTarget, *, database: str) -> _FakeServer: + databases.append(database) + return next(servers) + + monkeypatch.setattr(manager, "_new_server", make_server) + server = manager._connect_target_database( + SeekdbTarget(mode="embedded", database="everos", path=tmp_path) + ) + assert databases == ["everos", "test", "everos"] + server._cleanup() + + +def test_missing_database_permission_error_has_actionable_guidance( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + missing = _FakeServer(connection_error=OperationalError(1049, "unknown database")) + + class CreateDeniedServer(_FakeServer): + def _execute(self, sql: str) -> object: + self.executed.append(sql) + if sql.startswith("CREATE DATABASE"): + raise OperationalError(1044, "denied") + return [] + + admin = CreateDeniedServer() + servers = iter([missing, admin]) + monkeypatch.setattr( + manager, + "_new_server", + lambda target, *, database: next(servers), + ) + with pytest.raises(SeekdbConfigurationError, match=r"pre-create.*grant CREATE"): + manager._open_session( + SeekdbTarget(mode="remote", database="everos", host="db.example") + ) + + +def test_embedded_directory_lock_fails_before_engine_open( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + class BusyFcntl: + LOCK_EX = 2 + LOCK_NB = 4 + LOCK_UN = 8 + + @staticmethod + def flock(fd: int, operation: int) -> None: + raise BlockingIOError + + monkeypatch.setattr(manager.importlib, "import_module", lambda name: BusyFcntl) + with pytest.raises(SeekdbConfigurationError, match="already opened"): + manager._acquire_embedded_lock( + SeekdbTarget(mode="embedded", database="everos", path=tmp_path) + ) + + +def test_embedded_directory_lock_is_released_on_close_and_open_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import everos.infra.persistence.seekdb.seekdb_manager as manager + + class FakeLock: + def __init__(self) -> None: + self.releases = 0 + + def release(self) -> None: + self.releases += 1 + + target = SeekdbTarget(mode="embedded", database="everos", path=tmp_path) + held = FakeLock() + server = _FakeServer() + monkeypatch.setattr(manager, "_acquire_embedded_lock", lambda target: held) + monkeypatch.setattr(manager, "_connect_target_database", lambda target: server) + session = manager._open_session(target) + session.close() + assert held.releases == 1 + + failed = FakeLock() + monkeypatch.setattr(manager, "_acquire_embedded_lock", lambda target: failed) + + def fail_to_connect(target: SeekdbTarget) -> _FakeServer: + raise RuntimeError("engine failed") + + monkeypatch.setattr(manager, "_connect_target_database", fail_to_connect) + with pytest.raises(RuntimeError, match="engine failed"): + manager._open_session(target) + assert failed.releases == 1 + + +def test_table_name_validates_the_configured_prefix() -> None: + assert table_name("episode", SeekdbSettings(table_prefix="tenant_a")) == ( + "tenant_a_episode" + ) diff --git a/tests/unit/test_infra/test_seekdb/test_predicate.py b/tests/unit/test_infra/test_seekdb/test_predicate.py new file mode 100644 index 000000000..299095ee3 --- /dev/null +++ b/tests/unit/test_infra/test_seekdb/test_predicate.py @@ -0,0 +1,52 @@ +"""Cover every neutral predicate node in the SeekDB SQL renderer.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from everos.infra.persistence import predicate as predicates +from everos.infra.persistence.seekdb.predicate import render_predicate + + +def test_comparison_in_and_datetime_rendering() -> None: + moment = dt.datetime(1999, 1, 1, tzinfo=dt.UTC) + rendered = render_predicate( + predicates.all_of( + predicates.eq("owner_id", "o'reilly"), + predicates.gt("timestamp", moment), + predicates.one_of("entry_id", ["a", "b"]), + ), + datetime_fields={"timestamp"}, + ) + assert "`owner_id` = 'o''reilly'" in rendered + assert "`timestamp_ms` > 915148800000" in rendered + assert "`entry_id` IN ('a', 'b')" in rendered + + +def test_contains_null_and_nested_or_rendering() -> None: + rendered = render_predicate( + predicates.any_of( + predicates.contains("sender_ids", "user"), + predicates.is_null("subject_vector"), + ), + vector_fields={"subject_vector"}, + ) + assert "JSON_CONTAINS(`sender_ids`, '\"user\"')" in rendered + assert "`subject_vector` IS NULL" in rendered + assert " OR " in rendered + + +def test_is_null_uses_physical_datetime_column() -> None: + assert ( + render_predicate(predicates.is_null("timestamp"), datetime_fields={"timestamp"}) + == "`timestamp_ms` IS NULL" + ) + + +def test_invalid_fields_and_foreign_predicates_are_rejected() -> None: + with pytest.raises(ValueError, match="invalid SQL identifier"): + render_predicate(predicates.eq("x`; DELETE", "value")) + with pytest.raises(TypeError, match="neutral Predicate AST"): + render_predicate("owner_id = 1") # type: ignore[arg-type] diff --git a/tests/unit/test_infra/test_seekdb/test_repository.py b/tests/unit/test_infra/test_seekdb/test_repository.py new file mode 100644 index 000000000..0fa4dc8fe --- /dev/null +++ b/tests/unit/test_infra/test_seekdb/test_repository.py @@ -0,0 +1,272 @@ +"""Pin generated write and search SQL without requiring a SeekDB server.""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Sequence +from types import SimpleNamespace +from typing import Any + +import pytest + +from everos.config import load_settings +from everos.infra.persistence.index import Episode, eq +from everos.infra.persistence.index.schema import schema_for +from everos.infra.persistence.seekdb import episode_repo +from everos.infra.persistence.seekdb import repository as repository_module +from everos.infra.persistence.seekdb.errors import SeekdbSchemaMismatchError +from everos.infra.persistence.seekdb.repository import SeekdbRepoBase +from everos.infra.persistence.seekdb.schema import ( + build_create_table, + physical_columns, + physical_indexes, +) + + +class _FakeSession: + def __init__(self) -> None: + self.sql: list[str] = [] + self.rows: list[dict[str, Any]] = [] + self.row_batches: list[list[dict[str, Any]]] = [] + + def execute(self, sql: str, *, table: str | None = None) -> None: + assert table == "unit_episode" + self.sql.append(sql) + + def fetch_all( + self, + sql: str, + columns: Sequence[str], + *, + table: str | None = None, + ) -> list[dict[str, Any]]: + assert table == "unit_episode" + self.sql.append(sql) + if self.row_batches: + return self.row_batches.pop(0) + return self.rows + + def fetch_scalar(self, sql: str, *, table: str | None = None) -> int: + assert table == "unit_episode" + self.sql.append(sql) + return 0 + + +def _episode(**overrides: object) -> Episode: + values: dict[str, object] = { + "id": "u1_ep1", + "entry_id": "ep1", + "owner_id": "u1", + "owner_type": "user", + "session_id": "session", + "timestamp": dt.datetime(2026, 1, 1, tzinfo=dt.UTC), + "parent_id": "mc1", + "sender_ids": ["u1"], + "episode": "red apple memory", + "episode_tokens": "red apple memory", + "md_path": "users/u1/episodes/day.md", + "content_sha256": "a" * 64, + "vector": [1.0] + [0.0] * 1023, + } + values.update(overrides) + return Episode(**values) # type: ignore[arg-type] + + +@pytest.fixture(autouse=True) +def _fake_backend( + monkeypatch: pytest.MonkeyPatch, +) -> _FakeSession: + monkeypatch.setenv("EVEROS_SEEKDB__MODE", "remote") + monkeypatch.setenv("EVEROS_SEEKDB__HOST", "db.example") + monkeypatch.setenv("EVEROS_SEEKDB__TABLE_PREFIX", "unit") + load_settings.cache_clear() + SeekdbRepoBase._reset_locks_for_tests() + fake = _FakeSession() + + async def get_fake_session() -> _FakeSession: + return fake + + async def run_inline(fn: Any, /, *args: Any, **kwargs: Any) -> Any: + return fn(*args, **kwargs) + + monkeypatch.setattr(repository_module, "get_session", get_fake_session) + monkeypatch.setattr(repository_module, "run", run_inline) + SeekdbRepoBase._ready_tables.add(episode_repo.physical_table_name) + yield fake + SeekdbRepoBase._reset_locks_for_tests() + load_settings.cache_clear() + + +async def test_upsert_uses_on_duplicate_key_update(_fake_backend: _FakeSession) -> None: + await episode_repo.upsert([_episode()]) + sql = _fake_backend.sql[-1] + assert sql.startswith("INSERT INTO `unit_episode`") + assert "ON DUPLICATE KEY UPDATE" in sql + assert "`id`=VALUES(`id`)" not in sql + assert "`vector`=VALUES(`vector`)" in sql + + +async def test_add_is_insert_only_and_batches_at_64( + _fake_backend: _FakeSession, +) -> None: + await episode_repo.add([_episode(id=f"u1_ep_{number}") for number in range(65)]) + inserts = [sql for sql in _fake_backend.sql if sql.startswith("INSERT INTO")] + assert len(inserts) == 2 + assert all("ON DUPLICATE KEY UPDATE" not in sql for sql in inserts) + + +async def test_dense_search_is_filtered_clamped_and_null_safe( + _fake_backend: _FakeSession, +) -> None: + await episode_repo.dense_search( + [1.0] + [0.0] * 1023, + eq("owner_id", "u1"), + limit=100_000, + vector_field="subject_vector", + ) + sql = _fake_backend.sql[-1] + assert "cosine_distance(`subject_vector`" in sql + assert "`subject_vector` IS NOT NULL" in sql + assert "`owner_id` = 'u1'" in sql + assert "APPROXIMATE LIMIT 16384" in sql + + +async def test_sparse_search_queries_each_column_and_keeps_best_score( + _fake_backend: _FakeSession, +) -> None: + _fake_backend.rows = [{"id": "u1_ep1", "_score": 2.5}] + rows = await episode_repo.sparse_search( + ["red", "apple"], None, columns=["episode_tokens"], limit=10 + ) + assert "MATCH(`episode_tokens`) AGAINST" in _fake_backend.sql[-1] + assert rows == [{"id": "u1_ep1", "_score": 2.5}] + + +async def test_update_and_native_pagination_render_one_statement_each( + _fake_backend: _FakeSession, +) -> None: + await episode_repo.update({"subject": "updated"}, where=eq("id", "u1_ep1")) + assert _fake_backend.sql[-1] == ( + "UPDATE `unit_episode` SET `subject` = 'updated' WHERE `id` = 'u1_ep1'" + ) + + _fake_backend.rows = [] + await episode_repo.find_where_paginated( + eq("owner_id", "u1"), sort_by="timestamp", page=2, page_size=20 + ) + assert ( + "ORDER BY `timestamp_ms` DESC, `id` ASC LIMIT 20 OFFSET 20" + in (_fake_backend.sql[-1]) + ) + + +async def test_update_none_and_mutation_guards_are_explicit( + _fake_backend: _FakeSession, +) -> None: + await episode_repo.update({"subject": None}, where=eq("id", "u1_ep1")) + assert "SET `subject` = NULL" in _fake_backend.sql[-1] + with pytest.raises(TypeError, match="update requires"): + await episode_repo.update({"subject": "x"}, where=None) # type: ignore[arg-type] + with pytest.raises(TypeError, match="delete requires"): + await episode_repo.delete(None) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "sort_by", ["unknown", "vector", "sender_ids", "episode_tokens"] +) +async def test_pagination_rejects_unknown_and_non_scalar_sort_fields( + _fake_backend: _FakeSession, + sort_by: str, +) -> None: + with pytest.raises(ValueError): + await episode_repo.find_where_paginated(eq("owner_id", "u1"), sort_by=sort_by) + + +async def test_scan_uses_keyset_batches_beyond_one_thousand( + _fake_backend: _FakeSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + batches = [ + [{"id": f"id-{number:04d}"} for number in range(1000)], + [{"id": "id-1000"}], + ] + + def fetch_all( + sql: str, + columns: Sequence[str], + *, + table: str | None = None, + ) -> list[dict[str, Any]]: + assert table == "unit_episode" + _fake_backend.sql.append(sql) + return batches.pop(0) + + monkeypatch.setattr(_fake_backend, "fetch_all", fetch_all) + monkeypatch.setattr( + episode_repo, + "_model", + lambda row: SimpleNamespace(model_dump=lambda mode: {"id": row["id"]}), + ) + rows = await episode_repo.scan() + assert len(rows) == 1001 + assert "`id` > 'id-0999'" in _fake_backend.sql[-1] + assert batches == [] + + +async def test_verify_table_accepts_catalog_spelling_and_rejects_collation_drift( + _fake_backend: _FakeSession, +) -> None: + logical = schema_for(Episode) + columns = [] + for column in physical_columns(logical): + columns.append( + { + "COLUMN_NAME": column.name, + "COLUMN_TYPE": ( + "bigint(20)" if column.sql_type == "BIGINT" else column.sql_type + ), + "IS_NULLABLE": ( + "YES" if column.nullable and not column.primary else "NO" + ), + "COLUMN_KEY": "PRI" if column.primary else "", + "CHARACTER_SET_NAME": column.character_set, + "COLLATION_NAME": column.collation, + } + ) + indexes = [] + for index in physical_indexes(logical): + prefixes = index.prefix_lengths or (None,) * len(index.columns) + for position, (name, prefix) in enumerate( + zip(index.columns, prefixes, strict=True), start=1 + ): + indexes.append( + { + "INDEX_NAME": index.name, + "COLUMN_NAME": name, + "INDEX_TYPE": { + "btree": "BTREE", + "fulltext": "FULLTEXT", + "vector": "VECTOR", + }[index.kind], + "SEQ_IN_INDEX": position, + "SUB_PART": prefix, + } + ) + ddl = build_create_table("unit_episode", logical) + _fake_backend.row_batches = [ + columns, + indexes, + [{"Table": "unit_episode", "Create Table": ddl}], + ] + await episode_repo.verify_table() + + next(row for row in columns if row["COLUMN_NAME"] == "id")["COLLATION_NAME"] = ( + "utf8mb4_general_ci" + ) + _fake_backend.row_batches = [ + columns, + indexes, + [{"Table": "unit_episode", "Create Table": ddl}], + ] + with pytest.raises(SeekdbSchemaMismatchError, match="collation"): + await episode_repo.verify_table() diff --git a/tests/unit/test_infra/test_seekdb/test_schema.py b/tests/unit/test_infra/test_seekdb/test_schema.py new file mode 100644 index 000000000..6b1ac5841 --- /dev/null +++ b/tests/unit/test_infra/test_seekdb/test_schema.py @@ -0,0 +1,208 @@ +"""Verify SeekDB physical schemas and drift comparison for all seven tables.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Sequence + +from everos.infra.persistence.index import ALL_REPOS, Episode, UserProfile +from everos.infra.persistence.index.schema import schema_for +from everos.infra.persistence.seekdb.schema import ( + PhysicalColumn, + PhysicalIndex, + build_create_table, + column_drift, + index_drift, + normalize_sql_type, + physical_columns, + physical_indexes, +) + +_DDL_GOLDEN_SHA256 = { + "episode": "b4300451b98fc7d61d65f86d92c11f74ebebb2bb5f0c57ce49e5e9291554433f", + "atomic_fact": "3cabf5316b1e994c02172ace1f2f251debc3ed634942438cecb3db48b4f6309c", + "foresight": "e6c16cf6566a6d285f4d1db00e02b55037dc900228d9fe30deaf58cbcf0b7e6e", + "agent_case": "0f92b68f53c09a84ac815411c3577b94380301dd56ccaae3c63dc8b52a5ecc78", + "agent_skill": "c1a79ea2a7a2e68dc36a4e9ec2c20da88660854c1234eddac6595dc35b3f2b0f", + "user_profile": "364d7e68a065d52e7c0f60cc6f0fa8daf9860df6254a8bebf525b9ee41813bb1", + "knowledge_topic": ( + "ee501b9d525c5d357a49b7944ec3b6113f2fd670e5a67f9b48b78ee3f3cae4c8" + ), +} + + +def test_every_logical_field_has_one_seekdb_column() -> None: + for repo in ALL_REPOS: + logical = schema_for(repo.schema) + physical = physical_columns(logical) + assert len(physical) == len(logical.fields) + assert sum(column.primary for column in physical) == 1 + assert {index.name for index in physical_indexes(logical)} >= { + f"ft_{name}" for name in logical.bm25_fields + } + + +def test_all_seven_create_table_statements_match_reviewed_golden() -> None: + actual = {} + for repo in ALL_REPOS: + logical = schema_for(repo.schema) + ddl = build_create_table(f"unit_{logical.table_name}", logical) + actual[logical.table_name] = hashlib.sha256(ddl.encode()).hexdigest() + assert actual == _DDL_GOLDEN_SHA256 + + +def test_episode_ddl_has_bounded_keys_collation_and_search_indexes() -> None: + ddl = build_create_table("unit_episode", schema_for(Episode)) + assert "`timestamp_ms` BIGINT NOT NULL" in ddl + assert "`sender_ids` JSON NOT NULL" in ddl + assert "INDEX `ix_owner_scope` (`owner_id`(128), `app_id`(128)," in ddl + assert "FULLTEXT INDEX `ft_episode_tokens`" in ddl + assert ddl.count("VECTOR INDEX") == 2 + assert ddl.count("SYNC_MODE=immediate") == 2 + assert "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" in ddl + assert ddl.endswith("ORGANIZATION = HEAP;") + + +def test_vector_sync_mode_is_part_of_ddl_and_drift_contract() -> None: + logical = schema_for(Episode) + ddl = build_create_table("unit_episode", logical, "async") + assert ddl.count("SYNC_MODE=async") == 2 + missing, stale, incompatible = index_drift( + physical_indexes(logical, "immediate"), (), create_sql=ddl + ) + assert missing == [] + assert stale == [] + assert any("sync_mode 'async' != 'immediate'" in item for item in incompatible) + + +def test_vectorless_profile_does_not_get_a_dummy_vector() -> None: + ddl = build_create_table("unit_profile", schema_for(UserProfile)) + assert "VECTOR(" not in ddl + assert "VECTOR INDEX" not in ddl + + +def test_only_character_columns_carry_collation_expectations() -> None: + columns = {column.name: column for column in physical_columns(schema_for(Episode))} + assert columns["id"].collation == "utf8mb4_bin" + assert columns["episode_tokens"].collation == "utf8mb4_bin" + assert columns["sender_ids"].collation is None + assert columns["timestamp_ms"].collation is None + + +def test_integer_display_width_is_not_schema_drift() -> None: + assert normalize_sql_type(" BIGINT ( 20 ) ") == "bigint" + expected = physical_columns(schema_for(UserProfile)) + reported = [_catalog_column(column, integer_width=True) for column in expected] + assert column_drift(expected, reported) == ([], [], []) + + for row in reported: + for key in ( + "COLUMN_NAME", + "COLUMN_TYPE", + "IS_NULLABLE", + "COLUMN_KEY", + "CHARACTER_SET_NAME", + "COLLATION_NAME", + ): + if isinstance(row[key], str): + row[key] = row[key].encode() + assert column_drift(expected, reported) == ([], [], []) + + +def test_column_drift_reports_missing_stale_type_and_collation() -> None: + expected = physical_columns(schema_for(UserProfile)) + reported = [ + _catalog_column(column) for column in expected if column.name != "summary" + ] + id_row = next(row for row in reported if row["COLUMN_NAME"] == "id") + id_row["COLUMN_TYPE"] = "BIGINT" + id_row["COLLATION_NAME"] = "utf8mb4_general_ci" + reported.append( + { + "COLUMN_NAME": "stale", + "COLUMN_TYPE": "BIGINT", + "IS_NULLABLE": "YES", + "COLUMN_KEY": "", + "CHARACTER_SET_NAME": None, + "COLLATION_NAME": None, + } + ) + missing, stale, incompatible = column_drift(expected, reported) + assert missing == ["summary"] + assert stale == ["stale"] + assert any("id: type" in item for item in incompatible) + assert any("id: collation" in item for item in incompatible) + + +def test_index_drift_checks_stats_ddl_parameters_and_stale_indexes() -> None: + logical = schema_for(Episode) + expected = physical_indexes(logical) + reported = _catalog_indexes(expected) + ddl = build_create_table("unit_episode", logical) + assert index_drift(expected, reported, create_sql=ddl) == ([], [], []) + + wrong = ddl.replace("DISTANCE=cosine", "DISTANCE=l2", 1) + missing, stale, incompatible = index_drift(expected, reported, create_sql=wrong) + assert missing == [] + assert stale == [] + assert any("distance 'l2' != 'cosine'" in item for item in incompatible) + + wrong = ddl.replace( + "FULLTEXT INDEX `ft_episode_tokens`", + "INDEX `ft_episode_tokens`", + ) + _, _, incompatible = index_drift(expected, reported, create_sql=wrong) + assert any("DDL type 'btree' != 'fulltext'" in item for item in incompatible) + + with_stale = [ + *reported, + { + "INDEX_NAME": "ix_obsolete", + "COLUMN_NAME": "id", + "INDEX_TYPE": "BTREE", + "SEQ_IN_INDEX": 1, + "SUB_PART": None, + }, + ] + _, stale, _ = index_drift(expected, with_stale, create_sql=ddl) + assert stale == ["ix_obsolete"] + + +def _catalog_column( + column: PhysicalColumn, *, integer_width: bool = False +) -> dict[str, object]: + sql_type = column.sql_type + if integer_width and sql_type == "BIGINT": + sql_type = "bigint(20)" + return { + "COLUMN_NAME": column.name, + "COLUMN_TYPE": sql_type, + "IS_NULLABLE": ("YES" if column.nullable and not column.primary else "NO"), + "COLUMN_KEY": "PRI" if column.primary else "", + "CHARACTER_SET_NAME": column.character_set, + "COLLATION_NAME": column.collation, + } + + +def _catalog_indexes(indexes: Sequence[PhysicalIndex]) -> list[dict[str, object]]: + rows: list[dict[str, object]] = [] + for index in indexes: + prefixes = index.prefix_lengths or (None,) * len(index.columns) + index_type = { + "btree": "BTREE", + "fulltext": "FULLTEXT", + "vector": "VECTOR", + }[index.kind] + rows.extend( + { + "INDEX_NAME": index.name, + "COLUMN_NAME": column, + "INDEX_TYPE": index_type, + "SEQ_IN_INDEX": position, + "SUB_PART": prefix, + } + for position, (column, prefix) in enumerate( + zip(index.columns, prefixes, strict=True), start=1 + ) + ) + return rows diff --git a/tests/unit/test_infra/test_seekdb/test_sql.py b/tests/unit/test_infra/test_seekdb/test_sql.py new file mode 100644 index 000000000..bf5543c36 --- /dev/null +++ b/tests/unit/test_infra/test_seekdb/test_sql.py @@ -0,0 +1,35 @@ +"""Pin SeekDB identifier and literal escaping against injection-shaped input.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from everos.infra.persistence.seekdb.sql import ( + json_literal, + literal, + quote_identifier, +) + + +def test_identifier_validation_is_allow_listed() -> None: + assert quote_identifier("everos_episode") == "`everos_episode`" + for invalid in ("a.b", "bad-name", "x`; DROP TABLE t", ""): + with pytest.raises(ValueError, match="invalid SQL identifier"): + quote_identifier(invalid) + + +def test_literals_escape_mysql_control_and_quote_characters() -> None: + assert literal("o'reilly") == "'o''reilly'" + assert literal("a\\b") == "'a\\\\b'" + assert literal("'; DROP TABLE memories") == "'''; DROP TABLE memories'" + assert literal(True) == "TRUE" + assert literal(None) == "NULL" + assert literal(dt.datetime(1970, 1, 1, tzinfo=dt.UTC)) == "0" + + +def test_json_literals_are_compact_and_finite() -> None: + assert json_literal(["红 苹果", "quote'"]) == "'[\"红 苹果\",\"quote''\"]'" + with pytest.raises(ValueError, match="finite"): + literal(float("nan")) diff --git a/uv.lock b/uv.lock index 3b55b74de..14bd484ca 100644 --- a/uv.lock +++ b/uv.lock @@ -304,6 +304,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, ] +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -410,6 +419,81 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/21/0e/8459ca4413e1a21a06c97d134bfaf18adfd27cea068813dc0faae06cbf00/cssselect2-0.9.0-py3-none-any.whl", hash = "sha256:6a99e5f91f9a016a304dd929b0966ca464bcfda15177b6fb4a118fc0fb5d9563", size = 15453, upload-time = "2026-02-12T17:16:38.317Z" }, ] +[[package]] +name = "cuda-bindings" +version = "13.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/d0/76d0e45d98bf4933bf48eac6bbeb17464684540f69edec41fc37c7a422b0/cuda_bindings-13.4.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:84ee88862e2e6ac39a5434c061f7f4389fbefbc418487d0670c86601d517d038", size = 6479976, upload-time = "2026-09-10T01:16:54.622Z" }, + { url = "https://files.pythonhosted.org/packages/43/56/d7b219516980f3333e232c13d727e8f4dc59afc5381cbbcfbc1215014c81/cuda_bindings-13.4.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f444d7e488cbc47e79b7be0d1cfe97201f3e1f186a18ee10f2e4da3265975b16", size = 7170956, upload-time = "2026-09-10T01:16:56.854Z" }, + { url = "https://files.pythonhosted.org/packages/38/cf/165b4d449f94956c2a60930cf5dfeb27132ead60a7e7f2c37819df1cba07/cuda_bindings-13.4.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b601c0cbf0dffb648f68e56a60b320738a20210293f33896a1964a6438cc65f1", size = 6313772, upload-time = "2026-09-10T01:17:03.969Z" }, + { url = "https://files.pythonhosted.org/packages/d9/f9/cf021d1560541caa1f35f3e7e311d2678dbacb4fa6a4573b63470fe1ae00/cuda_bindings-13.4.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2c357698588b06ebd65811ee2013b6650dd0a10d16d899924aabad0d606d76", size = 6924300, upload-time = "2026-09-10T01:17:06.23Z" }, + { url = "https://files.pythonhosted.org/packages/58/17/74346b49114779920929ec0ea1361f0a0357262d6020cfdd017f670da638/cuda_bindings-13.4.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7df2dddb81feb15787e8c4a13b7aa3f4c23eabeec7746037e94dbbed065fc6c", size = 6406821, upload-time = "2026-09-10T01:17:12.344Z" }, + { url = "https://files.pythonhosted.org/packages/f6/77/2f9a38be7399a34e3703b6ce1be60bb2c56611d324750f8d544ff90b0473/cuda_bindings-13.4.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62df23df11074e9833bf348bbcf0b8eec2fcbded4f305c6fbaa3ed067433e97d", size = 6972525, upload-time = "2026-09-10T01:17:15.098Z" }, + { url = "https://files.pythonhosted.org/packages/7d/de/41197aebf91f6c5f82b35e06e3b4bbe08edddb335c4d6e53c1f1fe542e0f/cuda_bindings-13.4.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0947a6c491a622b076e9afcbc0444eefd9cbe001630558bbcba8d0a90dab7fe7", size = 6243282, upload-time = "2026-09-10T01:17:21.158Z" }, + { url = "https://files.pythonhosted.org/packages/be/d6/1b697092f53cfd4d721fbbe13e7d66dc72a9d508156af9be4c86607e4373/cuda_bindings-13.4.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e730e80d997b6037566033a79aae3995e1981654054536f512e5c091679c70", size = 6803245, upload-time = "2026-09-10T01:17:23.309Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.8.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/e6/22df83f82f9bc26cb1c42265cf14d34d4908dba2a0f261bd7b28244acb00/cuda_pathfinder-1.8.1-py3-none-any.whl", hash = "sha256:ae0137ff9e56ea97499bcbf54f5f2778ec25f3266715ac86da192a795af982a8", size = 62552, upload-time = "2026-09-02T16:55:28.64Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +curand = [ + { name = "nvidia-curand", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +cusolver = [ + { name = "nvidia-cublas", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusolver", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "(python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] + [[package]] name = "decorator" version = "5.2.1" @@ -625,6 +709,13 @@ otel = [ { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, ] +seekdb = [ + { name = "pyseekdb" }, +] +seekdb-embedded = [ + { name = "pylibseekdb", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "pyseekdb" }, +] [package.dev-dependencies] dev = [ @@ -668,7 +759,10 @@ requires-dist = [ { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, + { name = "pylibseekdb", marker = "(sys_platform == 'darwin' and extra == 'seekdb-embedded') or (sys_platform == 'linux' and extra == 'seekdb-embedded')", specifier = ">=1.4.0.post1,<1.5" }, { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=3.0.0" }, + { name = "pyseekdb", marker = "extra == 'seekdb'", specifier = ">=1.4.0.post1,<1.5" }, + { name = "pyseekdb", marker = "extra == 'seekdb-embedded'", specifier = ">=1.4.0.post1,<1.5" }, { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, @@ -679,7 +773,7 @@ requires-dist = [ { name = "watchdog", specifier = ">=4.0.0" }, { name = "watchfiles", specifier = ">=0.21.0" }, ] -provides-extras = ["multimodal", "otel", "milvus"] +provides-extras = ["multimodal", "otel", "milvus", "seekdb", "seekdb-embedded"] [package.metadata.requires-dev] dev = [ @@ -731,6 +825,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, ] +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl", hash = "sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", size = 26661, upload-time = "2025-12-19T23:16:13.622Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.75.0" @@ -929,6 +1040,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "hf-xet" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -993,6 +1128,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/0f/ed994dbade67a54407c28cab96ef845e0e6d25500be56aca6394f8bfc9dd/huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832", size = 792534, upload-time = "2026-05-21T18:40:00.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/79/621a7dbb80c70974f73a597275351ebe03ce5bc65cb5f8f4acb5859252bc/huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22", size = 668176, upload-time = "2026-05-21T18:39:58.596Z" }, +] + [[package]] name = "identify" version = "2.6.19" @@ -1100,6 +1255,18 @@ version = "0.42.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/c6/cb/18eeb235f833b726522d7ebed54f2278ce28ba9438e3135ab0278d9792a2/jieba-0.42.1.tar.gz", hash = "sha256:055ca12f62674fafed09427f176506079bc135638a14e23e25be909131928db2", size = 19214172, upload-time = "2020-01-20T14:27:23.5Z" } +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jiter" version = "0.14.0" @@ -1172,6 +1339,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/e9/1f9ada30cef7b05e74bb06f52127e7a724976c225f46adb65c37b1dadfb6/jiter-0.14.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67f00d94b281174144d6532a04b66a12cb866cbdc47c3af3bfe2973677f9861a", size = 349613, upload-time = "2026-04-10T14:28:40.066Z" }, ] +[[package]] +name = "joblib" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpickle", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/1d/537ab090f302b838943a1b56497dd53059b9a9b46a074936470173a2e207/joblib-1.6.0.tar.gz", hash = "sha256:2ccc96785b12046c08fd6d55839c12857831b54a3c1673ffadd2f04bfc4eda03", size = 327903, upload-time = "2026-08-31T09:39:04.122Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/53/84099323c2ec4be98d935f63c033ac4151ee83836ca1050ede3b3aadf155/joblib-1.6.0-py3-none-any.whl", hash = "sha256:3dbbf9f6e4b592a2357b854608e980fe6390d131d7a82f011a377ef2ebef7aba", size = 306115, upload-time = "2026-08-31T09:39:02.298Z" }, +] + [[package]] name = "lance-namespace" version = "0.9.0" @@ -1356,6 +1535,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "narwhals" +version = "2.26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/68/5351e34623d253423240ea7de3f8fc74fa8ab14b1ab3c0ec4ac8997413c9/narwhals-2.26.0.tar.gz", hash = "sha256:6b9cadca82f375c7e4cf584fdc86ca25da54827307a9c58f94547ee6104b82dd", size = 686970, upload-time = "2026-09-08T13:32:08.964Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/b5/1b84b2c784db76d69442334bc8b8748c840f13ca53be086f4f250ad4a0bc/narwhals-2.26.0-py3-none-any.whl", hash = "sha256:29326d74f107c347fd1009bd58e38d9f7c7c5b51e6de97bc93dbc325d9038b54", size = 474034, upload-time = "2026-09-08T13:32:07.159Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -1416,6 +1622,190 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, ] +[[package]] +name = "nvidia-cublas" +version = "13.1.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.24.0.43" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ba/791cffd048fe5b044e620df55267e3e95c0e6e07d50b41e377c03dfc910f/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:71f181cd810e90f9b6023b01186fe82d13d65f0ec098581ee201d39fad769e4b", size = 553099438, upload-time = "2026-07-02T16:27:42.58Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cusparse", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, + { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.30.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/21/a73174c6157101bdf1ffc22b517f76ff0082613989dd9bc8f43e8034caac/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:ca786ffa5a647c75d4d1f5cc72a6c4f537947e2ba8823d7c8aaf768e7a7b9f77", size = 215983881, upload-time = "2026-06-09T03:23:15.633Z" }, + { url = "https://files.pythonhosted.org/packages/3f/34/c500f90c7ae641b8e0f98965b36b8a7ac79cc8b296e8d251fe3eb592ee54/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:cefa7fdb9710efd0f39c5f1be1d61ff6fc9a996c451265bd7fbdcf9455ed4b50", size = 215965170, upload-time = "2026-06-09T03:23:39.73Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.4.52" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/c1/091f198d7f87e31d67fa9680eec8f8e4c6f889881f729b759db36ff01612/nvidia_nvjitlink-13.4.52-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:90401db7e5a580a5067a468b3086e0b65f3b96ab3c42524afd741efc8a0e150a", size = 42452221, upload-time = "2026-09-09T18:01:32.895Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/225ff51e80de170be880cb88e992193bc8134a51059cc0a3952f967f62c5/nvidia_nvjitlink-13.4.52-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3a589e3732140839349545efd0db67426523459b12e6952c3c73b6d55900f200", size = 40419746, upload-time = "2026-09-09T18:01:25.103Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + +[[package]] +name = "onnxruntime" +version = "1.30.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flatbuffers", marker = "python_full_version < '3.14'" }, + { name = "numpy", marker = "python_full_version < '3.14'" }, + { name = "packaging", marker = "python_full_version < '3.14'" }, + { name = "protobuf", marker = "python_full_version < '3.14'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/6f/48169f2e62b405bff5053cbd1d73fb5ce41ef7ecd13bb3bfcc191e689b8a/onnxruntime-1.30.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:001ed726c9bd5e2bc92faade7d37d889e9606a350b7d5529f0227df2e3bb57fd", size = 21544867, upload-time = "2026-09-10T16:31:19.876Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/cbc5b8f91963689fdd622f463508c01d0aa95d3f944747b1e0b1eb2160b8/onnxruntime-1.30.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6c32a000d5139a38ba9349030b0032e3331acb559d596b22738d9d2b343a2b83", size = 21345202, upload-time = "2026-09-10T16:31:23.361Z" }, + { url = "https://files.pythonhosted.org/packages/34/35/e7f862dbacbc99fadd9b14a614e49c99bf0f35fd9927a82f096e3de33531/onnxruntime-1.30.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:fa688e7891a6aa206636fe7372e27ee75fd17713289f6b4fc7b190e0a7de9328", size = 23585654, upload-time = "2026-09-10T16:31:26.65Z" }, + { url = "https://files.pythonhosted.org/packages/a6/13/0f1699f6de549c9324bc9112a2a85b14c517904cd11b562a654643b755a1/onnxruntime-1.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:f3501472571f1b1eee50e017851e7929f5ea37312d2d8c2494a19e8fc58b4a38", size = 14311470, upload-time = "2026-09-10T16:31:31.273Z" }, + { url = "https://files.pythonhosted.org/packages/a2/17/02b13e5f51461f0453b18ab854e2d0bc1b6ec353241b05a2ee6b79e35d87/onnxruntime-1.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:dc4c706f1935ebb62356e6a095b047859badd854482c40560888e95c328ed262", size = 14175072, upload-time = "2026-09-10T16:31:34.541Z" }, + { url = "https://files.pythonhosted.org/packages/f0/75/508454c5d01f31641dabc597fe559594c931a520a2673031179319d0afd8/onnxruntime-1.30.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:05e4fc41711d1f4abd19a9124b5be7a65a506cb2670a7f14b16162ef13c58134", size = 21544990, upload-time = "2026-09-10T16:31:37.685Z" }, + { url = "https://files.pythonhosted.org/packages/89/06/e603c71f43f4fe3fd156a053af79cbed6e27a2c649f0988a67d97fedd39f/onnxruntime-1.30.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5327cf6aa15a02bad805fac8bd6882a62571e8b72f6f2938a8f37e6bd1966ce9", size = 21344996, upload-time = "2026-09-10T16:31:40.915Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a1/ede48ab5dc54907a2999362777f541e132639fb06628ded1932058aa8a36/onnxruntime-1.30.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:86f940afc801ea9681a4da8af84fbe95e1d9ea7d80903952cc1bfad54faad38f", size = 23585560, upload-time = "2026-09-10T16:31:44.941Z" }, + { url = "https://files.pythonhosted.org/packages/3c/dd/c57c529dbc6dd55eca24b12cfbeab1b6a690de72083824eca689085f55b0/onnxruntime-1.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:4b63041bd623a9a9ac5e353948436c6fa7f43edd12d6b4a4ebc340bca959ba93", size = 14311378, upload-time = "2026-09-10T16:31:48.034Z" }, + { url = "https://files.pythonhosted.org/packages/11/2f/ef00b45b911e7f2115273a24ed1b43e73c8c6c9ec17f9bfa13f8492581f8/onnxruntime-1.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:c389b6887fc95e0fcb80e89b156bc2cb18e662c29df55e9326fe64140e7d7b4f", size = 14175134, upload-time = "2026-09-10T16:31:50.799Z" }, + { url = "https://files.pythonhosted.org/packages/92/0a/284fd6fe701c9a8aff39dbc119c37582ca40e9cff1ea05425a7cc8606a02/onnxruntime-1.30.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5224ba2b00284cb1c48b3edcd303c109245d66df9ec1b861858fd6de672a38e", size = 21349453, upload-time = "2026-09-10T16:31:53.898Z" }, + { url = "https://files.pythonhosted.org/packages/d6/72/4f4466f8fa1ec267a9ef5e2f3bd175c203d3da0facbdf4049dc93abbe91c/onnxruntime-1.30.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3f9e002417f1e3bbb31ed43dafa4f22ca1b2b68832244fdee192fcaa1ae19bca", size = 23579837, upload-time = "2026-09-10T16:31:56.82Z" }, + { url = "https://files.pythonhosted.org/packages/6a/03/05c9a9234688757d2876ddecf80bba908561ee11debf125bc1a427ae6f48/onnxruntime-1.30.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:8b6169c16a48429890d2f4a0c774ebf54dfe9066a998514aad0518a16d398547", size = 21545313, upload-time = "2026-09-10T16:31:59.654Z" }, + { url = "https://files.pythonhosted.org/packages/c6/bc/1069e58b24779ba9d2fd479db5ecb3a15a6f49b585107c898819c0789558/onnxruntime-1.30.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:d2184fddb6798136e7c478244391ca82443f5c757f59f15bb9e5ad2da5e03175", size = 21346830, upload-time = "2026-09-10T16:32:02.275Z" }, + { url = "https://files.pythonhosted.org/packages/f1/38/8138eed225c5bc6ddfc05879ecac7dacc63c34b9b6f99be72839c1f6dc49/onnxruntime-1.30.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:8b611d24db2954545ce6bd9acd4670183cb368e4642450de7a9ab6474eb374ec", size = 23586333, upload-time = "2026-09-10T16:32:04.996Z" }, + { url = "https://files.pythonhosted.org/packages/4d/0a/748b86000fbf9518b5dfc5bdc1b924eeaca976203740fd9d30649e568ee6/onnxruntime-1.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bdd1752a8502ac1a7ccc6e878d16db6943df574d54ed7f01058a6806ae05be4", size = 14675319, upload-time = "2026-09-10T16:32:07.5Z" }, + { url = "https://files.pythonhosted.org/packages/2f/db/db8e4c0cf6f70f1311060560630fd21763648e01dfbba5195a2592c052ef/onnxruntime-1.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:83d543843cbd352cfa9996a6c7b92f8a480f114a18f95ff2a1acaf6921e85b6d", size = 14569360, upload-time = "2026-09-10T16:32:09.896Z" }, + { url = "https://files.pythonhosted.org/packages/24/0a/ec0a9d656e39b43887c378a5388b20c3b1e1ee43bded84580b0936466694/onnxruntime-1.30.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:265de607ba6f9814264e1d5d413fa7069d70f48e3049b5e460b3b04bdfcef294", size = 21348433, upload-time = "2026-09-10T16:32:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/91/f0/40f74b7c00077e1e25627067ed98a70df1fef5c0e21b82849190d312554e/onnxruntime-1.30.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:67ad7f03433b6462c627d0f555dece80e6a26bc71e8542ced35cebd32142d1b7", size = 23579340, upload-time = "2026-09-10T16:32:15.532Z" }, +] + [[package]] name = "openai" version = "2.36.0" @@ -2046,6 +2436,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/97/03635143a12a5d941f545548b00f8ac39d35565321a2effb4154ed267338/pyinstrument-5.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:b6a71f5e7f53c86c9b476b30cf19509463a63581ef17ddbd8680fee37ae509db", size = 128164, upload-time = "2026-01-04T18:38:32.281Z" }, ] +[[package]] +name = "pylibseekdb" +version = "1.4.0.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pymysql", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/84/2c3c3b540ab23aeb46c14ac337e1cd4401fa5234fa24b0f110944dc93a36/pylibseekdb-1.4.0.post1-cp312-abi3-macosx_15_0_arm64.whl", hash = "sha256:ad34939175ee900e0e0af52d32868a50d5470200a34cba32558678b5edb8ed3f", size = 50711987, upload-time = "2026-09-14T07:44:12.633Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1a/3e9e68b8485009153fa0792b0dc1c176a151a4e815759fe7f0dc7b166a2d/pylibseekdb-1.4.0.post1-cp312-abi3-macosx_15_0_x86_64.whl", hash = "sha256:0a0a9a68639547f721f581f184faa793a25e70414e6bed2edf14f379224165d8", size = 58928002, upload-time = "2026-09-14T13:04:16.838Z" }, + { url = "https://files.pythonhosted.org/packages/2f/c8/4125f7926273e5f1217656e9a61715e5b67b8ce42c1e706ae2833b34d6b9/pylibseekdb-1.4.0.post1-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e2887533ab68a84197a222393e6d8fc8282274aa5e99af4f8259c5dfce147914", size = 49396173, upload-time = "2026-09-08T07:53:55.759Z" }, + { url = "https://files.pythonhosted.org/packages/83/06/c39545d88ddec475272991d1c5612f5d5463b843cf29553e5a4ac82ef09e/pylibseekdb-1.4.0.post1-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:58f2544cf7969a9b6cdf4ea1136e0d412443a2e839df656feec1b2712741ef85", size = 53973930, upload-time = "2026-09-08T07:53:07.949Z" }, +] + [[package]] name = "pymilvus" version = "3.0.1" @@ -2064,6 +2468,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/9d/7011887b29f452905745e8bd321f404068d5bfe78fe84e42c0b7cd81a065/pymilvus-3.0.1-py3-none-any.whl", hash = "sha256:c5a8d5c1fa1de7b416e3529d383d8cc2e7da2170433ffa4a2d9087e14f70171a", size = 386820, upload-time = "2026-07-29T14:55:44.279Z" }, ] +[[package]] +name = "pymysql" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/bc/1c6a92f385940f727daeecf3bacaf186e03875dff57197801046c583bcf0/pymysql-1.2.0.tar.gz", hash = "sha256:6c7b17ca686988104d7426c27895b455cdeea3e9d3ceb1270f0c3704fead8c33", size = 49021, upload-time = "2026-05-19T08:26:22.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/bd/2534e130295c8cfd4f0a2e31623baab7502278f1e97bcfe61db75656a77f/pymysql-1.2.0-py3-none-any.whl", hash = "sha256:62169ce6d5510f08e140c5e7990ee884a9764024e4a9a27b2cc11f1099322ae0", size = 45716, upload-time = "2026-05-19T08:26:20.974Z" }, +] + +[[package]] +name = "pyseekdb" +version = "1.4.0.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", marker = "python_full_version < '3.14'" }, + { name = "numpy" }, + { name = "onnxruntime", marker = "python_full_version < '3.14'" }, + { name = "pylibseekdb", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or sys_platform == 'linux'" }, + { name = "pymysql" }, + { name = "sentence-transformers", marker = "python_full_version >= '3.14'" }, + { name = "tenacity" }, + { name = "tokenizers", marker = "python_full_version < '3.14'" }, + { name = "tqdm", marker = "python_full_version < '3.14'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/87/d5dd862faa3d4adf3847c1ce19c3ea5ecd0dcfda9c2584a95bfd2b0fac0f/pyseekdb-1.4.0.post1-py3-none-any.whl", hash = "sha256:a3379f6962a0c01aa029d3e5a8f0c0f5a59b27a689b8aae1931d9ce5563f252c", size = 158375, upload-time = "2026-08-03T08:56:59.501Z" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -2369,6 +2801,181 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + +[[package]] +name = "scikit-learn" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib", marker = "python_full_version >= '3.14'" }, + { name = "narwhals", marker = "python_full_version >= '3.14'" }, + { name = "numpy", marker = "python_full_version >= '3.14'" }, + { name = "scipy", marker = "python_full_version >= '3.14'" }, + { name = "threadpoolctl", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d2/eb/eaf5e07fcc0da7149b0e084f24e54edd7441b9a89ce7e034032ae97fe3a0/scikit_learn-1.9.1.tar.gz", hash = "sha256:629cada3e33e2b9bf376cdc7614a47a4140b8aedc1d836579e359736fbd82977", size = 7786908, upload-time = "2026-09-10T18:34:04.679Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/a7/25f0a43d2fde306e8ef45f45121192f687b79beaf4bae8c21607c46c5e63/scikit_learn-1.9.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0c0f8b5d09b44101cea2767f300680bada1ea27f976fe4b48b83950a4f55a49a", size = 8775209, upload-time = "2026-09-10T18:32:42.804Z" }, + { url = "https://files.pythonhosted.org/packages/60/ea/57e57539ce175d774fc291ed091b0a6d756854b92cd92554c6bb4d0ae498/scikit_learn-1.9.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:8c14ce41d561f7749f990b41d6703fe02c4669fbc485e598e069e0a1967b488e", size = 8295541, upload-time = "2026-09-10T18:32:45.069Z" }, + { url = "https://files.pythonhosted.org/packages/78/2b/5721a174406bfba49bce20ae997b3b64cf355c3f623a2638284ab6a82156/scikit_learn-1.9.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4c20a6c017d820faa7ac8c783e3d0c6a9a2e297bf9f55332ca17cdf7fd4d04d", size = 8871592, upload-time = "2026-09-10T18:32:46.999Z" }, + { url = "https://files.pythonhosted.org/packages/8e/57/a50162f3d29feb979ab6347c6debda506dfb525bcff3c50dd17606651c7e/scikit_learn-1.9.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e5d7b18a5b9dca241a74695f3275fa4c895a9dadc72b3d8df5fa9d1083c9b83e", size = 9166043, upload-time = "2026-09-10T18:32:49.343Z" }, + { url = "https://files.pythonhosted.org/packages/72/8d/27c054166bac671770d1ea0ef7716134fe8119c0df224a58c89fd735a61c/scikit_learn-1.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:4b59abb30618121cc46b45972d6bf53a7128b4df4cd346c6ca6f4d5f9031e49c", size = 8262238, upload-time = "2026-09-10T18:32:51.679Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d6/493086006ea0c68ad62c40a8dece1961b61bf503f45400f133d47f56e5be/scikit_learn-1.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:d5945a2908be62350e2978344e62b56c1552c2ca4f844ebf6277c94944d647dd", size = 7902102, upload-time = "2026-09-10T18:32:53.67Z" }, + { url = "https://files.pythonhosted.org/packages/bb/8d/b60d5e7354ff0ff5cc9400e60273696589d87a30b8b2235886a76d80d062/scikit_learn-1.9.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c2b312fd8c02951a364fa120ea08c1cec10d863466bf1701b013152d7537835", size = 8739300, upload-time = "2026-09-10T18:32:55.483Z" }, + { url = "https://files.pythonhosted.org/packages/2f/81/3c6392c03665d2899457a76e535a9a6f597dddddf3220fd2e1d790da88c5/scikit_learn-1.9.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:61cd968ab831a76d0ecbaf0347ab2270268716da28f94fd022497e3d6f205f13", size = 8262364, upload-time = "2026-09-10T18:32:57.966Z" }, + { url = "https://files.pythonhosted.org/packages/0f/35/a15b8653499692879821301d48059376d6e68e8b65cd0f22d19b6ee83cd9/scikit_learn-1.9.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5990f9c69e431bfaddcde1a6d7c5355243e026bc9b9e560c13893b90dab53fb4", size = 8823698, upload-time = "2026-09-10T18:33:00.632Z" }, + { url = "https://files.pythonhosted.org/packages/23/e5/688703d357e5393f708d98eb189fd415ae69e39f6de03c6bd4005aef6118/scikit_learn-1.9.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:55e79d6e9b0923f1a978179822bd43d7f5543f45e970a00fe861f43486380aba", size = 9121732, upload-time = "2026-09-10T18:33:02.825Z" }, + { url = "https://files.pythonhosted.org/packages/96/45/a10add34c08184d373be9384660c75758128ca881ed27b503b6f6a742478/scikit_learn-1.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:2070f271e5375dc42c6bb93b461ab1c0aa5841d4009267e0cfd95a39dca94a43", size = 8237244, upload-time = "2026-09-10T18:33:05.26Z" }, + { url = "https://files.pythonhosted.org/packages/9e/08/7a89bcdadd1fff0d464d01056417b646c9abcbc54f7297a0a1203bba5ebb/scikit_learn-1.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:613f0a783ca05aa844a4e1ac42d48425058f2c52be73f40f8cd98b7cd111acd6", size = 7875398, upload-time = "2026-09-10T18:33:07.506Z" }, + { url = "https://files.pythonhosted.org/packages/64/e3/b58e45082dcf3dcf0eb1192ee03545ec43d8c98441dfe20e88afca8442ce/scikit_learn-1.9.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d5d117952769b563067656784e03c75a2d8235a7a05cf7fffa78a311e75aac08", size = 8747414, upload-time = "2026-09-10T18:33:10.698Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ed/d68115577c8b42b0442ebd8180945d4008880a33176094640e00b585128e/scikit_learn-1.9.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:8893bc6331f60f18d4ac75e12ed356e2dcf6a564bf767918b5b7ca54c8c8be49", size = 8279175, upload-time = "2026-09-10T18:33:12.846Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6c/06c7eb61a438e389cbf3f7210897069bec5a883dbe03c189ae792781e11a/scikit_learn-1.9.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5492cf2df5226691c32611de8734bcf42148c6547ae53c7f4e6b847793addc0", size = 8881706, upload-time = "2026-09-10T18:33:14.741Z" }, + { url = "https://files.pythonhosted.org/packages/86/4e/0bab75490ca4b85fad8388739c7ebc71d9db553f8c69e39943ee8db0aaae/scikit_learn-1.9.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:993d332ff80e62efae9e39603b7e872297c418d780f01a01855269a3489c950f", size = 9152030, upload-time = "2026-09-10T18:33:17.436Z" }, + { url = "https://files.pythonhosted.org/packages/9a/13/31c6f8ba1b7eecef9dd9558576c752d2ec5785fd0e456bb9fc59305be23f/scikit_learn-1.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:ca9051447455dae341d4d591eece7deb2d8e3d1020298fc87a81fc51e4da8f53", size = 8377024, upload-time = "2026-09-10T18:33:19.941Z" }, + { url = "https://files.pythonhosted.org/packages/62/e6/6d3cb8a45f5228f915acd66b819dd6b8232ccbe24532f51d278e3991df31/scikit_learn-1.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:90de6573f733a9fb79476ff1371af52a397d41c8b35f9146e20923db010d67b6", size = 8014196, upload-time = "2026-09-10T18:33:22.126Z" }, + { url = "https://files.pythonhosted.org/packages/66/6e/6befb2d5961490d18d9dbc16a5df37aa121d08bc9893316a6a363977a903/scikit_learn-1.9.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b5cad1624de8b75e5b9ccb7b0ce1ff1d01306340a3efc56d5529c5ba92392eb", size = 9066066, upload-time = "2026-09-10T18:33:24.396Z" }, + { url = "https://files.pythonhosted.org/packages/cb/18/11271f2f7db337db01f598e358721b1e83989407131272e5dd64214c28a8/scikit_learn-1.9.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:d137ce8a6142029fb5c35bd82f470c40cd9e760e5e2f7694b362c497c4ab3fa2", size = 8647654, upload-time = "2026-09-10T18:33:26.649Z" }, + { url = "https://files.pythonhosted.org/packages/e7/04/9c15d201e1b6a2e81b8215865df7646c5a360f560831769c8dc92ac1ab9a/scikit_learn-1.9.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:66f852f7325b5070bc28329005aca76055a2def78faac039548ae889aeaa45a6", size = 8889070, upload-time = "2026-09-10T18:33:28.95Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5a/4cb6c85160af4a639e87a3b7bf8b1c25cfc3b504c5af710ca416a6dcfc5f/scikit_learn-1.9.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:748bcb0a4cc04aec470652c9e5ec68450948e867387e7dfade647107ade68d25", size = 9131227, upload-time = "2026-09-10T18:33:31.008Z" }, + { url = "https://files.pythonhosted.org/packages/00/0e/361440972ae3d19b90ea88a84791138a51de0e8432a770ba741b2c8d9ced/scikit_learn-1.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:38cd925e893e5539be704d5edc64dbe081aacdab6b89d8c2977c1f6a7a453ce5", size = 8683001, upload-time = "2026-09-10T18:33:33.188Z" }, + { url = "https://files.pythonhosted.org/packages/e2/8f/a9f405c5c0e2df6f343a871b40c97fb32969e3ccc38e3033dd118f3c261e/scikit_learn-1.9.1-cp314-cp314t-win_arm64.whl", hash = "sha256:b01e5b01735d38474127ca3f49319b592506225a87793b27559816b5c75cea39", size = 8257996, upload-time = "2026-09-10T18:33:35.343Z" }, + { url = "https://files.pythonhosted.org/packages/e5/c5/74a83ea39cef7cd07f53e06cc1cf51e79f35df74f81db956835d59ec34b1/scikit_learn-1.9.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:8dec64f31a6e0ec826aca6c1b39a51e16d946e400d4c0904316f3ca72ccfb825", size = 8747991, upload-time = "2026-09-10T18:33:37.4Z" }, + { url = "https://files.pythonhosted.org/packages/64/c9/cc93e8a7fe204e43d70e96e5eb89871643be20025eea05eb4fdaf19afe39/scikit_learn-1.9.1-cp315-cp315-macosx_12_0_arm64.whl", hash = "sha256:e1b468241f4a7a9a7a0d6479ad3cc47681cc151a4046c530f2777c3d68f08942", size = 8279340, upload-time = "2026-09-10T18:33:39.705Z" }, + { url = "https://files.pythonhosted.org/packages/a2/61/0c6080f0d356fb966053009e7f25e9bff6cf74b0b16195ccf0c3757d1ae8/scikit_learn-1.9.1-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ca869d0080a5723cde2d5a8b54a2da1ff7e68735a9e9adb3da1243183a0fa01", size = 8881917, upload-time = "2026-09-10T18:33:41.912Z" }, + { url = "https://files.pythonhosted.org/packages/47/bb/98a31f10fffbd39edcc2f8bf4119b29652248bb110b7d45c84e68aa293ab/scikit_learn-1.9.1-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6754b7cabfc3df0b1f7b38f7a344f559bbae9d82f0ac5e3d48cccbd19fdcefdf", size = 9152805, upload-time = "2026-09-10T18:33:44.23Z" }, + { url = "https://files.pythonhosted.org/packages/f6/48/858ceff52213cfd97c0a069362071756bcb70b9fac9771b38d87c4cf7f17/scikit_learn-1.9.1-cp315-cp315-win_amd64.whl", hash = "sha256:52cfdb1fed3a34362dbc0bd96f2e761a66fd5724d6901629f5a558f1f3bd9849", size = 8376719, upload-time = "2026-09-10T18:33:46.492Z" }, + { url = "https://files.pythonhosted.org/packages/2a/1e/5337a871bdea53effbd154b61429df048f2665653251de74a3bd8a6dea9e/scikit_learn-1.9.1-cp315-cp315-win_arm64.whl", hash = "sha256:ae6571a4828c6f5019bcd2b4125e5b18c0af3dbc9c99726c891f45f41335ec8e", size = 8014341, upload-time = "2026-09-10T18:33:48.762Z" }, + { url = "https://files.pythonhosted.org/packages/0e/35/150383a42d83ec4c7b39f9c50bd68408ecf04c19fc30ea5198fa42e67d9c/scikit_learn-1.9.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:48fefd8eb42bd4eec3e2d348149368ccd6d71987e20c30706a56a24eb86a6e73", size = 9059953, upload-time = "2026-09-10T18:33:50.951Z" }, + { url = "https://files.pythonhosted.org/packages/9f/dd/aa0d738808540f7eaacfab93e01982db8ef1c1c7473ef0ad38193e6aebd1/scikit_learn-1.9.1-cp315-cp315t-macosx_12_0_arm64.whl", hash = "sha256:09f4d73049cd63575157f6b1060e06a8c83a4bd3488dbfaeedf35ccba7aad712", size = 8646602, upload-time = "2026-09-10T18:33:53.23Z" }, + { url = "https://files.pythonhosted.org/packages/7e/cc/687ae4214c2f598906c3b9fa5f86fbaf834b35e20360625528ff1b713f06/scikit_learn-1.9.1-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3da53831534214322d9cb240fa6f390b36cf69eba727a6d4bd3238677630d70", size = 8880366, upload-time = "2026-09-10T18:33:55.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/03/82215cb78ad1c513a4498777571fb28444621ad26ef636287551767b7732/scikit_learn-1.9.1-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:caae15634feceafa2612566b109a3082d3293167fac388eedaf77bff66b51983", size = 9129640, upload-time = "2026-09-10T18:33:58.186Z" }, + { url = "https://files.pythonhosted.org/packages/d4/90/4b7af4efd7909a4a0524a9f18457e4eb2eb60616eff2c7627eda8e3cdceb/scikit_learn-1.9.1-cp315-cp315t-win_amd64.whl", hash = "sha256:ffbcbbbb44202fbe9bc64bced25a145759adb9ef010b3d37a8064958ac13df2a", size = 8678083, upload-time = "2026-09-10T18:34:00.354Z" }, + { url = "https://files.pythonhosted.org/packages/31/27/068e484d4b83004302e0d9cfc1faca69bcc010d76fbb66a642446095af1b/scikit_learn-1.9.1-cp315-cp315t-win_arm64.whl", hash = "sha256:800dd22dd87fe97dcea484c24e85dd93cf1734d86bd74e668ad18f7967f4d1b5", size = 8259434, upload-time = "2026-09-10T18:34:02.678Z" }, +] + +[[package]] +name = "scipy" +version = "1.18.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/f7/240c110c08693826b4513a52f5717d62ec7c7af72f2920821247c03b17b3/scipy-1.18.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:457fd7a2a8edeb044ab6ffbc0aa03ff6cd18491356e5e0c834d76ce621b916d1", size = 31111061, upload-time = "2026-08-21T23:23:44.522Z" }, + { url = "https://files.pythonhosted.org/packages/05/4a/78c6285577c375e7cf27277ea8ee6961224327f1e1a0c44af5f17f23635c/scipy-1.18.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:e708533e8b2ae2497d65346538a7dcc92814410b25b81432eac66de0f2af8265", size = 28733332, upload-time = "2026-08-21T23:23:50.015Z" }, + { url = "https://files.pythonhosted.org/packages/a5/f6/a5b82f8abbe14d134691b8b903696f701d25a081353a29dc655c364d9e62/scipy-1.18.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:7bbf207c4453ce1ad2e00b17313852b33310b83090c2311bdaf97f93c0380d12", size = 20475078, upload-time = "2026-08-21T23:23:54.138Z" }, + { url = "https://files.pythonhosted.org/packages/23/22/0858a0bbd6b3e825ceb8cd9baf9eaf3b2f2b1d77727eb6be40500bcdc92f/scipy-1.18.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:78c0665edead396b1abb4897c41a5c1d9bf090c8a637a4c20a61678e0a264e66", size = 23108904, upload-time = "2026-08-21T23:23:57.824Z" }, + { url = "https://files.pythonhosted.org/packages/75/9a/2e71719f31eaefe0e3a1706c4a1ded94e664bfd95ffca2b219a671faee01/scipy-1.18.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c085faa2cfa879c5141df483f836f4d691045a078224a670fa570fa01612d89", size = 34025113, upload-time = "2026-08-21T23:24:02.209Z" }, + { url = "https://files.pythonhosted.org/packages/df/64/ff35eb9e54894cf471ff4716abd3c81eb0a0626869217ce3e6ba4ccf17d7/scipy-1.18.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218", size = 35344199, upload-time = "2026-08-21T23:24:07.844Z" }, + { url = "https://files.pythonhosted.org/packages/d3/af/c5538be1792f7034c12c7db6ee67cace58253c7b87b122d68253eaf5de89/scipy-1.18.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c35d74ce0e193ff740c2f2be2ac913ddc232fe6c1ff40b26cfecb9c670c63314", size = 35639587, upload-time = "2026-08-21T23:24:13.05Z" }, + { url = "https://files.pythonhosted.org/packages/91/4c/075e4f66471bac101141ac739e9e135549be1bae584571bd03a530c056e1/scipy-1.18.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1", size = 37480330, upload-time = "2026-08-21T23:24:19.608Z" }, + { url = "https://files.pythonhosted.org/packages/39/e7/979fd14e75008623df31ba70d6bb144700f68feadcea042021c06a05bf82/scipy-1.18.1-cp312-cp312-win_amd64.whl", hash = "sha256:5e4d44984abc0020154ea81b247adeddcc3ac5527b975ff798bd1ba0adc513c2", size = 36658278, upload-time = "2026-08-21T23:24:25.463Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0b/e1525354ff9d7d5feb6d1b31af6d14072e5c91e9607b421fa1ec889660b3/scipy-1.18.1-cp312-cp312-win_arm64.whl", hash = "sha256:d65d448389b8436493abcf629cc94ad0cf32aecaf06e1acca1de53cc795f2f12", size = 24400588, upload-time = "2026-08-21T23:24:30.579Z" }, + { url = "https://files.pythonhosted.org/packages/b6/55/4540ee0f9c42a9ad7109d0d1a8cc70de54c3572b01c6693a2b1c70e90ceb/scipy-1.18.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:3ab3523da44749156e1f68b464dc56af11ae4cbc5c739a49d05f32b982eca9f3", size = 31089958, upload-time = "2026-08-21T23:24:35.8Z" }, + { url = "https://files.pythonhosted.org/packages/2a/f5/769f36d14922b8071a43e95d24d18b6bdafad10d7f5cf647867e1ac052bc/scipy-1.18.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6fb6a55cc0ba97b59a1f288fb86dc6fce8bdfc0fffcbfd015e3a954bf2a2d93", size = 28715106, upload-time = "2026-08-21T23:24:40.775Z" }, + { url = "https://files.pythonhosted.org/packages/9a/d7/21d890274f75ea37a8209d5519e72da3da90302e3b9fb8397a0918386a62/scipy-1.18.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ea324d9dd34c38bfb9bec8ca4d1b407db97dbb74029f566b8e322b1b6fe56fe6", size = 20456846, upload-time = "2026-08-21T23:24:45.066Z" }, + { url = "https://files.pythonhosted.org/packages/ec/01/798430ecea2e78ec7c02663d5f71c007bb6abeca931080debd40d7fa55ea/scipy-1.18.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:75b00eb8fb802090aa903f4ea1c7f5a584779f967361e68b7e98e531cc2d7174", size = 23087986, upload-time = "2026-08-21T23:24:49.539Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5f/4634e9d35c68496e4e34cb6946eafab044458e6cedab42b40b6588e475b6/scipy-1.18.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d416b16cccfd70fbf62400e84d0bb2f4e6af519a45557f1692c749b37f14b315", size = 33998146, upload-time = "2026-08-21T23:24:54.714Z" }, + { url = "https://files.pythonhosted.org/packages/41/48/6450ed9243315322bbc19ac57b9b70d66a20bf1d38d124c96bc4bf6af9ea/scipy-1.18.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9", size = 35312578, upload-time = "2026-08-21T23:25:00.44Z" }, + { url = "https://files.pythonhosted.org/packages/00/bd/bf5a4be6a3525676499f6dff307991739ff6fdcad1481b1aeb6745339f58/scipy-1.18.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c825cef2f49e46753726a7181a8e199804a912b29519ada542c6ebc654951899", size = 35612621, upload-time = "2026-08-21T23:25:06.144Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4e/3c45c33e00a77996c4b1cb707929f833ba7b1d522ee29f882512c330676d/scipy-1.18.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07", size = 37457323, upload-time = "2026-08-21T23:25:12.483Z" }, + { url = "https://files.pythonhosted.org/packages/93/0e/e0348fbc0dbab65c114cf78957e7dfeb49f8e8b556b4d930cc12ff195e18/scipy-1.18.1-cp313-cp313-win_amd64.whl", hash = "sha256:559ed65f60c1af5a03f3912605a1b5114f522c7c32fb23c3376ae8f03219fe28", size = 36622841, upload-time = "2026-08-21T23:25:18.722Z" }, + { url = "https://files.pythonhosted.org/packages/50/a8/6a77f5f267c555108f0a864b6db714363dab567a8266422a79a385f9232b/scipy-1.18.1-cp313-cp313-win_arm64.whl", hash = "sha256:cd479fc04dd9401e3b4f49e76518768ef99c4f517a98c284eb091fd725719adf", size = 24399315, upload-time = "2026-08-21T23:25:23.458Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/d8eb4e280ddb56a4ab2c6f02ee49b56b23f6e977cf0802fd6d68dbef14f5/scipy-1.18.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:83de5453a7799afc9048b4616bd085cef126e36412f0ea2f6370c36a2a3a51e7", size = 31090936, upload-time = "2026-08-21T23:25:28.686Z" }, + { url = "https://files.pythonhosted.org/packages/2a/49/59ea385dc3a62ff498ddf3cfff7c2b41b0f9f9d3c4122b3f1dcb6d6327fe/scipy-1.18.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:9554bcc6d715ee87a633a3cc8e7703c6628b100dd29cb8a2efc4c0533c7ff729", size = 28725221, upload-time = "2026-08-21T23:25:33.244Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/6b0c288c50942d78193696c9f15f9a0874f5178aa0ddf40f83d9924b3e8d/scipy-1.18.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc", size = 20466839, upload-time = "2026-08-21T23:25:37.516Z" }, + { url = "https://files.pythonhosted.org/packages/4b/e0/54fd3793c729e3b936782f181b59cbb1205bf250ab605a16cb1ba61cdd5e/scipy-1.18.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:88f0e784020649f88ea48c9f5ddfa403bf9205820667c0914740b392035afb82", size = 23089121, upload-time = "2026-08-21T23:25:42.019Z" }, + { url = "https://files.pythonhosted.org/packages/0b/56/030af62bea3cf878e0028515dff78c123b01633606a879b63f42d2db99cc/scipy-1.18.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d3ab0e8c69a17dd3559eab8cbb88f258e285c94d572c2719033f90f83290c89", size = 34053851, upload-time = "2026-08-21T23:25:47.998Z" }, + { url = "https://files.pythonhosted.org/packages/6b/89/2a844506d49651e9aa1af6ef95b6bd8031cb1d5a4375edec6155037e04cf/scipy-1.18.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac0333bdf38309aa3dcbe7e3fa7ea29e7a2c37c6ea306a757b700ded8e4596ad", size = 35329183, upload-time = "2026-08-21T23:25:53.522Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/c7370c3640e92ac9613cbf26cb3f729f9b12ddf1727b55b94b53b24d6f48/scipy-1.18.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:911de823097db8b63f034299d12662db93344e6ffa0b881cbb57748974b70168", size = 35672551, upload-time = "2026-08-21T23:25:59.387Z" }, + { url = "https://files.pythonhosted.org/packages/24/16/ec8536f351421f8bf60a1120930638f83790f4710b8230446aca3d6159d4/scipy-1.18.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:95298364e251be3e60249facbeeca03631d3bb7584f85879516ec55ac717b81f", size = 37469416, upload-time = "2026-08-21T23:26:05.432Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/d73da0d28f16c45bb9b0a5691b91610b0275c5ef0eb5e43c87cf2dc1bf31/scipy-1.18.1-cp314-cp314-win_amd64.whl", hash = "sha256:78a0d7c918e74a232394117160e7e3db503377572a45bcef8826e4ab8a35feba", size = 37362755, upload-time = "2026-08-21T23:26:11.366Z" }, + { url = "https://files.pythonhosted.org/packages/89/25/e996e4dc74e10e227b1e14db5eaf6608bb6dd33884a64851c38f18dd4249/scipy-1.18.1-cp314-cp314-win_arm64.whl", hash = "sha256:cbf38d043c1aa4ab306e1ada6ab6eddacc3322a20b7af1b30bc93254b366fe09", size = 25036090, upload-time = "2026-08-21T23:26:15.887Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c9/c00213f92309d753b48903e6a451b87eb52ff5b7a16e789d1568bbf221c4/scipy-1.18.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0fcb3c93519f27bb4f0c4b0f7802cdcaca7fcf93267b75edda2e9f4e8a55cbd7", size = 31485550, upload-time = "2026-08-21T23:26:20.776Z" }, + { url = "https://files.pythonhosted.org/packages/74/b2/e3067c487982d4eeab2938928529410370c06fea84a4d3f4925e7d96647d/scipy-1.18.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ddef79fb382df40104a19bb7151b3b23e57c1778fcf857c71ceecd9bd264513f", size = 29174642, upload-time = "2026-08-21T23:26:25.395Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ab/374c9fe2d1ec014e576c781a4b5d8e1ba340e8f6b4638c16f711d2b194f0/scipy-1.18.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:0e82073ecc7acc6436fac4b31674109c7e1d3e596789767eda01258a8c9e8123", size = 20916357, upload-time = "2026-08-21T23:26:30.112Z" }, + { url = "https://files.pythonhosted.org/packages/90/38/223915c88a17317cafbf8ca2a42b11c265a9fb1e804aa665544132b5fe8a/scipy-1.18.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:8bcf3c1ba5d6456e2effd30fcbd3459b044d683fcdac79a2e6830f0bdf7de487", size = 23482611, upload-time = "2026-08-21T23:26:34.846Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d1/db0948da8ca57a80b36520ef0a768b967d99f3af65f4b6f1bf6362ad4dd4/scipy-1.18.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cfbf154f2ba187f2ed6cce2639efff7d105f1140573642c0161615b6d91d6a87", size = 34143202, upload-time = "2026-08-21T23:26:40.4Z" }, + { url = "https://files.pythonhosted.org/packages/87/53/39d046cc7574ed6acacb6bd5723e220107ece80bff12faaf3efc4ddeede4/scipy-1.18.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1d33a7836f7ddc1993427966a0823468ec41bcbdb1a9f9942d1d7e57f803ba3", size = 35380876, upload-time = "2026-08-21T23:26:46.1Z" }, + { url = "https://files.pythonhosted.org/packages/f9/da/32e0e799d875a85ca57d9bde6c78148afcc0e38276df683d95854eadc8c3/scipy-1.18.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f4b8bc363b6d65ee2152bec57568e3c52639bb34c46057b09857a307ed5e21d", size = 35770885, upload-time = "2026-08-21T23:26:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/88/2e/f97a666d362fee68b18f41c9c30ed502ca5c98b549749bfcb52a8b74d1eb/scipy-1.18.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11c423f1049c5755ad4409af52a9ada1cff96fe9b50795d4af3619f292901239", size = 37525424, upload-time = "2026-08-21T23:26:56.751Z" }, + { url = "https://files.pythonhosted.org/packages/ca/d5/a9e765a84654ebba8479a1fd1b059ced1af72b168a3b2a3a46540ea38d20/scipy-1.18.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c24acac1e18912761c4700239bbc1fd32f615af690f1584d49b35859be51324d", size = 37416961, upload-time = "2026-08-21T23:27:01.546Z" }, + { url = "https://files.pythonhosted.org/packages/ee/16/e79e0d1c63ef698879d85439d37e9fb434e3b804e506a6991038d086ebd9/scipy-1.18.1-cp314-cp314t-win_arm64.whl", hash = "sha256:9f2897bf7737392ad0d5213ea7b6add72a4edf5679b3153106aeb88b6507b3b9", size = 25331848, upload-time = "2026-08-21T23:27:05.884Z" }, + { url = "https://files.pythonhosted.org/packages/be/4f/1bd37c883b67163e2ca1f60977a399500e6879c15defecac62831c8d078d/scipy-1.18.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:eb0dfcf4e28a99c12c999744a2ff67c9b06200e20401c7c88186e33552a46331", size = 31091484, upload-time = "2026-08-21T23:27:11.051Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c5/ba929d7feb9b2332f96827c12e0e924b61973b59b4dea383b603372c65ce/scipy-1.18.1-cp315-cp315-macosx_12_0_arm64.whl", hash = "sha256:30f464bee641fa8e282577c7dce027308403213c6ca8270bba73285c91024bc5", size = 28725057, upload-time = "2026-08-21T23:27:15.9Z" }, + { url = "https://files.pythonhosted.org/packages/a4/19/68f1c50f609d955d230e66d25d02bd3e1e167ec540232135354fb9a4b9e3/scipy-1.18.1-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:1bca3b943fc2567ea49cd02c99abde49da4d5178ec46f624bd8255cda8755beb", size = 20466734, upload-time = "2026-08-21T23:27:20.044Z" }, + { url = "https://files.pythonhosted.org/packages/ef/6d/319fa29b73d1802fa80b32a6eaf3f5be456ef81526da2716a9493bcb5501/scipy-1.18.1-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:c9d18a33309122074ea483dd92dd444189166b8b2ec429fe9ed5ac73c7a0aa23", size = 23089664, upload-time = "2026-08-21T23:27:24.345Z" }, + { url = "https://files.pythonhosted.org/packages/b7/db/30992f9b51a63de671daf3888ffd18378b6cb9ec9f2c972264238ffa7fd6/scipy-1.18.1-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82f201b4c878551d48558337aab270d3c6cca5507b8737c8d8a608d234cccde0", size = 34054035, upload-time = "2026-08-21T23:27:29.409Z" }, + { url = "https://files.pythonhosted.org/packages/91/d4/bf3e735dc0b9d5a8ff45079d2540e17d3aff7a2f0048dd8f552ffd031d2b/scipy-1.18.1-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5", size = 35333883, upload-time = "2026-08-21T23:27:34.293Z" }, + { url = "https://files.pythonhosted.org/packages/19/93/12d78ce9f871fe945fca588d32644e6e63f553c2a35c564d73f3b22a3313/scipy-1.18.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ceb30a00ce7c92d459819443d29ca486d882b83fb6738bdcbb2a1cce94ac5daa", size = 35673124, upload-time = "2026-08-21T23:27:39.059Z" }, + { url = "https://files.pythonhosted.org/packages/70/cd/886219313a1012a48e6ae0ec4f302c837151beb92e1ff0d709ef8fdfc488/scipy-1.18.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f29633129f9fa7e88a3f0fca835de2d030bfc9643f7799e1a0c46cee24d38fc7", size = 37470753, upload-time = "2026-08-21T23:27:44.435Z" }, + { url = "https://files.pythonhosted.org/packages/17/6c/a776888ce618bee54fbde26172f0f46ac1da70d27b63861797fe78e1904b/scipy-1.18.1-cp315-cp315-win_amd64.whl", hash = "sha256:92c14f5bdbfb6216315ce33e78080474082de8b3830122ba97809bfbe65f75c0", size = 37361483, upload-time = "2026-08-21T23:27:49.334Z" }, + { url = "https://files.pythonhosted.org/packages/ab/09/97b651691322ebee97999b017ffc18a15a0b815103844c97e8da9d469731/scipy-1.18.1-cp315-cp315-win_arm64.whl", hash = "sha256:e402cf31eb68f453dbb2d36fc6d722b33f24a55d68b2ae1d92fa6305ca71c298", size = 25035883, upload-time = "2026-08-21T23:27:53.596Z" }, + { url = "https://files.pythonhosted.org/packages/ed/0f/9ec20467bbabd0d44e2a77d0fd3d124f884b4d67df92af82c91d2d6a486f/scipy-1.18.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:2a0b02f9fc46f8520330c23d45e6560db7e3a0d927232139427637f98943e11d", size = 31474926, upload-time = "2026-08-21T23:27:57.993Z" }, + { url = "https://files.pythonhosted.org/packages/8a/58/dcb79161e56efbedc50079fcd2f5fe427a0ebb53022eb476aa73c015ad8f/scipy-1.18.1-cp315-cp315t-macosx_12_0_arm64.whl", hash = "sha256:1d73131e358976663dd969e1fb4ed1404b815cd977eaaedc3b3a133ba2d81c35", size = 29164940, upload-time = "2026-08-21T23:28:03.062Z" }, + { url = "https://files.pythonhosted.org/packages/71/d3/1eeea80c817fcb8ef7bd4a05a58824977a0e57a375cfc3d7ea7c911c01ad/scipy-1.18.1-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:bff0b729edd992766136b34e39cc76bc2fad905aa58897ee72a9cd000a6d8443", size = 20906742, upload-time = "2026-08-21T23:28:07.642Z" }, + { url = "https://files.pythonhosted.org/packages/54/46/e59350428b6099301a20128108c995e2eb175a43f383af9a346e38824f9b/scipy-1.18.1-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:10ac20c69d880f77f375db44c22e3e6a644f9fefa291d4cd2fb9790a89fc99fd", size = 23472183, upload-time = "2026-08-21T23:28:12.109Z" }, + { url = "https://files.pythonhosted.org/packages/89/31/cc91623fa98f0621766a0f0aaaadb2c66de74a7ea7e3837164f6e4354260/scipy-1.18.1-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33a834464fdabc0f26a45508df31b3cc5d028e04dbf6c5ed398541418e0a12fe", size = 34130796, upload-time = "2026-08-21T23:28:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/fc/3e/8572ef536957ddb8aa81bb4090d9e25f257e3b4e05d97deb54319deb8a3a/scipy-1.18.1-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49023963c193dacee096301452f223ee24d86ec5807f8df93c0f7221d119e305", size = 35374253, upload-time = "2026-08-21T23:28:23.732Z" }, + { url = "https://files.pythonhosted.org/packages/b5/c6/59fdeffb4f1435299f93d9dc8140b43ad2916e6cfc944be6c3041fcec86d/scipy-1.18.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:d84a09d0dad90ba6525d8ac1c2334b33e64bf3ccfe9e841f02feb867a22681e4", size = 35758543, upload-time = "2026-08-21T23:28:29.431Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d9/135be205d9de8783193aff9cc3bf483a03a38e4b29432c954e8cb66ac14e/scipy-1.18.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:179ce34a8d0fe273d8883ba59e17e052247d08973dfcb743ca52bb1cce2d60b0", size = 37521946, upload-time = "2026-08-21T23:28:35.245Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a2/5b7d5270621ab7cfa3f7766067bf95dc360b5efb6394694e8143b4156e2b/scipy-1.18.1-cp315-cp315t-win_amd64.whl", hash = "sha256:5632e3ae3d09197c446310cd5187de63e28448ce22f0f67b2b93d97503c0c230", size = 37408295, upload-time = "2026-08-21T23:28:40.724Z" }, + { url = "https://files.pythonhosted.org/packages/63/ad/741c19fcb66755ff953daf9243af8480e4bf3d7fbe57583c178c7d2b6b51/scipy-1.18.1-cp315-cp315t-win_arm64.whl", hash = "sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a", size = 25319710, upload-time = "2026-08-21T23:28:45.713Z" }, +] + +[[package]] +name = "sentence-transformers" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub", marker = "python_full_version >= '3.14'" }, + { name = "numpy", marker = "python_full_version >= '3.14'" }, + { name = "scikit-learn", marker = "python_full_version >= '3.14'" }, + { name = "scipy", marker = "python_full_version >= '3.14'" }, + { name = "tokenizers", marker = "python_full_version >= '3.14'" }, + { name = "torch", marker = "python_full_version >= '3.14'" }, + { name = "tqdm", marker = "python_full_version >= '3.14'" }, + { name = "transformers", marker = "python_full_version >= '3.14'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/ef/87681678d0aa91a1e77f0ffdd2e5d900aaf358aac644dd14bead8d95ec4c/sentence_transformers-6.0.1.tar.gz", hash = "sha256:1c3b8d9403f87ad0c879638554f36cc85744f38a6a70d150fd7e964ca0e9935b", size = 575395, upload-time = "2026-08-31T07:50:43.518Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/ab/dc6031faf1b6bba08076fbf457b21a5c51ba7928a77916915e093c16b8c7/sentence_transformers-6.0.1-py3-none-any.whl", hash = "sha256:b8888d72c707ba33c63aa30845850702dd5acadf1dd0d051436380bcebe4fd0f", size = 739832, upload-time = "2026-08-31T07:50:41.635Z" }, +] + +[[package]] +name = "setuptools" +version = "84.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -2496,6 +3103,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a9/18/489c97b834dfff9cf2fc2507cede4bcd4b11e67f84bc462acd1992496f86/structlog-26.1.0-py3-none-any.whl", hash = "sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e", size = 73764, upload-time = "2026-06-06T07:33:38.046Z" }, ] +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "textual" version = "8.2.8" @@ -2513,6 +3141,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/be/35261223d9416a0751cdff1c7b4a6f881387218a12d439fe22fefebc8c04/textual-8.2.8-py3-none-any.whl", hash = "sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a", size = 731418, upload-time = "2026-06-30T06:51:26.364Z" }, ] +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + [[package]] name = "tiktoken" version = "0.12.0" @@ -2572,6 +3209,72 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/45/c7b5c3168458db837e8ceab06dc77824e18202679d0463f0e8f002143a97/tinycss2-1.5.1-py3-none-any.whl", hash = "sha256:3415ba0f5839c062696996998176c4a3751d18b7edaaeeb658c9ce21ec150661", size = 28404, upload-time = "2025-11-23T10:29:08.676Z" }, ] +[[package]] +name = "tokenizers" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/1e/bc6587c5ab643b2e17776cace9070a2ae73549c86bffac9934a600bf3c31/tokenizers-0.23.2.tar.gz", hash = "sha256:7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac", size = 385745, upload-time = "2026-09-03T08:55:42.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/ed/8a443528baa6fac8dfe8c3b75b038c63ac92bb539bcabe311e227c718173/tokenizers-0.23.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:85a9a357a3764aecc904ee76bdaf8cf1ad8e5a67a1b929a487c4a39b49ed0e90", size = 3148852, upload-time = "2026-09-03T08:55:30.874Z" }, + { url = "https://files.pythonhosted.org/packages/67/49/22da045a91732384d3a3771816bf188dc5a1f702c32e635afa7c679c0bef/tokenizers-0.23.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:986670e43691469dcee610ea0f846f91a8f84e91fc6f7a48d4c064414c0ec2bf", size = 3101593, upload-time = "2026-09-03T08:55:28.587Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4d/8f569ed49372a3ed8e57099bd515055fd48d7c95912c4307cda6973c2168/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a37039b5dfc4af84eb3ef0a92f4307e28936c8f9adccba2629d36f652e9bf7a2", size = 3516830, upload-time = "2026-09-03T08:55:14.741Z" }, + { url = "https://files.pythonhosted.org/packages/2a/de/e2f14c8919d5bf51874051d00d6c7b7e0e8bde6c6a2dbeddda7f642896ff/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b7e37ba198f24150f523e1242e83c4970de4a525480586be5dcc24d9add32c5", size = 3407975, upload-time = "2026-09-03T08:55:16.842Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bd/93c69152d02ef06ce47aed8b2bf4952dcf733c935a62791873932b2934d9/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43e4f2071e3cc8d5d86421c874aebc82659bb51a68bcdef5a0da75ee89511ccb", size = 3748165, upload-time = "2026-09-03T08:55:24.769Z" }, + { url = "https://files.pythonhosted.org/packages/2d/b7/56b84b80bc96942bba8eb23751a9e8a1fce4faaf4390425e7083f721c98c/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:325fee2e0418a9dc6c9ecf736a5f5f0db7875183ace9549ae339da76f7a1fbb7", size = 4024165, upload-time = "2026-09-03T08:55:18.806Z" }, + { url = "https://files.pythonhosted.org/packages/9b/8a/0175e216f005c2fe08238292663aa41e4c802b216e71047a69a0e9fc6fa3/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:950d7c9426fa72406a0ffeacdbc0bb9985f5db20eb8b263f29c79aaf83105703", size = 3591899, upload-time = "2026-09-03T08:55:22.752Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ca/ca6b93c7820df123b2662a9469e8facc826ccc94e98fdd0d615f6431e73a/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305", size = 3386843, upload-time = "2026-09-03T08:55:26.584Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a4/4f9106d317b14a80aefea9f0e3a8d07ef25f856a7607eb7f5ab894281fcb/tokenizers-0.23.2-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:12f0835dc2ee694746a76adf7b1567d4346a4a502ebe93fb1f5f80ea49799b78", size = 3577314, upload-time = "2026-09-03T08:55:20.825Z" }, + { url = "https://files.pythonhosted.org/packages/8d/6a/1552b70fb0d9ab074fd3fc961435d01364e79c9058481822c3af6e8d402c/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:eb2f9c8a24da020ea8c11a01a19c1c2547912d92121ae4a01cfbca46125dee40", size = 9967367, upload-time = "2026-09-03T08:55:33.188Z" }, + { url = "https://files.pythonhosted.org/packages/06/01/3ccb3a956c7528b2507b8a9714155c4baf86af593039db6ea375dd0c96c3/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f486f402f6f9abee5bb032553736813af0c710a86b2e0ca592634c55cea1f835", size = 9811886, upload-time = "2026-09-03T08:55:35.642Z" }, + { url = "https://files.pythonhosted.org/packages/fa/73/7038e612d48bda1599457f712f6bd3854eae1a9dc9c13aa47f835349db48/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:bef235815a067b2648caf6dcc7a71091b0b0fff9ee8057f6451eb9335fae52ef", size = 10146224, upload-time = "2026-09-03T08:55:38.391Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d8/8e9e4e0b287a338d8f88976729628c9d22e8a54cfaf9777018a7f7cb58a0/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5c56bda1511921587789163e524d196ed8284174ac23abd7685d5ea8da6c4718", size = 10256304, upload-time = "2026-09-03T08:55:40.977Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1f/c79a01f671a49728ebb0b61f7ff9ea45663b66cab40bc0858e9859b25c16/tokenizers-0.23.2-cp310-abi3-win32.whl", hash = "sha256:debf978920d93ba9c219bd67cc4bbfaf912c9039e41e7a28b91ec15e3728c95a", size = 2592809, upload-time = "2026-09-03T08:55:48.02Z" }, + { url = "https://files.pythonhosted.org/packages/db/f7/0a69ac6b82dbccf3f71add938a161c497952749294b8dd6dfe03a819dc40/tokenizers-0.23.2-cp310-abi3-win_amd64.whl", hash = "sha256:2e96f5699d5249c9c64aa8412e044f727aae3a4098cf830f9901ec1afc361cde", size = 2863236, upload-time = "2026-09-03T08:55:46.193Z" }, + { url = "https://files.pythonhosted.org/packages/d7/b0/dee84cb44175be1b4c35bd2f770727494e78f0bb38e571a623ade94dbebb/tokenizers-0.23.2-cp310-abi3-win_arm64.whl", hash = "sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa", size = 2729352, upload-time = "2026-09-03T08:55:44.345Z" }, +] + +[[package]] +name = "torch" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", marker = "python_full_version == '3.14.*' and sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "filelock", marker = "python_full_version >= '3.14'" }, + { name = "fsspec", marker = "python_full_version >= '3.14'" }, + { name = "jinja2", marker = "python_full_version >= '3.14'" }, + { name = "networkx", marker = "python_full_version >= '3.14'" }, + { name = "nvidia-cudnn-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "setuptools", marker = "python_full_version >= '3.14'" }, + { name = "sympy", marker = "python_full_version >= '3.14'" }, + { name = "triton", marker = "python_full_version == '3.14.*' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.14'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/76/bb4770f56cf6d8971671dbcbb7493e5a6a15ad2825f4e359b02c27c38297/torch-2.14.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c1f844f1c750e87df4b68bc3afbc0e2b0c7ef19d7b8f666e48bdcf6a0c4f0056", size = 127303200, upload-time = "2026-09-02T13:43:20.311Z" }, + { url = "https://files.pythonhosted.org/packages/fe/7b/ec44bacf2c8886b85ba4ca2285e8b09f2dff5d9c99e6a031326954082cb5/torch-2.14.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ada340e62591d06a2bcc2d68170f20f45f0b0665d372dc510a8ed7eb3b1d609a", size = 454010251, upload-time = "2026-09-02T13:44:24.927Z" }, + { url = "https://files.pythonhosted.org/packages/15/71/49399acd41f750a906c686bd23c08a2001ccb8dd25f2971003c2ed89c1dd/torch-2.14.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:fecffb58f51fd643d213acd68da21cc3fc19bea05a3bc64b4ee55128f47a4963", size = 554620488, upload-time = "2026-09-02T13:44:49.442Z" }, + { url = "https://files.pythonhosted.org/packages/be/16/9489b137112040f9911d7527e452854f21cc4e499ce0da79864e6a7451a7/torch-2.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:cad84f41bbdf3dcf333ce394aeeaf25237c4d94fd6b659ba5eb813c829978823", size = 124114011, upload-time = "2026-09-02T13:43:55.034Z" }, + { url = "https://files.pythonhosted.org/packages/ca/40/0db773452c2a62b37761d3f418acf933d381f9e87077036fb57c2a386c37/torch-2.14.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9d4b1022a5d9b71282ec67ad0d9e7235870096b8a246dc1c32d6ea1fc83dc998", size = 127311393, upload-time = "2026-09-02T13:43:59.607Z" }, + { url = "https://files.pythonhosted.org/packages/13/36/537fd9da2adad49e7b2bb20741398625bee548493158274e87369a8eed56/torch-2.14.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:731b9ebdea402b8b1996d4c2ae613b16660e559b19e47bdc45d970568bc91c53", size = 454010525, upload-time = "2026-09-02T13:45:03.719Z" }, + { url = "https://files.pythonhosted.org/packages/21/f1/39bd13b21f57d1982b7f3ddf663f01c7266e2957714880744eba9e8c8d11/torch-2.14.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:84bf384779a10c02fc3c6bdbab71a9cb66b0dd93c652d1ed5d6dfc0cb37e5962", size = 554619993, upload-time = "2026-09-02T13:45:28.209Z" }, + { url = "https://files.pythonhosted.org/packages/89/a8/683d9c44737554b67ca76dd2db4f42258a0f014246cb511293e51e0154bd/torch-2.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0e7cf18cb0d8bd666b6120932e29c7aef3502b61a08b44da4839580c539a7cdb", size = 124113865, upload-time = "2026-09-02T13:44:33.705Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/1241e7db5ccc2455f8735bd6b1becfad39916206ad18001c4c0014d139e2/torch-2.14.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:860423e970f2ce02c4476e8e2d1350131b1c5c5a5e4912180e78b50b53241efa", size = 127321431, upload-time = "2026-09-02T13:44:38.236Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/bbeba65e3fbb4b8f61ea19cf4ebea9f4268cc6fe64e6f7d38bfb6cc152eb/torch-2.14.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:b985c7defeb8d28691b7513ebaecc64946c5f68ec770e9f4ddba39688864f46a", size = 454027631, upload-time = "2026-09-02T13:45:43.73Z" }, + { url = "https://files.pythonhosted.org/packages/50/75/8d2b9a7e724759470c209489b79260cac537f091cc9aac8001c8d2bc845c/torch-2.14.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b2cd92bce63d40bf6fc2e5d840fc2f0063bd180a242daa761cb6088cc2f46e27", size = 554623549, upload-time = "2026-09-02T13:46:04.252Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1e/a5475c00b0555e686333e6b4036f2213e7cbea021a772ce6f9ced4dcbd2f/torch-2.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:44b044b9f6f633d982839422a57433d6a1da520037fd88e0c8a47efde589b3b8", size = 124110863, upload-time = "2026-09-02T13:45:12.764Z" }, + { url = "https://files.pythonhosted.org/packages/a5/49/bbaee76337742a42d2c5b0296ea62252567050e1d821bc2283b2e72ba6fb/torch-2.14.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:553aec938d37d77b783bcf801e638cad068870e7643c47eadac21eb180f551ed", size = 127653463, upload-time = "2026-09-02T13:45:17.1Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/6cc7a511bab384fbe8f4b8ddeecdd22e724b2162d6f15076f07a0a7ef15b/torch-2.14.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cd8cd8f714d511ccdca907282d1da3d9be8322d4e1520b9c3bce39a5c1318a4b", size = 454009927, upload-time = "2026-09-02T13:46:19.637Z" }, + { url = "https://files.pythonhosted.org/packages/b6/da/0f04fe15fd05bf3f613a359b14acb44c35ae06fe69da0aefa8b5cdb4f9f9/torch-2.14.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d2526f71e6638133b97b3cd2881ece3df521460a4727030e8ae2a72a7d3ae31d", size = 554580530, upload-time = "2026-09-02T13:46:34.818Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c3/72ae1f02747b1f012e1975743e48cd608f83095d7f9ce58de78b79248b35/torch-2.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:731784e3914843c6bcc7aba3987ff7610ac57dbbc816a5d6b9b62e04c240a641", size = 124400194, upload-time = "2026-09-02T13:45:53.555Z" }, +] + [[package]] name = "tqdm" version = "4.67.3" @@ -2593,6 +3296,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/98/a9937a969d018a23badfea0b381f66783649d48e0ea6c41923265c3cbeb3/traitlets-5.15.0-py3-none-any.whl", hash = "sha256:fb36a18867a6803deab09f3c5e0fa81bb7b26a5c9e82501c9933f759166eff40", size = 85877, upload-time = "2026-05-06T08:05:55.853Z" }, ] +[[package]] +name = "transformers" +version = "5.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub", marker = "python_full_version >= '3.14'" }, + { name = "numpy", marker = "python_full_version >= '3.14'" }, + { name = "packaging", marker = "python_full_version >= '3.14'" }, + { name = "pyyaml", marker = "python_full_version >= '3.14'" }, + { name = "regex", marker = "python_full_version >= '3.14'" }, + { name = "safetensors", marker = "python_full_version >= '3.14'" }, + { name = "tokenizers", marker = "python_full_version >= '3.14'" }, + { name = "tqdm", marker = "python_full_version >= '3.14'" }, + { name = "typer", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/9e/750649904a065007a838981785b2bd8d9ff26154c6c341ac67d0b7f82c68/transformers-5.17.0.tar.gz", hash = "sha256:a153be279169b55b92d8000bf4af294aed684503d091cca7804da2dd8a9de000", size = 9817878, upload-time = "2026-09-09T15:39:56.886Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/d0/c502b60d684adbd98a8dc7d5bb866842772b816ac4354e4608be240041ae/transformers-5.17.0-py3-none-any.whl", hash = "sha256:78ec1ce21579b38dfb83950a0658cd119f87212a2fcfdff478096ce9d6c03801", size = 12295140, upload-time = "2026-09-09T15:39:53.746Z" }, +] + +[[package]] +name = "triton" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/07/0f8cd8e8db0472334253efdaaab3d0819fea27aa99bf0e7f1aeea4ceb5ae/triton-3.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9c404c69ed4a39e8ec632eaf6b9fe058a060bf98979c177f6ef666f06bb8d50", size = 226474486, upload-time = "2026-08-28T16:08:18.29Z" }, + { url = "https://files.pythonhosted.org/packages/c1/09/b7012e5bfae67640f268aa584caa80fe1674f6b0da949046b679972c33e3/triton-3.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91ffa46d095b252248297292dd22bcbacd53a125a0c2eefbbbf74925a320bc3", size = 247972921, upload-time = "2026-08-28T15:55:53.157Z" }, + { url = "https://files.pythonhosted.org/packages/87/4d/4c564374bcdadb166fccbf3e45aee0d4a473f88d341761bd2fefe3b8e8c1/triton-3.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7004666652f500ed854a86988e4b3d69d247188b5d2092b5df1e44f4a954099", size = 226476793, upload-time = "2026-08-28T16:08:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/3394d5548404c1cabd1dadadd28d0b3f9478db1dff8180da53bb3f0a1e19/triton-3.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0497218e26b7d79773ad9c2a3fa3b539ee69f587a13fac2e552b1d322a8015", size = 247975122, upload-time = "2026-08-28T15:56:04.112Z" }, + { url = "https://files.pythonhosted.org/packages/b8/59/bf0e9493118bb353ab59a5d6a65db3618d9b314417cc1459f0121e0ec5c9/triton-3.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f6b48d0591929a3867973acac3dccd4e058585f91bfb41022de496c9ffab304", size = 226488654, upload-time = "2026-08-28T16:08:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/93/d9/08c75f3459f19ad00425b564058e40efa4bcd79b816064cf27499303ea42/triton-3.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:387dae4cb0089a7b6ba1a428ae0782b65c4c58f57d94617cb22ca8593d8ccbca", size = 247972313, upload-time = "2026-08-28T15:56:14.007Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/429c5592181cfb7361a0a8e0bff218e7224b726709d75da2472b3e819f70/triton-3.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b84e7d512490ba529111260fa6f7cad8b254a6bb5fbdf41d5ef9a5e57f52d0a", size = 226591133, upload-time = "2026-08-28T16:09:02.271Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d1/aa8a3e935c37efee7945984fdb64d7e0851bf6d920afd97b2d21f9d23360/triton-3.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74217bb56ed8692759227758e4c4b3bd2d608a209c1a7a081bf361fb4c2c1bf9", size = 248077577, upload-time = "2026-08-28T15:56:24.94Z" }, +] + [[package]] name = "typer" version = "0.26.8"
本地三件套✅ Markdown + SQLite + LanceDB；不需要 MongoDB、Elasticsearch 或 Redis✅ 默认 Markdown + SQLite + LanceDB；可选 Milvus 或 SeekDB backend ❌ 常依赖 managed service、vector DB、graph DB 或 server stack