From cf5897c03e62def46844bbd3235a1d09e3186afd Mon Sep 17 00:00:00 2001 From: anmathad <280455309+anumathad-o@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:09:53 +0200 Subject: [PATCH 1/3] Improve VecDB onboarding, examples, notebooks, and documentation for version 1.0.2 --- CHANGELOG.rst | 12 ++ README.md | 321 +++++++++++++++++------------------- src/oracle_vecdb/version.py | 2 +- 3 files changed, 164 insertions(+), 171 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d5d8045..e7e28ba 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,18 @@ All notable changes to this project will be documented in this file. The format is based on the `Keep a Changelog `__, and this project adheres to `Semantic Versioning `__. +1.0.2 - 2026-08-19 +------------------ + +Changed +~~~~~~~ + +- Documentation-only release; no SDK API or runtime changes. +- Refreshed the README with direct links to runnable geospatial and semantic + search, semantic code search, RAG document chat, and VecDB notebook examples. +- Improved the getting-started journey from SDK installation to hands-on + applications and workflows. + 1.0.1 - 2026-08-10 ------------------ diff --git a/README.md b/README.md index 687a3ec..66def76 100644 --- a/README.md +++ b/README.md @@ -1,122 +1,61 @@ -# Oracle VecDB Python SDK  βš‘️ +# Oracle VecDB Python SDK -

- PyPI - Python Versions - Status -

+**Build vector search, RAG, and AI applications on Oracle AI Database - from Python.** +Keep vectors alongside your operational data, combine semantic similarity with relational and spatial filtering, and build retrieval applications without introducing a separate vector database. -## πŸš€ About +[![PyPI](https://img.shields.io/pypi/v/oracle-vecdb)](https://pypi.org/project/oracle-vecdb/) +[![Python](https://img.shields.io/pypi/pyversions/oracle-vecdb)](https://pypi.org/project/oracle-vecdb/) +[![License](https://img.shields.io/github/license/oracle/vecdb-python-sdk)](LICENSE.txt) -Oracle VecDB Python SDK provides a Python-native interface for building vector search and AI applications with Oracle AI Database 23.26.3 and later. It supports both Autonomous AI Vector Database deployments and customer-managed Oracle AI Database instances exposed through ORDS 26.2.2 or later. +**⭐ [Star `oracle/vecdb-python-sdk`](https://github.com/oracle/vecdb-python-sdk) to follow the project and help more developers discover it.** -The SDK provides straightforward APIs for creating and managing vector tables and indexes, executing vector similarity searches, and invoking inference operationsβ€”allowing developers to integrate Oracle AI Database vector capabilities into Python applications with minimal setup and boilerplate. +**[Quickstart](#-quickstart) Β· [Sample Apps](#-see-what-you-can-build) Β· [Notebooks](#-hands-on-notebooks) Β· [Docs](#-documentation) Β· [Releases](https://github.com/oracle/vecdb-python-sdk/releases)** -## ✨ Highlights - -- πŸ” Typed client with simple auth + configuration -- πŸ“¦ Manage vector tables, vector indexes, and metadata programmatically -- 🧠 Run embeddings & inference flows via Oracle AI Database models -- πŸ”„ Integrate vector search, filtering, and RAG-style pipelines quickly - -## πŸ“¦ Installation - -```bash -python -m pip install --upgrade oracle-vecdb -``` +--- ## πŸš€ Quickstart -See the [Oracle VecDB documentation](https://docs.oracle.com/en/cloud/paas/autonomous-vector-database/vcapi/quickstart.html) -for installation, pre-requisites, and the complete API reference. +### Requirements -This quickstart connects to Oracle VecDB, creates a table with integrated -embeddings, loads sample records, and runs a filtered similarity search. -Most SDK methods return typed response models. Import stable SDK response types -from `oracle_vecdb.data_types`, and use `.model_dump()` or `.to_dict()` when -you need a plain dictionary representation. +- **Python:** 3.10+ +- **Oracle AI Database:** 23.26.3+ +- **ORDS:** 26.2.2+ -### 1. Configure the client +### Installation -VecDB `rest_url` has this form but it might change based on the setup: +Install with `pip` or `uv`: -```text -https://:/ords//_/db-api/stable/vecdb/ +```bash +# pip +pip install oracle-vecdb + +# uv +uv add oracle-vecdb ``` -**Note:** Ensure TLS is enabled and that the endpoint is reachable from your environment. +### Connect ```python from oracle_vecdb import OracleVecDB, Configuration config = Configuration( rest_url="https://:/ords//_/db-api/stable/vecdb/", - # choose one auth method - access_token="", - # or username="", password="", + access_token="", ) vecdb = OracleVecDB(config) ``` -For all constructor parameters and object attributes, see the -[Oracle VecDB documentation](https://docs.oracle.com/en/cloud/paas/autonomous-vector-database/vcapi/configuration.html). - -### 2. Create an integrated embedding vector table - -Create a table that generates embeddings from text stored in metadata. The -configured model must already be available in Oracle AI Database. - -```python -vecdb.create_vector_table( - name="demo", - table_params={"auto_generate_id": True}, - embed_params={ - "model": "all_MiniLM_L12_v2", # must be preloaded via Vector Database Console or load_model() - "embed_metadata_jsonpath": "content", # JSON field in metadata to extract text from for embedding - }, -) -``` - -### 3. Load integrated embedding records +### Run a semantic search with metadata filtering -When an integrated embedding vector table is configured, provide text in the -metadata field selected by `embed_metadata_jsonpath`. The database generates -the vector during the upsert. - -```python -vecdb.upsert_vectors( - table_name="demo", - vectors=[ - { - "metadata": { - "title": "Comedy movie review", - "content": "A lighthearted comedy with fast-paced jokes.", # text to embed - "genre": "comedy", - } - }, - { - "metadata": { - "title": "Drama movie review", - "content": "An emotional family drama with strong performances.", - "genre": "drama", - } - }, - ], -) -``` - -### 4. Run a text query with filtering - -A text query uses the table's configured embedding model to generate the query -vector. +> This example assumes a vector table named demo already exists and contains data. See the [full quickstart](https://docs.oracle.com/en/cloud/paas/autonomous-vector-database/vcapi/quickstart.html) for create_vector_table() and upsert(). ```python results = vecdb.query( table_name="demo", - query_by={"text": "family drama"}, # uses integrated embeddings for the query text + query_by={"text": "family film"}, # uses integrated embeddings for the query text filters={"genre": {"$eq": "drama"}}, - top_k=1, + top_k=3, ) for index in range(len(results)): @@ -125,92 +64,120 @@ for index in range(len(results)): print(row["id"], row["distance"], row["metadata"]) ``` -### πŸ“₯ Ingestion Options +### ⚑ **Integrated embeddings. Automatic vector indexing. Semantic search + structured filtering. One Python SDK.** -#### Bring your own vectors +[Read the full quickstart β†’](https://docs.oracle.com/en/cloud/paas/autonomous-vector-database/vcapi/quickstart.html) -For precomputed embeddings, omit `embed_params` when creating the vector table -and provide `dense_vector` values in each record. +--- -```python -vecdb.create_vector_table(name="demo_byov") -vecdb.upsert_vectors( - table_name="demo_byov", - vectors=[ - {"id": "1", "dense_vector": [0.1, 0.1], "metadata": {"genre": "comedy"}}, - {"id": "2", "dense_vector": [0.2, 0.2], "metadata": {"genre": "drama"}}, - ], -) -results = vecdb.query( - table_name="demo_byov", - query_by={"vector": [0.15, 0.1]}, - filters={"genre": {"$eq": "drama"}}, - top_k=1, -) +# πŸ§ͺ See what you can build -for index in range(len(results)): - item = results[index] - row = item if isinstance(item, dict) else item.model_dump() - print(row["metadata"]["genre"]) -``` +Complete applications built with `oracle-vecdb` are available in the [Oracle AI Developer Hub](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/apps/vecdb). -### πŸ”§ Indexing and tuning +## 🌲 Semantic + Geospatial Search -#### Create indexes after loading data +**Combine vector similarity with spatial filtering in one application.** -Create the table first and build its index explicitly when the data-loading -workflow is complete. +Semantic search plus geographic and structured constraints, powered by Oracle AI Database. -```python -vecdb.create_vector_table( - name="demo_manual", - index_params={"vector_index_params": {"auto_index": False}}, -) +![Ask the Parks β€” semantic, metadata, and spatial search with Oracle VecDB](https://raw.githubusercontent.com/oracle-devrel/oracle-ai-developer-hub/main/apps/vecdb/vecdb_ask_parks/static/assets/ask_the_parks_demo.gif) -vecdb.create_index(table_name="demo_manual") -``` +**Oracle Spatial Β· Vector Search Β· Oracle VecDB** -#### Create an HNSW index +[View the sample app β†’](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/apps/vecdb/vecdb_ask_parks) -Use `INMEMORY GRAPH` organization for an HNSW (Hierarchical Navigable Small -World) vector index. +--- -```python -vecdb.create_vector_table( - name="demo_hnsw", - index_params={ - "vector_index_params": { - "auto_index": True, - "organization": "INMEMORY GRAPH", # HNSW-style index organization - "distance_metric": "COSINE", - "advanced_params": { - "neighbors": 32, # higher = better recall, more memory - "efConstruction": 200, # higher = better recall, slower index build - }, - }, - }, -) -``` +## πŸ’» Semantic Code Search -#### Query-time HNSW tuning +**Search source code by meaning, not just keywords.** -Use `advanced_options` to adjust HNSW runtime search behavior. `efsearch` is -HNSW-only; use it to control the candidate pool size and balance recall against -query latency without rebuilding the index. +Use natural-language queries to find relevant functions, files, and surrounding code. -```python -results = vecdb.query( - table_name="demo", - query_by={"text": "family drama"}, - filters={"genre": {"$eq": "drama"}}, - top_k=1, - advanced_options={ - "idx_parameters": { - "efsearch": 64, # number of candidates explored (higher = better recall, higher latency) - } - }, -) -``` +![Semantic Code Search - natural-language query, ranked code results, repository navigation, and highlighted source code](https://raw.githubusercontent.com/oracle-devrel/oracle-ai-developer-hub/main/apps/vecdb/semantic_code_search/images/semantic_code_search.gif) + +**FastAPI Β· React Β· Jina Embeddings Β· Oracle VecDB** + +[View the sample app β†’](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/apps/vecdb/semantic_code_search) + +--- + +## πŸ€– RAG Document Chatbot + +**Upload documents and ask grounded questions over their content.** + +Chunk documents, generate embeddings, retrieve relevant context, and pass it to an LLM for grounded answers. + +![Document Chatbot UI showing uploaded documents, a user question, retrieved context, and a grounded answer](https://raw.githubusercontent.com/oracle-devrel/oracle-ai-developer-hub/main/apps/vecdb/doc_chatbot/images/doc_chat_bot.gif) + +**Streamlit Β· OpenAI / Ollama Β· Oracle VecDB** + +[View the sample app β†’](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/apps/vecdb/doc_chatbot) + +--- + +**More examples:** Multi-Modal Product Search, Product Recommendations Β· hands-on notebooks + +[Explore all sample applications β†’](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/apps/vecdb) + +--- + +# πŸ’‘ Why Oracle VecDB? + +Modern AI applications often need vector search plus the structured data around each result. + +Oracle VecDB lets Python applications use vector search alongside relational, spatial, and all other capabilities of the Oracle AI Database. + +Use Oracle VecDB to: + +- πŸ”Ž Run semantic and similarity search +- 🌍 Combine vector search with spatial and structured queries +- πŸ€– Build RAG applications and AI agents +- 🧠 Use integrated embeddings or bring your own vectors +- ⚑ Create vector indexes automatically by default +- πŸŽ›οΈ Tune HNSW and embedding settings when needed + +If you're building enterprise AI apps, the data you need is probably already in an Oracle AI Database, VecDB can reduce the need to move or synchronize that data into a separate vector database. + +--- + +# πŸ““ Hands-on notebooks + +Learn Oracle VecDB hands-on. The Oracle AI Developer Hub includes runnable notebooks that take you from first query to production-oriented tuning. + +## 🧠 Embeddings & RAG + +- **Integrated embeddings** β€” generate embeddings as part of the VecDB workflow +- **Bring Your Own Vectors** β€” use embeddings from your preferred model or provider +- **Gemini RAG** β€” build retrieval-augmented generation with Gemini +- **OCI Generative AI embeddings** β€” use OCI-hosted embedding models with VecDB +- **Oracle Private AI Services Container** - use in an air-gapped environment with OpenAI-style inference layer + +[Explore embeddings & RAG notebooks β†’](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/notebooks/vecdb) + +## πŸ”Ž Search & filtering + +- **Semantic search** β€” retrieve results by meaning rather than keywords +- **Metadata filtering** β€” combine vector similarity with structured constraints +- **Search diagnostics** β€” inspect and understand vector-search behavior +- **Financial-data search** β€” apply vector retrieval to structured financial datasets + +[Explore search notebooks β†’](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/notebooks/vecdb) + +## ⚑ Performance & scale + +- **HNSW tuning** β€” understand and tune vector-index search parameters +- **Bulk vector loading** β€” compare approaches for loading larger datasets +- **Index management** β€” create, inspect, and manage vector indexes +- **Maintenance workflows** β€” operate vector tables and indexes over time + +[Explore performance notebooks β†’](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/notebooks/vecdb) + +> **New to Oracle VecDB?** Start with integrated embeddings and semantic search, then move on to filtering and HNSW tuning. + +[Browse all VecDB notebooks β†’](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/notebooks/vecdb) + +--- ## Examples @@ -218,20 +185,18 @@ results = vecdb.query( - [Sample notebooks](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/notebooks/vecdb) – Guided notebooks for setup, table/index workflows, vector search, and inference via the SDK. - [Sample applications](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/apps/vecdb) – Oracle AI Developer Hub apps showcasing ingestion, embeddings, search, filtering, and FastAPI + React/Vite integration using this SDK. -## Dependencies and Interoperability - -- Python 3.10 or later. -- Oracle AI Database 23.26.3 or later with ORDS 26.2.2+ enabled. -- An Oracle ORDS VecDB endpoint configured with either bearer-token or HTTP Basic authentication. +--- -The SDK can be used in applications, notebooks, retrieval-augmented generation -(RAG) pipelines, and other Python services that need Oracle vector search. -For setup instructions and guidance on getting started with the VecDB APIs, see the [Oracle VecDB documentation](https://docs.oracle.com/en/cloud/paas/autonomous-vector-database/vcapi/overview.html) +# πŸ“š Documentation -## πŸ“š Documentation and Resources +- **[Getting Started](https://docs.oracle.com/en/cloud/paas/autonomous-vector-database/vcapi/quickstart.html)** +- **[Python API Reference](https://docs.oracle.com/en/cloud/paas/autonomous-vector-database/vcapi/python-api-reference.html)** +- **[Sample Applications](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/apps/vecdb)** +- **[Hands-on Notebooks](https://github.com/oracle-devrel/oracle-ai-developer-hub/tree/main/notebooks/vecdb)** +- **[GitHub Releases](https://github.com/oracle/vecdb-python-sdk/releases)** +- **[Locally-managed REST](https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/26.2/)** -- [Oracle VecDB documentation](https://docs.oracle.com/en/cloud/paas/autonomous-vector-database/vcapi/overview.html) - for detailed API documentation, including features, usage, and reference information. -- [Customer-managed Oracle AI Database (26ai+) requirements](https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/26.2/) – DB 23.26.3+ with ORDS 26.2.2+, plus TLS/ORDS notes for handling self-signed certificates. +--- ## Help @@ -239,14 +204,30 @@ Questions can be asked in [GitHub Discussions](https://github.com/oracle/vecdb-p Problem reports can be raised in [GitHub Issues](https://github.com/oracle/vecdb-python-sdk/issues). +--- + ## 🀝 Contributing This project welcomes contributions from the community. Before submitting a pull request, please [review our contribution guide](./CONTRIBUTING.md) +[Open an issue β†’](https://github.com/oracle/vecdb-python-sdk/issues) + +--- + ## πŸ” Security Please consult the [security guide](./SECURITY.md) for our responsible security vulnerability disclosure process +--- + ## πŸ“„ License See [LICENSE.txt](./LICENSE.txt), [THIRD_PARTY_LICENSE.txt](./THIRD_PARTY_LICENSE.txt), and [NOTICE.txt](./NOTICE.txt). + +--- + +## ⭐ Like Oracle VecDB? + +**[Star `oracle/vecdb-python-sdk` β†’](https://github.com/oracle/vecdb-python-sdk)** + +It helps you follow the project and helps other Python and AI developers discover it. diff --git a/src/oracle_vecdb/version.py b/src/oracle_vecdb/version.py index e6296a6..bbbbcb3 100644 --- a/src/oracle_vecdb/version.py +++ b/src/oracle_vecdb/version.py @@ -1,4 +1,4 @@ """Single source of truth for SDK and generated ORDS versions.""" -SDK_VERSION = "1.0.1" +SDK_VERSION = "1.0.2" ORDS_RELEASE_VERSION = "26.2.2" From 2f728d86e04413db7f840f3e5cbbf78af6c31f1d Mon Sep 17 00:00:00 2001 From: anmathad <280455309+anumathad-o@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:49:14 +0200 Subject: [PATCH 2/3] Update security.md template --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index fb42c94..2ca8102 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -35,4 +35,4 @@ sufficiently hardened for production use. [1]: mailto:secalert_us@oracle.com [2]: https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html [3]: https://www.oracle.com/security-alerts/encryptionkey.html -[4]: https://www.oracle.com/security-alerts/ \ No newline at end of file +[4]: https://www.oracle.com/security-alerts/ From 00a6b5bc81208150ce6d1ded03f13fbc55d2e71a Mon Sep 17 00:00:00 2001 From: anumathad-o <280455309+anumathad-o@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:34:38 +0200 Subject: [PATCH 3/3] chore(release): SDK v1.0.3 bug fixes --- .gitignore | 2 +- .gitmessage | 32 ++ AGENTS.md | 10 + CHANGELOG.rst | 37 ++ CONTRIBUTING.md | 17 + README.md | 2 +- SECURITY.md | 2 +- src/oracle_vecdb/client.py | 41 +- src/oracle_vecdb/configuration.py | 24 + src/oracle_vecdb/default_settings.py | 76 ++++ src/oracle_vecdb/error_messages.py | 37 +- src/oracle_vecdb/ords.py | 91 +++- src/oracle_vecdb/parameter_validation.py | 479 ++++++++++++++++++++ src/oracle_vecdb/validation.py | 153 ++++++- src/oracle_vecdb/vecdb_errors.py | 33 ++ src/oracle_vecdb/vecdb_exception.py | 259 +++++++++-- src/oracle_vecdb/version.py | 2 +- tests/client/test_client_facade_contract.py | 205 ++++++++- tests/client/test_configuration_facade.py | 19 +- tests/data_types/test_responses.py | 44 ++ tests/internal/test_vecdb_errors.py | 16 +- tests/services/test_ords.py | 137 ++++++ tests/services/test_ords_exceptions.py | 186 ++++++++ tests/test_parameter_validation.py | 268 +++++++++++ tox.ini | 2 +- 25 files changed, 2106 insertions(+), 68 deletions(-) create mode 100644 .gitmessage create mode 100644 src/oracle_vecdb/default_settings.py create mode 100644 src/oracle_vecdb/parameter_validation.py create mode 100644 tests/test_parameter_validation.py diff --git a/.gitignore b/.gitignore index 3df1a05..24b39bc 100644 --- a/.gitignore +++ b/.gitignore @@ -71,4 +71,4 @@ target/ .ipynb_checkpoints # Oracle Parfait analysis cache -__parfait__/ \ No newline at end of file +__parfait__/ diff --git a/.gitmessage b/.gitmessage new file mode 100644 index 0000000..921f3a2 --- /dev/null +++ b/.gitmessage @@ -0,0 +1,32 @@ +# Based on Conventional Commits 1.0.0: +# Conventional Commit format: +# [optional scope][!]: +# +# Examples: +# feat(query): add metadata filters +# fix: handle empty query results +# docs!: remove the legacy authentication flow +# +# Common types: +# feat New user-visible capability +# fix Corrected behavior or defect +# docs User documentation change +# perf User-visible performance improvement +# refactor Internal restructuring without intended behavior change +# test Test-only change +# build Build or packaging change +# ci Continuous-integration change +# chore Routine maintenance +# revert Revert a previous change +# style Formatting-only change +# +# Keep the description concise and imperative. Add an optional body after a +# blank line to explain motivation or implementation details. +# +# Use a footer for issue references or breaking-change details, for example: +# BREAKING CHANGE: describe the required migration +# +# To omit a redundant entry related to an unreleased feature, use: +# Changelog: skip +# To omit an unreleased feature and the revert that removes it, use: +# Changelog: retract diff --git a/AGENTS.md b/AGENTS.md index 42b1d8f..cf5399c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,16 @@ filters = { Use only filter operators documented in `docs/source/rest_api.rst` or verified by SDK tests. Do not copy filter syntax from another vector database product. +## Validation + +- For SDK source or test changes, run `make check` and `git diff --check` + before handoff. +- Run focused tests for any changed development tool. Run `make build` when + packaging or build metadata changes. +- Run `make integration_test` only when the change can affect live VecDB + behavior and the required test environment is configured. +- Report any required validation that was not run. + ## Quick Start - Use the `README.md` Quickstart as the source of truth for runnable diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e7e28ba..bc29560 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,43 @@ All notable changes to this project will be documented in this file. The format is based on the `Keep a Changelog `__, and this project adheres to `Semantic Versioning `__. +1.0.3 - 2026-09-07 +------------------ + +Added +~~~~~ + +- Added verified public-operation defaults for omitted arguments, including + ``create_vector_table.table_params.auto_generate_id``, + ``list_vectors.limit``, and ``query.include_vectors``. +- Added transport-neutral validation for parameter values + and cross-field combinations before requests are sent to ORDS. +- Added explicit authentication-mode detection for unauthenticated, basic, + and bearer configurations, including validation of incomplete or conflicting + credentials. + +Changed +~~~~~~~ + +- Resource-name validation now rejects blank values and transport-unsafe NUL + or double-quote characters while leaving database-specific identifier rules + to Oracle Database. +- Upsert vector field names are normalized case-insensitively; unknown fields + and duplicate fields with different casing now produce clear validation + errors. +- Improved validation and error messages for vector index organizations, + distribution settings, quantization, metadata paths, query modes, and other + parameter dependencies. +- Improved ORDS exception normalization and diagnostics by redacting sensitive + request and response data, preserving useful error categories, and avoiding + duplicate raw transport exception context. + +Fixed +~~~~~ + +- Fixed graph-index requests that omitted required distribution parameters from + reaching the service with an invalid request shape. + 1.0.2 - 2026-08-19 ------------------ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 637430b..26f4ca1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,22 @@ git commit --signoff Only pull requests from committers that can be verified as having signed the OCA can be accepted. +### Commit messages + +Use [Conventional Commits][conventional-commits] for commit subjects so future +release automation can derive changelog entries from commit history. Configure +the repository template once after cloning: + +```bash +git config --local commit.template .gitmessage +``` + +Use this format and retain the OCA sign-off with `git commit -s`: + +```text +[optional scope][!]: +``` + ## Pull request process 1. Ensure there is an issue created to track and discuss the fix or enhancement @@ -53,3 +69,4 @@ like more specific guidelines, see the [Contributor Covenant Code of Conduct][CO [OCA]: https://oca.opensource.oracle.com [COC]: https://www.contributor-covenant.org/version/1/4/code-of-conduct/ +[conventional-commits]: https://www.conventionalcommits.org/en/v1.0.0/ diff --git a/README.md b/README.md index 66def76..b4e2512 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Keep vectors alongside your operational data, combine semantic similarity with r [![PyPI](https://img.shields.io/pypi/v/oracle-vecdb)](https://pypi.org/project/oracle-vecdb/) [![Python](https://img.shields.io/pypi/pyversions/oracle-vecdb)](https://pypi.org/project/oracle-vecdb/) -[![License](https://img.shields.io/github/license/oracle/vecdb-python-sdk)](LICENSE.txt) +[![License](https://img.shields.io/github/license/oracle/vecdb-python-sdk)](./LICENSE.txt) **⭐ [Star `oracle/vecdb-python-sdk`](https://github.com/oracle/vecdb-python-sdk) to follow the project and help more developers discover it.** diff --git a/SECURITY.md b/SECURITY.md index 2ca8102..fb42c94 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -35,4 +35,4 @@ sufficiently hardened for production use. [1]: mailto:secalert_us@oracle.com [2]: https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html [3]: https://www.oracle.com/security-alerts/encryptionkey.html -[4]: https://www.oracle.com/security-alerts/ +[4]: https://www.oracle.com/security-alerts/ \ No newline at end of file diff --git a/src/oracle_vecdb/client.py b/src/oracle_vecdb/client.py index c8dc21f..8f95d30 100644 --- a/src/oracle_vecdb/client.py +++ b/src/oracle_vecdb/client.py @@ -43,7 +43,11 @@ VectorDebugFlags, VectorEmbedInputItem, ) -from .validation import validate_resource_names +from .validation import ( + set_default_arguments, + validate_common_spec_arguments, + validate_resource_names, +) from .vecdb_exception import VecDBException from .vecdb_errors import ( InvalidTableNameFormatError, @@ -196,6 +200,7 @@ def __setattr__(self, name: str, value: Any) -> None: def _get_active_service(self) -> VecDBServiceProtocol: return self._get_ords_service() + @set_default_arguments def describe_vector_database(self) -> DatabaseSummaryResponse: """ Get summary statistics for the entire vector database service. @@ -230,6 +235,7 @@ def describe_vector_database(self) -> DatabaseSummaryResponse: """ return self._get_active_service().describe_vector_database() + @set_default_arguments def list_vector_tables( self, limit: Optional[int] = None, offset: Optional[int] = None ) -> VectorTableCollectionResponse: @@ -327,6 +333,8 @@ def list_vector_tables( limit=limit, offset=offset ) + @set_default_arguments + @validate_common_spec_arguments @validate_resource_names(name=InvalidTableNameFormatError) def create_vector_table( self, @@ -490,6 +498,7 @@ def create_vector_table( debug_flags=debug_flags, ) + @set_default_arguments @validate_resource_names(name=InvalidTableNameFormatError) def describe_vector_table(self, name: str) -> VectorTableResponse: """ @@ -563,6 +572,7 @@ def describe_vector_table(self, name: str) -> VectorTableResponse: name=name, ) + @set_default_arguments @validate_resource_names(name=InvalidTableNameFormatError) def drop_vector_table(self, name: str) -> DropVectorTableResponse: """ @@ -611,6 +621,7 @@ def drop_vector_table(self, name: str) -> DropVectorTableResponse: name=name, ) + @set_default_arguments @validate_resource_names(name=InvalidTableNameFormatError) def update_vector_table_annotation( self, @@ -678,6 +689,7 @@ def update_vector_table_annotation( debug_flags=debug_flags, ) + @set_default_arguments @validate_resource_names(model_name=InvalidModelNameFormatError) def generate_embedding( self, @@ -757,6 +769,7 @@ def generate_embedding( debug_flags=debug_flags, ) + @set_default_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def upsert_vectors( self, @@ -962,6 +975,8 @@ def _submit_upsert_batch( has_count = True return total, has_count + @set_default_arguments + @validate_common_spec_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def list_vectors( self, @@ -1071,6 +1086,7 @@ def list_vectors( ) # VectorApi methods + @set_default_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def delete_vectors( self, @@ -1129,6 +1145,7 @@ def delete_vectors( debug_flags=debug_flags, ) + @set_default_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def load_vectors( self, @@ -1226,6 +1243,7 @@ def load_vectors( debug_flags=debug_flags, ) + @set_default_arguments def list_vector_load_jobs( self, limit: Optional[int] = None, offset: Optional[int] = None ) -> JobCollectionResponse: @@ -1281,6 +1299,7 @@ def list_vector_load_jobs( limit=limit, offset=offset ) + @set_default_arguments @validate_resource_names(load_job_name=InvalidLoadJobNameFormatError) def describe_vector_load_job(self, load_job_name: str) -> JobResponse: """ @@ -1336,6 +1355,7 @@ def describe_vector_load_job(self, load_job_name: str) -> JobResponse: load_job_name=load_job_name, ) + @set_default_arguments @validate_resource_names(load_job_name=InvalidLoadJobNameFormatError) def get_vector_load_job_log(self, load_job_name: str) -> JobLogResponse: """ @@ -1395,6 +1415,8 @@ def get_vector_load_job_log(self, load_job_name: str) -> JobLogResponse: ) # SearchApi methods + @set_default_arguments + @validate_common_spec_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def query( self, @@ -1589,6 +1611,8 @@ def query( # SummaryApi methods + @set_default_arguments + @validate_common_spec_arguments def rerank( self, query: str, @@ -1699,6 +1723,8 @@ def metadata_for(result): # ModelApi methods # IndexApi methods + @set_default_arguments + @validate_common_spec_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def create_index( self, @@ -1729,7 +1755,7 @@ def create_index( may include ``auto_index``, ``include_paths``, and ``exclude_paths``. Example: - ``{'vector_index_params': {'organization': 'INMEMORY GRAPH', 'distance_metric': 'COSINE', 'advanced_params': {'neighbors': 32, 'efConstruction': 200}}, 'parallel_creation': 4}`` + ``{'vector_index_params': {'organization': 'INMEMORY GRAPH', 'distance_metric': 'COSINE', 'distribute_params': {'distribute_method': 'AUTO'}, 'advanced_params': {'neighbors': 32, 'efConstruction': 200}}, 'parallel_creation': 4}`` :type index_params: dict, optional :param debug_flags: Debug configuration for detailed logging. @@ -1832,6 +1858,7 @@ def create_index( debug_flags=debug_flags, ) + @set_default_arguments def list_index_jobs( self, limit: Optional[int] = None, offset: Optional[int] = None ) -> JobCollectionResponse: @@ -1878,6 +1905,7 @@ def list_index_jobs( limit=limit, offset=offset ) + @set_default_arguments @validate_resource_names(index_job_name=InvalidIndexJobNameFormatError) def describe_index_job(self, index_job_name: str) -> JobResponse: """ @@ -1941,6 +1969,7 @@ def describe_index_job(self, index_job_name: str) -> JobResponse: index_job_name=index_job_name, ) + @set_default_arguments @validate_resource_names(index_job_name=InvalidIndexJobNameFormatError) def get_index_job_log(self, index_job_name: str) -> JobLogResponse: """ @@ -2002,6 +2031,8 @@ def get_index_job_log(self, index_job_name: str) -> JobLogResponse: index_job_name=index_job_name, ) + @set_default_arguments + @validate_common_spec_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def rebuild_index( self, @@ -2061,6 +2092,7 @@ def rebuild_index( # InferenceApi methods + @set_default_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def describe_index(self, table_name: str) -> IndexDescriptionResponse: """ @@ -2099,6 +2131,7 @@ def describe_index(self, table_name: str) -> IndexDescriptionResponse: table_name=table_name, ) + @set_default_arguments @validate_resource_names(table_name=InvalidTableNameFormatError) def drop_index( self, @@ -2160,6 +2193,7 @@ def drop_index( debug_flags=debug_flags, ) + @set_default_arguments def list_models( self, limit: Optional[int] = None, offset: Optional[int] = None ) -> ModelCollectionResponse: @@ -2229,6 +2263,7 @@ def list_models( limit=limit, offset=offset ) + @set_default_arguments @validate_resource_names(model_name=InvalidModelNameFormatError) def load_model( self, @@ -2322,6 +2357,7 @@ def load_model( debug_flags=debug_flags, ) + @set_default_arguments @validate_resource_names(model_name=InvalidModelNameFormatError) def describe_model(self, model_name: str) -> ModelResponse: """ @@ -2388,6 +2424,7 @@ def describe_model(self, model_name: str) -> ModelResponse: model_name=model_name, ) + @set_default_arguments @validate_resource_names(model_name=InvalidModelNameFormatError) def drop_model(self, model_name: str) -> DropModelResponse: """ diff --git a/src/oracle_vecdb/configuration.py b/src/oracle_vecdb/configuration.py index d22f3f1..4d98687 100644 --- a/src/oracle_vecdb/configuration.py +++ b/src/oracle_vecdb/configuration.py @@ -62,6 +62,7 @@ r"(stable|\d+(?:\.\d+)*)/?$" ) RUNTIME_BASE_PATH_PLACEHOLDER = "https://REPLACED_AT_RUNTIME" +SUPPORTED_AUTHENTICATION_MODES = ("none", "basic", "bearer") ServerVariablesT = Dict[str, str] @@ -369,6 +370,24 @@ def __init__( if password is None and env_password: password = env_password + configured_auth_modes = [] + if username is not None or password is not None: + if username is None or password is None: + raise ValueError( + "Basic authentication requires both username and password." + ) + configured_auth_modes.append("basic") + if access_token is not None: + configured_auth_modes.append("bearer") + if len(configured_auth_modes) > 1: + raise ValueError( + "Conflicting authentication settings: choose exactly one " + "authentication mode (username/password or access_token)." + ) + self._authentication_mode = ( + configured_auth_modes[0] if configured_auth_modes else "none" + ) + if self._rest_service_configured: self._validate_base_path(self._base_path) """Default Base url @@ -713,6 +732,11 @@ def auth_settings(self) -> AuthSettings: ) return auth + @property + def authentication_mode(self) -> str: + """Return the single authentication mode selected for this client.""" + return self._authentication_mode + def to_debug_report(self) -> str: """Gets the essential information for debugging. diff --git a/src/oracle_vecdb/default_settings.py b/src/oracle_vecdb/default_settings.py new file mode 100644 index 0000000..7380502 --- /dev/null +++ b/src/oracle_vecdb/default_settings.py @@ -0,0 +1,76 @@ +## +## Copyright (c) 2026 Oracle and/or its affiliates. +## Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +## + +"""PL/SQL-grounded default arguments shared by VecDB client transports. + +The public facade can use this catalog before delegating to either ORDS or a +native SQL*Net backend. Entries use public Python method and parameter names, +not names from legacy PL/SQL overloads or transport-specific request fields. +""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any, ClassVar, Mapping + + +class DefaultSettings: + """Return independent copies of verified public-operation defaults. + + The values below come from the current canonical ``DBMS_VECTOR_DATABASE`` + entry points in ``prvtvectordb.sql``. Only defaults that are meaningful to + materialize in the public Python call are included. ``None`` defaults are + deliberately omitted: the future decorator must preserve the distinction + between an omitted value and a caller explicitly supplying ``None``. + """ + + # This catalog contains unconditional, sensible defaults for public + # operations. They can be applied without inspecting another argument + # in the same request (for example, list_vectors.limit). + # + # Defaults such as distribute_method="AUTO" do not belong here because + # they are operation-dependent nested values: they are meaningful only + # after the caller selects an INMEMORY GRAPH index organization. + _DEFAULT_ARGUMENTS: ClassVar[Mapping[str, Mapping[str, Any]]] = { + # create_vector_table(..., table_params JSON DEFAULT NULL) normalizes + # an omitted table_params value to auto_generate_id=false. + "create_vector_table": { + "table_params": {"auto_generate_id": False}, + }, + # list_vectors(..., limit IN NUMBER DEFAULT 15, ...). + "list_vectors": { + "limit": 15, + }, + # OracleVecDB.query maps to DBMS_VECTOR_DATABASE.search, whose + # include_vectors parameter defaults to FALSE. + "query": { + "include_vectors": False, + }, + } + + @classmethod + def get_default_args_for(cls, function_name: str) -> dict[str, Any]: + """Return defaults for ``function_name`` without sharing mutables. + + Unknown operations intentionally return an empty dictionary so the + decorator can be applied to every public facade method safely. + """ + return deepcopy(dict(cls._DEFAULT_ARGUMENTS.get(function_name, {}))) + + @classmethod + def apply_operation_aware_defaults( + cls, + function_name: str, + arguments: Mapping[str, Any], + ) -> dict[str, Any]: + """Return arguments with any operation-aware defaults materialized. + + Index distribution is intentionally not defaulted. Callers selecting + ``INMEMORY GRAPH`` must explicitly choose a valid + ``distribute_method`` so the SDK does not hide an incomplete request. + The hook remains for future verified operation-aware defaults. + """ + del function_name + return dict(arguments) diff --git a/src/oracle_vecdb/error_messages.py b/src/oracle_vecdb/error_messages.py index ee10066..10043b8 100644 --- a/src/oracle_vecdb/error_messages.py +++ b/src/oracle_vecdb/error_messages.py @@ -14,24 +14,24 @@ "action": "Use https://:/ords//_/db-api/(stable|)/vecdb/.", }, "VECDB-003": { - "message": "Invalid table name format: '{table_name}'.", - "cause": "The table name does not match the required format.", - "action": "Use only letters, digits, and underscore (_).", + "message": "Invalid table name: '{table_name}'.", + "cause": "The table name must be non-empty text and must not contain NUL or double-quote characters.", + "action": 'Provide a non-empty table name without NUL (`\\x00`) or double-quote (`"`) characters. Other character and length restrictions are determined by the configured Oracle Database.', }, "VECDB-004": { "message": "Invalid model name format: '{model_name}'.", - "cause": "The model name does not match the required format.", - "action": "Use only letters, digits, and underscore (_).", + "cause": "The model name must be non-empty text and must not contain NUL or double-quote characters.", + "action": 'Provide a non-empty model name without NUL (`\\x00`) or double-quote (`"`) characters. Other character and length restrictions are determined by the configured Oracle Database.', }, "VECDB-005": { "message": "Invalid load job name format: '{load_job_name}'.", - "cause": "The load job name does not match the required format.", - "action": "Use only letters, digits, and underscore (_).", + "cause": "The load job name must be non-empty text and must not contain NUL or double-quote characters.", + "action": 'Provide a non-empty load job name without NUL (`\\x00`) or double-quote (`"`) characters. Other character and length restrictions are determined by the configured Oracle Database.', }, "VECDB-006": { "message": "Invalid index job name format: '{index_job_name}'.", - "cause": "The index job name does not match the required format.", - "action": "Use only letters, digits, and underscore (_).", + "cause": "The index job name must be non-empty text and must not contain NUL or double-quote characters.", + "action": 'Provide a non-empty index job name without NUL (`\\x00`) or double-quote (`"`) characters. Other character and length restrictions are determined by the configured Oracle Database.', }, "VECDB-007": { "message": ( @@ -68,4 +68,23 @@ "cause": "The index job has not finished yet.", "action": "Wait until the index job reaches a terminal state before fetching its log.", }, + "VECDB-012": { + "message": ( + "CommonSpec defaults for '{function_name}' do not match public " + "parameters: {parameter_names}." + ), + "cause": ( + "The CommonSpec catalog references a parameter that the decorated " + "function does not accept." + ), + "action": ( + "Update CommonSpec or the function signature so their parameter " + "names match." + ), + }, + "VECDB-013": { + "message": "Invalid value or combination for {parameter_name}: {detail}", + "cause": "The request violates a deterministic parameter constraint defined by the VecDB PL/SQL API.", + "action": "Correct the parameter value or combination and retry the operation.", + }, } diff --git a/src/oracle_vecdb/ords.py b/src/oracle_vecdb/ords.py index 5600143..6a26634 100644 --- a/src/oracle_vecdb/ords.py +++ b/src/oracle_vecdb/ords.py @@ -146,10 +146,11 @@ def call_with_context(*args: Any, **kwargs: Any) -> Any: service_name=type(self).__name__, error=error, ) - # The wrapper already contains the complete original ORDS - # traceback. Suppress implicit exception chaining so reports - # contain one clear template instead of the same failure twice. - raise wrapped from None + # Raise after leaving the handler. This avoids attaching the raw + # transport exception as ``wrapped.__context__``; the wrapper + # already contains its sanitized diagnostics. + # Suppress implicit chaining so reports contain one clear template. + raise wrapped from None return call_with_context @@ -241,11 +242,22 @@ def _validate_index_params_fields(index_params: Any) -> None: "Unsupported vector_index_params field(s): " f"{', '.join(sorted(unknown))}" ) + organization = vector.get("organization") + if organization == "INMEMORY GRAPH" and ( + "distribute_params" not in vector + or vector["distribute_params"] is None + ): + raise ValueError( + "vector_index_params.distribute_params cannot be None for " + "organization 'INMEMORY GRAPH'; it must contain a valid " + "distribute_method. Valid values are: 'ROWID RANGE', " + "'SIMILARITY', 'PARTITION', " + "'SUBPARTITION', 'DISTRIBUTE', 'AUTO'" + ) if "distribute_params" in vector: distribute = vector["distribute_params"] - # The OpenAPI contract declares distribute_params nullable. - # Validate its required child field only when an object was - # supplied; an explicit null is a valid request value. + # ``distribute_params`` is nullable for non-graph index + # organizations, but graph indexes require a method. if distribute is None: pass elif not isinstance(distribute, dict): @@ -271,6 +283,67 @@ def _validate_index_params_fields(index_params: Any) -> None: f"{', '.join(sorted(unknown))}" ) + @staticmethod + def _normalize_upsert_vector_fields( + vector: Dict[str, Any], + ) -> Dict[str, Any]: + """Normalize upsert record field names without changing field values.""" + model = _models.UpsertVectorsRequestVectorsInner + supported_fields: Dict[str, str] = {} + + for field_name, field_info in getattr( + model, "model_fields", {} + ).items(): + canonical_name = str(field_name) + supported_fields[canonical_name.casefold()] = canonical_name + alias = getattr(field_info, "alias", None) + if alias: + supported_fields[str(alias).casefold()] = canonical_name + + for field_name in getattr(model, "__properties", []): + canonical_name = str(field_name) + supported_fields.setdefault( + canonical_name.casefold(), canonical_name + ) + + normalized: Dict[str, Any] = {} + original_names: Dict[str, str] = {} + duplicate_names: Dict[str, list[str]] = {} + unknown_names: list[str] = [] + + for field_name, value in vector.items(): + if not isinstance(field_name, str): + unknown_names.append(repr(field_name)) + continue + matched_name = supported_fields.get(field_name.casefold()) + if matched_name is None: + unknown_names.append(field_name) + continue + canonical_name = matched_name + if canonical_name in normalized: + duplicate_names.setdefault( + canonical_name, [original_names[canonical_name]] + ).append(field_name) + continue + normalized[canonical_name] = value + original_names[canonical_name] = field_name + + if unknown_names: + raise ValueError( + "Unknown upsert vector field(s): " + f"{', '.join(sorted(unknown_names))}" + ) + if duplicate_names: + details = ", ".join( + f"{canonical_name} ({', '.join(names)})" + for canonical_name, names in sorted(duplicate_names.items()) + ) + raise ValueError( + "Duplicate upsert vector field(s) with different casing: " + f"{details}" + ) + return normalized + @ORDSResponseHandler def describe_vector_database(self) -> DatabaseSummaryResponse: return DatabaseSummaryResponse.from_internal( @@ -434,7 +507,9 @@ def upsert_vectors( vector if isinstance(vector, _models.UpsertVectorsRequestVectorsInner) else _models.UpsertVectorsRequestVectorsInner( - **cast(Dict[str, Any], vector) + **self._normalize_upsert_vector_fields( + cast(Dict[str, Any], vector) + ) ) ) for vector in (vectors or []) diff --git a/src/oracle_vecdb/parameter_validation.py b/src/oracle_vecdb/parameter_validation.py new file mode 100644 index 0000000..241747f --- /dev/null +++ b/src/oracle_vecdb/parameter_validation.py @@ -0,0 +1,479 @@ +"""Transport-neutral validation for PL/SQL parameter dependencies. + +The public facade accepts Python mappings that become JSON for either ORDS or +the future native SDK. This module validates deterministic request shape, +enums, ranges, and cross-field dependencies before a transport is selected. +Database-state checks (for example, whether a table exists) remain in PL/SQL. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from enum import Enum +from typing import Any + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + model_validator, +) + + +class IndexOrganization(str, Enum): + """Organizations accepted by ``validate_index_params`` in PL/SQL.""" + + PARTITIONS = "PARTITIONS" + INMEMORY_GRAPH = "INMEMORY GRAPH" + + +class DistributeMethod(str, Enum): + """Values accepted by the PL/SQL distribute-method validation.""" + + ROWID_RANGE = "ROWID RANGE" + SIMILARITY = "SIMILARITY" + PARTITION = "PARTITION" + SUBPARTITION = "SUBPARTITION" + DISTRIBUTE = "DISTRIBUTE" + AUTO = "AUTO" + + +class QuantizationType(str, Enum): + NONE = "NONE" + SCALAR = "SCALAR" + + +class IndexType(str, Enum): + VECTOR = "vector" + METADATA = "metadata" + ALL = "all" + + +class DistanceMetric(str, Enum): + MANHATTAN = "MANHATTAN" + HAMMING = "HAMMING" + DOT = "DOT" + COSINE = "COSINE" + EUCLIDEAN = "EUCLIDEAN" + EUCLIDEAN_SQUARED = "EUCLIDEAN_SQUARED" + JACCARD = "JACCARD" + L2_SQUARED = "L2_SQUARED" + + +class _StrictModel(BaseModel): + """Mirror PL/SQL JSON schemas by rejecting unknown object keys.""" + + # JSON inputs represent enums as strings. Model-wide strict mode would + # incorrectly require callers to construct Python Enum instances, so enum + # membership is validated normally while object shape stays strict. + model_config = ConfigDict(extra="forbid") + + +class TableParams(_StrictModel): + auto_generate_id: bool | None = None + + +class EmbedParams(_StrictModel): + model: str = Field(min_length=1, max_length=128) + embed_metadata_jsonpath: str = Field(min_length=1, max_length=128) + + +class DistributeParams(_StrictModel): + distribute_method: DistributeMethod + service_name: str | None = Field(default=None, min_length=1, max_length=128) + + +class MetadataIndexParams(_StrictModel): + auto_index: bool | None = None + include_paths: list[str] | None = None + exclude_paths: list[str] | None = None + + @model_validator(mode="after") + def validate_metadata_paths(self) -> "MetadataIndexParams": + # PL/SQL supports only non-empty dot paths or "*" for metadata MVIs. + # Array syntax such as ``tags[*]`` is explicitly rejected there. + for parameter_name, paths in ( + ("include_paths", self.include_paths), + ("exclude_paths", self.exclude_paths), + ): + for path in paths or []: + if not path.strip() or "[" in path or "]" in path: + raise ValueError( + f"metadata_index_params.{parameter_name} must contain " + "non-empty paths without array syntax" + ) + + # A wildcard on both sides gives no unambiguous metadata-index policy, + # so PL/SQL rejects this before index reconciliation begins. + if "*" in (self.include_paths or []) and "*" in ( + self.exclude_paths or [] + ): + raise ValueError( + "metadata_index_params.include_paths and exclude_paths " + "cannot both contain '*'" + ) + return self + + +class VectorIndexParams(_StrictModel): + auto_index: bool | None = None + organization: IndexOrganization | None = None + distance_metric: DistanceMetric | None = None + accuracy: int | None = Field(default=None, ge=0, le=100) + quantization_type: QuantizationType | None = None + compression_ratio: int | None = None + online_build: bool | None = None + distribute_params: DistributeParams | None = None + advanced_params: dict[str, Any] | None = None + + @model_validator(mode="after") + def validate_dependencies(self) -> "VectorIndexParams": + # PL/SQL uses PARTITIONS when organization is omitted. Cross-field + # validation must use that effective value, not merely the raw input. + organization = self.organization or IndexOrganization.PARTITIONS + + # When distribute_params is not present reject the request as + # it would otherwise reach PL/SQL without a valid method. + if ( + organization == IndexOrganization.INMEMORY_GRAPH + and self.distribute_params is None + ): + raise ValueError( + "vector_index_params.distribute_params cannot be None for " + "organization 'INMEMORY GRAPH'; it must contain a valid " + "distribute_method. Valid values are: " + + ", ".join(f"'{method.value}'" for method in DistributeMethod) + ) + + # Distribution is implemented by the HNSW / INMEMORY GRAPH path. IVF + # PARTITIONS indexes must not receive a distribute_params object. + if ( + self.distribute_params is not None + and organization != IndexOrganization.INMEMORY_GRAPH + ): + raise ValueError( + "vector_index_params.distribute_params is supported only for " + "organization 'INMEMORY GRAPH'" + ) + + # Online index construction is likewise an HNSW-only capability. + if ( + self.online_build is True + and organization != IndexOrganization.INMEMORY_GRAPH + ): + raise ValueError( + "vector_index_params.online_build is supported only for " + "organization 'INMEMORY GRAPH'" + ) + + # SCALAR quantization and its compression ratio are a pair in PL/SQL: + # neither setting is meaningful without the other. + if self.quantization_type == QuantizationType.SCALAR: + if self.compression_ratio is None: + raise ValueError( + "vector_index_params.compression_ratio is required when " + "quantization_type is 'SCALAR'" + ) + if self.compression_ratio not in {2, 4, 8}: + raise ValueError( + "vector_index_params.compression_ratio must be one of 2, 4, or 8" + ) + elif self.compression_ratio is not None: + raise ValueError( + "vector_index_params.compression_ratio requires " + "quantization_type 'SCALAR'" + ) + + # advanced_params has two disjoint schemas. Selecting IVF exposes + # only partitions; selecting HNSW exposes graph-construction controls. + if self.advanced_params is not None: + if organization == IndexOrganization.PARTITIONS: + _validate_ivf_advanced_params(self.advanced_params) + else: + _validate_hnsw_advanced_params(self.advanced_params) + return self + + +class IndexParams(_StrictModel): + vector_index_params: VectorIndexParams | None = None + metadata_index_params: MetadataIndexParams | None = None + parallel_creation: int | None = Field(default=None, ge=1) + + +class IndexActionParams(IndexParams): + """The rebuild action adds the PL/SQL-only ``index_type`` selector.""" + + index_type: IndexType | None = None + + +class LegacyIndexParams(_StrictModel): + """Older flat vector-index request shape still accepted by PL/SQL.""" + + indexing: str | None = None + organization: IndexOrganization | None = None + distance_metric: DistanceMetric | None = None + distance: DistanceMetric | None = None + accuracy: int | None = Field(default=None, ge=0, le=100) + quantization_type: QuantizationType | None = None + compression_ratio: int | None = None + distribute_params: DistributeParams | None = None + advanced_params: dict[str, Any] | None = None + parallel_creation: int | None = Field(default=None, ge=1) + + @model_validator(mode="after") + def validate_legacy_dependencies(self) -> "LegacyIndexParams": + if self.indexing is not None and self.indexing.lower() not in { + "auto", + "manual", + }: + raise ValueError("indexing must be one of AUTO or MANUAL") + if ( + self.distance_metric is not None + and self.distance is not None + and self.distance_metric != self.distance + ): + raise ValueError("distance and distance_metric must match") + return self + + +class QueryBy(_StrictModel): + text: str | None = None + id: str | None = None + vector: list[float] | None = None + + @model_validator(mode="after") + def validate_single_mode(self) -> "QueryBy": + # Search resolves exactly one query source. Sending multiple modes is + # ambiguous, while sending none gives the database nothing to resolve. + modes = sum( + value is not None for value in (self.text, self.id, self.vector) + ) + if modes != 1: + raise ValueError( + "query_by must contain exactly one of text, id, or vector" + ) + return self + + +class QueryAdvancedParams(_StrictModel): + rescore_factor: int | None = Field(default=None, ge=1, le=100) + + +class QueryAdvancedOptions(_StrictModel): + distance_metric: DistanceMetric | None = None + accuracy: int | None = Field(default=None, ge=0, le=100) + advanced_params: QueryAdvancedParams | None = None + idx_parameters: dict[str, Any] | None = None + + +class RerankModelParams(_StrictModel): + top_n: int | None = Field(default=None, gt=0) + + +def _validate_ivf_advanced_params(value: Mapping[str, Any]) -> None: + """Validate the organization-specific advanced options for IVF.""" + + class IVFAdvancedParams(_StrictModel): + partitions: int | None = Field(default=None, ge=1, le=10_000_000) + + try: + IVFAdvancedParams.model_validate(value) + except ValidationError as validation_error: + # Prefix nested Pydantic locations so the public error identifies the + # JSON container the caller must correct. + raise ValueError( + f"vector_index_params.advanced_params is invalid: {validation_error}" + ) from validation_error + + +def _validate_hnsw_advanced_params(value: Mapping[str, Any]) -> None: + """Validate the organization-specific advanced options for HNSW.""" + + class HNSWAdvancedParams(_StrictModel): + neighbors: int | None = Field(default=None, ge=1, le=2048) + efConstruction: int | None = Field(default=None, ge=1, le=65535) + rescore_factor: int | None = Field(default=None, ge=1, le=100) + algorithm: str | None = None + + @model_validator(mode="after") + def validate_algorithm(self) -> "HNSWAdvancedParams": + if ( + self.algorithm is not None + and self.algorithm != "uniform_quantization" + ): + raise ValueError("algorithm must be 'uniform_quantization'") + return self + + try: + HNSWAdvancedParams.model_validate(value) + except ValidationError as validation_error: + # Keep the same public JSON path for graph-specific nested failures. + raise ValueError( + f"vector_index_params.advanced_params is invalid: {validation_error}" + ) from validation_error + + +def _require_mapping(value: Any, parameter_name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{parameter_name} must be a JSON object") + return value + + +def validate_table_params(value: Any) -> None: + TableParams.model_validate(_require_mapping(value, "table_params")) + + +def validate_embed_params(value: Any) -> None: + EmbedParams.model_validate(_require_mapping(value, "embed_params")) + + +def validate_index_params(value: Any) -> None: + params = _require_mapping(value, "index_params") + if "vector_index_params" in params or "metadata_index_params" in params: + IndexParams.model_validate(params) + return + + # The package converts legacy flat parameters to the current nested shape + # before applying the same dependency rules. Do the conversion on a new + # dictionary only; the decorator must never rewrite the caller's request. + legacy = LegacyIndexParams.model_validate(params) + vector_params: dict[str, Any] = {} + if legacy.indexing is not None: + vector_params["auto_index"] = legacy.indexing.lower() == "auto" + for key in ( + "organization", + "accuracy", + "quantization_type", + "compression_ratio", + "distribute_params", + "advanced_params", + ): + value = getattr(legacy, key) + if value is not None: + vector_params[key] = value + distance = legacy.distance_metric or legacy.distance + if distance is not None: + vector_params["distance_metric"] = distance + current: dict[str, Any] = {} + if vector_params: + current["vector_index_params"] = vector_params + if legacy.parallel_creation is not None: + current["parallel_creation"] = legacy.parallel_creation + IndexParams.model_validate(current) + + +def validate_rebuild_index_params(value: Any) -> None: + raw_params = _require_mapping(value, "index_params") + if ( + "vector_index_params" not in raw_params + and "metadata_index_params" not in raw_params + ): + # Action APIs preserve the flat payload's optional index_type selector + # while converting the remaining legacy fields before validation. + index_type = raw_params.get("index_type") + if index_type is not None: + IndexType(index_type) + validate_index_params( + { + key: item + for key, item in raw_params.items() + if key != "index_type" + } + ) + return + + params = IndexActionParams.model_validate(raw_params) + # Rebuild uses existing metadata indexes; it may select paths, but PL/SQL + # rejects metadata auto_index because that flag only belongs to creation. + if ( + params.metadata_index_params + and params.metadata_index_params.auto_index is not None + ): + raise ValueError( + "metadata_index_params.auto_index is not supported for rebuild_index" + ) + + +def validate_query_by(value: Any) -> None: + QueryBy.model_validate(_require_mapping(value, "query_by")) + + +def validate_query_advanced_options(value: Any) -> None: + QueryAdvancedOptions.model_validate( + _require_mapping(value, "advanced_options") + ) + + +def validate_positive_top_k(value: Any) -> None: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or value <= 0 + ): + raise ValueError("top_k must be greater than zero") + + +def validate_list_vectors_arguments(arguments: Mapping[str, Any]) -> None: + limit = arguments.get("limit") + offset = arguments.get("offset") + ids = arguments.get("ids") + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError("limit must be a positive integer") + if offset is not None and ( + isinstance(offset, bool) + or not isinstance(offset, (int, float)) + or offset < 0 + ): + raise ValueError("offset must be greater than or equal to zero") + if ids is not None and ( + not isinstance(ids, list) + or not all(isinstance(item, str) for item in ids) + ): + raise ValueError("ids must be a JSON array of strings") + + +def validate_rerank_model_params(value: Any) -> None: + RerankModelParams.model_validate(_require_mapping(value, "model_params")) + + +ParameterValidator = Any + + +OPERATION_PARAMETER_VALIDATORS: dict[str, dict[str, ParameterValidator]] = { + "create_vector_table": { + "embed_params": validate_embed_params, + "index_params": validate_index_params, + }, + "create_index": {"index_params": validate_index_params}, + "rebuild_index": {"index_params": validate_rebuild_index_params}, + "query": { + "query_by": validate_query_by, + "top_k": validate_positive_top_k, + "advanced_options": validate_query_advanced_options, + }, + "rerank": {"model_params": validate_rerank_model_params}, +} + + +def validate_operation_arguments( + operation: str, arguments: Mapping[str, Any] +) -> None: + """Validate JSON parameters and operation-level scalar dependencies.""" + for parameter_name, validator in OPERATION_PARAMETER_VALIDATORS.get( + operation, {} + ).items(): + value = arguments.get(parameter_name) + # A missing value and explicit None both map to PL/SQL DEFAULT NULL + # where applicable, so no local JSON validation is needed in that case. + if value is not None: + validator(value) + + if operation == "list_vectors": + validate_list_vectors_arguments(arguments) + + +__all__ = [ + "OPERATION_PARAMETER_VALIDATORS", + "validate_operation_arguments", +] diff --git a/src/oracle_vecdb/validation.py b/src/oracle_vecdb/validation.py index 92732c3..38b8485 100644 --- a/src/oracle_vecdb/validation.py +++ b/src/oracle_vecdb/validation.py @@ -6,26 +6,160 @@ from __future__ import annotations import inspect +from collections.abc import Mapping +from copy import deepcopy from functools import wraps -from typing import Callable, ParamSpec, TypeVar, cast +from typing import Any, Callable, ParamSpec, TypeVar, cast -from pydantic import BaseModel, ValidationError, field_validator +from pydantic import BaseModel, StrictStr, ValidationError, field_validator +from .default_settings import DefaultSettings +from .parameter_validation import validate_operation_arguments from .vecdb_exception import VecDBException +from .vecdb_errors import ( + DefaultSettingsParameterMismatchError, + InvalidParameterCombinationError, +) P = ParamSpec("P") R = TypeVar("R") ResourceErrorFactory = Callable[[str], BaseException] +def _merge_default_mapping( + defaults: Mapping[str, Any], supplied: Mapping[str, Any] +) -> dict[str, Any]: + """Merge nested defaults without mutating either input mapping. + + Values from ``supplied`` take precedence, including an explicit ``None``. + Nested mappings are merged recursively so callers can override one option + while retaining DefaultSettings values for sibling options. + """ + result = deepcopy(dict(defaults)) + for key, supplied_value in supplied.items(): + default_value = result.get(key) + if isinstance(default_value, Mapping) and isinstance( + supplied_value, Mapping + ): + result[key] = _merge_default_mapping(default_value, supplied_value) + else: + result[key] = deepcopy(supplied_value) + return result + + +def set_default_arguments( + function: Callable[P, R], +) -> Callable[P, R]: + """Apply verified DefaultSettings defaults to omitted public arguments. + + Explicit caller values, including ``None``, always take precedence. + Dictionary values are merged recursively so a caller can override one + nested option without discarding unspecified DefaultSettings defaults. + """ + signature = inspect.signature(function) + defaults = DefaultSettings.get_default_args_for(function.__name__) + unknown_parameters = set(defaults).difference(signature.parameters) + if unknown_parameters: + error = DefaultSettingsParameterMismatchError( + function_name=function.__name__, + parameter_names=sorted(unknown_parameters), + ) + raise VecDBException.from_service_error( + operation=function.__name__, + arguments={ + "kwargs": {"parameter_names": sorted(unknown_parameters)} + }, + service_name="common_spec", + error=error, + ) from error + + @wraps(function) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + arguments = signature.bind(*args, **kwargs) + for parameter_name, default in defaults.items(): + if parameter_name not in arguments.arguments: + # ``bind`` records only arguments supplied by the caller. + # Insert a private copy only when this parameter was omitted. + arguments.arguments[parameter_name] = deepcopy(default) + elif isinstance(default, Mapping) and isinstance( + arguments.arguments[parameter_name], Mapping + ): + # For nested option dictionaries, retain unspecified DefaultSettings + # values while giving every caller-provided key precedence. + arguments.arguments[parameter_name] = _merge_default_mapping( + default, + arguments.arguments[parameter_name], + ) + + # Apply any future operation-aware defaults after the public arguments + # have been bound and unconditional defaults materialized. Index + # distribution is deliberately validated as explicitly supplied. + arguments.arguments.update( + DefaultSettings.apply_operation_aware_defaults( + function.__name__, + arguments.arguments, + ) + ) + return function(*arguments.args, **arguments.kwargs) + + return wrapper + + +def validate_common_spec_arguments( + function: Callable[P, R], +) -> Callable[P, R]: + """Validate PL/SQL-derived argument dependencies before delegation. + + This decorator is transport-neutral: it protects the common facade before + either ORDS or a future native SDK receives the request. It deliberately + validates only deterministic request rules; database-state checks remain + the responsibility of PL/SQL. + """ + signature = inspect.signature(function) + + @wraps(function) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + arguments = signature.bind(*args, **kwargs) + try: + validate_operation_arguments(function.__name__, arguments.arguments) + except (ValidationError, ValueError, TypeError) as validation_error: + error = InvalidParameterCombinationError( + parameter_name="request parameters", + detail=str(validation_error), + ) + # Do not attach the full nested request: it may contain credentials, + # signed URLs, query text, or other sensitive application data. + raise VecDBException.from_service_error( + operation=function.__name__, + arguments={"kwargs": {"request": ""}}, + service_name="validation", + error=error, + ) from validation_error + return function(*arguments.args, **arguments.kwargs) + + return wrapper + + class ResourceName(BaseModel): - resource_name: str + """Conservative resource-name checks shared by public SDK methods. + + Character, case, and length rules belong to the configured Oracle Database: + they can vary with the database version and identifier configuration. The + SDK therefore rejects only values that are not safe, meaningful text for a + resource identifier on every supported transport. + """ + + resource_name: StrictStr @field_validator("resource_name", mode="after") @classmethod def validate_resource_name(cls, value: str) -> str: if value is None or len(value.strip()) == 0: raise ValueError("Input value cannot be None, empty, or blank") + if "\x00" in value or '"' in value: + raise ValueError( + "Input value cannot contain NUL or double-quote characters" + ) return value @@ -36,13 +170,16 @@ def validate_resource_name( parameter_name: str, error_factory: ResourceErrorFactory, ) -> str: - """Validate a public resource name using the common error contract. + """Apply only transport-safe validation to a public resource name. The public facade exposes :class:`VecDBException` for both local and - service validation failures. ``original_exception`` retains the - resource-specific SDK error so existing callers can distinguish table, - model, and job-name validation failures without importing implementation - details from the generated client. + service validation failures. ``original_exception_type`` and + ``is_original_exception()`` retain the resource-specific SDK error + category so existing callers can distinguish table, model, and job-name + validation failures without retaining the original error payload. + + Database-specific grammar, including other allowed punctuation, case, + Unicode, and length, is deliberately delegated to the Oracle Database. """ try: return ResourceName(resource_name=value).resource_name diff --git a/src/oracle_vecdb/vecdb_errors.py b/src/oracle_vecdb/vecdb_errors.py index 2ce13b6..594fdcf 100644 --- a/src/oracle_vecdb/vecdb_errors.py +++ b/src/oracle_vecdb/vecdb_errors.py @@ -171,3 +171,36 @@ def __init__(self, index_job_name, state, *, locale=None): locale=locale, params={"index_job_name": index_job_name, "state": state}, ) + + +class DefaultSettingsParameterMismatchError(VecDBError): + """Raised when DefaultSettings references a missing public parameter.""" + + def __init__( + self, + function_name: str, + parameter_names: list[str], + *, + locale=None, + ): + super().__init__( + "", + error_code="VECDB-012", + locale=locale, + params={ + "function_name": function_name, + "parameter_names": ", ".join(parameter_names), + }, + ) + + +class InvalidParameterCombinationError(VecDBError): + """Raised when JSON request options violate PL/SQL dependency rules.""" + + def __init__(self, parameter_name: str, detail: str, *, locale=None): + super().__init__( + "", + error_code="VECDB-013", + locale=locale, + params={"parameter_name": parameter_name, "detail": detail}, + ) diff --git a/src/oracle_vecdb/vecdb_exception.py b/src/oracle_vecdb/vecdb_exception.py index 3474e8a..30547a1 100644 --- a/src/oracle_vecdb/vecdb_exception.py +++ b/src/oracle_vecdb/vecdb_exception.py @@ -10,7 +10,92 @@ import json import re import traceback -from typing import Any, Dict, Optional +from collections.abc import Mapping +from typing import Any, Dict, Optional, cast + +_REDACTED = "" + +# These names identify values that must not survive in an exception object. +# Keep this list exact (after normalization) so useful fields such as +# ``request_id``, ``connection_id``, and ``query_id`` remain available. +_SENSITIVE_VALUE_KEYS = { + "authorization", + "proxy_authorization", + "cookie", + "set_cookie", + "x_api_key", + "api_key", + "apikey", + "password", + "passwd", + "pwd", + "secret", + "client_secret", + "token", + "bearer_token", + "oauth_token", + "access_token", + "refresh_token", + "id_token", + "credential", + "credentials", + "auth", + "authentication", + "header", + "headers", + "private_key", + "username", + "database", + "database_name", + "db", + "schema", + "schema_name", + "connection", + "connection_string", + "dsn", + "sql", + "query", + "query_by", + "filter", + "filters", + "document", + "documents", + "metadata", + "payload", + "vector", + "vectors", + "embedding", + "embeddings", + "url", +} + + +def _normalized_key(value: Any) -> str: + return re.sub(r"[^a-z0-9]+", "_", str(value).lower()).strip("_") + + +def _is_sensitive_key(value: Any) -> bool: + normalized = _normalized_key(value) + return normalized in _SENSITIVE_VALUE_KEYS or normalized.endswith( + ("_token", "_secret", "_password", "_credential", "_api_key") + ) + + +class _SanitizedServiceError(Exception): + """Safe, displayable copy of a transport exception.""" + + def __init__(self, class_name: str, **fields: Any) -> None: + self.class_name = class_name + for name, value in fields.items(): + setattr(self, name, value) + super().__init__(self._safe_text(fields.get("reason"))) + + @staticmethod + def _safe_text(value: Any) -> str: + return value if isinstance(value, str) else str(value or "") + + def __str__(self) -> str: + return self._safe_text(getattr(self, "reason", None)) class VecDBException(Exception): @@ -34,16 +119,18 @@ def __init__( stack_trace: Optional[str] = None, ) -> None: self.status = status - self.reason = reason - self.body = body - self.data = data - self.headers = headers + self.reason = self._redact_value(reason) + self.body = self._redact_value(body) + self.data = self._redact_value(data) + self.headers = self._redact_value(headers) self.operation = operation - self.arguments = arguments + self.arguments = self._sanitize_arguments(arguments) self.service_name = service_name - self.service_error = service_error + self.service_error = self._safe_service_error(service_error) self.service_error_class_name = service_error_class_name - self.original_exception = service_error + # Keep only a sanitized snapshot. Retaining the transport exception + # would also retain its raw response, headers, and args. + self.original_exception = self.service_error self.original_exception_type = ( type(service_error) if service_error is not None else None ) @@ -121,6 +208,88 @@ def from_service_error( # a safe fallback for other service adapters. return cls(**values) + @staticmethod + def _safe_service_error( + error: Optional[BaseException], + ) -> Optional[BaseException]: + if error is None or isinstance(error, _SanitizedServiceError): + return error + fields = { + name: VecDBException._redact_value(getattr(error, name, None)) + for name in ( + "status", + "reason", + "body", + "data", + "headers", + "error_code", + "error_message", + "error_type", + "error_instance", + ) + if hasattr(error, name) + } + if "reason" not in fields: + fields["reason"] = VecDBException._redact_value(str(error)) + snapshot = _SanitizedServiceError(type(error).__name__, **fields) + # Preserve the useful ``isinstance`` compatibility promised by the + # old public attribute without keeping the original object alive. + try: + safe_type = type( + f"Sanitized{type(error).__name__}", + (_SanitizedServiceError, type(error)), + {"__module__": __name__}, + ) + safe_error = cast(Any, safe_type).__new__(safe_type) + safe_error.__dict__.update(snapshot.__dict__) + Exception.__init__(safe_error, str(snapshot)) + return safe_error + except TypeError: + if type(error).__module__.split(".", 1)[0] == "pydantic_core": + safe_type = type( + "SanitizedValidationError", + (_SanitizedServiceError, ValueError), + {"__module__": __name__}, + ) + safe_error = cast(Any, safe_type).__new__(safe_type) + safe_error.__dict__.update(snapshot.__dict__) + Exception.__init__(safe_error, str(snapshot)) + return safe_error + return snapshot + + @staticmethod + def _redact_value(value: Any, key: str = "") -> Any: + """Recursively remove secrets from values retained for diagnostics.""" + if _is_sensitive_key(key): + return _REDACTED + if isinstance(value, Mapping): + return { + item_key: VecDBException._redact_value(item, str(item_key)) + for item_key, item in value.items() + } + if isinstance(value, (list, tuple)): + sanitized = [VecDBException._redact_value(item) for item in value] + return tuple(sanitized) if isinstance(value, tuple) else sanitized + if isinstance(value, (bytes, bytearray, memoryview)): + value = bytes(value).decode("utf-8", errors="replace") + if isinstance(value, str): + try: + parsed = json.loads(value) + except (TypeError, ValueError): + parsed = None + if isinstance(parsed, (dict, list)): + sanitized = VecDBException._redact_value(parsed) + return json.dumps(sanitized) + return VecDBException._redact_diagnostic_text(value) + for serializer_name in ("model_dump", "to_dict"): + serializer = getattr(value, serializer_name, None) + if callable(serializer): + try: + return VecDBException._redact_value(serializer(), key) + except (TypeError, ValueError): + break + return value + @staticmethod def _sanitize_arguments(arguments: Any) -> Any: """Retain only safe resource identifiers in request diagnostics.""" @@ -138,38 +307,34 @@ def _sanitize_arguments(arguments: Any) -> Any: "vector_index_params", "debug_flags", } - sensitive_keys = ( - "auth", - "credential", - "document", - "filter", - "header", - "metadata", - "password", - "payload", - "secret", - "token", - "url", - "vector", - ) def sanitize( value: Any, key: str = "", safe_context: bool = False ) -> Any: - key_lower = key.lower() - if any(term in key_lower for term in sensitive_keys): - return "" - is_safe_key = safe_context or key_lower in safe_keys + key_normalized = _normalized_key(key) + if _is_sensitive_key(key): + return _REDACTED + is_safe_key = safe_context or key_normalized in { + _normalized_key(safe_key) for safe_key in safe_keys + } if isinstance(value, dict): - if not is_safe_key and key_lower not in {"", "args", "kwargs"}: - return "" + if not is_safe_key and key_normalized not in { + "", + "args", + "kwargs", + }: + return _REDACTED return { item_key: sanitize(item_value, str(item_key), is_safe_key) for item_key, item_value in value.items() } if isinstance(value, (list, tuple)): - if not is_safe_key and key_lower not in {"", "args", "kwargs"}: - return "" + if not is_safe_key and key_normalized not in { + "", + "args", + "kwargs", + }: + return _REDACTED sanitized = [ sanitize(item, safe_context=is_safe_key) for item in value ] @@ -182,7 +347,7 @@ def sanitize( return "" # Positional arguments have no reliable semantic key and may be # URLs, credentials, or payloads. - return value if is_safe_key else "" + return value if is_safe_key else _REDACTED return sanitize(arguments) @@ -476,13 +641,43 @@ def _redact_diagnostic_text(value: Any) -> Any: r"\1", redacted, ) + redacted = re.sub( + r"(?i)([\"']?(?:password|passwd|credential|username)[\"']?" + r"\s*[:=]\s*[\"']?)[^\s,;\"'}]+", + r"\1", + redacted, + ) + redacted = re.sub( + r"(?i)(input_value\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^,\]\n]+)", + r"\1", + redacted, + ) + redacted = re.sub( + r"(?i)([\"'](?:password|passwd|pwd|secret|token|credential|" + r"authorization|proxy[-_]authorization|cookie|set[-_]cookie|" + r"(?:x[-_])?api[-_]?key|username|database|database[-_]name|" + r"schema|schema[-_]name|connection|connection[-_]string|dsn|" + r"sql|query|query[-_]by|filter|filters|document|documents|" + r"metadata|payload|vector|vectors|embedding|embeddings|url)" + r"[\"']\s*:\s*)([\"'])[^\"']*\2", + r"\1\2\2", + redacted, + ) + redacted = re.sub( + r"(?i)(\b(?:database|database[-_]name|schema|schema[-_]name|" + r"connection[-_]string|dsn)\s*[:=]\s*[\"']?)[^\s,;\"']+", + r"\1", + redacted, + ) return redacted def is_original_exception( self, exception_type: type[BaseException] ) -> bool: """Return whether the wrapped exception is an instance of ``exception_type``.""" - return isinstance(self.original_exception, exception_type) + return self.original_exception_type is not None and issubclass( + self.original_exception_type, exception_type + ) def __str__(self) -> str: return self.format() diff --git a/src/oracle_vecdb/version.py b/src/oracle_vecdb/version.py index bbbbcb3..d538cfd 100644 --- a/src/oracle_vecdb/version.py +++ b/src/oracle_vecdb/version.py @@ -1,4 +1,4 @@ """Single source of truth for SDK and generated ORDS versions.""" -SDK_VERSION = "1.0.2" +SDK_VERSION = "1.0.3" ORDS_RELEASE_VERSION = "26.2.2" diff --git a/tests/client/test_client_facade_contract.py b/tests/client/test_client_facade_contract.py index 2a4d77c..24ffb01 100644 --- a/tests/client/test_client_facade_contract.py +++ b/tests/client/test_client_facade_contract.py @@ -13,6 +13,7 @@ import oracle_vecdb.client as client_module from oracle_vecdb.client import OracleVecDB +from oracle_vecdb.default_settings import DefaultSettings from oracle_vecdb.configuration import Configuration from oracle_vecdb.data_types import UpsertVectorsResponse from oracle_vecdb.service_protocol import VecDBServiceProtocol @@ -105,12 +106,149 @@ def test_query_forwards_output_selector_to_active_backend(mocker): top_k=3, filters=None, advanced_options=None, - include_vectors=None, + include_vectors=False, output_selector=["category", "price"], debug_flags=None, ) +def test_common_spec_applies_defaults_for_omitted_arguments(mocker): + """Verify omitted facade arguments are populated from DefaultSettings. + + This test currently uses the mocked ORDS backend, but it verifies the + transport-agnostic OracleVecDB facade contract that a native backend must + also receive. ``table_params`` and ``include_vectors`` are intentionally + omitted by the caller and must therefore be obtained from DefaultSettings. + """ + client, active_backend, _ = _make_client(mocker) + + client.create_vector_table(name="docs") + client.query( + table_name="docs", + query_by={"text": "hi"}, + top_k=3, + ) + + assert active_backend.calls[0] == ( # nosec B101 + "create_vector_table", + (), + { + "name": "docs", + "comment": None, + "annotations": None, + "table_params": {"auto_generate_id": False}, + "embed_params": None, + "index_params": None, + "debug_flags": None, + }, + ) + assert active_backend.calls[1][2]["include_vectors"] is False # nosec B101 + + +def test_common_spec_rejects_graph_index_without_distribution_method(mocker): + client, active_backend, _ = _make_client(mocker) + + with pytest.raises( + VecDBException, + match="distribute_params cannot be None.*INMEMORY GRAPH", + ): + client.create_index( + table_name="docs", + index_params={ + "vector_index_params": {"organization": "INMEMORY GRAPH"} + }, + ) + + assert active_backend.calls == [] # nosec B101 + + +def test_common_spec_rejects_graph_index_with_null_distribution_params(mocker): + client, active_backend, _ = _make_client(mocker) + + with pytest.raises( + VecDBException, + match="distribute_params cannot be None.*INMEMORY GRAPH", + ): + client.create_index( + table_name="docs", + index_params={ + "vector_index_params": { + "organization": "INMEMORY GRAPH", + "distribute_params": None, + } + }, + ) + + assert active_backend.calls == [] # nosec B101 + + +def test_common_spec_honors_scalar_and_nested_caller_overrides(mocker): + """Verify DefaultSettings defaults do not override explicit caller values. + + The mocked backend is ORDS, while the behavior belongs to the shared + facade contract and applies equally to a future native backend. DefaultSettings + supplies missing nested ``table_params`` keys, but explicit caller values + such as ``include_vectors=True`` always take precedence. + """ + client, active_backend, _ = _make_client(mocker) + + client.create_vector_table( + name="docs", + table_params={"custom_option": {"source": "caller"}}, + ) + client.query( + table_name="docs", + query_by={"text": "hi"}, + top_k=3, + include_vectors=True, + ) + + assert active_backend.calls[0][2]["table_params"] == { # nosec B101 + "auto_generate_id": False, + "custom_option": {"source": "caller"}, + } + assert active_backend.calls[1][2]["include_vectors"] is True # nosec B101 + + +def test_common_spec_honors_explicit_none_and_ignores_unknown_operations( + mocker, +): + """Verify explicit ``None`` is preserved and unknown specs are a no-op. + + This uses the mocked ORDS backend to exercise facade behavior that must + remain transport-neutral for native calls. Only omitted arguments are + obtained from DefaultSettings; explicitly passed ``None`` and operations with + no DefaultSettings entry must reach the backend unchanged. + + This test specifically verifies that ``DefaultSettings`` does not override anything: + + - ``table_params=None`` was explicitly supplied, so it remains ``None``. + - ``include_vectors=None`` was explicitly supplied, so it remains ``None``. + - ``list_models()`` has no ``DefaultSettings`` entry, so its ``limit`` and ``offset`` remain their Python defaults of ``None``. + """ + client, active_backend, _ = _make_client(mocker) + + client.create_vector_table(name="docs", table_params=None) + client.query( + table_name="docs", + query_by={"text": "hi"}, + top_k=3, + include_vectors=None, + ) + client.list_models() + + assert active_backend.calls[0][2]["table_params"] is None # nosec B101 + assert active_backend.calls[1][2]["include_vectors"] is None # nosec B101 + assert active_backend.calls[2] == ( # nosec B101 + "list_models", + (), + {"limit": None, "offset": None}, + ) + assert ( + DefaultSettings.get_default_args_for("list_models") == {} + ) # nosec B101 + + @pytest.mark.parametrize( "method_name", [ @@ -404,6 +542,10 @@ def test_facade_delegates_public_methods_to_active_backend( bound.apply_defaults() expected_kwargs = dict(bound.arguments) expected_kwargs.pop("self", None) + if method_name == "create_vector_table": + expected_kwargs["table_params"] = {"auto_generate_id": False} + elif method_name == "query": + expected_kwargs["include_vectors"] = False result = getattr(client, method_name)(*args, **kwargs) assert result == { @@ -526,6 +668,67 @@ def test_facade_rejects_invalid_resource_names( assert active_backend.calls == [] # nosec B101 +@pytest.mark.parametrize( + "method_name,error_type", + [ + ("describe_vector_table", InvalidTableNameFormatError), + ("describe_model", InvalidModelNameFormatError), + ("describe_vector_load_job", InvalidLoadJobNameFormatError), + ("describe_index_job", InvalidIndexJobNameFormatError), + ], +) +@pytest.mark.parametrize( + "unsafe_name", [b"docs", "\x00", "docs\x00old", 'docs"old'] +) +def test_facade_rejects_unsafe_resource_name_values( + mocker, method_name, error_type, unsafe_name +): + """Reject names that cannot safely retain a resource's text identity.""" + client, active_backend, _ = _make_client(mocker) + + with pytest.raises(VecDBException) as exception: + getattr(client, method_name)(unsafe_name) + + assert exception.value.is_original_exception(error_type) # nosec B101 + assert active_backend.calls == [] # nosec B101 + + +@pytest.mark.parametrize( + "method_name,resource_parameter", + [ + ("describe_vector_table", "name"), + ("describe_model", "model_name"), + ("describe_vector_load_job", "load_job_name"), + ("describe_index_job", "index_job_name"), + ], +) +@pytest.mark.parametrize( + "resource_name", + [ + "my-table", + "my.table", + "my table", + "1st_name", + "Delta_名", + "my$name", + "my#name", + "my/name", + "my'name", + ], +) +def test_facade_delegates_database_defined_resource_name_grammar( + mocker, method_name, resource_parameter, resource_name +): + """Leave database-specific characters and identifier rules to VecDB.""" + client, active_backend, _ = _make_client(mocker) + + getattr(client, method_name)(resource_name) + + assert active_backend.calls == [ + (method_name, (), {resource_parameter: resource_name}) + ] # nosec B101 + + @pytest.mark.parametrize( "method_name,kwargs", [ diff --git a/tests/client/test_configuration_facade.py b/tests/client/test_configuration_facade.py index abeb532..4ac3c42 100644 --- a/tests/client/test_configuration_facade.py +++ b/tests/client/test_configuration_facade.py @@ -498,8 +498,6 @@ def test_debug_true_does_not_log_configured_credentials(caplog, monkeypatch): cfg = Configuration( rest_url=VALID_HOST, access_token=access_token, - username="test-user", - password="test-password", # nosec B106 ) with caplog.at_level("DEBUG"): @@ -510,6 +508,23 @@ def test_debug_true_does_not_log_configured_credentials(caplog, monkeypatch): assert "test-password" not in caplog.text # nosec B101 +@pytest.mark.parametrize( + "kwargs", + [ + { + "username": "user", + "password": "pass", + "access_token": "token", + }, # nosec B105 + {"username": "user", "password": None}, # nosec B105 + {"username": None, "password": "pass"}, # nosec B105 + ], +) +def test_configuration_rejects_conflicting_or_partial_authentication(kwargs): + with pytest.raises(ValueError, match="authentication"): + Configuration(rest_url=VALID_HOST, **kwargs) + + def test_configuration_debug_constructor_sets_debug(monkeypatch): _reset_env_vars(monkeypatch) cfg = Configuration(rest_url=VALID_HOST, debug=True) diff --git a/tests/data_types/test_responses.py b/tests/data_types/test_responses.py index b8e2de3..c5f3cb2 100644 --- a/tests/data_types/test_responses.py +++ b/tests/data_types/test_responses.py @@ -71,6 +71,9 @@ def test_message_response_rejects_missing_message_field(): with pytest.raises(ValueError, match="missing required field 'message'"): DeleteVectorsResponse.from_internal({"status": "ok"}) + with pytest.raises(ValueError, match="missing required field 'message'"): + DeleteVectorsResponse.from_internal(AttributeItem(status="ok")) + def test_rerank_response_wraps_result_items(): wrapped = RerankResponse.from_internal([AttributeItem(index=1, score=0.97)]) @@ -181,6 +184,39 @@ def test_vector_table_indexes_normalize_generated_objects(): ] # nosec B101 +def test_index_details_normalizes_existing_sequence_and_serialized_values(): + existing = IndexDetailsResponse(dense_idx_name="existing") + + assert ( + IndexDetailsResponse.from_internal(existing) is existing + ) # nosec B101 + assert ( + IndexDetailsResponse.from_internal( + [("dense_idx_name", "from-sequence")] + ).dense_idx_name + == "from-sequence" + ) # nosec B101 + + assert ( + IndexDetailsResponse.from_internal( + AttributeItem( + dense_idx_name="from-attributes", + indexed_metadata_json_paths=["active"], + ) + ).dense_idx_name + == "from-attributes" + ) # nosec B101 + + class SerializedIndexes: + def to_dict(self): + return {"dense_idx_name": "from-serializer"} + + assert ( + IndexDetailsResponse.from_internal(SerializedIndexes()).dense_idx_name + == "from-serializer" + ) # nosec B101 + + def test_query_result_item_normalizes_existing_and_object_values(): existing = QueryResultItem(id="vec-existing", distance=0.1) object_item = AttributeItem( @@ -230,6 +266,14 @@ def test_query_response_from_generated_query_vectors_response(): assert response.items[0].distance == 0.5 # nosec B101 +def test_query_response_rejects_generated_or_object_response_without_results(): + with pytest.raises(ValueError, match="missing required field 'results'"): + QueryResponse.from_internal(QueryVectors200Response(results=None)) + + with pytest.raises(ValueError, match="missing required field 'results'"): + QueryResponse.from_internal(AttributeItem()) + + def test_rerank_result_and_response_normalize_dict_and_existing_values(): existing_item = RerankResultItem(index=0, score=0.5) existing_response = RerankResponse(items=[existing_item]) diff --git a/tests/internal/test_vecdb_errors.py b/tests/internal/test_vecdb_errors.py index a412c9b..abac0fe 100644 --- a/tests/internal/test_vecdb_errors.py +++ b/tests/internal/test_vecdb_errors.py @@ -13,7 +13,11 @@ InvalidIndexJobNameFormatError, InvalidHostFormatError, InvalidLoadJobNameFormatError, + InvalidLoadJobLogError, + InvalidIndexJobLogError, InvalidModelNameFormatError, + DefaultSettingsParameterMismatchError, + ResourceNotFoundError, InvalidTableNameFormatError, VecDBError, ) @@ -124,7 +128,7 @@ def test_error_messages_fallback_to_english_for_requested_locale(): def test_error_messages_fallback_to_english_for_unknown_locale(): err = InvalidTableNameFormatError("bad table", locale="fr-FR") - assert "Invalid table name format" in err.get_error() # nosec B101 + assert "Invalid table name" in err.get_error() # nosec B101 assert "VECDB-003" in err.get_error() # nosec B101 @@ -148,3 +152,13 @@ def test_resource_name_errors_include_codes_causes_and_actions( assert value in message # nosec B101 assert "Cause:" in message # nosec B101 assert "Action:" in message # nosec B101 + assert "configured Oracle Database" in message # nosec B101 + assert "double-quote" in message # nosec B101 + assert "only letters, digits, and underscore" not in message # nosec B101 + + +def test_job_and_default_setting_errors_construct_stable_messages(): + ResourceNotFoundError("missing-resource") + InvalidLoadJobLogError("load-job", "RUNNING") + InvalidIndexJobLogError("index-job", "RUNNING") + DefaultSettingsParameterMismatchError("query", ["missing_parameter"]) diff --git a/tests/services/test_ords.py b/tests/services/test_ords.py index b6d9c42..5f5d5c8 100644 --- a/tests/services/test_ords.py +++ b/tests/services/test_ords.py @@ -161,6 +161,28 @@ def execute(self): assert endpoint.calls == 2 # nosec B101 +def test_ords_service_does_not_attach_raw_transport_error_as_context(): + service = _make_service() + + class LeakyTransportError(Exception): + status = 401 + reason = "Authorization: Bearer token-value" + body = '{"message": "bad token"}' + headers = {"Authorization": "Bearer token-value"} + + def fail(_request): + raise LeakyTransportError() + + service.table_api.create_vector_table = fail + + with pytest.raises(VecDBException) as error: + service.create_vector_table(name="docs") + + assert error.value.__context__ is None # nosec B101 + assert "token-value" not in repr(vars(error.value)) # nosec B101 + assert "bad token" in str(error.value) # nosec B101 + + def test_create_ords_service_wires_generated_api_delegates(monkeypatch): class FakeApiClient: def __init__(self, config): @@ -242,6 +264,52 @@ def test_generated_index_params_accepts_documented_distribution_method(): } # nosec B101 +def test_ords_service_rejects_graph_index_without_distribution_method(): + service = _make_service() + + with pytest.raises( + VecDBException, + match="distribute_params cannot be None.*INMEMORY GRAPH", + ): + service.create_index( + "docs", + { + "vector_index_params": { + "auto_index": True, + "organization": "INMEMORY GRAPH", + "distance_metric": "COSINE", + "accuracy": 95, + "advanced_params": { + "neighbors": 10, + "efConstruction": 100, + }, + } + }, + ) + + assert service.index_api.calls == [] # nosec B101 + + +def test_ords_service_rejects_graph_index_with_null_distribution_params(): + service = _make_service() + + with pytest.raises( + VecDBException, + match="distribute_params cannot be None.*INMEMORY GRAPH", + ): + service.create_index( + "docs", + { + "vector_index_params": { + "organization": "INMEMORY GRAPH", + "distribute_params": None, + } + }, + ) + + assert service.index_api.calls == [] # nosec B101 + + def test_ords_debug_flag_conversion(): service = _make_service() @@ -464,6 +532,75 @@ def test_ords_service_maps_table_inference_and_vector_requests(): assert upsert_request.vectors[1] is vector_item # nosec B101 +@pytest.mark.parametrize( + "vector, expected", + [ + ( + {"ID": "v1", "DENSE_VECTOR": [0.1, 0.2], "METADATA": {"a": 1}}, + {"id": "v1", "dense_vector": [0.1, 0.2], "metadata": {"a": 1}}, + ), + ( + {"iD": "v2", "dense_Vector": [0.3, 0.4], "mEtAdAtA": {"b": 2}}, + {"id": "v2", "dense_vector": [0.3, 0.4], "metadata": {"b": 2}}, + ), + ], +) +def test_ords_service_normalizes_upsert_field_names_case_insensitively( + vector, expected +): + service = _make_service() + + service.upsert_vectors("docs", [vector]) + + request = _first_keyword_request( + service.vector_api, "upsert_vectors", "upsert_vectors_request" + ) + assert request.vectors[0].to_dict() == expected # nosec B101 + + +def test_ords_service_normalizes_all_records_in_batched_upsert(): + service = _make_service() + + service.upsert_vectors( + "docs", + [ + {"ID": "v1", "DENSE_VECTOR": [0.1], "METADATA": {"a": 1}}, + {"id": "v2", "Dense_Vector": [0.2], "metadata": {"b": 2}}, + ], + ) + + request = _first_keyword_request( + service.vector_api, "upsert_vectors", "upsert_vectors_request" + ) + assert [item.to_dict() for item in request.vectors] == [ # nosec B101 + {"id": "v1", "dense_vector": [0.1], "metadata": {"a": 1}}, + {"id": "v2", "dense_vector": [0.2], "metadata": {"b": 2}}, + ] + + +def test_ords_service_rejects_duplicate_case_insensitive_upsert_fields(): + service = _make_service() + + with pytest.raises(VecDBException, match="Duplicate upsert vector field"): + service.upsert_vectors("docs", [{"id": "v1", "ID": "v2"}]) + + assert service.vector_api.calls == [] # nosec B101 + + +def test_ords_service_rejects_unknown_upsert_fields(): + service = _make_service() + + with pytest.raises( + VecDBException, match="Unknown upsert vector field.*EXTRA" + ): + service.upsert_vectors( + "docs", + [{"ID": "v1", "DENSE_VECTOR": [0.1], "EXTRA": "unexpected"}], + ) + + assert service.vector_api.calls == [] # nosec B101 + + @pytest.mark.parametrize( "kwargs, message", [ diff --git a/tests/services/test_ords_exceptions.py b/tests/services/test_ords_exceptions.py index 0117492..cd89a29 100644 --- a/tests/services/test_ords_exceptions.py +++ b/tests/services/test_ords_exceptions.py @@ -1,9 +1,12 @@ import json +import builtins from types import SimpleNamespace import pytest +from pydantic import BaseModel, ValidationError from oracle_vecdb import VecDBException from oracle_vecdb.services.ords.exceptions import ApiException +import oracle_vecdb.vecdb_exception as vecdb_exception_module from oracle_vecdb.vecdb_exception import guidance_for_status @@ -289,6 +292,107 @@ def test_exception_response_payload_supports_bytes_and_invalid_text(): ) # nosec B101 +def test_exception_response_payload_supports_object_serializers(): + class SerializedResponse: + def to_dict(self): + return {"message": "serialized"} + + assert VecDBException._response_payload( # nosec B101 + None, SerializedResponse() + ) == {"message": "serialized"} + + +def test_exception_redaction_handles_collections_bytes_and_serializers(): + sensitive_key = "token" + sensitive_value = "hidden" + redacted_value = "" + + class SerializedValue: + def model_dump(self): + return {"safe": "value"} + + class BrokenSerializer: + def model_dump(self): + raise TypeError("unsupported serializer") + + assert VecDBException._redact_value( # nosec B101 + ["value", {sensitive_key: sensitive_value}] + ) == ["value", {sensitive_key: redacted_value}] + assert VecDBException._redact_value(("value", "other")) == ( # nosec B101 + "value", + "other", + ) + assert ( + VecDBException._redact_value( # nosec B101 + memoryview(b"Authorization: Bearer token-value") + ) + == "Authorization: Bearer " + ) + assert VecDBException._redact_value(SerializedValue()) == { # nosec B101 + "safe": "value" + } + VecDBException._redact_value(BrokenSerializer()) + + +def test_exception_formats_pydantic_validation_details(): + class RequiredModel(BaseModel): + value: int + + with pytest.raises(ValidationError) as raised: + RequiredModel.model_validate({}) + + error = VecDBException(status=422) + error.service_error = raised.value + error.service_error_class_name = "ValidationError" + + rendered = error._format_service_error() + assert '"errors"' in rendered # nosec B101 + assert '"value"' in rendered # nosec B101 + + +def test_exception_formatting_falls_back_for_invalid_validation_details(): + class BrokenValidationError(Exception): + __module__ = "pydantic_core" + + def errors(self): + raise TypeError("unsupported validation details") + + def __str__(self): + return "" + + error = VecDBException(status=422) + error.service_error = BrokenValidationError("fallback details") + error.service_error_class_name = "BrokenValidationError" + + rendered = error._format_service_error() + assert "fallback details" in rendered # nosec B101 + + +def test_exception_wrapper_falls_back_when_dynamic_subclassing_fails( + monkeypatch, +): + real_type = builtins.type + + def reject_dynamic_types(*args): + if len(args) == 3: + raise TypeError("dynamic type unavailable") + return real_type(*args) + + monkeypatch.setattr( + vecdb_exception_module, "type", reject_dynamic_types, raising=False + ) + + class LocalError(Exception): + pass + + error = VecDBException.from_service_error( + "query", {}, "ORDSService", LocalError("details") + ) + + assert isinstance(error, VecDBException) # nosec B101 + assert error.service_error_class_name == "LocalError" # nosec B101 + + def test_not_found_payload_without_code_is_stable(): error = VecDBException.from_service_error( "drop_vector_table", @@ -412,3 +516,85 @@ def __init__(self): assert error.code == "LocalError" # nosec B101 assert error.exception_type == "LocalError" # nosec B101 assert error.original_exception_type is LocalError # nosec B101 + + +def test_wrapped_exception_does_not_retain_credentials_or_sensitive_payload(): + class TransportError(Exception): + status = 401 + reason = "Authorization: Bearer token-value" + data = { + "database": "customer_db", + "access_token": "token-value", + } # nosec B105 + headers = { + "Authorization": "Bearer token-value", + "Cookie": "session=session-value", + "X-Request-ID": "request-id", + } + + error = VecDBException.from_service_error( + "query", + {"kwargs": {"token": "token-value"}}, # nosec B105 + "ORDSService", + TransportError(), + ) + rendered = f"{error}\n{error!r}\n{error.format(include_trace=True)}" + + for secret in ("token-value", "session-value"): + assert secret not in rendered # nosec B101 + assert secret not in repr(error.headers) # nosec B101 + assert error.original_exception_type is TransportError # nosec B101 + assert error.is_original_exception(TransportError) # nosec B101 + assert error.headers["X-Request-ID"] == "request-id" # nosec B101 + + +def test_redaction_preserves_actionable_service_diagnostics(): + error = VecDBException.from_service_error( + "describe_vector_table", + {"kwargs": {"table_name": "DOCS"}}, + "ORDSService", + ServiceError( + status=400, + reason="Bad Request", + body=json.dumps( + { + "code": "TABLE_INVALID", + "message": "ORA-00942: table DOCS does not exist in database=customer_database", + "type": "tag:oracle.com,2020:error/BadRequest", + "instance": "ecid-123", + "o:errorCode": "ORDS-25001", + "action": "Verify the table name and schema", + "requestId": "request-123", + "connectionId": "connection-123", + "queryId": "query-123", + "database": "customer_database", + } + ), + data={ + "database": "customer_database", + "table": "DOCS", + "request_id": "request-123", + "connection_id": "connection-123", + "query_id": "query-123", + }, + headers={ + "Content-Type": "application/problem+json", + "X-Request-ID": "request-123", + "Authorization": "Bearer token-value", + }, + ), + ) + + rendered = error.format(include_trace=True) + assert "TABLE_INVALID" in rendered # nosec B101 + assert "ORA-00942" in rendered # nosec B101 + assert "DOCS" in rendered # nosec B101 + assert "ORDS-25001" in rendered # nosec B101 + assert "Verify the table name and schema" in rendered # nosec B101 + assert "ecid-123" in rendered # nosec B101 + assert "connection-123" in rendered # nosec B101 + assert "query-123" in rendered # nosec B101 + assert "customer_database" not in rendered # nosec B101 + assert "customer_database" not in repr(vars(error)) # nosec B101 + assert "request-123" in repr(error.data) # nosec B101 + assert error.headers["X-Request-ID"] == "request-123" # nosec B101 diff --git a/tests/test_parameter_validation.py b/tests/test_parameter_validation.py new file mode 100644 index 0000000..190fc5d --- /dev/null +++ b/tests/test_parameter_validation.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from oracle_vecdb.parameter_validation import ( + MetadataIndexParams, + VectorIndexParams, + validate_embed_params, + validate_index_params, + validate_list_vectors_arguments, + validate_operation_arguments, + validate_positive_top_k, + validate_query_advanced_options, + validate_query_by, + validate_rebuild_index_params, + validate_rerank_model_params, + validate_table_params, +) + + +@pytest.mark.parametrize( + "field, value", + [ + ("include_paths", [" "]), + ("exclude_paths", ["tags[*]"]), + ("include_paths", ["tags]"]), + ], +) +def test_metadata_index_rejects_empty_or_array_paths(field, value): + with pytest.raises(ValidationError): + MetadataIndexParams.model_validate({field: value}) + + +def test_metadata_index_rejects_conflicting_wildcards(): + with pytest.raises(ValidationError, match="cannot both contain"): + MetadataIndexParams.model_validate( + {"include_paths": ["*"], "exclude_paths": ["*"]} + ) + + +def test_metadata_index_accepts_distinct_paths(): + MetadataIndexParams.model_validate( + {"include_paths": ["profile.name", "*"], "exclude_paths": ["tags"]} + ) + + +@pytest.mark.parametrize( + "payload, message", + [ + ( + { + "organization": "PARTITIONS", + "distribute_params": {"distribute_method": "AUTO"}, + }, + "supported only", + ), + ( + {"organization": "PARTITIONS", "online_build": True}, + "supported only", + ), + ( + {"organization": "PARTITIONS", "quantization_type": "SCALAR"}, + "compression_ratio is required", + ), + ( + { + "organization": "PARTITIONS", + "quantization_type": "SCALAR", + "compression_ratio": 3, + }, + "one of 2, 4, or 8", + ), + ( + {"organization": "PARTITIONS", "compression_ratio": 2}, + "requires", + ), + ], +) +def test_vector_index_rejects_invalid_dependencies(payload, message): + with pytest.raises(ValidationError, match=message): + VectorIndexParams.model_validate(payload) + + +def test_vector_index_accepts_ivf_advanced_params(): + VectorIndexParams.model_validate( + { + "organization": "PARTITIONS", + "advanced_params": {"partitions": 8}, + } + ) + + +def test_vector_index_rejects_invalid_ivf_advanced_params(): + with pytest.raises(ValidationError, match="advanced_params is invalid"): + VectorIndexParams.model_validate( + {"organization": "PARTITIONS", "advanced_params": {"neighbors": 4}} + ) + + +def test_vector_index_accepts_hnsw_advanced_params(): + VectorIndexParams.model_validate( + { + "organization": "INMEMORY GRAPH", + "distribute_params": {"distribute_method": "AUTO"}, + "online_build": True, + "advanced_params": { + "neighbors": 4, + "efConstruction": 16, + "rescore_factor": 2, + "algorithm": "uniform_quantization", + }, + } + ) + + +def test_vector_index_rejects_invalid_hnsw_algorithm(): + with pytest.raises(ValidationError, match="algorithm must be"): + VectorIndexParams.model_validate( + { + "organization": "INMEMORY GRAPH", + "distribute_params": {"distribute_method": "AUTO"}, + "advanced_params": {"algorithm": "unsupported"}, + } + ) + + +def test_validate_index_params_converts_legacy_flat_shape(): + validate_index_params( + { + "indexing": "AUTO", + "organization": "PARTITIONS", + "distance": "COSINE", + "accuracy": 90, + "advanced_params": {"partitions": 8}, + "parallel_creation": 2, + } + ) + validate_index_params({}) + + +def test_validate_index_params_accepts_current_nested_shape(): + validate_index_params( + {"metadata_index_params": {"include_paths": ["profile.name"]}} + ) + + +@pytest.mark.parametrize( + "payload, message", + [ + ({"indexing": "invalid"}, "indexing must be one"), + ( + {"distance_metric": "COSINE", "distance": "HAMMING"}, + "distance and distance_metric must match", + ), + ], +) +def test_validate_index_params_rejects_invalid_legacy_shape(payload, message): + with pytest.raises(ValidationError, match=message): + validate_index_params(payload) + + +def test_validate_rebuild_index_params_converts_legacy_shape(): + validate_rebuild_index_params( + { + "index_type": "vector", + "organization": "PARTITIONS", + "distance_metric": "COSINE", + "advanced_params": {"partitions": 4}, + } + ) + + +def test_validate_rebuild_index_params_accepts_nested_shape(): + validate_rebuild_index_params( + { + "vector_index_params": {"organization": "PARTITIONS"}, + "index_type": "vector", + } + ) + + +def test_validate_rebuild_index_params_rejects_metadata_auto_index(): + with pytest.raises(ValueError, match="not supported for rebuild_index"): + validate_rebuild_index_params( + {"metadata_index_params": {"auto_index": True}} + ) + + +def test_validate_rebuild_index_params_rejects_unknown_index_type(): + with pytest.raises(ValueError): + validate_rebuild_index_params({"index_type": "unsupported"}) + + +@pytest.mark.parametrize("value", [None, {}, {"text": "one", "id": "two"}]) +def test_validate_query_by_requires_one_query_mode(value): + with pytest.raises((ValidationError, ValueError)): + validate_query_by(value) + + +def test_validate_query_and_rerank_models_accept_documented_shapes(): + validate_query_by({"vector": [0.1, 0.2]}) + validate_query_advanced_options( + { + "distance_metric": "COSINE", + "accuracy": 80, + "advanced_params": {"rescore_factor": 2}, + "idx_parameters": {"efSearch": 16}, + } + ) + validate_rerank_model_params({"top_n": 3}) + + +@pytest.mark.parametrize("value", [True, 0, -1, "3"]) +def test_validate_positive_top_k_rejects_non_positive_or_non_numeric(value): + with pytest.raises(ValueError, match="greater than zero"): + validate_positive_top_k(value) + + +def test_validate_list_vectors_accepts_pagination_and_ids(): + validate_list_vectors_arguments({"limit": 10, "offset": 0, "ids": ["one"]}) + validate_list_vectors_arguments({"limit": 1, "offset": 1.5}) + + +@pytest.mark.parametrize( + "arguments, message", + [ + ({"limit": True}, "limit must be"), + ({"limit": 0}, "limit must be"), + ({"limit": 1, "offset": -1}, "offset must be"), + ({"limit": 1, "offset": "1"}, "offset must be"), + ({"limit": 1, "ids": ("one",)}, "ids must be"), + ({"limit": 1, "ids": [1]}, "ids must be"), + ], +) +def test_validate_list_vectors_rejects_invalid_arguments(arguments, message): + with pytest.raises(ValueError, match=message): + validate_list_vectors_arguments(arguments) + + +@pytest.mark.parametrize( + "validator, name", + [ + (validate_table_params, "table_params"), + (validate_embed_params, "embed_params"), + (validate_index_params, "index_params"), + (validate_query_by, "query_by"), + (validate_query_advanced_options, "advanced_options"), + (validate_rerank_model_params, "model_params"), + ], +) +def test_parameter_validators_reject_non_mapping_values(validator, name): + with pytest.raises(ValueError, match=f"{name} must be a JSON object"): + validator(None) + + +def test_validate_operation_arguments_skips_missing_optional_values(): + validate_operation_arguments( + "query", + {"query_by": None, "top_k": None, "advanced_options": None}, + ) + validate_operation_arguments("unknown_operation", {}) + + +def test_validate_operation_arguments_dispatches_list_vectors_validation(): + validate_operation_arguments("list_vectors", {"limit": 5, "offset": 0}) + with pytest.raises(ValueError, match="limit must be"): + validate_operation_arguments("list_vectors", {"limit": 0}) diff --git a/tox.ini b/tox.ini index deef92a..7579679 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ commands = --color=yes \ --cov=oracle_vecdb \ --cov-report=term-missing \ - --cov-fail-under=90 \ + --cov-fail-under=95 \ --ignore={toxinidir}/dev-tools --ignore={toxinidir}/examples --ignore={toxinidir}/__parfait__ \ {env:UNIT_TEST_JUNIT_FLAG:} \ {env:COVERAGE_XML_FLAG:} \