From f88a15ecb2b24a1298b0cdca454f9cca99a1c609 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 2 Jul 2026 16:26:20 +1000 Subject: [PATCH 1/5] test(benchmark): add gated EQL v3 benchmark schema and setup task Port tests/benchmark/sql/benchmark-schema.sql to EQL v3 as benchmark-schema-v3.sql. The equality-only encrypted benchmark column becomes eql_v3.text_eq (hm term); the eql_v2.add_column call is dropped because EQL v3 has no database-side configuration (the proxy-side Encrypt config replaces it). Applied via the new (opt-in) benchmark:setup:v3 task. The pgbench transaction scripts are version-agnostic and remain shared with v2. GATED: the eql-mapper cannot speak EQL v3 yet, so this schema is not used by the default benchmark tasks. --- tests/benchmark/mise.toml | 13 ++++++++ tests/benchmark/sql/benchmark-schema-v3.sql | 34 +++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/benchmark/sql/benchmark-schema-v3.sql diff --git a/tests/benchmark/mise.toml b/tests/benchmark/mise.toml index c3d8b073..297d069f 100644 --- a/tests/benchmark/mise.toml +++ b/tests/benchmark/mise.toml @@ -145,6 +145,19 @@ cat sql/benchmark-schema.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql p docker compose run --rm postgres${CONTAINER_SUFFIX:-} pgbench --host=host.docker.internal --port=${CS_DATABASE__PORT} --scale=1 -i --no-vacuum """ +# GATED: EQL v3 variant of benchmark:setup. Opt-in while the eql-mapper cannot +# speak v3. Requires EQL v3 to be installed first: +# CS_EQL_V3_PATH=../encrypt-query-language/release mise run postgres:eql:v3:setup +# The pgbench transaction scripts are version-agnostic and shared with v2. +[tasks."benchmark:setup:v3"] +description = "Apply the EQL v3 benchmark schema (requires postgres:eql:v3:setup)" +run = """ +cat sql/benchmark-schema-v3.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql -v ON_ERROR_STOP=1 postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD_ESCAPED_FOR_TESTS}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- + +# Initialize pgbench +docker compose run --rm postgres${CONTAINER_SUFFIX:-} pgbench --host=host.docker.internal --port=${CS_DATABASE__PORT} --scale=1 -i --no-vacuum +""" + [tasks."benchmark:plot"] alias = 'b' description = "Plot graphs from benchmark results" diff --git a/tests/benchmark/sql/benchmark-schema-v3.sql b/tests/benchmark/sql/benchmark-schema-v3.sql new file mode 100644 index 00000000..367e7202 --- /dev/null +++ b/tests/benchmark/sql/benchmark-schema-v3.sql @@ -0,0 +1,34 @@ +-- EQL v3 port of benchmark-schema.sql. +-- +-- GATED: the proxy's eql-mapper cannot speak EQL v3 yet, so this schema is +-- opt-in via `mise run benchmark:setup:v3` (requires EQL v3 installed: +-- `CS_EQL_V3_PATH=... mise run postgres:eql:v3:setup`). +-- +-- Differences from the v2 benchmark schema: +-- +-- * `email eql_v2_encrypted` becomes `email eql_v3.text_eq`: the encrypted +-- benchmark transaction only exercises equality (`WHERE email = $1`), and +-- `eql_v3.text_eq` is the v3 text domain that requires the `hm` +-- (hash-equality) term. +-- +-- * The `eql_v2.add_column` call (and the `eql_v2_configuration` table it +-- populates) has no v3 equivalent: EQL v3 has no database-side +-- configuration. The proxy-side Encrypt config replaces it, and the +-- fail-closed domain CHECK constraints validate stored payloads. +-- +-- The pgbench transaction scripts (transaction-*.sql) are version-agnostic +-- plain SQL over column names and are shared with the v2 benchmark. + +DROP TABLE IF EXISTS benchmark_plaintext; +CREATE TABLE benchmark_plaintext ( + id serial primary key, + username text, + email text +); + +DROP TABLE IF EXISTS benchmark_encrypted; +CREATE TABLE benchmark_encrypted ( + id serial primary key, + username text, + email eql_v3.text_eq +); From dc14e5cf7a523d6d1a328089dd83ebb84d7a365c Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 2 Jul 2026 16:26:32 +1000 Subject: [PATCH 2/5] test(sql): add gated EQL v3 integration test fixture Port tests/sql/schema.sql to EQL v3 as schema-v3.sql (with matching schema-v3-uninstall.sql), applied by the new opt-in postgres:setup:v3 task after postgres:eql:v3:setup. Column domains are derived from each column's v2 add_search_config calls: encrypted_text unique+match+ore -> eql_v3.text_search encrypted_bool unique+ore -> eql_v3.bool (storage-only) encrypted_int2/4/8 unique+ore -> eql_v3.int{2,4,8}_ord_ore encrypted_float8 unique+ore -> eql_v3.float8_ord_ore encrypted_date unique+ore -> eql_v3.date_ord_ore encrypted_jsonb(_filtered) ste_vec -> eql_v3.json encrypted_unconfigured (none) -> eql_v3.text (storage-only) The per-test ORE/OPE fixture tables get the matching *_ord_ore / *_ord_ope domains (ORE text keeps its match term via eql_v3.text_search; OPE text uses eql_v3.text_ord_ope). All add_search_config / add_encrypted_constraint calls are dropped: v3 has no database-side configuration, and the fail-closed domain CHECKs replace the encrypted constraint. The fixture reuses the v2 table names so the existing integration suite can ride on it unchanged once the mapper speaks v3; applying it replaces the v2 fixture (restore with postgres:setup). GATED: not applied by postgres:setup while the eql-mapper cannot speak EQL v3. --- DEVELOPMENT.md | 18 +++ mise.toml | 15 ++ tests/sql/schema-v3-uninstall.sql | 14 ++ tests/sql/schema-v3.sql | 232 ++++++++++++++++++++++++++++++ 4 files changed, 279 insertions(+) create mode 100644 tests/sql/schema-v3-uninstall.sql create mode 100644 tests/sql/schema-v3.sql diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 2e34f8a2..c9cf2668 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -443,6 +443,24 @@ CS_EQL_V3_PATH=../encrypt-query-language/release mise run postgres:eql:v3:setup Note: unlike `postgres:setup`, this does **not** apply `tests/sql/schema.sql` — that fixture is still EQL v2. Use `postgres:eql:v3:teardown` to just uninstall EQL v3. +##### EQL v3 test fixtures (gated) + +The EQL v3 ports of the test fixtures are opt-in while the eql-mapper cannot speak v3: + +```shell +# Install EQL v3 and apply the v3 test schema (tests/sql/schema-v3.sql). +# This REPLACES the v2 fixture tables (same table names). +CS_EQL_V3_PATH=../encrypt-query-language/release mise run postgres:setup:v3 + +# Run the gated EQL v3 integration tests (all `#[ignore]`d by default) +cargo nextest run -p cipherstash-proxy-integration -E 'test(eql_v3)' --run-ignored all + +# Restore the v2 fixture afterwards +mise run postgres:setup +``` + +The v3 benchmark schema (`tests/benchmark/sql/benchmark-schema-v3.sql`) is applied with `mise run benchmark:setup:v3` from `tests/benchmark` (it also requires `postgres:eql:v3:setup` to have been run). + #### Convention: PostgreSQL ports PostgreSQL port numbers are 4 digits: diff --git a/mise.toml b/mise.toml index cc4390a9..3358e099 100644 --- a/mise.toml +++ b/mise.toml @@ -629,6 +629,21 @@ mise run postgres:fail_if_not_running cat "{{config_root}}/cipherstash-encrypt-v3.sql" | docker exec -i postgres${CONTAINER_SUFFIX:-} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD_ESCAPED_FOR_TESTS}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- """ +# GATED: the EQL v3 test fixture is opt-in while the eql-mapper cannot speak +# v3. This task is NOT part of `postgres:setup`; it replaces the v2 fixture +# tables with their v3-domain equivalents (same table names). Re-run +# `mise run postgres:setup` to restore the v2 fixture. +[tasks."postgres:setup:v3"] +depends = ["postgres:eql:v3:setup"] +description = "Installs EQL v3 and applies the v3 test schema (tests/sql/schema-v3.sql) to the database" +run = """ +#!/bin/bash +cd tests +mise run postgres:fail_if_not_running +cat sql/schema-v3-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX:-} psql -v ON_ERROR_STOP=1 postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD_ESCAPED_FOR_TESTS}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat sql/schema-v3.sql | docker exec -i postgres${CONTAINER_SUFFIX:-} psql -v ON_ERROR_STOP=1 postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD_ESCAPED_FOR_TESTS}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +""" + [tasks."test:integration:lang:python"] dir = "{{config_root}}/tests" description = "Runs python tests" diff --git a/tests/sql/schema-v3-uninstall.sql b/tests/sql/schema-v3-uninstall.sql new file mode 100644 index 00000000..9b57b9ba --- /dev/null +++ b/tests/sql/schema-v3-uninstall.sql @@ -0,0 +1,14 @@ +-- Uninstall for tests/sql/schema-v3.sql (EQL v3 fixture). +-- +-- Mirrors schema-uninstall.sql: drops the shared fixture tables. The v3 +-- fixture has no configuration table of its own (v3 configuration is +-- client-side), so unlike the v2 uninstall there is no +-- `public.eql_v2_configuration` to drop here. + +-- Regular old table +DROP TABLE IF EXISTS plaintext; + +-- Exciting cipherstash table +DROP TABLE IF EXISTS encrypted; + +DROP TABLE IF EXISTS unconfigured; diff --git a/tests/sql/schema-v3.sql b/tests/sql/schema-v3.sql new file mode 100644 index 00000000..5b23b347 --- /dev/null +++ b/tests/sql/schema-v3.sql @@ -0,0 +1,232 @@ +-- EQL v3 port of tests/sql/schema.sql. +-- +-- GATED: the proxy's eql-mapper cannot speak EQL v3 yet, so this fixture is +-- NOT applied by `postgres:setup` (which still applies the v2 schema.sql). +-- Apply it explicitly with `mise run postgres:setup:v3` after installing a +-- locally built EQL v3 (`CS_EQL_V3_PATH=... mise run postgres:eql:v3:setup`). +-- +-- Key differences from the v2 fixture: +-- +-- * There is no single `eql_v2_encrypted` type in v3. Each column uses a +-- per-scalar-per-capability jsonb domain (`eql_v3.[_]`) +-- chosen from the column's v2 `eql_v2.add_search_config` calls: +-- - unique -> `hm` term (`_eq`) +-- - ore -> `ob` term (`_ord_ore`) +-- - ope -> `op` term (`_ord_ope`) +-- - match -> `bf` term (`_match`, text only) +-- - unique + match + ore (text) -> `_search` +-- - ste_vec (jsonb) -> `eql_v3.json` +-- +-- * There are no `eql_v2.add_search_config` / `eql_v2.add_encrypted_constraint` +-- equivalents: v3 has no database-side configuration. Index/term +-- configuration lives client-side (the proxy's Encrypt config), and the +-- domains are fail-closed - their CHECK constraints reject payloads that +-- are missing the required terms, replacing `add_encrypted_constraint`. +-- +-- * Ordering does not use an on-column operator class. It uses functional +-- btree indexes on term extractors, created where a test needs one, e.g.: +-- CREATE INDEX ON encrypted (eql_v3.ord_term(encrypted_text)); +-- CREATE INDEX ON encrypted (eql_v3.ord_ope_term(encrypted_int4)); +-- and GIN containment for SteVec documents: +-- CREATE INDEX ON encrypted USING GIN ((eql_v3.to_ste_vec_query(encrypted_jsonb)::jsonb) jsonb_path_ops); +-- +-- This fixture reuses the v2 table names (`encrypted`, `unconfigured`, ...) so +-- the existing integration tests can ride on it unchanged once the mapper +-- speaks v3. Applying it therefore REPLACES the v2 fixture tables; re-run +-- `mise run postgres:setup` to restore the v2 fixture. + +-- The v2 fixture truncates its configuration table here. v3 has no +-- database-side configuration table, but stale v2 configuration must not be +-- left pointing at tables whose columns are now v3 domains. +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'eql_v2_configuration' + ) THEN + TRUNCATE TABLE public.eql_v2_configuration; + END IF; +END $$; + +-- Regular old table +DROP TABLE IF EXISTS plaintext; +CREATE TABLE plaintext ( + id bigint, + plaintext text, + PRIMARY KEY(id) +); + +DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'domain_type_with_check') THEN + CREATE DOMAIN domain_type_with_check AS VARCHAR(2) CHECK (VALUE ~ '^[A-Z]{2}$'); + END IF; + END +$$; + + +-- Exciting cipherstash table. +-- +-- Column -> domain mapping (from the v2 add_search_config calls): +-- encrypted_text unique + match + ore -> eql_v3.text_search (hm + ob + bf) +-- encrypted_bool unique + ore -> eql_v3.bool (storage-only: v3 bool +-- has no query capability domains) +-- encrypted_int2 unique + ore -> eql_v3.int2_ord_ore +-- encrypted_int4 unique + ore -> eql_v3.int4_ord_ore +-- encrypted_int8 unique + ore -> eql_v3.int8_ord_ore +-- encrypted_float8 unique + ore -> eql_v3.float8_ord_ore +-- encrypted_date unique + ore -> eql_v3.date_ord_ore +-- encrypted_jsonb ste_vec -> eql_v3.json +-- encrypted_jsonb_filtered ste_vec + term_filters -> eql_v3.json (term +-- filters are client-side in v3) +-- +-- Note: the non-text `_ord_ore` domains only REQUIRE the `ob` term; the `hm` +-- term written for `unique` is carried as an additional payload field and +-- extracted with `eql_v3.eq_term` where equality is needed. +DROP TABLE IF EXISTS encrypted; +CREATE TABLE encrypted ( + id bigint, + plaintext text, + plaintext_date date, + plaintext_domain domain_type_with_check, + encrypted_text eql_v3.text_search, + encrypted_bool eql_v3.bool, + encrypted_int2 eql_v3.int2_ord_ore, + encrypted_int4 eql_v3.int4_ord_ore, + encrypted_int8 eql_v3.int8_ord_ore, + encrypted_float8 eql_v3.float8_ord_ore, + encrypted_date eql_v3.date_ord_ore, + encrypted_jsonb eql_v3.json, + encrypted_jsonb_filtered eql_v3.json, + PRIMARY KEY(id) +); + +-- "Unconfigured" in v3 means the proxy has no client-side Encrypt config for +-- the column. Database-side it is just a storage-only domain (no query terms +-- required); `eql_v3.text` is used as the representative storage envelope. +DROP TABLE IF EXISTS unconfigured; +CREATE TABLE unconfigured ( + id bigint, + encrypted_unconfigured eql_v3.text, + PRIMARY KEY(id) +); + + +-- Per-test encrypted index fixture tables. +-- +-- Each integration test that exercises ORE/OPE range or order operators gets +-- its own table. This eliminates parallel-test races on a shared `encrypted` +-- table without having to mark tests `#[serial]`. +-- +-- The schema mirrors `encrypted` minus the jsonb columns (these tests never +-- touch jsonb). `kind` is `ore` or `ope`, selecting the `_ord_ore` or +-- `_ord_ope` domain per column; ORE text columns additionally carry a match +-- (bf) term, so they use `eql_v3.text_search` while OPE text columns use +-- `eql_v3.text_ord_ope`. `encrypted_bool` is storage-only in v3 (see above). +DO $$ +DECLARE + spec record; + tn text; + text_domain text; +BEGIN + FOR spec IN + -- map_ore_index_where (one per column type) + map_ore_index_order (one per test fn) + SELECT 'ore'::text AS kind, unnest(ARRAY[ + 'encrypted_ore_where_int2', + 'encrypted_ore_where_int4', + 'encrypted_ore_where_int8', + 'encrypted_ore_where_float8', + 'encrypted_ore_where_date', + 'encrypted_ore_where_text', + 'encrypted_ore_where_bool', + 'encrypted_ore_order_text', + 'encrypted_ore_order_text_desc', + 'encrypted_ore_order_nulls_last', + 'encrypted_ore_order_nulls_first', + 'encrypted_ore_order_qualified', + 'encrypted_ore_order_qualified_alias', + 'encrypted_ore_order_no_select_projection', + 'encrypted_ore_order_plaintext_column', + 'encrypted_ore_order_plaintext_and_eql', + 'encrypted_ore_order_simple_protocol', + 'encrypted_ore_order_int2', + 'encrypted_ore_order_int2_desc', + 'encrypted_ore_order_int4', + 'encrypted_ore_order_int4_desc', + 'encrypted_ore_order_int8', + 'encrypted_ore_order_int8_desc', + 'encrypted_ore_order_float8', + 'encrypted_ore_order_float8_desc' + ]) AS table_name + UNION ALL + -- map_ope_index_where (one per column type) + map_ope_index_order (one per test fn) + SELECT 'ope'::text AS kind, unnest(ARRAY[ + 'encrypted_ope_where_int2', + 'encrypted_ope_where_int4', + 'encrypted_ope_where_int8', + 'encrypted_ope_where_float8', + 'encrypted_ope_where_date', + 'encrypted_ope_where_text', + 'encrypted_ope_where_bool', + 'encrypted_ope_order_text_asc', + 'encrypted_ope_order_text_desc', + 'encrypted_ope_order_int4_asc', + 'encrypted_ope_order_int4_desc', + 'encrypted_ope_order_nulls_last', + 'encrypted_ope_order_nulls_first' + ]) AS table_name + LOOP + tn := spec.table_name; + + IF spec.kind = 'ore' THEN + text_domain := 'eql_v3.text_search'; + ELSE + text_domain := 'eql_v3.text_ord_ope'; + END IF; + + EXECUTE format('DROP TABLE IF EXISTS %I CASCADE', tn); + EXECUTE format( + 'CREATE TABLE %I ( + id bigint, + plaintext text, + plaintext_date date, + encrypted_text %s, + encrypted_bool eql_v3.bool, + encrypted_int2 eql_v3.int2_ord_%s, + encrypted_int4 eql_v3.int4_ord_%s, + encrypted_int8 eql_v3.int8_ord_%s, + encrypted_float8 eql_v3.float8_ord_%s, + encrypted_date eql_v3.date_ord_%s, + PRIMARY KEY(id) + )', tn, text_domain, spec.kind, spec.kind, spec.kind, spec.kind, spec.kind); + END LOOP; +END $$; + + +-- This is the exact same schema as above but using a database-generated primary key. +-- It is required to remove flake from the Elixir integration test suite. +-- TODO: port all the rest of our integration tests to this schema. +DROP TABLE IF EXISTS encrypted_elixir; +CREATE TABLE encrypted_elixir ( + id serial, + plaintext text, + plaintext_date date, + plaintext_domain domain_type_with_check, + encrypted_text eql_v3.text_search, + encrypted_bool eql_v3.bool, + encrypted_int2 eql_v3.int2_ord_ore, + encrypted_int4 eql_v3.int4_ord_ore, + encrypted_int8 eql_v3.int8_ord_ore, + encrypted_float8 eql_v3.float8_ord_ore, + encrypted_date eql_v3.date_ord_ore, + encrypted_jsonb eql_v3.json, + encrypted_jsonb_filtered eql_v3.json, + PRIMARY KEY(id) +); + +DROP TABLE IF EXISTS unconfigured_elixir; +CREATE TABLE unconfigured_elixir ( + id serial, + encrypted_unconfigured eql_v3.text, + PRIMARY KEY(id) +); From 51feccc802731ce2516f2174513fe3872108a95e Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 2 Jul 2026 16:26:44 +1000 Subject: [PATCH 3/5] test(integration): add gated EQL v3 variants for payload-coupled tests Add an eql_v3 module with #[ignore = "blocked on eql-mapper v3"] variants for the tests whose SQL/payload surface changes shape between EQL v2 and v3: - disable_mapping: raw column values are v3 envelopes ({v: 3, i: {t,c}, c, }, no k discriminator) - indexing: on-column eql_v2.encrypted_operator_class is replaced by functional btree indexes on term extractors (eql_v3.ord_term) - jsonb_containment: eql_v2.jsonb_contains() has no v3 function; containment is the @> / <@ operators on eql_v3.json - match_index: LIKE/ILIKE has no v3 equivalent; the match (bloom) index only supports containment. Deliberately no port of the LIKE tests - the placeholder documents the replacement surface - regression_cast: the single ::eql_v2_encrypted cast becomes a per-domain cast (::text::jsonb::eql_v3.text_search; the leading ::text keeps the bound parameter described as text) The rest of the integration suite intentionally has no v3 duplicates: it rides on the fixture and the mapper, and will be enabled wholesale by the eql-mapper v3 project. The gated tests never run by default (cargo nextest run is unchanged); run them with --run-ignored all after mise run postgres:setup:v3. --- .../src/eql_v3/disable_mapping.rs | 63 +++++++++++++++++++ .../src/eql_v3/indexing.rs | 51 +++++++++++++++ .../src/eql_v3/jsonb_containment.rs | 50 +++++++++++++++ .../src/eql_v3/match_index.rs | 41 ++++++++++++ .../src/eql_v3/mod.rs | 31 +++++++++ .../src/eql_v3/regression_cast.rs | 62 ++++++++++++++++++ .../cipherstash-proxy-integration/src/lib.rs | 1 + 7 files changed, 299 insertions(+) create mode 100644 packages/cipherstash-proxy-integration/src/eql_v3/disable_mapping.rs create mode 100644 packages/cipherstash-proxy-integration/src/eql_v3/indexing.rs create mode 100644 packages/cipherstash-proxy-integration/src/eql_v3/jsonb_containment.rs create mode 100644 packages/cipherstash-proxy-integration/src/eql_v3/match_index.rs create mode 100644 packages/cipherstash-proxy-integration/src/eql_v3/mod.rs create mode 100644 packages/cipherstash-proxy-integration/src/eql_v3/regression_cast.rs diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/disable_mapping.rs b/packages/cipherstash-proxy-integration/src/eql_v3/disable_mapping.rs new file mode 100644 index 00000000..2bb179a2 --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/eql_v3/disable_mapping.rs @@ -0,0 +1,63 @@ +//! EQL v3 variant of `disable_mapping.rs`. +//! +//! With `SET CIPHERSTASH.UNSAFE_DISABLE_MAPPING = true` the proxy passes raw +//! column values through. In v2 those are `eql_v2_encrypted` composites; in v3 +//! each column is a jsonb-backed `eql_v3.*` domain, so the raw value is the v3 +//! envelope `{v: 3, i: {t, c}, c, }` with no `k` discriminator. + +#[cfg(test)] +mod tests { + use crate::common::{ + clear, connect_with_tls, execute_query, query_with_client, random_id, + simple_query_with_client, trace, PROXY, + }; + use serde_json::{json, Value}; + + /// + /// Tests mapping is disabled when the `SET CIPHERSTASH.UNSAFE_DISABLE_MAPPING` command is used + /// and the raw value is an EQL v3 envelope. + /// + #[tokio::test] + #[ignore = "blocked on eql-mapper v3"] + async fn select_with_set_unsafe_disable_mapping_returns_v3_envelope() { + trace(); + + clear().await; + + let client = connect_with_tls(PROXY).await; + + let id = random_id(); + let encrypted_text = "hello".to_string(); + let expected = vec![encrypted_text.to_owned()]; + + let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; + execute_query(sql, &[&id, &encrypted_text]).await; + + let sql = "SET CIPHERSTASH.UNSAFE_DISABLE_MAPPING = true"; + client.query(sql, &[]).await.unwrap(); + + // Data should not be decrypted: the raw jsonb envelope comes back + let select_sql = "SELECT encrypted_text FROM encrypted"; + let rows = simple_query_with_client::(select_sql, &client).await; + + assert_eq!(rows.len(), 1); + + for s in rows { + let envelope: Value = serde_json::from_str(&s).unwrap(); + // v3 envelope: version 3, identifier, ciphertext - and no v2 `k` discriminator + assert_eq!(envelope["v"], json!(3)); + assert_eq!(envelope["i"]["c"], json!("encrypted_text")); + assert!(envelope.get("c").is_some()); + assert!(envelope.get("k").is_none()); + } + + // --------------------- + // Turn mapping back on and regular services resume + + let sql = "SET CIPHERSTASH.UNSAFE_DISABLE_MAPPING = false"; + client.query(sql, &[]).await.unwrap(); + + let actual = query_with_client::(select_sql, &client).await; + assert_eq!(expected, actual); + } +} diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/indexing.rs b/packages/cipherstash-proxy-integration/src/eql_v3/indexing.rs new file mode 100644 index 00000000..69a3155b --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/eql_v3/indexing.rs @@ -0,0 +1,51 @@ +//! EQL v3 variant of `select/indexing.rs`. +//! +//! v2 indexes the encrypted column with an on-column operator class: +//! +//! ```sql +//! CREATE INDEX ON encrypted (encrypted_text eql_v2.encrypted_operator_class); +//! ``` +//! +//! v3 has no on-column operator class. Ordering/range queries engage +//! functional btree indexes on the term extractors instead: +//! +//! ```sql +//! CREATE INDEX ON encrypted (eql_v3.ord_term(encrypted_text)); +//! CREATE INDEX ON encrypted (eql_v3.ord_ope_term(encrypted_int4)); +//! ``` + +#[cfg(test)] +mod tests { + use crate::common::{connect_with_tls, insert, query_by, random_id, trace, PROXY}; + use tracing::info; + + /// + /// Tests a range query with a functional index on the ORE term extractor. + /// + #[tokio::test] + #[ignore = "blocked on eql-mapper v3"] + async fn select_with_functional_index() { + trace(); + + for n in 1..=10 { + let id = random_id(); + + let encrypted_text = format!("hello_{}", n); + + let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; + insert(sql, &[&id, &encrypted_text]).await; + } + + let client = connect_with_tls(PROXY).await; + + let sql = "CREATE INDEX IF NOT EXISTS encrypted_text_ord_term_idx ON encrypted (eql_v3.ord_term(encrypted_text))"; + let _ = client.simple_query(sql).await; + + let sql = "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= $1"; + + let encrypted_text = "hello_10".to_string(); + let result = query_by::(sql, &encrypted_text).await; + + info!("Result: {:?}", result); + } +} diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/jsonb_containment.rs b/packages/cipherstash-proxy-integration/src/eql_v3/jsonb_containment.rs new file mode 100644 index 00000000..af5c71ba --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/eql_v3/jsonb_containment.rs @@ -0,0 +1,50 @@ +//! EQL v3 variant of the explicit-function test in +//! `select/jsonb_containment_index.rs`. +//! +//! v2 exposes SteVec containment both as the `@>` operator and as an explicit +//! `eql_v2.jsonb_contains()` function. v3 has no `jsonb_contains` function: +//! containment is expressed only through the `@>` / `<@` operators on +//! `eql_v3.json`, with `eql_v3.jsonb_query` needles (see +//! `eql_v3.to_ste_vec_query` for the functional GIN index expression). +//! +//! The operator-shaped containment tests in `select/jsonb_containment_index.rs` +//! ride on the mapper and the fixture, and are not duplicated here. + +#[cfg(test)] +mod tests { + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; + use serde_json::json; + + /// v2: `WHERE eql_v2.jsonb_contains(encrypted_jsonb, $1)`. + /// v3: `WHERE encrypted_jsonb @> $1` - there is no function-call form. + #[tokio::test] + #[ignore = "blocked on eql-mapper v3"] + async fn jsonb_containment_operator_replaces_jsonb_contains_function() { + trace(); + + clear().await; + + let client = connect_with_tls(PROXY).await; + + // 20 rows of {"string": "value_N", "number": n} with N = n % 10, + // giving exactly 2 rows per "value_N". + for n in 1..=20_i64 { + let id = random_id(); + let encrypted_jsonb = json!({ + "string": format!("value_{}", n % 10), + "number": n, + }); + + let sql = "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)"; + client.query(sql, &[&id, &encrypted_jsonb]).await.unwrap(); + } + + let search_value = json!({"string": "value_1"}); + let sql = "SELECT COUNT(*) FROM encrypted WHERE encrypted_jsonb @> $1"; + + let rows = client.query(sql, &[&search_value]).await.unwrap(); + let count: i64 = rows[0].get(0); + + assert_eq!(count, 2); + } +} diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/match_index.rs b/packages/cipherstash-proxy-integration/src/eql_v3/match_index.rs new file mode 100644 index 00000000..91bf6b57 --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/eql_v3/match_index.rs @@ -0,0 +1,41 @@ +//! EQL v3 replacement surface for the v2 match-index tests. +//! +//! v2's `map_match_index.rs` exercises `LIKE` over the match index. EQL v3 +//! does NOT support `LIKE`/`ILIKE`: the match index (bloom filter, `bf` term) +//! only supports containment, via `@>` / `<@` (`eql_v3.contains`). There is +//! deliberately no v3 port of the LIKE/ILIKE tests; this placeholder documents +//! and exercises the replacement operator surface. + +#[cfg(test)] +mod tests { + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; + + /// v2: `WHERE encrypted_text LIKE $1` (match index). + /// v3: `WHERE encrypted_text @> $1` - bloom containment on the `bf` term + /// of `eql_v3.text_search` / `eql_v3.text_match`. + #[tokio::test] + #[ignore = "blocked on eql-mapper v3"] + async fn match_index_bloom_containment_replaces_like() { + trace(); + + clear().await; + + let client = connect_with_tls(PROXY).await; + + let id = random_id(); + let encrypted_text = "hello@cipherstash.com"; + + let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; + client.query(sql, &[&id, &encrypted_text]).await.unwrap(); + + let sql = "SELECT id, encrypted_text FROM encrypted WHERE encrypted_text @> $1"; + let rows = client.query(sql, &[&"hello@"]).await.unwrap(); + + assert_eq!(rows.len(), 1); + + for row in rows { + let result: String = row.get("encrypted_text"); + assert_eq!(encrypted_text, result); + } + } +} diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/mod.rs b/packages/cipherstash-proxy-integration/src/eql_v3/mod.rs new file mode 100644 index 00000000..f078aa22 --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/eql_v3/mod.rs @@ -0,0 +1,31 @@ +//! EQL v3 test variants. +//! +//! GATED: every test in this module is `#[ignore = "blocked on eql-mapper v3"]` +//! because the proxy's eql-mapper cannot speak EQL v3 yet (that redesign is +//! out of scope here). The tests exist so the v3 SQL surface is documented and +//! compile-checked, and so they can be switched on when the mapper lands. +//! +//! Running them requires the EQL v3 fixture in place of the default v2 fixture: +//! +//! ```shell +//! CS_EQL_V3_PATH=../encrypt-query-language/release mise run postgres:setup:v3 +//! cargo nextest run -p cipherstash-proxy-integration -E 'test(eql_v3)' --run-ignored all +//! mise run postgres:setup # restore the v2 fixture afterwards +//! ``` +//! +//! The bulk of the integration suite is intentionally NOT duplicated here: it +//! rides on the fixture and the mapper, and will be enabled wholesale by the +//! eql-mapper v3 project. These modules cover only the payload/SQL-surface +//! coupled minority whose shape changes between v2 and v3: +//! +//! * `disable_mapping` - raw column values are v3 envelopes (`v: 3`, no `k`) +//! * `indexing` - on-column operator class -> functional term index +//! * `jsonb_containment` - `eql_v2.jsonb_contains()` -> `@>` on `eql_v3.json` +//! * `match_index` - LIKE/ILIKE -> bloom containment (`@>` / `<@`) +//! * `regression_cast` - `::eql_v2_encrypted` -> per-domain casts + +mod disable_mapping; +mod indexing; +mod jsonb_containment; +mod match_index; +mod regression_cast; diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/regression_cast.rs b/packages/cipherstash-proxy-integration/src/eql_v3/regression_cast.rs new file mode 100644 index 00000000..ef89f892 --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/eql_v3/regression_cast.rs @@ -0,0 +1,62 @@ +//! EQL v3 variant of the direct-insert cast surface in `eql_regression.rs`. +//! +//! The v2 regression harness reinserts captured ciphertexts with a single +//! `$2::eql_v2_encrypted` cast. v3 has no single encrypted type: direct +//! inserts cast to the column's specific domain (here `eql_v3.text_search`), +//! and the fail-closed domain CHECK validates the envelope on the way in. +//! +//! The full fixture-file regression flow (generate on main, replay on branch) +//! stays v2-only until the mapper speaks v3 and v3 fixtures can be generated. + +#[cfg(test)] +mod tests { + use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; + + fn get_database_port() -> u16 { + std::env::var("CS_DATABASE__PORT") + .map(|s| s.parse().unwrap()) + .unwrap_or(5617) // Default to TLS port + } + + /// + /// Captures a proxy-encrypted v3 ciphertext, reinserts it directly with a + /// per-domain cast, and decrypts it back through the proxy. + /// + #[tokio::test] + #[ignore = "blocked on eql-mapper v3"] + async fn insert_v3_ciphertext_directly_and_decrypt_via_proxy() { + trace(); + + clear().await; + + let id = random_id(); + let plaintext = "regression".to_string(); + + // Insert via proxy (encrypts to a v3 envelope) + let proxy_client = connect_with_tls(PROXY).await; + let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; + proxy_client.query(sql, &[&id, &plaintext]).await.unwrap(); + + // Read the raw ciphertext directly from the database (bypassing proxy) + let db_client = connect_with_tls(get_database_port()).await; + let sql = "SELECT encrypted_text::text FROM encrypted WHERE id = $1"; + let rows = db_client.query(sql, &[&id]).await.unwrap(); + let ciphertext: String = rows[0].get(0); + + // Reinsert the ciphertext directly, casting to the column's v3 domain + // (v2 used a single `::eql_v2_encrypted` cast here). The leading + // `::text` keeps the bound parameter described as text - a bare + // `$2::eql_v3.text_search` makes Postgres describe the parameter as + // the jsonb-backed domain, which a text binding cannot satisfy. + let new_id = random_id(); + let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2::text::jsonb::eql_v3.text_search)"; + db_client.query(sql, &[&new_id, &ciphertext]).await.unwrap(); + + // Decrypt via proxy + let sql = "SELECT encrypted_text FROM encrypted WHERE id = $1"; + let rows = proxy_client.query(sql, &[&new_id]).await.unwrap(); + let decrypted: String = rows[0].get(0); + + assert_eq!(plaintext, decrypted); + } +} diff --git a/packages/cipherstash-proxy-integration/src/lib.rs b/packages/cipherstash-proxy-integration/src/lib.rs index 8050045d..0c849c70 100644 --- a/packages/cipherstash-proxy-integration/src/lib.rs +++ b/packages/cipherstash-proxy-integration/src/lib.rs @@ -6,6 +6,7 @@ mod disable_mapping; mod empty_result; mod encryption_sanity; mod eql_regression; +mod eql_v3; mod extended_protocol_error_messages; mod insert; mod map_concat; From ddd3913315ca643fe2e6c85ddf074db3cf391c89 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 2 Jul 2026 16:48:49 +1000 Subject: [PATCH 4/5] test(benchmark): guard v3 schema against stale v2 config; document CASCADE footgun Review feedback: - benchmark-schema-v3.sql now carries the same guarded eql_v2_configuration truncate as schema-v3.sql, so a stale eql_v2.add_column row from a prior v2 benchmark:setup never survives into a v3 setup pointing at a v3-domain column. - Document that benchmark:setup:v3 must be re-run after any postgres:eql:v3:setup/teardown: the CASCADE uninstall silently drops eql_v3-typed columns from tables it doesn't recreate, and benchmark_encrypted is not recreated by postgres:setup:v3. --- DEVELOPMENT.md | 3 +++ tests/benchmark/mise.toml | 4 ++++ tests/benchmark/sql/benchmark-schema-v3.sql | 14 ++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c9cf2668..01dd7788 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -461,6 +461,9 @@ mise run postgres:setup The v3 benchmark schema (`tests/benchmark/sql/benchmark-schema-v3.sql`) is applied with `mise run benchmark:setup:v3` from `tests/benchmark` (it also requires `postgres:eql:v3:setup` to have been run). +> [!IMPORTANT] +> Re-run `benchmark:setup:v3` after any `postgres:eql:v3:setup` / `postgres:eql:v3:teardown`: the CASCADE uninstall silently drops `eql_v3`-typed columns from tables it doesn't recreate (`postgres:setup:v3` recreates the test fixture tables, but not `benchmark_encrypted`). + #### Convention: PostgreSQL ports PostgreSQL port numbers are 4 digits: diff --git a/tests/benchmark/mise.toml b/tests/benchmark/mise.toml index 297d069f..fbed71e9 100644 --- a/tests/benchmark/mise.toml +++ b/tests/benchmark/mise.toml @@ -149,6 +149,10 @@ docker compose run --rm postgres${CONTAINER_SUFFIX:-} pgbench --host=host.docker # speak v3. Requires EQL v3 to be installed first: # CS_EQL_V3_PATH=../encrypt-query-language/release mise run postgres:eql:v3:setup # The pgbench transaction scripts are version-agnostic and shared with v2. +# +# Re-run this task after any postgres:eql:v3:setup/teardown: the CASCADE +# uninstall silently drops eql_v3-typed columns from tables it doesn't +# recreate (benchmark_encrypted is not recreated by postgres:setup:v3). [tasks."benchmark:setup:v3"] description = "Apply the EQL v3 benchmark schema (requires postgres:eql:v3:setup)" run = """ diff --git a/tests/benchmark/sql/benchmark-schema-v3.sql b/tests/benchmark/sql/benchmark-schema-v3.sql index 367e7202..758c9d4b 100644 --- a/tests/benchmark/sql/benchmark-schema-v3.sql +++ b/tests/benchmark/sql/benchmark-schema-v3.sql @@ -19,6 +19,20 @@ -- The pgbench transaction scripts (transaction-*.sql) are version-agnostic -- plain SQL over column names and are shared with the v2 benchmark. +-- The v2 benchmark schema truncates its configuration table here. v3 has no +-- database-side configuration table, but stale v2 configuration (e.g. the +-- eql_v2.add_column row from a prior v2 benchmark:setup) must not be left +-- pointing at tables whose columns are now v3 domains. +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'eql_v2_configuration' + ) THEN + TRUNCATE TABLE public.eql_v2_configuration; + END IF; +END $$; + DROP TABLE IF EXISTS benchmark_plaintext; CREATE TABLE benchmark_plaintext ( id serial primary key, From 693988c72f361c7e85655ab15472d41e522bdd87 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 2 Jul 2026 16:48:58 +1000 Subject: [PATCH 5/5] test(integration): harden gated eql_v3 tests per review Review feedback: - regression_cast: reuse common::get_database_port instead of a local panicky re-declaration - disable_mapping, jsonb_containment, match_index: scope queries to the inserted ids so the tests are parallel-safe on the shared encrypted table when they are un-ignored (mirrors the id-range scoping the v2 jsonb containment test uses) - indexing: stop discarding CREATE INDEX errors and assert the EXPLAIN ANALYZE result is non-empty instead of only logging it --- .../src/eql_v3/disable_mapping.rs | 9 +++++---- .../cipherstash-proxy-integration/src/eql_v3/indexing.rs | 5 ++++- .../src/eql_v3/jsonb_containment.rs | 8 ++++++-- .../src/eql_v3/match_index.rs | 5 +++-- .../src/eql_v3/regression_cast.rs | 8 +------- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/disable_mapping.rs b/packages/cipherstash-proxy-integration/src/eql_v3/disable_mapping.rs index 2bb179a2..fa5d3aba 100644 --- a/packages/cipherstash-proxy-integration/src/eql_v3/disable_mapping.rs +++ b/packages/cipherstash-proxy-integration/src/eql_v3/disable_mapping.rs @@ -36,9 +36,10 @@ mod tests { let sql = "SET CIPHERSTASH.UNSAFE_DISABLE_MAPPING = true"; client.query(sql, &[]).await.unwrap(); - // Data should not be decrypted: the raw jsonb envelope comes back - let select_sql = "SELECT encrypted_text FROM encrypted"; - let rows = simple_query_with_client::(select_sql, &client).await; + // Data should not be decrypted: the raw jsonb envelope comes back. + // Scoped by id so the test stays parallel-safe on the shared table. + let select_sql = format!("SELECT encrypted_text FROM encrypted WHERE id = {id}"); + let rows = simple_query_with_client::(&select_sql, &client).await; assert_eq!(rows.len(), 1); @@ -57,7 +58,7 @@ mod tests { let sql = "SET CIPHERSTASH.UNSAFE_DISABLE_MAPPING = false"; client.query(sql, &[]).await.unwrap(); - let actual = query_with_client::(select_sql, &client).await; + let actual = query_with_client::(&select_sql, &client).await; assert_eq!(expected, actual); } } diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/indexing.rs b/packages/cipherstash-proxy-integration/src/eql_v3/indexing.rs index 69a3155b..ea0eb9d5 100644 --- a/packages/cipherstash-proxy-integration/src/eql_v3/indexing.rs +++ b/packages/cipherstash-proxy-integration/src/eql_v3/indexing.rs @@ -39,7 +39,7 @@ mod tests { let client = connect_with_tls(PROXY).await; let sql = "CREATE INDEX IF NOT EXISTS encrypted_text_ord_term_idx ON encrypted (eql_v3.ord_term(encrypted_text))"; - let _ = client.simple_query(sql).await; + client.simple_query(sql).await.unwrap(); let sql = "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= $1"; @@ -47,5 +47,8 @@ mod tests { let result = query_by::(sql, &encrypted_text).await; info!("Result: {:?}", result); + + // EXPLAIN ANALYZE returns the executed plan as rows of text + assert!(!result.is_empty()); } } diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/jsonb_containment.rs b/packages/cipherstash-proxy-integration/src/eql_v3/jsonb_containment.rs index af5c71ba..b67f99bc 100644 --- a/packages/cipherstash-proxy-integration/src/eql_v3/jsonb_containment.rs +++ b/packages/cipherstash-proxy-integration/src/eql_v3/jsonb_containment.rs @@ -28,8 +28,10 @@ mod tests { // 20 rows of {"string": "value_N", "number": n} with N = n % 10, // giving exactly 2 rows per "value_N". + let mut ids = Vec::with_capacity(20); for n in 1..=20_i64 { let id = random_id(); + ids.push(id); let encrypted_jsonb = json!({ "string": format!("value_{}", n % 10), "number": n, @@ -39,10 +41,12 @@ mod tests { client.query(sql, &[&id, &encrypted_jsonb]).await.unwrap(); } + // Scoped to the inserted ids so the test stays parallel-safe on the + // shared table (mirrors the id-range scoping in the v2 test). let search_value = json!({"string": "value_1"}); - let sql = "SELECT COUNT(*) FROM encrypted WHERE encrypted_jsonb @> $1"; + let sql = "SELECT COUNT(*) FROM encrypted WHERE encrypted_jsonb @> $1 AND id = ANY($2)"; - let rows = client.query(sql, &[&search_value]).await.unwrap(); + let rows = client.query(sql, &[&search_value, &ids]).await.unwrap(); let count: i64 = rows[0].get(0); assert_eq!(count, 2); diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/match_index.rs b/packages/cipherstash-proxy-integration/src/eql_v3/match_index.rs index 91bf6b57..7cba31b8 100644 --- a/packages/cipherstash-proxy-integration/src/eql_v3/match_index.rs +++ b/packages/cipherstash-proxy-integration/src/eql_v3/match_index.rs @@ -28,8 +28,9 @@ mod tests { let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; client.query(sql, &[&id, &encrypted_text]).await.unwrap(); - let sql = "SELECT id, encrypted_text FROM encrypted WHERE encrypted_text @> $1"; - let rows = client.query(sql, &[&"hello@"]).await.unwrap(); + // Scoped by id so the test stays parallel-safe on the shared table + let sql = "SELECT id, encrypted_text FROM encrypted WHERE encrypted_text @> $1 AND id = $2"; + let rows = client.query(sql, &[&"hello@", &id]).await.unwrap(); assert_eq!(rows.len(), 1); diff --git a/packages/cipherstash-proxy-integration/src/eql_v3/regression_cast.rs b/packages/cipherstash-proxy-integration/src/eql_v3/regression_cast.rs index ef89f892..72153549 100644 --- a/packages/cipherstash-proxy-integration/src/eql_v3/regression_cast.rs +++ b/packages/cipherstash-proxy-integration/src/eql_v3/regression_cast.rs @@ -10,13 +10,7 @@ #[cfg(test)] mod tests { - use crate::common::{clear, connect_with_tls, random_id, trace, PROXY}; - - fn get_database_port() -> u16 { - std::env::var("CS_DATABASE__PORT") - .map(|s| s.parse().unwrap()) - .unwrap_or(5617) // Default to TLS port - } + use crate::common::{clear, connect_with_tls, get_database_port, random_id, trace, PROXY}; /// /// Captures a proxy-encrypted v3 ciphertext, reinserts it directly with a