From a1945c340d68228c46a2992d88105d82c13c950a Mon Sep 17 00:00:00 2001 From: "vericto-bot[bot]" <351306506+vericto-bot[bot]@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:59:05 +0000 Subject: [PATCH] feat(offline): offline_migration for the ci profile (vericto CLI 2.3.0) Vericto CLI 2.3.0 caps schema DDL in the ci profile only for migrations: a vericto serve ci request without "migration": true is checked like check --offline --sql, so DROP TABLE is BLOCKED (was FLAGGED). TRUNCATE and DROP SCHEMA / DATABASE are never capped (engine 5.1.0). - Vericto(offline_migration=True), OfflineEvaluator(migration=...) and OfflineEngine.check(migration=...) send the marker (ci profile only; runtime, a non-bool or online raise VerictoConfigError). The field is sent only when set; verdict.raw["migration"] says what was sent. - vericto-dbt check --offline does not mark dbt SQL as a migration: schema DDL in a model or hook now blocks with CLI 2.3.0+. Documented, with the remedies (--no-hooks, --select, --fail-on). - README, docstrings and CHANGELOG (Unreleased) describe the scope; the fake serve follows CLI 2.3.0; parity with a migrations/ file against the real CLI (2.1.0, 2.2.0, 2.3.0). - CI: the offline job pins vericto CLI 2.3.0 (was 2.1.0). --- .github/workflows/ci.yml | 6 ++-- CHANGELOG.md | 46 +++++++++++++++++++++++++ README.md | 49 ++++++++++++++++++++++----- src/vericto/dbt.py | 10 ++++-- src/vericto/guard.py | 17 ++++++++-- src/vericto/offline.py | 71 +++++++++++++++++++++++++++++++++------ tests/fake_serve.py | 31 ++++++++++++----- tests/test_dbt.py | 22 ++++++++++++ tests/test_offline.py | 69 +++++++++++++++++++++++++++++++++---- tests/test_offline_cli.py | 51 ++++++++++++++++++++++++++-- 10 files changed, 327 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84efbca..235c414 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,9 +166,9 @@ jobs: runs-on: ${{ github.event.repository.private && fromJSON(vars.CI_RUNS_ON || '["self-hosted","linux","x64","vericto-spot"]') || fromJSON(vars.CI_RUNS_ON_PUBLIC || '"ubuntu-latest"') }} timeout-minutes: 10 env: - VERICTO_CLI_VERSION: "2.1.0" - # sha256 of vericto-cli-x86_64-unknown-linux-musl.tar.xz in the v2.1.0 release. - VERICTO_CLI_SHA256: "8c041e05bc0f2260300f3e987d366f18e0b9fbefaacc089959d53c60f3f37f32" + VERICTO_CLI_VERSION: "2.3.0" + # sha256 of vericto-cli-x86_64-unknown-linux-musl.tar.xz in the v2.3.0 release. + VERICTO_CLI_SHA256: "edc257e9bbbf83cda194c324b3df112f54576bde54a3c8bbace405a19ff613fa" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 71b49b7..ef4aad8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,52 @@ All notable changes to this package are documented here. The format follows ## [Unreleased] +Offline `ci` checks follow Vericto CLI 2.3.0 and engine 5.1.0: schema DDL blocks +unless the SQL is marked as a migration. + +### Added + +- **`offline_migration=True`** (`Vericto(...)`, `OfflineEvaluator(migration=...)`, + `OfflineEngine.check(..., migration=...)`): marks the checked SQL as a versioned + migration. The SDK sends `"migration": true` in the `vericto serve` request, so + the schema DDL rules (DROP TABLE, ALTER TABLE, DROP INDEX / TRIGGER, the + Snowflake and SQLite schema rules) are capped at `FLAGGED`, as for a file under + `migrations/` in `vericto check --offline`. It needs `offline_profile="ci"` + (with `"runtime"`, or a value that is not a bool, `VerictoConfigError`) and + offline mode. The field is sent only when set, so requests without it are + unchanged. `verdict.raw["migration"]` says what was sent. + +### Changed + +- **Behaviour change with Vericto CLI 2.3.0+: offline `ci` checks block schema + DDL unless the request is a migration.** CLI 2.3.0 caps schema DDL only for + migration inputs: `vericto check --offline --sql "DROP TABLE users"` and a + `vericto serve` `ci` request without `"migration": true` are now `BLOCKED` + (were `FLAGGED`). `Vericto(offline=True, offline_profile="ci")` therefore + blocks `DROP TABLE` with CLI 2.3.0+; pass `offline_migration=True` for SQL that + is a migration. CLI 2.1.0 and 2.2.0 ignore the marker and cap every `ci` check. + TRUNCATE (VERICTO-011) and DROP SCHEMA / DATABASE (VERICTO-012) are never + capped with engine 5.1.0 (CLI 2.3.0): they block even in a migration. +- **`vericto-dbt check --offline`**: dbt models and hooks are not migrations, so + with CLI 2.3.0+ schema DDL in them (a `DROP TABLE` / `ALTER TABLE … DROP + COLUMN` post-hook) is `BLOCKED` and exits `1`, and TRUNCATE / DROP SCHEMA + hooks block. vericto-dbt does not mark its SQL as a migration. Remedies: + `--no-hooks`, `--select`, or `--fail-on`. The offline note in the report says + so. Online, the CI-check API decides. +- README: the profile table, the offline example (now `offline_migration=True`), + the dbt section and the Spanish summary describe the migration scope. + +### Tests + +- The fake `vericto serve` follows CLI 2.3.0 (`DROP` flagged only for a `ci` + migration, `TRUNCATE` always blocks, a non-bool `migration` is + `INVALID_REQUEST`; `FAKE_SERVE_MODE=cap_all` is CLI 2.2.0). New tests: the + `ci` profile with and without the marker, the marker sent only when set, the + configuration errors, CLI 2.2.0 behaviour, and a dbt `DROP TABLE` post-hook + that blocks offline. `tests/test_offline_cli.py` adds a parity case per input: + `offline_migration=True` equals `vericto check --offline` of the same SQL as + `migrations/0001_case.sql` (passes with CLI 2.1.0, 2.2.0 and 2.3.0). + ## [0.8.0] - 2026-11-15 Offline mode: check SQL with no account and no network, through the engine built diff --git a/README.md b/README.md index 35735d2..ddd939a 100644 --- a/README.md +++ b/README.md @@ -197,15 +197,31 @@ with vericto.Vericto(offline=True) as guard: # or VERICTO_OFFLINE=1 print(exc.verdict.rule_code, exc.suggested_fix) # VERICTO-001 and a safer statement conn.close() -# Exactly what `vericto check --offline` answers (schema DDL is flagged, not blocked): -with vericto.Vericto(offline=True, offline_profile="ci") as guard: +# Exactly what `vericto check --offline` answers for a file under migrations/ +# (schema DDL is flagged, not blocked): +with vericto.Vericto(offline=True, offline_profile="ci", offline_migration=True) as guard: assert guard.check("DROP TABLE users", "postgres").decision is vericto.Decision.FLAGGED ``` | `offline_profile` | Matches | `DROP TABLE users` | |---|---|---| | `"runtime"` (default) | the Runtime API, MCP `check_sql` and the proxy's local mode, for a workspace with no overrides | `BLOCKED` (VERICTO-010) | -| `"ci"` | `vericto check --offline` and the CI-check API: the DDL rules VERICTO-010 to 019 are capped at `FLAGGED` | `FLAGGED` | +| `"ci"` | `vericto check --offline --sql` (CLI 2.3.0+): schema DDL blocks, as under `"runtime"` | `BLOCKED` (VERICTO-010) | +| `"ci"` + `offline_migration=True` | `vericto check --offline` of a migration file: the schema DDL rules (DROP TABLE, ALTER TABLE, DROP INDEX / TRIGGER: VERICTO-010, 013, 015 to 019, and the Snowflake / SQLite ones) are capped at `FLAGGED` | `FLAGGED` | + +**Schema DDL in `ci` checks blocks unless the SQL is a migration** (Vericto CLI +2.3.0, engine 5.1.0). `vericto check --offline` caps schema DDL only for +migration files (a `migrations/`, `migration/` or `migrate/` directory, a Flyway +`V1__name.sql` or golang-migrate `1_name.up.sql` file, or a path in +`.vericto.toml`'s `migration_paths`), and `vericto serve` caps it only for a `ci` +request marked `"migration": true`. The SDK sends that marker when you pass +`offline_migration=True` (it needs `offline_profile="ci"`); pass it when the SQL +you check is a versioned migration, for example in a migration runner. Without it +a `DROP TABLE` is `BLOCKED` under both profiles. TRUNCATE (VERICTO-011) and DROP +SCHEMA / DATABASE (VERICTO-012) are never capped: they block even in a migration. +CLI 2.1.0 and 2.2.0 ignore the marker and cap every `ci` check, so with them +`DROP TABLE` is `FLAGGED` under `"ci"` either way; `verdict.raw["migration"]` +says what the SDK sent. How it behaves: @@ -232,6 +248,8 @@ How it behaves: logged. - `vericto-dbt check --offline` (or `VERICTO_OFFLINE=1`) checks a dbt project with the `ci` profile, with no key; Airflow workers use it with `VERICTO_OFFLINE=1`. + dbt models and hooks are not migrations, so with CLI 2.3.0+ schema DDL in them + blocks (see [dbt](#dbt)). Offline checks use the 42 standard rules at their default severities. A workspace adds custom rules, severity overrides, sensitive-column masking, agent @@ -513,7 +531,7 @@ All inherit from `VerictoError`. and `supported_dialects()`. When the caller gives `context` or `end_user`, the guard passes them as keyword arguments of the same names (an evaluator without them raises `TypeError` then, rather than dropping them). This release ships -`RuntimeApiEvaluator` and `OfflineEvaluator(profile=..., cli_path=..., timeout=...)`, +`RuntimeApiEvaluator` and `OfflineEvaluator(profile=..., migration=..., cli_path=..., timeout=...)`, what `Vericto(offline=True)` builds (see [Offline mode](#offline-mode-no-account)). An evaluator with a `close()` method is closed by `Vericto.close()`. @@ -723,9 +741,16 @@ imported: only its artifacts are read. tags. Runs carry GitHub/GitLab provenance and appear in the dashboard's CI reports. - **Offline** (`--offline` or `VERICTO_OFFLINE=1`): each statement is checked by the local `vericto` CLI (2.1.0+, see [Offline mode](#offline-mode-no-account)) - with the `ci` profile, so the verdicts equal `vericto check --offline`. No key, + with the `ci` profile, so the verdicts equal `vericto check --offline` of the + same SQL outside a migration directory. No key, no network, no allowance spent; workspace rules and sensitive-column tags do not - apply. Combined with `VERICTO_API_KEY`, `--api-url` or `--database-id` it exits + apply. **With CLI 2.3.0+ (engine 5.1.0), schema DDL in a model or hook is + `BLOCKED`, not `FLAGGED`:** dbt SQL is not a versioned migration, it runs on + every `dbt run`. A `post-hook` with `DROP TABLE` / `ALTER TABLE … DROP COLUMN` + now exits `1`, and `TRUNCATE` / `DROP SCHEMA` hooks block with any CLI on + engine 5.1.0. Remedies: `--no-hooks`, `--select` to leave the node out, or + `--fail-on` (exit code only). Online, the CI-check API decides; it applies the + same rules once the hosted evaluator runs engine 5.1.0. Combined with `VERICTO_API_KEY`, `--api-url` or `--database-id` it exits `3` ("choose one"); a local engine failure exits `4`. - SARIF points at the model file (line 1: compiled lines do not map to the Jinja source; the message gives the compiled line). GitHub Actions example: @@ -1016,10 +1041,18 @@ nada. Los wrappers e integraciones funcionan igual y el resultado es el mismo `Verdict`, con `mode="offline"`, `engine_version` y `ruleset_version` (`standard@engine-`). Perfiles: `offline_profile="runtime"` (por defecto, como la Runtime API: `DROP TABLE` queda `BLOCKED`) o `"ci"` (igual que -`vericto check --offline`: el DDL de esquema queda `FLAGGED`). Combinarlo con una +`vericto check --offline`). Desde la CLI 2.3.0 (motor 5.1.0), el perfil `"ci"` +limita el DDL de esquema a `FLAGGED` solo en migraciones: sin marcar, `DROP TABLE` +queda `BLOCKED`, como con `--sql`; con `offline_migration=True` (requiere +`offline_profile="ci"`) el SDK envía `"migration": true` y el DDL de esquema queda +`FLAGGED`, como en un archivo de `migrations/`. TRUNCATE y DROP SCHEMA / DATABASE se +bloquean siempre. Las CLI 2.1.0 y 2.2.0 ignoran la marca y limitan todo chequeo +`ci`. Combinarlo con una clave de API o con otro evaluador lanza `VerictoConfigError` ("choose one"). Si el proceso muere, se reinicia una vez; si vuelve a fallar, se aplica `fail_mode` como -con la API. `vericto-dbt check --offline` usa el perfil `ci`. Sin conexión se usan +con la API. `vericto-dbt check --offline` usa el perfil `ci` sin marca de +migración: con la CLI 2.3.0+, el DDL de esquema en un modelo o hook de dbt queda +`BLOCKED`. Sin conexión se usan las 42 reglas estándar: las reglas propias, el enmascarado de columnas sensibles, las allowlists, las políticas de negocio y la auditoría requieren un workspace. El motor corre en el proceso aparte de la CLI, con licencia AGPL-3.0-only; este diff --git a/src/vericto/dbt.py b/src/vericto/dbt.py index 5870ad7..c8b6a77 100644 --- a/src/vericto/dbt.py +++ b/src/vericto/dbt.py @@ -22,8 +22,11 @@ ``--offline`` (or ``VERICTO_OFFLINE=1``) evaluates with the local engine of the ``vericto`` CLI instead (2.1.0+, ``vericto serve --stdio``): no key, no network, the standard rules under the ``ci`` profile, so the verdicts equal ``vericto check ---offline``. Workspace rules, sensitive-column tags and the CI-check allowance do -not apply. +--offline`` of the same SQL outside a migration directory. Workspace rules, +sensitive-column tags and the CI-check allowance do not apply. dbt SQL is not a +versioned migration, so with CLI 2.3.0+ schema DDL in a model or hook (a +``DROP TABLE`` post-hook) is BLOCKED, not FLAGGED, and TRUNCATE / DROP SCHEMA / +DATABASE block with engine 5.1.0+. Exit codes (as vericto-cli): 0 nothing at or above ``--fail-on``; 1 a finding at or above it; 2 usage error (no manifest, unsupported adapter, nothing compiled); @@ -722,7 +725,8 @@ def run_check(args: argparse.Namespace, out: IO[str], err: IO[str], client: Opti if report.mode == "offline": report.notes.append( f"Offline: the standard rules of the local engine ({report.ruleset_version or 'vericto serve'}), " - "ci profile. Workspace rules and sensitive-column tags do not apply." + "ci profile, not a migration (schema DDL blocks with vericto CLI 2.3.0+). " + "Workspace rules and sensitive-column tags do not apply." ) if args.format == "json": diff --git a/src/vericto/guard.py b/src/vericto/guard.py index 92c1022..aaecbc5 100644 --- a/src/vericto/guard.py +++ b/src/vericto/guard.py @@ -73,6 +73,12 @@ class Vericto: an ``evaluator``, it raises :class:`~vericto.VerictoConfigError`. offline_profile: ``"runtime"`` (default: ``DROP TABLE`` is ``BLOCKED``, as on the Runtime API) or ``"ci"`` (exactly ``vericto check --offline``). + offline_migration: with ``offline_profile="ci"``, ``True`` checks the SQL as a + versioned migration: schema DDL (DROP TABLE, ALTER TABLE, ...) is capped + at ``FLAGGED``, as for a file under ``migrations/``. Default ``False``: + checked like ``vericto check --offline --sql``, so ``DROP TABLE`` is + ``BLOCKED`` (Vericto CLI 2.3.0+; 2.1.0 and 2.2.0 cap every ``ci`` check). + TRUNCATE and DROP SCHEMA / DATABASE block either way. offline_cli_path: the ``vericto`` binary (default ``VERICTO_CLI``, then PATH). The guard owns its evaluator: :meth:`close` (or ``with Vericto(...) as guard:``) @@ -96,6 +102,7 @@ def __init__( offline: Optional[bool] = None, offline_profile: Optional[str] = None, offline_cli_path: Optional[str] = None, + offline_migration: Optional[bool] = None, ) -> None: if offline_requested(offline): online = [ @@ -118,11 +125,15 @@ def __init__( "its own rules." ) evaluator = OfflineEvaluator( - profile=offline_profile or DEFAULT_PROFILE, cli_path=offline_cli_path, timeout=timeout + profile=offline_profile or DEFAULT_PROFILE, + migration=False if offline_migration is None else offline_migration, + cli_path=offline_cli_path, + timeout=timeout, ) - elif offline_profile is not None or offline_cli_path is not None: + elif offline_profile is not None or offline_cli_path is not None or offline_migration is not None: raise VerictoConfigError( - "offline_profile and offline_cli_path apply to offline mode only: pass offline=True too." + "offline_profile, offline_cli_path and offline_migration apply to offline mode only: " + "pass offline=True too." ) if evaluator is None: evaluator = RuntimeApiEvaluator( diff --git a/src/vericto/offline.py b/src/vericto/offline.py index 3b4de61..2c2c42e 100644 --- a/src/vericto/offline.py +++ b/src/vericto/offline.py @@ -18,8 +18,15 @@ * ``"runtime"`` (default): what the Runtime API answers for a workspace with no overrides. ``DROP TABLE`` is ``BLOCKED``. -* ``"ci"``: exactly ``vericto check --offline``. Schema DDL (VERICTO-010 to 019) is - capped at ``FLAGGED``. +* ``"ci"``: exactly ``vericto check --offline``. Since Vericto CLI 2.3.0 the schema + DDL cap applies to migrations only, as in ``check --offline``: a request is + checked like ``--sql`` (``DROP TABLE`` is ``BLOCKED``, as under ``"runtime"``) + unless it is marked as a migration (``migration=True``, the serve request's + ``"migration": true``), which caps the schema DDL rules (DROP TABLE, ALTER TABLE, + DROP INDEX / TRIGGER, the Snowflake and SQLite schema rules) at ``FLAGGED``, as + for a file under ``migrations/``. TRUNCATE and DROP SCHEMA / DATABASE are never + capped (engine 5.1.0). CLI 2.1.0 and 2.2.0 ignore the marker and cap every + ``ci`` request. What needs a workspace is not available offline: custom rules, severity overrides, sensitive-column masking, agent allowlists, business policies and the audit trail. @@ -354,6 +361,16 @@ def _fail_all(self, reason: str) -> None: slot.event.set() +def _check_migration(migration: bool, profile: str) -> None: + """``migration`` is a bool, and means something only under the ``ci`` profile.""" + if not isinstance(migration, bool): + raise VerictoConfigError(f"offline_migration must be True or False, got {migration!r}.") + if migration and profile != "ci": + raise VerictoConfigError( + "offline_migration=True needs offline_profile='ci': the runtime profile has no schema-migration cap." + ) + + class OfflineEngine: """The ``vericto serve --stdio`` process behind offline mode, started on demand. @@ -364,6 +381,9 @@ class OfflineEngine: profile: ``"runtime"`` (default) or ``"ci"`` (see the module docs). cli_path: the ``vericto`` binary; default ``VERICTO_CLI``, then PATH. timeout: seconds to wait for one answer (default 30). + + :meth:`check` takes ``migration=True`` to mark the SQL as a versioned migration + (``ci`` profile only). """ def __init__( @@ -393,14 +413,21 @@ def __repr__(self) -> str: def ruleset_version(self) -> Optional[str]: return f"standard@engine-{self.engine_version}" if self.engine_version else None - def check(self, sql: str, dialect: str) -> Dict[str, Any]: + def check(self, sql: str, dialect: str, *, migration: bool = False) -> Dict[str, Any]: """The ``result`` document for ``sql`` (``check --offline --format json``). + ``migration=True`` (``ci`` profile only) marks ``sql`` as a versioned + migration: the schema DDL rules are capped at ``FLAGGED``, as for a file + under ``migrations/`` (CLI 2.3.0+; older CLIs cap every ``ci`` request). + Raises :class:`_ServeError` for an error answer, :class:`VerictoConfigError` when the CLI is unusable, and :class:`VerictoUnavailable` when the process failed twice (or timed out). """ - payload = {"op": "check", "sql": sql, "dialect": dialect, "profile": self.profile} + _check_migration(migration, self.profile) + payload: Dict[str, Any] = {"op": "check", "sql": sql, "dialect": dialect, "profile": self.profile} + if migration: + payload["migration"] = True last = "no attempt made" for attempt in range(2): proc = self._process(restart=attempt > 0) @@ -562,6 +589,12 @@ class OfflineEvaluator: Args: profile: ``"runtime"`` (default): a ``DROP TABLE`` is ``BLOCKED``, as on the Runtime API. ``"ci"``: exactly ``vericto check --offline``. + migration: ``True`` (``ci`` profile only) checks every statement as part of a + versioned migration: schema DDL (DROP TABLE, ALTER TABLE, ...) is capped + at ``FLAGGED``, as for a file under ``migrations/``. ``False`` + (default): checked like ``vericto check --offline --sql``, so a + ``DROP TABLE`` is ``BLOCKED`` (CLI 2.3.0+; CLI 2.1.0 and 2.2.0 cap every + ``ci`` check). TRUNCATE and DROP SCHEMA / DATABASE always block. cli_path: the ``vericto`` binary (default ``VERICTO_CLI``, then PATH). timeout: seconds to wait for one verdict (default 30). @@ -581,13 +614,22 @@ class OfflineEvaluator: mode = "offline" def __init__( - self, *, profile: str = DEFAULT_PROFILE, cli_path: Optional[str] = None, timeout: Optional[float] = None + self, + *, + profile: str = DEFAULT_PROFILE, + migration: bool = False, + cli_path: Optional[str] = None, + timeout: Optional[float] = None, ) -> None: + if profile in PROFILES: + _check_migration(migration, profile) self.engine = OfflineEngine(profile=profile, cli_path=cli_path, timeout=timeout) + self.migration = migration self._noted_policy_fields = False def __repr__(self) -> str: - return f"OfflineEvaluator(profile={self.profile!r}, cli_path={self.engine.cli_path!r})" + extra = ", migration=True" if self.migration else "" + return f"OfflineEvaluator(profile={self.profile!r}{extra}, cli_path={self.engine.cli_path!r})" @property def profile(self) -> str: @@ -605,14 +647,14 @@ def evaluate( logger.info("vericto: offline mode has no business policies; context and end_user have no effect.") self._noted_policy_fields = True try: - result = self.engine.check(sql, dialect) + result = self.engine.check(sql, dialect, migration=self.migration) except _ServeError as exc: if exc.code in _REQUEST_ERRORS: raise _config_error(exc, dialect) from None raise VerictoUnavailable( f"{_WHERE} failed on this statement ({exc.code}): {exc.message or 'no detail'}.", code=exc.code ) from None - return verdict_from_result(result, sql, dialect, self.profile) + return verdict_from_result(result, sql, dialect, self.profile, migration=self.migration) def supported_dialects(self) -> Optional[Sequence[str]]: return KNOWN_DIALECTS @@ -622,9 +664,12 @@ def close(self) -> None: self.engine.close() -def verdict_from_result(result: Mapping[str, Any], sql: str, dialect: str, profile: str) -> Verdict: +def verdict_from_result( + result: Mapping[str, Any], sql: str, dialect: str, profile: str, *, migration: bool = False +) -> Verdict: """Map a ``check --offline --format json`` document (one query) to a :class:`Verdict`, - through the same :meth:`Verdict.from_response` as an online answer.""" + through the same :meth:`Verdict.from_response` as an online answer. ``raw`` carries + ``profile`` and ``migration``.""" query: Mapping[str, Any] = result["queries"][0] summary = result.get("summary") summary_fields: Mapping[str, Any] = summary if isinstance(summary, Mapping) else {} @@ -641,6 +686,7 @@ def verdict_from_result(result: Mapping[str, Any], sql: str, dialect: str, profi "mode": "offline", "engine_version": engine_version, "profile": profile, + "migration": migration, # The whole document, as `vericto check --offline --format json` prints it. "offline_result": dict(result), } @@ -650,7 +696,10 @@ def verdict_from_result(result: Mapping[str, Any], sql: str, dialect: str, profi class OfflineCiClient: """A stand-in for :class:`~vericto.ci_api.CiCheckClient` that checks with the local engine, for ``vericto-dbt check --offline``. Uses the ``ci`` profile by default, - so verdicts equal ``vericto check --offline``.""" + so verdicts equal ``vericto check --offline`` of the same SQL outside a migration + directory: dbt models and hooks are not versioned migrations, so with CLI 2.3.0+ + schema DDL in them (a ``DROP TABLE`` post-hook) is ``BLOCKED``, and TRUNCATE / + DROP SCHEMA / DATABASE block with engine 5.1.0+.""" def __init__(self, *, profile: str = "ci", cli_path: Optional[str] = None, timeout: Optional[float] = None) -> None: self.engine = OfflineEngine(profile=profile, cli_path=cli_path, timeout=timeout) diff --git a/tests/fake_serve.py b/tests/fake_serve.py index d56b862..33ad06d 100644 --- a/tests/fake_serve.py +++ b/tests/fake_serve.py @@ -3,13 +3,17 @@ """A stand-in for ``vericto serve --stdio`` (protocol 1), for the offline unit tests. -It is not the engine: its rules are string checks. ``DROP`` is VERICTO-010 (BLOCKED -under the ``runtime`` profile, FLAGGED under ``ci``), ``DELETE`` without ``WHERE`` is +It is not the engine: its rules are string checks. ``TRUNCATE`` is VERICTO-011 +(BLOCKED under both profiles, engine 5.1.0). ``DROP`` is VERICTO-010: BLOCKED, except +under ``ci`` with ``"migration": true`` (CLI 2.3.0), where it is FLAGGED; a +``migration`` that is not a boolean is INVALID_REQUEST. ``DELETE`` without ``WHERE`` is VERICTO-001 (BLOCKED), ``SELECT *`` is VERICTO-060 (MONITORED), ``GARBAGE`` is a PARSE_ERROR, anything else is ALLOWED. Several statements give the strongest one. ``FAKE_SERVE_MODE`` changes the behavior (comma-separated): +* ``cap_all``: CLI 2.1.0 / 2.2.0, which ignore ``migration`` and flag ``DROP`` in every + ``ci`` request. * ``old``: the hello says CLI 2.0.0. ``protocol2``: protocol 2. ``no_hello``: exit 2 before the hello, as a CLI without ``serve`` does. ``silent``: never says hello. * ``crash_once``: exit 1 on the earliest request whose SQL contains ``CRASH`` (state in @@ -37,12 +41,14 @@ _RANK = {"BLOCKED": 0, "FLAGGED": 1, "PARSE_ERROR": 2, "MONITORED": 3, "ALLOWED": 4} -def _verdict(statement: str, profile: str) -> Tuple[str, Optional[str], Optional[str], Optional[str]]: +def _verdict(statement: str, capped: bool) -> Tuple[str, Optional[str], Optional[str], Optional[str]]: upper = statement.upper() if "GARBAGE" in upper: return "PARSE_ERROR", None, None, None + if "TRUNCATE" in upper: + return "BLOCKED", "block", "VERICTO-011", "critical" if "DROP" in upper: - status, action = ("BLOCKED", "block") if profile == "runtime" else ("FLAGGED", "flag") + status, action = ("FLAGGED", "flag") if capped else ("BLOCKED", "block") return status, action, "VERICTO-010", "critical" if upper.lstrip().startswith("DELETE") and "WHERE" not in upper: return "BLOCKED", "block", "VERICTO-001", "critical" @@ -51,8 +57,9 @@ def _verdict(statement: str, profile: str) -> Tuple[str, Optional[str], Optional return "ALLOWED", None, None, None -def result_for(sql: str, profile: str) -> Dict[str, Any]: - found = [_verdict(s, profile) for s in sql.split(";") if s.strip()] or [("ALLOWED", None, None, None)] +def result_for(sql: str, capped: bool) -> Dict[str, Any]: + """The result document; ``capped``: the schema DDL cap applies (a ``ci`` migration).""" + found = [_verdict(s, capped) for s in sql.split(";") if s.strip()] or [("ALLOWED", None, None, None)] status, action, rule, severity = min(found, key=lambda v: _RANK[v[0]]) counts = {k: 0 for k in ("allowed", "blocked", "flagged", "monitored", "parse_errors")} counts["parse_errors" if status == "PARSE_ERROR" else status.lower()] += 1 @@ -105,7 +112,7 @@ def main() -> int: "profiles": ["runtime", "ci"], "type": "hello", "protocol": 2 if "protocol2" in modes else 1, - "cli_version": "2.0.0" if "old" in modes else "2.1.0", + "cli_version": "2.0.0" if "old" in modes else ("2.2.0" if "cap_all" in modes else "2.3.0"), "engine_version": ENGINE, "future_field": {"ignored": True}, } @@ -136,6 +143,12 @@ def main() -> int: message = f"unsupported dialect '{request.get('dialect')}'" _emit({"id": rid, "ok": False, "error": {"code": "UNSUPPORTED_DIALECT", "message": message}}) continue + migration = request.get("migration") + if migration is not None and not isinstance(migration, bool): + _emit( + {"id": rid, "ok": False, "error": {"code": "INVALID_REQUEST", "message": "migration must be a boolean"}} + ) + continue if "CRASH" in sql and ("crash" in modes or "crash_once" in modes): state = Path(os.environ.get("FAKE_SERVE_STATE", "/nonexistent")) if "crash" in modes or not state.exists(): @@ -156,7 +169,9 @@ def main() -> int: if "bad_result" in modes: _emit({"id": rid, "ok": True, "result": {"queries": []}}) continue - answer = {"id": rid, "ok": True, "result": result_for(sql, request.get("profile") or "runtime")} + ci = (request.get("profile") or "runtime") == "ci" + capped = ci and (migration is True or "cap_all" in modes) + answer = {"id": rid, "ok": True, "result": result_for(sql, capped)} if "reverse" in modes and "PAIR" in sql.upper(): held.append(answer) if len(held) == 2: diff --git a/tests/test_dbt.py b/tests/test_dbt.py index e14d653..8e4c207 100644 --- a/tests/test_dbt.py +++ b/tests/test_dbt.py @@ -651,3 +651,25 @@ def change(m: Dict[str, Any]) -> None: assert report["ruleset_version"] == "standard@engine-5.0.0" [model] = [r for r in report["results"] if r["unique_id"] == target and r["kind"] == "model"] assert (model["status"], model["rule_code"]) == ("BLOCKED", "VERICTO-001") + + +def test_offline_hook_ddl_blocks_because_dbt_sql_is_not_a_migration( + project: Path, fake_cli: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # vericto CLI 2.3.0: the ci profile caps schema DDL only for migrations, and + # vericto-dbt never marks its SQL as one, so a DROP TABLE post-hook blocks. + monkeypatch.setenv("VERICTO_CLI", fake_cli) + target = "model.jaffle.customers" + + def change(m: Dict[str, Any]) -> None: + m["nodes"][target]["config"]["post-hook"] = [{"sql": "DROP TABLE IF EXISTS analytics.customers_old"}] + + edit_manifest(project, change) + out = tmp_path / "offline.json" + assert dbt.main(["check", "--project-dir", str(project), "--offline", "--format", "json", "-o", str(out)]) == 1 + report = json.loads(out.read_text()) + [hook] = [r for r in report["results"] if r["unique_id"] == target and r["kind"] == "post-hook"] + assert (hook["status"], hook["rule_code"]) == ("BLOCKED", "VERICTO-010") + assert any("not a migration" in n for n in report["notes"]) + # The same project passes with --no-hooks (the documented remedy). + assert dbt.main(["check", "--project-dir", str(project), "--offline", "--no-hooks", "-q"]) == 0 diff --git a/tests/test_offline.py b/tests/test_offline.py index e52eadc..a2791e4 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -101,11 +101,65 @@ def test_offline_verdicts_carry_mode_engine_and_ruleset(offline_guard: Vericto) assert offline_guard.prepare("SELECT 1", "sqlite") == "SELECT 1" -def test_ci_profile_caps_ddl(fake_cli: str) -> None: - with Vericto(offline=True, offline_profile="ci", offline_cli_path=fake_cli) as guard: +def test_ci_profile_caps_ddl_only_in_a_migration(fake_cli: str) -> None: + # CLI 2.3.0: a ci check is a `--sql` check unless it is marked as a migration. + with ( + Vericto(offline=True, offline_profile="ci", offline_cli_path=fake_cli) as guard, + pytest.raises(VerictoBlocked) as info, + ): + guard.enforce("DROP TABLE users", "postgres") + assert info.value.verdict.rule_code == "VERICTO-010" + assert (info.value.verdict.raw["profile"], info.value.verdict.raw["migration"]) == ("ci", False) + + with Vericto(offline=True, offline_profile="ci", offline_migration=True, offline_cli_path=fake_cli) as guard: v = guard.enforce("DROP TABLE users", "postgres") - assert v.decision is Decision.FLAGGED - assert v.raw["profile"] == "ci" + assert v.decision is Decision.FLAGGED + assert (v.raw["profile"], v.raw["migration"]) == ("ci", True) + # Never capped (engine 5.1.0): TRUNCATE blocks in a migration too. + with pytest.raises(VerictoBlocked) as info: + guard.enforce("TRUNCATE users", "postgres") + assert info.value.verdict.rule_code == "VERICTO-011" + + +def test_migration_is_sent_only_when_set(fake_cli: str) -> None: + evaluator = OfflineEvaluator(profile="ci", cli_path=fake_cli) + sent: List[Dict[str, Any]] = [] + try: + proc = evaluator.engine._process(restart=False) + original = proc.request + + def spy(payload: Dict[str, Any], timeout: float) -> Dict[str, Any]: + sent.append(dict(payload)) + return original(payload, timeout) + + proc.request = spy # type: ignore[method-assign] + evaluator.evaluate("SELECT 1", "postgres") + evaluator.engine.check("SELECT 1", "postgres", migration=True) + finally: + evaluator.close() + assert "migration" not in sent[0] + assert sent[1]["migration"] is True + + +def test_cli_before_2_3_caps_every_ci_check(fake_cli: str, mode: Any) -> None: + mode("cap_all") + with Vericto(offline=True, offline_profile="ci", offline_cli_path=fake_cli) as guard: + assert guard.check("DROP TABLE users", "postgres").decision is Decision.FLAGGED + + +def test_migration_needs_the_ci_profile_and_a_bool(fake_cli: str, pids: Path) -> None: + with pytest.raises(VerictoConfigError, match="needs offline_profile='ci'"): + Vericto(offline=True, offline_migration=True, offline_cli_path=fake_cli) + with pytest.raises(VerictoConfigError, match="needs offline_profile='ci'"): + OfflineEvaluator(profile="runtime", migration=True, cli_path=fake_cli) + with pytest.raises(VerictoConfigError, match="True or False"): + OfflineEvaluator(profile="ci", migration="yes", cli_path=fake_cli) # type: ignore[arg-type] + with pytest.raises(VerictoConfigError, match="needs offline_profile='ci'"): + OfflineEvaluator(cli_path=fake_cli).engine.check("DROP TABLE t", "postgres", migration=True) + # offline_migration=False is the default and allowed with either profile. + with Vericto(offline=True, offline_migration=False, offline_cli_path=fake_cli) as guard: + assert guard.check("DROP TABLE users", "postgres").blocked + assert "migration=True" in repr(OfflineEvaluator(profile="ci", migration=True, cli_path=fake_cli)) def test_online_verdicts_say_online(fake: Any, guard: Vericto) -> None: @@ -152,7 +206,7 @@ def test_offline_with_an_evaluator_or_a_key_variable_is_choose_one( def test_an_offline_evaluator_can_be_passed_directly(fake_cli: str) -> None: - with Vericto(OfflineEvaluator(profile="ci", cli_path=fake_cli)) as guard: + with Vericto(OfflineEvaluator(profile="ci", migration=True, cli_path=fake_cli)) as guard: assert guard.offline assert guard.check("DROP TABLE t", "postgres").decision is Decision.FLAGGED @@ -160,6 +214,8 @@ def test_an_offline_evaluator_can_be_passed_directly(fake_cli: str) -> None: def test_offline_options_need_offline(fake_cli: str) -> None: with pytest.raises(VerictoConfigError, match="offline=True"): Vericto(api_key=API_KEY, database_id=DATABASE_ID, offline_profile="ci") + with pytest.raises(VerictoConfigError, match="offline_migration apply to offline mode only"): + Vericto(api_key=API_KEY, database_id=DATABASE_ID, offline_migration=True) with pytest.raises(VerictoConfigError, match="offline_profile must be"): Vericto(offline=True, offline_cli_path=fake_cli, offline_profile="strict") with pytest.raises(VerictoConfigError, match="timeout"): @@ -535,7 +591,8 @@ def test_offline_ci_client_shapes(fake_cli: str, mode: Any) -> None: assert not client.config().check_files body = client.check_queries(["SELECT 1", "DROP TABLE t"], "postgres") assert [q["line"] for q in body["queries"]] == [1, 2] - assert body["queries"][1]["status"] == "FLAGGED" # ci profile + # ci profile, not a migration (CLI 2.3.0): checked like `check --offline --sql`. + assert body["queries"][1]["status"] == "BLOCKED" files = client.check_files([CiFile("a.sql", "SELECT 1")], "postgres") assert (files["queries"][0]["file_index"], files["queries"][0]["line"]) == (1, 1) with pytest.raises(CiRequestError): diff --git a/tests/test_offline_cli.py b/tests/test_offline_cli.py index b10d863..f356fe0 100644 --- a/tests/test_offline_cli.py +++ b/tests/test_offline_cli.py @@ -8,9 +8,11 @@ CI job that installs the CLI), which turns the skip into a failure. Parity: for each case, the verdict of ``Vericto(offline=True, offline_profile="ci")`` -and its whole result document equal ``vericto check --offline --format json`` from -the same binary. Under the ``runtime`` profile, the schema DDL rules (VERICTO-010 to -019) block instead of flagging, and nothing else changes. +and its whole result document equal ``vericto check --offline --format json --sql`` +from the same binary, and with ``offline_migration=True`` they equal the same check of +a file under ``migrations/``. The schema DDL rules (VERICTO-010, 013, 015 to 019) are +capped at FLAGGED in a migration only (CLI 2.3.0; 2.1.0 and 2.2.0 cap every ``ci`` +check); under the ``runtime`` profile they always block, and nothing else changes. """ from __future__ import annotations @@ -90,6 +92,32 @@ def cli_check(sql: str, dialect: str, cwd: Path) -> Dict[str, Any]: return doc +def cli_check_migration(sql: str, dialect: str, cwd: Path) -> Dict[str, Any]: + """``vericto check --offline --format json`` for the SQL as a file under ``migrations/``.""" + (cwd / "migrations").mkdir(exist_ok=True) + (cwd / "migrations" / "0001_case.sql").write_text(sql, encoding="utf-8") + env = {k: v for k, v in os.environ.items() if not k.startswith("VERICTO_")} + proc = subprocess.run( # noqa: S603 - the CLI under test + [str(CLI), "check", "--offline", "--format", "json", "--dialect", dialect, "migrations/0001_case.sql"], + capture_output=True, + text=True, + timeout=60, + check=False, + cwd=cwd, + env=env, + ) + assert proc.returncode in (0, 1), proc.stderr + doc = json.loads(proc.stdout[proc.stdout.index("{") :]) + assert isinstance(doc, dict) + return doc + + +@pytest.fixture(scope="module") +def migration_guard() -> Iterator[Vericto]: + with Vericto(offline=True, offline_profile="ci", offline_migration=True, offline_cli_path=CLI) as guard: + yield guard + + @pytest.fixture(scope="module") def ci_guard() -> Iterator[Vericto]: with Vericto(offline=True, offline_profile="ci", offline_cli_path=CLI) as guard: @@ -127,6 +155,23 @@ def test_parity_with_check_offline( assert (runtime.decision, runtime.rule_code) == (ci.decision, ci.rule_code) +@pytest.mark.parametrize(("case", "dialect", "sql", "capped"), CASES, ids=[c[0] for c in CASES]) +def test_migration_parity_with_a_migration_file( + migration_guard: Vericto, tmp_path: Path, case: str, dialect: str, sql: str, capped: bool +) -> None: + expected = cli_check_migration(sql, dialect, tmp_path) + (query,) = expected["queries"] + v = migration_guard.check(sql, dialect) + # The file's document names the file; everything the verdict is built from is equal. + got = v.raw["offline_result"] + assert got["summary"] == expected["summary"] + assert got["exit_code"] == expected["exit_code"] + for key in ("status", "action", "rule_code", "severity", "ast_node_path", "suggested_fix"): + assert got["queries"][0][key] == query[key], key + assert v.decision.value == query["status"] + assert v.raw["migration"] is True + + def test_runtime_profile_blocks_drop_table(runtime_guard: Vericto) -> None: with pytest.raises(VerictoBlocked) as info: runtime_guard.enforce("DROP TABLE users", "postgres")