From 8e599ed5331a989af175ddb71c5bab31773ee20a Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Fri, 2 Oct 2026 17:32:14 -0400 Subject: [PATCH 01/33] [design] Prove waiting recovery boundaries Preserve accepted-but-unacknowledged mutation evidence and the retention counterexample. Keep the unconditional recovery contract blocked for a decision rather than implementing silent replay. Solves: DRY-1398 evidence and recovery decision record Tests: 16 local probes; 3 crash repeats; 565 tests on Python 3.10-3.14 (cherry picked from commit 0ab51016552e68c2e6e2ee90a3eb2983433a27d0) (cherry picked from commit 924ffd5279e86d6611477d4e873111334932810b) --- docs/design/2026-10-02-ch-migrate-1.0.md | 5 +- docs/design/2026-10-02-waiting.md | 452 ++++++++++++++ .../spikes/2026-10-02-waiting/resume_fence.py | 49 ++ .../2026-10-02-waiting/submit_without_ack.py | 30 + tests/integration/waiting_spike_probe.py | 558 ++++++++++++++++++ 5 files changed, 1093 insertions(+), 1 deletion(-) create mode 100644 docs/design/2026-10-02-waiting.md create mode 100644 docs/design/spikes/2026-10-02-waiting/resume_fence.py create mode 100644 docs/design/spikes/2026-10-02-waiting/submit_without_ack.py create mode 100644 tests/integration/waiting_spike_probe.py diff --git a/docs/design/2026-10-02-ch-migrate-1.0.md b/docs/design/2026-10-02-ch-migrate-1.0.md index 614aba1..7a4953d 100644 --- a/docs/design/2026-10-02-ch-migrate-1.0.md +++ b/docs/design/2026-10-02-ch-migrate-1.0.md @@ -308,7 +308,10 @@ is unaffected; Python 3.9 is not supported, so 0.6 requires 3.10. 3. Does `bootstrap` (roles, users, grants) sit cleanly on the new connection path? 4. Does the dialect support Python 3.9? If not, it is ruled out for 1.0 (see Compatibility). -**A design note on waiting, before the 0.7 plan:** +**A design note on waiting, before the 0.7 plan.** Experiments and proposed rules: +[waiting design](2026-10-02-waiting.md). Accepted-but-unacknowledged mutation recovery +is demonstrated; the missing/expired-evidence case needs a decision before implementation. +The note does not weaken capabilities 4 or 5 without that decision. 5. On a re-run, how does `up` recognise a mutation it started earlier, without mistaking someone else's mutation for its own? diff --git a/docs/design/2026-10-02-waiting.md b/docs/design/2026-10-02-waiting.md new file mode 100644 index 0000000..17edee5 --- /dev/null +++ b/docs/design/2026-10-02-waiting.md @@ -0,0 +1,452 @@ +# Waiting for mutations and distributed DDL + +**DRY-1398 · 2026-10-02 · Design blocked on the recovery contract.** + +A write-ahead intent plus a marker carried into ClickHouse's mutation record can +recover an accepted `UPDATE` after the client dies without reading its response. +The experiment passes three times: the resumed process observes the same mutation +ID, waits, and the counter ends at 1, not 2. + +That is **not an unconditional replay guarantee**. Completed mutation records can +expire while the client is dead. An intent without a surviving marker cannot tell +us whether the request never arrived, is still being submitted, or completed and +was removed from the system table. Reissuing can apply `x = x + 1` twice; skipping +can omit it. No command-normalization or time-window heuristic fixes that. + +**Decision needed:** permit an explicit, nonzero, fail-closed result when durable +completion evidence is unavailable, with operator reconciliation rather than +automatic replay. That is the recommendation, not an accepted change to +[capabilities 4 and 5](2026-10-02-ch-migrate-1.0.md). Until that exception is +accepted, DRY-1399 and DRY-1400 must not implement a narrower guarantee and call it +complete. The current product definition remains authoritative. + +## Reproduce the evidence + +The explicit probe uses the existing owned-container fixtures. It creates its own +databases, containers, network, passwords, and subprocesses; it does not use a +shared service. Its filename keeps it out of normal pytest collection. + +Run from the repository root: + +```bash +env -u CH_MIGRATE_IT_URL uv run --locked --python 3.12 pytest -q -s \ + -m integration tests/integration/waiting_spike_probe.py + +# Independent second pass: three fresh projects and three killed processes. +env -u CH_MIGRATE_IT_URL uv run --locked --python 3.12 pytest -q -s \ + -m integration tests/integration/waiting_spike_probe.py \ + -k kill_before_receipt +``` + +The checked-in [submitter](spikes/2026-10-02-waiting/submit_without_ack.py) sends the +HTTP body but deliberately never reads the response. The +[resumer](spikes/2026-10-02-waiting/resume_fence.py) reads only the pre-send intent +and server records. It never reissues the original mutation. The probe kills the +submitter only after an independent connection observes the pending mutation, so +the test proves the difficult **accepted but not acknowledged** case. It does not +pretend that it proves every possible interruption point. + +Observed server: ClickHouse 26.3.39.7. The single-node and one-shard/two-replica +fixtures use the `26.3` image. Final reproduction: **16 passed**; independent +kill-and-re-run second pass: **3 passed**. The local CI command +`uv run --locked --python pytest -q` passed with **565 tests** on each +of Python 3.10, 3.11, 3.12, 3.13 and 3.14. These are local results, not remote CI. +The SQL below uses `db` and short marker names instead of random identifiers. + +## Q5. Recognize our work without claiming another writer's work + +### What the server preserves + +The metadata probe queries `system.columns` and submits these two requests with +distinct `query_id` and `log_comment` settings: + +```sql +ALTER TABLE db.ownership UPDATE x = x + 1 +WHERE id = 1 /* chm_marker_comment */; +ALTER TABLE db.ownership UPDATE x = x + 1 +WHERE id = 2 AND 'chm_marker_expression' = 'chm_marker_expression'; +SELECT mutation_id, command, is_done +FROM system.mutations WHERE database = 'db' AND table = 'ownership'; +``` + +Observed with merges stopped: + +```text +mutation_2.txt | (UPDATE x = x + 1 WHERE id = 1) | 0 +mutation_3.txt | (UPDATE x = x + 1 WHERE (id = 2) AND + ('chm_marker_expression' = 'chm_marker_expression')) | 0 +``` + +`system.mutations` has `mutation_id` and `command`, but no query ID or log comment. +The comment disappears. The literal expression survives. `system.query_log` +preserves query IDs and log comments, but the experiment's mutation-related +`ProfileEvents` map is empty: it supplies no query-to-mutation receipt. Query logs +are also asynchronously flushed and retained for a bounded period. + +**Rejected identities:** normalized SQL, table plus timestamp, and before/after ID +set difference. Another writer can issue the same statement in the same window. +Those techniques can identify candidates or preceding blockers, not ownership. +`formatQuery()` would normalize spelling, not make a command unique. + +### A durable marker within the submitted operation + +For an `ALTER UPDATE`, a combined empty delete preserves the original predicate +and adds a unique marker: + +```sql +ALTER TABLE db.counter + UPDATE x = x + 1 WHERE 1, + DELETE WHERE 0 AND 'chm_step_TOKEN' = 'chm_step_TOKEN' +SETTINGS mutations_sync = 0, alter_sync = 0; +``` + +Observed: two command rows, **one mutation ID**. Counting rows in +`system.mutations` is not counting distinct mutations. + +```text +mutation_2.txt | (UPDATE x = x + 1 WHERE 1) | 0 +mutation_2.txt | (DELETE WHERE 0 AND ('chm_step_TOKEN' = 'chm_step_TOKEN')) | 0 +``` + +The three killed-client trials each printed: + +```text +resumed mutation_2.txt; mutation_ids=['mutation_2.txt']; command_rows=2; x=1 +``` + +The original and final sets of mutation IDs were equal. The resumer remained +running while merges were stopped and completed after `SYSTEM START MERGES`. + +For schema changes, **do not assume the marker shares the mutation ID**: + +| Combined `ALTER` action | Observed command IDs | +|---|---| +| `MODIFY COLUMN x UInt32, DELETE WHERE ` | schema `mutation_2.txt`, marker `mutation_3.txt` | +| `MATERIALIZE COLUMN y, DELETE WHERE ` | both `mutation_2.txt` | +| `CLEAR COLUMN y, DELETE WHERE ` | schema `mutation_2.txt`, marker `mutation_3.txt` | +| `RENAME COLUMN y TO renamed, DELETE WHERE ` | schema `mutation_2.txt`, marker `mutation_3.txt` | + +Thus a marker is a **completion barrier**, not necessarily the original mutation. +Wait for its command rows and all preceding outstanding work on that table; never +infer the original command's completion from a different table's marker. Inspect +failures in preceding mutations rather than reporting only the empty delete. +Persist the table UUID so that dropping and recreating a name cannot make an old +intent apply to a different table. + +The empty-marker cost probe used 10,000 rows with a payload column. It retained all +10,000 rows. `system.part_log` reported a `MutatePart` event with `read_rows = 0`, +`read_bytes = 0`, and `MutationUntouchedParts = 1`, but also file reads and buffer +allocations in `ProfileEvents`. The marker is not free. It also requires DELETE +privilege. Predicate markers avoid that extra command for suitable UPDATE/DELETE +statements, but require syntax-aware insertion and their own corpus proof. + +### Smallest proposed state + +One append-only server-side journal, deployed with the same replication policy as +the version table, stores: + +- revision and statement position; +- exact statement/parameter digest, target database/table and UUID; +- unique attempt token and operation kind; +- phase: intent, accepted receipt, completed; +- receipt: mutation marker/IDs or DDL queue entry, plus required hosts. + +Write and acknowledge intent **before** sending SQL. Never derive the intent only +after submission. Record completion before moving to the next statement. An +unfinished revision reuses its attempt, validates its digest and target UUID, +skips completed work, and attaches to known unfinished work. A changed migration +or replaced target fails before submission. Successful downgrade must invalidate +that revision's old completed journal, so a subsequent upgrade actually runs. + +The insertion-deduplication probes establish only a useful primitive, not a lock: +with a stable `insert_deduplication_token`, two single-node inserts retained the +first owner; concurrent replicated inserts through both nodes retained the same +one owner on both. Deduplication has a retention window. Do not turn this result +into an unbounded exactly-once or concurrent-run claim. Concurrent migrators need +serialization; this spike has not proved a portable fencing lock. + +### The unresolved boundary + +If the process dies after preparing intent but before receiving a durable receipt, +look up the marker. If it is present, attach without submitting again. If it is +absent, **absence is not completion and is not permission to retry**. + +There are at least three indistinguishable histories at this boundary: + +1. Intent was committed; SQL never reached the server. +2. SQL reached the server but has not registered its marker yet. +3. SQL completed; the process died before recording completion; the server later + removed the completed marker under its retention policy. + +The retention probe sets `finished_mutations_to_keep = 1`, `old_parts_lifetime = 0`, +`cleanup_delay_period = 1` and `cleanup_delay_period_random_add = 0` on its disposable +table, completes a marked increment and then another mutation. Observed: +only `mutation_3.txt (DELETE WHERE 0)` remains, while the counter is still 1. +The first trial without accelerated cleanup retained both records for 30 seconds; +retention is asynchronous, not immediate after the next mutation. +A resumed journal reader has no receipt to distinguish +that history from an unsubmitted operation. Unfinished records surviving a +normal mid-wait SIGKILL do not establish that completed records survive forever. + +**Recommended exception:** fail with `outcome unknown; statement not reissued; +revision unchanged`, include the intent token and target, and require the operator +to reconcile the server outcome. No automatic stamp, retry flag, blind journal +deletion, or background KILL. Choosing the supported reconciliation interface is +part of accepting this exception. Until then, this is a blocker, not a fallback. + +## Q6. Foreign mutations ahead of ours + +The probe submits a failing foreign mutation, then a second update: + +```sql +ALTER TABLE db.foreign_probe UPDATE x = throwIf(x = 5) WHERE 1; +ALTER TABLE db.foreign_probe UPDATE x = x + 1 WHERE id = 1; +``` + +Observed: + +```text +mutation_2.txt | is_done=0 | parts_to_do=1 | Code: 395 ... throwIf ... +mutation_3.txt | is_done=0 | parts_to_do=1 | latest_fail_reason='' +rows: [(1, 1), (5, 5)] +``` + +Recommendation: before creating more work on that table, report and wait for +healthy preceding foreign mutations. A preceding failure exits nonzero with its +host, reason and exact operator-only `KILL MUTATION` command. Do not label it as +our mutation or insert another mutation behind a known permanent failure. A +foreign mutation arriving after our completion barrier is not a reason to hold +our revision open. An operator's KILL is not proof our work succeeded. + +## Q7. Every replica, not merely the connected replica + +The cluster probe creates a ReplicatedMergeTree, inserts a row, and synchronizes +node 2. It stops merges, fetches and replication queues **on node 2 only**, submits +an increment through node 1, then pulls the mutation metadata on node 2. + +```text +node 1: mutation_id=0000000000, is_done=1, parts_to_do=0 +node 2: mutation_id=0000000000, is_done=0, parts_to_do=1 +``` + +After restarting those operations and synchronizing node 2, its counter is 1. +A local-only query would have reported success too early. + +Rules for self-hosted replicas: + +- Snapshot the required topology before submission. Use the configured cluster or + a Replicated database's generated cluster, and verify coverage against the + table's replica metadata. Missing topology is an error, not permission to wait + only locally. +- Query every required host's `system.mutations`, for example with + `clusterAllReplicas`. Identify hosts as well as mutation IDs; IDs are not global + across unrelated tables/shards. +- Require `is_done = 1` for every command row of every required mutation on every + required replica. `parts_to_do = 0` is progress, not completion: a long insert + may still add a part that must be mutated. +- A host omitted by `skip_unavailable_shards` is missing evidence, not success. + Track required hosts separately. A missing record is likewise not a completed + record unless the journal already stores that host's completion. +- Persist observed completion so later retention cannot erase an already-proved + result. The crash before persisting that result remains Q5's blocker. + +## Q8. Distributed DDL and a down host + +The probe stops node 2 and submits: + +```sql +CREATE TABLE db.ddl_owned ON CLUSTER it_cluster (id UInt64) ENGINE = Memory +SETTINGS distributed_ddl_task_timeout = 0, + distributed_ddl_output_mode = 'none', log_comment = 'chm_ddl_TOKEN'; +SELECT entry, host, settings['log_comment'], status +FROM system.distributed_ddl_queue +WHERE settings['log_comment'] = 'chm_ddl_TOKEN'; +``` + +Observed one entry, `query-0000000001`, with both target hosts and the same marker: +node 1 `Active`, node 2 `Inactive`. ClickHouse inserted a UUID into the stored CREATE +query. Exact comparison with the original SQL would not identify it reliably. +The `settings['log_comment']` marker survived and is a better identity. + +Rules: + +- Write intent first. Set a unique, journaled `log_comment` for the request. Preserve + any caller comment in the journal rather than treating it as the identity. +- Submit with `distributed_ddl_task_timeout = 0` and output mode `none`, then poll + the queue by marker/entry. The server's synchronous client timeout is not the + migration's completion test. +- Success requires every recorded target host to be `Finished` with + `exception_code = 0`. `Finished` alone can mean failure. Missing rows, + `Inactive`, `Active`, `Removing` and `Unknown` are not success. +- A down host stays pending. Default wait has no deadline; `--timeout` exits + nonzero naming the host, queue entry and status. It does not delete queue work. + Once the host returns, rerun attaches to the same entry. +- Any nonzero exception exits nonzero with host, code and text. The failure probe + pre-adds `conflict String` only on node 2, then enqueues `ADD COLUMN conflict + UInt64` on both. This produces a host-specific schema error without a network + timeout or an intentionally stuck mutation. + Observed queue results: node 1 `Finished / 0`, node 2 `Finished / 15` + with `Cannot add column conflict: column with this name already exists`. +- For mutation-producing `ALTER ... ON CLUSTER`, finish both barriers: all hosts + executed DDL **and** all required mutations finished. Neither subsumes the other. +- Queue retention creates the same unknown-outcome boundary as Q5. A missing + entry must not trigger blind DDL resubmission. + +## Q9. Cloud: reasoned, not tested + +DRY-1413 has no `service ready` comment. No Cloud connection or credential lookup +was made for this note. These conclusions come from official documentation, not +the local replicated harness: + +- The [system.mutations documentation](https://clickhouse.com/docs/reference/system-tables/mutations) + says each Cloud node exposes all cluster mutations, so `clusterAllReplicas` is + unnecessary for this table. +- The [SharedMergeTree reference](https://clickhouse.com/docs/products/cloud/features/infrastructure/shared-merge-tree) + describes shared mutation work, not a per-replica rewrite. It also states that + `system.replication_queue` is not the SharedMergeTree introspection API. +- Cloud converts MergeTree-family tables to SharedMergeTree-family tables. The + journal should follow that policy, not introduce a self-hosted Keeper path. +- `ON CLUSTER` is unnecessary on Cloud. Skip the distributed-DDL mechanism for + ordinary Cloud DDL; do not invent a cluster or infer completion from a + self-hosted replica queue. + +The 0.7 acceptance ticket must prove mutation visibility, marker preservation, +retention behavior, session continuity and journal persistence on the authorized +Cloud service. Local success cannot close these checks. + +## Which statements create mutation records + +The corpus experiment runs each statement on a fresh seeded MergeTree table and +queries `system.mutations` afterward. It leaves completed records in place for +inspection. + +| Statement | Observed record on 26.3.39.7 | +|---|---| +| `ALTER ... UPDATE` | `UPDATE ...` | +| `ALTER ... DELETE` | `DELETE ...` | +| lightweight `DELETE FROM` | `UPDATE _row_exists = 0 ...` | +| lightweight `UPDATE ... SET` | no mutation record; patch data already visible | +| `MODIFY COLUMN x UInt32` | `MODIFY COLUMN x UInt32` | +| `DROP COLUMN y` | `DROP COLUMN y` | +| `CLEAR COLUMN y` | `CLEAR COLUMN y` | +| `MATERIALIZE COLUMN y` | `MATERIALIZE COLUMN y` | +| `RENAME COLUMN y TO z` | `RENAME COLUMN y TO z` | +| `MODIFY TTL ts + INTERVAL 1 DAY` | `MATERIALIZE TTL` | +| `UPDATE x = throwIf(x = 5)` | pending record with failure code 395 | + +The first lightweight UPDATE attempt failed with code 48 because the required +storage columns were disabled. With `enable_block_number_column = 1`, +`enable_block_offset_column = 1`, and `allow_experimental_lightweight_update = 1`, +it returned rows `[(1, 11), (2, 20)]` and **no** `system.mutations` rows. It must not +be modeled as a background mutation that is complete merely because a lookup is +empty. Wait for its synchronous request; patch-merge completion is not a mutation +completion signal. Do not silently rewrite it as a heavyweight UPDATE. + +`MODIFY TTL` can produce work under a different command string. Generic command +matching would miss it. The implementation corpus must cover query/session +settings that change materialization, including `materialize_ttl_after_modify`. + +## Execution, sessions, version state and output + +### Hook at the actual cursor boundary + +Use SQLAlchemy `before_cursor_execute` with `retval=True` and +`after_cursor_execute` on the migration connection, as established by DRY-1391. +This covers `op.execute`, `op.get_bind()` and `run_sql`'s `exec_driver_sql` path. +Do not use only `ChMigrateImpl._exec`. Guard journal, polling and progress queries +against recursive instrumentation. Offline SQL generation does not connect or +invent journal receipts. + +Poll on the same migration session, using bounded requests and an interval less +than its idle timeout. This both preserves session `SET`s and keeps the session +alive. Propagate DRY-1392's expired-session failure; never silently recreate it. +A separate monitoring client alone does not keep the migration session alive. + +### Do not hold a request open to wait + +The managed request uses asynchronous mutation settings and polls afterward. +The settings probe submits a stopped mutation with both `mutations_sync = 2` and +then `mutations_sync = 0`; it returns while both command rows remain unfinished. +That proves the last duplicate value wins on this server, not that text appending +is a general SQL-rewriting strategy. Parse and replace managed settings instead +of relying on duplicate-setting precedence. Do not permanently reset unrelated +session settings. + +### The version table's delete + +DRY-1412 inserts the new head before deleting the previous one. Its delete is a +real mutation and currently uses the official implementation's synchronous wait. +The waiting implementation must put that delete through the same asynchronous +polling/error surface, without weakening the insert-first safety invariant. +After interruption, both heads may be present; that is a fail-closed overlap, not +permission to replay the revision. Attach to the known version-delete receipt +before evaluating the next revision. Never emit “applied” until the delete barrier +finishes. This is not a promise of transactional rollback of the inserted head. + +### Progress and exits + +Proposed stable messages (data substituted, not a literal output fixture): + +- Progress: `Waiting: db.table mutation ID host HOST; N parts; Ns elapsed`. +- Failure, exit 1: `Mutation failed: ... REASON`, followed by an operator-only + `KILL MUTATION WHERE database = 'db' AND table = 'table' AND mutation_id = 'ID'`. + On clusters, name the affected host; do not suggest an unscoped kill. +- Timeout, exit 1: `Timed out; server work continues; revision not completed`, + followed by every pending mutation/host/DDL entry and its last known state. +- Foreign failure, exit 1: `Blocked by foreign mutation ...`, reason and command. +- Unknown outcome, proposed exit 1: `Outcome unknown; statement not reissued; + revision unchanged`, intent token, target UUID and reconciliation requirement. +- Configuration/permission/topology error: exit 1 with the actual error; never a + zero-work success. Existing CLI usage errors remain Click's exit 2. + +Use a live line only on a TTY. In CI, emit an initial plain line, state changes, +periodic summaries, and a final result to stderr. Use monotonic elapsed time and +one optional `--timeout SECONDS` budget for the invocation's waits, not a fresh +budget for each host. No default timeout; no automatic mutation/DDL cancellation. + +## Numbered implementation test matrix + +These are required future checks, **not claimed implementation results**. Rows +involving unknown outcomes depend on the decision above. The experiment names +make the subset already demonstrated reproducible. + +| Row | Owner | Required observable result | +|---|---|---| +| W01 | DRY-1399 | Every mutation-producing corpus statement waits; schema mutations and TTL are not missed; lightweight UPDATE is not falsely identified as a background mutation. | +| W02 | DRY-1399 | `throwIf` failure exits nonzero with reason, table, ID and operator KILL; no automatic KILL; pre-revision head unchanged. | +| W03 | DRY-1399 | SIGKILL while waiting; rerun attaches to the same ID and increments once. Run three independent trials. | +| W04 | DRY-1399 | Kill after accepted send but before reading receipt; recovery uses pre-send intent and marker; no second command. | +| W05 | DRY-1399 | Timeout reports outstanding IDs, parts and hosts; work remains on the server; rerun finishes without replay. | +| W06 | DRY-1399 | A paused second replica prevents success even after the first finishes; returning replica permits completion. | +| W07 | DRY-1399 | Missing required host/record is not success, including a query using skip-unavailable behavior. | +| W08 | DRY-1399 | Healthy foreign predecessor is reported and waited; failed predecessor is named; trailing foreign work is not adopted as ours. | +| W09 | DRY-1399 | Same SQL from another writer is not owned; table recreation and changed migration digest refuse before replay. | +| W10 | DRY-1399 | Expired/missing ownership evidence never causes silent success or duplicate execution; recovery behavior requires the explicit decision. | +| W11 | DRY-1399 | Polls longer than a short session timeout preserve a prior SET; an actually expired session fails loudly. | +| W12 | DRY-1399 | Python op.execute, op.get_bind().execute/exec_driver_sql and run_sql all use the hook; polling does not recurse. | +| W13 | DRY-1399 | Interrupted/failed version DELETE uses the same diagnostics, preserves insert-first safety, and never re-executes an applied revision. | +| W14 | DRY-1399 | TTY and CI progress identify real state; timeout budget is not reset per mutation; zero parts alone does not mean done. | +| W15 | DRY-1399 | Completed earlier statements are not repeated on resume; downgrade then upgrade does not reuse an old completed journal. | +| W16 | DRY-1400 | CREATE then ALTER ON CLUSTER finishes on all hosts before success and before version completion. | +| W17 | DRY-1400 | Stopped node remains pending; timeout names it; restart and rerun attach to the same queue entry. | +| W18 | DRY-1400 | One-host schema conflict exits nonzero with host/code/text even when queue status is Finished. | +| W19 | DRY-1400 | Accepted DDL with lost client receipt is found by journaled log_comment, not exact normalized query text. | +| W20 | DRY-1400 | Mutation-producing ON CLUSTER ALTER satisfies both the DDL and mutation barriers. | +| W21 | DRY-1400 | Missing/expired queue entry is not success or automatic resubmission; uses the agreed unknown-outcome rule. | +| W22 | DRY-1400 | Single-node/no-ON-CLUSTER statements do not query a fictitious distributed DDL queue. | +| W23 | DRY-1401 | Authorized Cloud acceptance proves SharedMergeTree mutation visibility, markers, journal persistence and same-session polling; no self-hosted replica assumptions. | + +## Sources and limits + +- [system.mutations](https://clickhouse.com/docs/reference/system-tables/mutations): + command rows, IDs, completion versus parts, Cloud visibility. +- [system.distributed_ddl_queue](https://clickhouse.com/docs/reference/system-tables/distributed_ddl_queue): + settings, per-host status and exception columns. +- [SharedMergeTree](https://clickhouse.com/docs/products/cloud/features/infrastructure/shared-merge-tree): + shared mutation work and metadata visibility. +- Local SQL/output: `tests/integration/waiting_spike_probe.py` and the two scripts + linked above. These probes are not the `up` implementation and do not imply that + its acceptance tests pass yet. + +This note does not authorize Cloud use, release a version, change accepted product +behavior, or claim an unbounded exactly-once protocol from bounded server records. diff --git a/docs/design/spikes/2026-10-02-waiting/resume_fence.py b/docs/design/spikes/2026-10-02-waiting/resume_fence.py new file mode 100644 index 0000000..668a9cd --- /dev/null +++ b/docs/design/spikes/2026-10-02-waiting/resume_fence.py @@ -0,0 +1,49 @@ +"""Recover only from the pre-send intent and server marker; never resubmit SQL.""" + +import time +from pathlib import Path + +from clickhouse_alembic.config import get_env_config +from clickhouse_alembic.connection import get_client + +config = get_env_config("it", Path("config.yaml")) +client = get_client(config) +database = config["database"] +try: + token, expected_uuid = client.query( + f"SELECT token, table_uuid FROM {database}.waiting_intent" + ).result_rows[0] + actual_uuid = client.query( + "SELECT uuid FROM system.tables WHERE database = {db:String} AND name = 'counter'", + parameters={"db": database}, + ).result_rows[0][0] + assert str(actual_uuid) == expected_uuid + markers = client.query( + "SELECT mutation_id FROM system.mutations WHERE database = {db:String} " + "AND table = 'counter' AND position(command, {token:String}) > 0", + parameters={"db": database, "token": token}, + ).result_rows + assert len(markers) == 1 + marker = markers[0][0] + Path("resumed_marker").write_text(marker) + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + rows = client.query( + "SELECT mutation_id, command, is_done FROM system.mutations " + "WHERE database = {db:String} AND table = 'counter' ORDER BY mutation_id", + parameters={"db": database}, + ).result_rows + if all(row[2] for row in rows): + value = client.command(f"SELECT x FROM {database}.counter") + ids = {row[0] for row in rows} + print( + f"resumed {marker}; mutation_ids={sorted(ids)}; command_rows={len(rows)}; x={value}", + flush=True, + ) + assert value == 1 and ids == {marker} and len(rows) == 2 + break + time.sleep(0.05) + else: + raise TimeoutError("Experiment did not finish") +finally: + client.close() diff --git a/docs/design/spikes/2026-10-02-waiting/submit_without_ack.py b/docs/design/spikes/2026-10-02-waiting/submit_without_ack.py new file mode 100644 index 0000000..c8d2742 --- /dev/null +++ b/docs/design/spikes/2026-10-02-waiting/submit_without_ack.py @@ -0,0 +1,30 @@ +"""Submit one owned ALTER over HTTP without ever reading its response.""" + +import http.client +import sys +import time +from pathlib import Path +from urllib.parse import urlencode + +from clickhouse_alembic.config import get_env_config + +config = get_env_config("it", Path("config.yaml")) +token = sys.argv[1] +query = ( + f"ALTER TABLE {config['database']}.counter UPDATE x = x + 1 WHERE 1, " + f"DELETE WHERE 0 AND '{token}' = '{token}'" +) +connection_type = ( + http.client.HTTPSConnection if config.get("secure") else http.client.HTTPConnection +) +connection = connection_type(config["host"], config["port"], timeout=30) +parameters = urlencode({"mutations_sync": 0, "alter_sync": 0, "query_id": token}) +connection.request( + "POST", + "/?" + parameters, + body=query.encode(), + headers={"X-ClickHouse-User": config["migration_user"], "X-ClickHouse-Key": config["password"]}, +) +# Deliberately never call getresponse() or write a mutation-id receipt. +Path("submitted_without_ack").write_text("sent") +time.sleep(60) diff --git a/tests/integration/waiting_spike_probe.py b/tests/integration/waiting_spike_probe.py new file mode 100644 index 0000000..7865f00 --- /dev/null +++ b/tests/integration/waiting_spike_probe.py @@ -0,0 +1,558 @@ +"""Explicit local experiments for the waiting design; not auto-collected.""" + +import secrets +import subprocess +import sys +import time +from pathlib import Path + +import clickhouse_connect +import pytest + +pytestmark = pytest.mark.integration + + +def test_waiting_metadata_and_mutation_markers(project): + for table in ("mutations", "distributed_ddl_queue", "query_log", "part_log"): + columns = project.client.query( + "SELECT name, type FROM system.columns WHERE database = 'system' AND table = {table:String} " + "AND (position(name, 'query') > 0 OR position(name, 'mutation') > 0 OR name IN ('command', 'entry', 'settings')) " + "ORDER BY position", + parameters={"table": table}, + ).result_rows + print(table, columns) + target = f"{project.database}.ownership" + project.client.command( + f"CREATE TABLE {target} (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id" + ) + project.client.command(f"INSERT INTO {target} VALUES (1, 0), (2, 0)") + project.client.command(f"SYSTEM STOP MERGES {target}") + try: + statements = [ + f"ALTER TABLE {target} UPDATE x = x + 1 WHERE id = 1 /* chm_marker_comment */", + f"ALTER TABLE {target} UPDATE x = x + 1 WHERE id = 2 AND 'chm_marker_expression' = 'chm_marker_expression'", + ] + for sql in statements: + query_id = "chm_probe_" + secrets.token_hex(8) + project.client.command( + sql, settings={"mutations_sync": 0, "query_id": query_id, "log_comment": query_id} + ) + print("submitted", query_id) + rows = project.client.query( + "SELECT mutation_id, command, is_done FROM system.mutations " + "WHERE database = {db:String} AND table = 'ownership' ORDER BY mutation_id", + parameters={"db": project.database}, + ).result_rows + print("mutation commands", rows) + project.client.command("SYSTEM FLUSH LOGS") + events = project.client.query( + "SELECT query_id, type, log_comment, query FROM system.query_log " + "WHERE startsWith(query_id, 'chm_probe_') AND position(query, {db:String}) > 0 " + "ORDER BY event_time_microseconds", + parameters={"db": project.database}, + ).result_rows + profiles = project.client.query( + "SELECT query_id, mapFilter((k, v) -> positionCaseInsensitive(k, 'mutation') > 0, ProfileEvents) " + "FROM system.query_log WHERE type = 'QueryFinish' AND startsWith(query_id, 'chm_probe_') " + "AND position(query, {db:String}) > 0", + parameters={"db": project.database}, + ).result_rows + print("mutation profile events", profiles) + print("submission events", events) + finally: + project.client.command(f"SYSTEM START MERGES {target}") + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + values = project.client.query(f"SELECT id, x FROM {target} ORDER BY id").result_rows + if values == [(1, 1), (2, 1)]: + break + time.sleep(0.05) + assert values == [(1, 1), (2, 1)] + + +def test_waiting_schema_mutation_marker_options(project, clickhouse_server, request): + clauses = [ + "MODIFY COLUMN x UInt32, DELETE WHERE 0 AND 'chm_schema_marker' = 'chm_schema_marker'", + "MATERIALIZE COLUMN y, DELETE WHERE 0 AND 'chm_schema_marker' = 'chm_schema_marker'", + "CLEAR COLUMN y, DELETE WHERE 0 AND 'chm_schema_marker' = 'chm_schema_marker'", + "RENAME COLUMN y TO renamed, DELETE WHERE 0 AND 'chm_schema_marker' = 'chm_schema_marker'", + ] + for index, clause in enumerate(clauses): + name = f"schema_probe_{index}" + table = f"{project.database}.{name}" + project.client.command( + f"CREATE TABLE {table} (id UInt64, x UInt64, y UInt64 DEFAULT x + 1) " + "ENGINE = MergeTree ORDER BY id" + ) + project.client.command(f"INSERT INTO {table} (id, x) VALUES (1, 10), (2, 20)") + project.client.command(f"SYSTEM STOP MERGES {table}") + server = clickhouse_server + worker = clickhouse_connect.get_client( + host=server.host, + port=server.port, + username=server.user, + password=server.password, + secure=server.secure, + send_receive_timeout=2, + ) + request.addfinalizer(worker.close) + query_id = "chm_schema_" + secrets.token_hex(8) + try: + try: + worker.command( + f"ALTER TABLE {table} {clause}", + settings={"mutations_sync": 0, "alter_sync": 0, "query_id": query_id}, + ) + print("combined marker request", clause, "returned") + except Exception as error: + print( + "combined marker request", + clause, + type(error).__name__, + str(error).splitlines()[0], + ) + finally: + project.client.command(f"SYSTEM START MERGES {table}") + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + active = project.client.query( + "SELECT count() FROM system.processes WHERE query_id = {id:String}", + parameters={"id": query_id}, + ).result_rows[0][0] + if not active: + break + time.sleep(0.05) + result = project.client.query( + "SELECT mutation_id, command, is_done FROM system.mutations WHERE database = {db:String} AND table = {table:String}", + parameters={"db": project.database, "table": name}, + ).result_rows + print("combined marker durable commands", clause, result) + + +def test_waiting_ddl_settings_identity(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + marker = "chm_ddl_" + secrets.token_hex(8) + cluster.stop_node(2) + try: + cluster.clients[1].command( + f"CREATE TABLE {project.database}.ddl_owned ON CLUSTER {cluster.name} (id UInt64) ENGINE = Memory", + settings={ + "distributed_ddl_task_timeout": 0, + "distributed_ddl_output_mode": "none", + "log_comment": marker, + }, + ) + rows = ( + cluster.clients[1] + .query( + "SELECT entry, host, query, settings['log_comment'], status " + "FROM system.distributed_ddl_queue WHERE position(query, {db:String}) > 0 " + "AND position(query, 'ddl_owned') > 0 ORDER BY host", + parameters={"db": project.database}, + ) + .result_rows + ) + print("DDL durable marker", marker, rows) + assert len(rows) == 2 and all(row[3] == marker for row in rows) + finally: + cluster.start_node(2) + + +def test_waiting_actual_mutation_corpus(project): + cases = [ + ("update", "ALTER TABLE {table} UPDATE x = x + 1 WHERE id = 1"), + ("delete", "ALTER TABLE {table} DELETE WHERE id = 1"), + ("lightweight_delete", "DELETE FROM {table} WHERE id = 1"), + ("lightweight_update", "UPDATE {table} SET x = x + 1 WHERE id = 1"), + ("modify_type", "ALTER TABLE {table} MODIFY COLUMN x UInt32"), + ("drop_column", "ALTER TABLE {table} DROP COLUMN y"), + ("clear_column", "ALTER TABLE {table} CLEAR COLUMN y"), + ("materialize_column", "ALTER TABLE {table} MATERIALIZE COLUMN y"), + ("rename_column", "ALTER TABLE {table} RENAME COLUMN y TO z"), + ("modify_ttl", "ALTER TABLE {table} MODIFY TTL ts + INTERVAL 1 DAY"), + ("failure", "ALTER TABLE {table} UPDATE x = throwIf(x = 5) WHERE 1"), + ] + for label, template in cases: + name = "corpus_" + label + table = f"{project.database}.{name}" + project.client.command( + f"CREATE TABLE {table} (id UInt64, x UInt64, y UInt64 DEFAULT x + 1, ts DateTime DEFAULT now()) " + "ENGINE = MergeTree ORDER BY id" + ) + project.client.command(f"INSERT INTO {table} (id, x) VALUES (1, 1), (5, 5)") + settings = {"mutations_sync": 0, "alter_sync": 0, "lightweight_deletes_sync": 0} + if label == "lightweight_update": + settings["allow_experimental_lightweight_update"] = 1 + try: + project.client.command(template.format(table=table), settings=settings) + outcome = "accepted" + except Exception as error: + outcome = type(error).__name__ + ": " + str(error).splitlines()[0] + deadline = time.monotonic() + 5 + while True: + mutations = project.client.query( + "SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason " + "FROM system.mutations WHERE database = {db:String} AND table = {table:String}", + parameters={"db": project.database, "table": name}, + ).result_rows + if ( + not mutations + or all(row[2] or row[4] for row in mutations) + or time.monotonic() > deadline + ): + break + time.sleep(0.05) + print("mutation corpus", label, outcome, mutations) + + +def test_waiting_single_node_deduplicated_intent(project): + table = f"{project.database}.intents" + project.client.command( + f"CREATE TABLE {table} (step String, owner String) ENGINE = MergeTree ORDER BY step " + "SETTINGS non_replicated_deduplication_window = 10000" + ) + for owner in ("first", "second"): + project.client.insert( + table, + [["step1", owner]], + column_names=["step", "owner"], + settings={"insert_deduplication_token": "same-step", "insert_deduplicate": 1}, + ) + rows = project.client.query(f"SELECT step, owner FROM {table}").result_rows + print("single intent dedup", rows) + assert rows == [("step1", "first")] + + +def test_waiting_cluster_deduplicated_intent(cluster_project, clickhouse_cluster): + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + + project, cluster = cluster_project, clickhouse_cluster + table = f"{project.database}.intents" + cluster.clients[1].command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (step String, owner String) " + f"ENGINE = ReplicatedMergeTree('/clickhouse/intents/{project.database}', '{{replica}}') ORDER BY step" + ) + barrier = Barrier(2) + + def insert_owner(node): + barrier.wait() + cluster.clients[node].insert( + table, + [["step1", f"node{node}"]], + column_names=["step", "owner"], + settings={ + "insert_deduplication_token": "same-step", + "insert_deduplicate": 1, + "insert_quorum": "auto", + }, + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(insert_owner, (1, 2))) + for node, client in cluster.clients.items(): + client.command(f"SYSTEM SYNC REPLICA {table} LIGHTWEIGHT") + rows = client.query(f"SELECT step, owner FROM {table}").result_rows + print("cluster intent dedup", node, rows) + assert len(rows) == 1 and rows[0][1] in ("node1", "node2") + + +def test_waiting_lightweight_update_with_required_storage(project): + table = f"{project.database}.patch_probe" + project.client.command( + f"CREATE TABLE {table} (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id " + "SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1" + ) + project.client.command(f"INSERT INTO {table} VALUES (1, 10), (2, 20)") + project.client.command( + f"UPDATE {table} SET x = x + 1 WHERE id = 1", + settings={"allow_experimental_lightweight_update": 1, "mutations_sync": 0}, + ) + mutations = project.client.query( + "SELECT mutation_id, command, is_done FROM system.mutations WHERE database = {db:String} AND table = 'patch_probe'", + parameters={"db": project.database}, + ).result_rows + values = project.client.query(f"SELECT id, x FROM {table} ORDER BY id").result_rows + print("lightweight update enabled", mutations, values) + assert values == [(1, 11), (2, 20)] + + +def test_waiting_empty_fence_cost_and_retention(project): + table = f"{project.database}.fence_cost" + project.client.command( + f"CREATE TABLE {table} (id UInt64, x UInt64, payload String) ENGINE = MergeTree ORDER BY id" + ) + project.client.command( + f"INSERT INTO {table} SELECT number, number, repeat('payload', 100) FROM numbers(10000)" + ) + project.client.command( + f"ALTER TABLE {table} DELETE WHERE 0 AND 'chm_cost_marker' = 'chm_cost_marker'", + settings={"mutations_sync": 2}, + ) + project.client.command("SYSTEM FLUSH LOGS") + names = { + row[0] + for row in project.client.query( + "SELECT name FROM system.columns WHERE database = 'system' AND table = 'part_log'" + ).result_rows + } + fields = [ + name + for name in ("event_type", "query_id", "rows", "read_rows", "read_bytes", "ProfileEvents") + if name in names + ] + if fields: + events = project.client.query( + "SELECT " + ", ".join(fields) + " FROM system.part_log " + "WHERE database = {db:String} AND table = 'fence_cost' AND toString(event_type) LIKE '%Mutat%'", + parameters={"db": project.database}, + ).result_rows + print("empty fence physical cost", fields, events) + mutations = project.client.query( + "SELECT command, is_done, parts_to_do FROM system.mutations WHERE database = {db:String} AND table = 'fence_cost'", + parameters={"db": project.database}, + ).result_rows + print("empty fence retained", mutations) + assert project.client.command(f"SELECT count() FROM {table}") == 10000 + + +@pytest.mark.parametrize("trial", range(3)) +def test_waiting_kill_before_receipt_and_resume_from_intent(project, trial): + table = f"{project.database}.counter" + project.client.command(f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY tuple()") + project.client.command(f"INSERT INTO {table} VALUES (0)") + table_uuid = str( + project.client.query( + "SELECT uuid FROM system.tables WHERE database = {db:String} AND name = 'counter'", + parameters={"db": project.database}, + ).result_rows[0][0] + ) + project.client.command( + f"CREATE TABLE {project.database}.waiting_intent (token String, table_uuid String) " + "ENGINE = MergeTree ORDER BY token SETTINGS non_replicated_deduplication_window = 10000" + ) + token = "chm_recovery_" + secrets.token_hex(8) + project.client.insert( + f"{project.database}.waiting_intent", + [[token, table_uuid]], + column_names=["token", "table_uuid"], + settings={"insert_deduplication_token": "revision-step", "insert_deduplicate": 1}, + ) + project.client.command(f"SYSTEM STOP MERGES {table}") + scripts = Path(__file__).parents[2] / "docs/design/spikes/2026-10-02-waiting" + submitted = subprocess.Popen( + [sys.executable, str(scripts / "submit_without_ack.py"), token], + cwd=project.root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + resumed = None + try: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + rows = project.client.query( + "SELECT mutation_id, command, is_done FROM system.mutations " + "WHERE database = {db:String} AND table = 'counter' ORDER BY mutation_id", + parameters={"db": project.database}, + ).result_rows + if any(token in row[1] for row in rows): + break + time.sleep(0.05) + assert len(rows) == 2 and any(token in row[1] for row in rows), rows + original_ids = [row[0] for row in rows] + print("pre-receipt crash", trial, rows) + submitted.kill() + submitted.communicate(timeout=5) + resumed = subprocess.Popen( + [sys.executable, str(scripts / "resume_fence.py")], + cwd=project.root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + deadline = time.monotonic() + 10 + while not (project.root / "resumed_marker").exists() and time.monotonic() < deadline: + time.sleep(0.05) + assert (project.root / "resumed_marker").exists() + assert resumed.poll() is None + project.client.command(f"SYSTEM START MERGES {table}") + output = resumed.communicate(timeout=20)[0] + assert resumed.returncode == 0, output + print("recovered", trial, output.strip()) + ids = [ + row[0] + for row in project.client.query( + "SELECT mutation_id FROM system.mutations WHERE database = {db:String} " + "AND table = 'counter' ORDER BY mutation_id", + parameters={"db": project.database}, + ).result_rows + ] + assert ids == original_ids + assert project.client.command(f"SELECT x FROM {table}") == 1 + finally: + for process in (submitted, resumed): + if process is not None and process.poll() is None: + process.kill() + process.communicate(timeout=5) + project.client.command(f"SYSTEM START MERGES {table}") + + +def test_waiting_foreign_failure_blocks_later_mutation(project): + table = f"{project.database}.foreign_probe" + project.client.command( + f"CREATE TABLE {table} (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id" + ) + project.client.command(f"INSERT INTO {table} VALUES (1, 1), (5, 5)") + project.client.command( + f"ALTER TABLE {table} UPDATE x = throwIf(x = 5) WHERE 1", settings={"mutations_sync": 0} + ) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + rows = project.client.query( + "SELECT mutation_id, command, is_done, latest_fail_reason FROM system.mutations " + "WHERE database = {db:String} AND table = 'foreign_probe' ORDER BY mutation_id", + parameters={"db": project.database}, + ).result_rows + if rows and rows[0][3]: + break + time.sleep(0.05) + assert rows[0][3] + project.client.command( + f"ALTER TABLE {table} UPDATE x = x + 1 WHERE id = 1", settings={"mutations_sync": 0} + ) + rows = project.client.query( + "SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason FROM system.mutations " + "WHERE database = {db:String} AND table = 'foreign_probe' ORDER BY mutation_id", + parameters={"db": project.database}, + ).result_rows + print("foreign failure and queued follower", rows) + assert len(rows) == 2 and all(not row[2] for row in rows) + assert project.client.query(f"SELECT id, x FROM {table} ORDER BY id").result_rows == [ + (1, 1), + (5, 5), + ] + + +def test_waiting_requires_every_replica(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + table = f"{project.database}.replica_wait" + cluster.clients[1].command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64, x UInt64) " + f"ENGINE = ReplicatedMergeTree('/clickhouse/wait/{project.database}', '{{replica}}') ORDER BY id" + ) + cluster.clients[1].command(f"INSERT INTO {table} VALUES (1, 0)") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") + for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): + cluster.clients[2].command(f"SYSTEM STOP {action} {table}") + try: + cluster.clients[1].command( + f"ALTER TABLE {table} UPDATE x = x + 1 WHERE 1", settings={"mutations_sync": 0} + ) + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table} PULL") + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + states = {} + for node, client in cluster.clients.items(): + states[node] = client.query( + "SELECT mutation_id, is_done, parts_to_do FROM system.mutations " + "WHERE database = {db:String} AND table = 'replica_wait'", + parameters={"db": project.database}, + ).result_rows + if states[1] and states[2] and states[1][0][1] and not states[2][0][1]: + break + time.sleep(0.05) + print("replica completion divergence", states) + assert states[1][0][1] == 1 and states[2][0][1] == 0 + assert states[1][0][0] == states[2][0][0] + finally: + for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): + cluster.clients[2].command(f"SYSTEM START {action} {table}") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") + assert cluster.clients[2].query(f"SELECT x FROM {table}").result_rows == [(1,)] + + +def test_waiting_managed_sync_settings_override(project): + table = f"{project.database}.sync_settings" + project.client.command(f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY tuple()") + project.client.command(f"INSERT INTO {table} VALUES (0)") + project.client.command(f"SYSTEM STOP MERGES {table}") + try: + project.client.command( + f"ALTER TABLE {table} UPDATE x = x + 1 WHERE 1, " + "DELETE WHERE 0 AND 'chm_settings_marker' = 'chm_settings_marker' " + "SETTINGS mutations_sync = 2, mutations_sync = 0, alter_sync = 0" + ) + rows = project.client.query( + "SELECT mutation_id, command, is_done FROM system.mutations " + "WHERE database = {db:String} AND table = 'sync_settings'", + parameters={"db": project.database}, + ).result_rows + print("managed sync override", rows) + assert rows and all(row[2] == 0 for row in rows) + finally: + project.client.command(f"SYSTEM START MERGES {table}") + + +def test_waiting_completed_ownership_marker_can_expire(project): + table = f"{project.database}.retention_probe" + project.client.command( + f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY tuple() " + "SETTINGS finished_mutations_to_keep = 1, old_parts_lifetime = 0, " + "cleanup_delay_period = 1, cleanup_delay_period_random_add = 0" + ) + project.client.command(f"INSERT INTO {table} VALUES (0)") + project.client.command( + f"ALTER TABLE {table} UPDATE x = x + 1 WHERE " "'chm_expired_owner' = 'chm_expired_owner'", + settings={"mutations_sync": 2}, + ) + project.client.command( + f"ALTER TABLE {table} DELETE WHERE 0", + settings={"mutations_sync": 2}, + ) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + rows = project.client.query( + "SELECT mutation_id, command FROM system.mutations " + "WHERE database = {db:String} AND table = 'retention_probe'", + parameters={"db": project.database}, + ).result_rows + if not any("chm_expired_owner" in row[1] for row in rows): + break + time.sleep(0.1) + print("expired ownership marker", rows, "x =", project.client.command(f"SELECT x FROM {table}")) + assert rows and not any("chm_expired_owner" in row[1] for row in rows) + assert project.client.command(f"SELECT x FROM {table}") == 1 + + +def test_waiting_ddl_failure_names_host(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + table = f"{project.database}.ddl_failure" + for client in cluster.clients.values(): + client.command(f"CREATE TABLE {table} (id UInt64, x String) ENGINE = MergeTree ORDER BY id") + cluster.clients[2].command(f"ALTER TABLE {table} ADD COLUMN conflict String") + token = "chm_ddl_failure_" + secrets.token_hex(8) + cluster.clients[1].command( + f"ALTER TABLE {table} ON CLUSTER {cluster.name} ADD COLUMN conflict UInt64", + settings={ + "distributed_ddl_task_timeout": 0, + "distributed_ddl_output_mode": "none", + "log_comment": token, + }, + ) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + rows = ( + cluster.clients[1] + .query( + "SELECT host, status, exception_code, exception_text FROM system.distributed_ddl_queue " + "WHERE settings['log_comment'] = {token:String} ORDER BY host", + parameters={"token": token}, + ) + .result_rows + ) + if len(rows) == 2 and any(row[2] for row in rows): + break + time.sleep(0.05) + print("DDL host exception", rows) + assert len(rows) == 2 and any(row[2] and row[3] for row in rows) From 8473e08f3cc8ce48c24452c7d1e3a539645da6c4 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Sun, 4 Oct 2026 11:21:35 -0400 Subject: [PATCH 02/33] [design] Adopt fail-closed unknown outcomes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record Dan’s approved recovery exception without treating missing ownership evidence as completion or permission to replay a write. Keep normal reattachment and the local-only execution boundaries. Solves: DRY-1398 Tests: 16 local probes; 3 crash repeats; 641 tests on Python 3.10-3.14 (cherry picked from commit 22b4c1a57c1a9c41e02e8c0b65eb6e0fa263ef27) (cherry picked from commit f1f42b58af9facb9264a1c49ebd065ad93632241) --- docs/design/2026-10-02-ch-migrate-1.0.md | 15 ++++--- docs/design/2026-10-02-waiting.md | 54 ++++++++++++++---------- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/docs/design/2026-10-02-ch-migrate-1.0.md b/docs/design/2026-10-02-ch-migrate-1.0.md index 7a4953d..f658184 100644 --- a/docs/design/2026-10-02-ch-migrate-1.0.md +++ b/docs/design/2026-10-02-ch-migrate-1.0.md @@ -152,12 +152,17 @@ started. the same: the migration is not recorded as applied, ClickHouse keeps doing the work, and the next `up` finds the unfinished mutation and waits on it instead of issuing the statement again. Issuing `UPDATE … SET n = n + 1` twice applies it twice. +- **Recovery exception approved on 2026-10-04:** if an interrupted statement's + outcome cannot be proven (for example, ownership evidence has expired), `up` + exits nonzero without reissuing the statement or marking the migration complete. + It identifies the unresolved work and requires operator reconciliation. Known + unfinished work still reattaches automatically. - A failing mutation does not end on its own: ClickHouse retries it and holds back every later mutation on the same table. `up` reads `latest_fail_reason`, fails the migration, and prints the `KILL MUTATION` statement for the operator. ch-migrate never kills a mutation itself. -The exact rules, starting with how `up` recognises a mutation it started on an earlier run, are -settled in a short design note before the 0.7 plan (see Open questions). +The execution, ownership and recovery rules are in the +[waiting design](2026-10-02-waiting.md). **Accept:** integration tests show that a migration whose mutation fails exits non-zero with the failure reason and leaves the version table unchanged; and that killing `up` while a mutation @@ -308,10 +313,10 @@ is unaffected; Python 3.9 is not supported, so 0.6 requires 3.10. 3. Does `bootstrap` (roles, users, grants) sit cleanly on the new connection path? 4. Does the dialect support Python 3.9? If not, it is ruled out for 1.0 (see Compatibility). -**A design note on waiting, before the 0.7 plan.** Experiments and proposed rules: +**A design note on waiting, before the 0.7 plan.** Answered in the [waiting design](2026-10-02-waiting.md). Accepted-but-unacknowledged mutation recovery -is demonstrated; the missing/expired-evidence case needs a decision before implementation. -The note does not weaken capabilities 4 or 5 without that decision. +is demonstrated. Dan approved explicit failure and operator reconciliation when +missing or expired evidence prevents establishing the outcome, on 2026-10-04. 5. On a re-run, how does `up` recognise a mutation it started earlier, without mistaking someone else's mutation for its own? diff --git a/docs/design/2026-10-02-waiting.md b/docs/design/2026-10-02-waiting.md index 17edee5..0632aa4 100644 --- a/docs/design/2026-10-02-waiting.md +++ b/docs/design/2026-10-02-waiting.md @@ -1,6 +1,6 @@ # Waiting for mutations and distributed DDL -**DRY-1398 · 2026-10-02 · Design blocked on the recovery contract.** +**DRY-1398 · 2026-10-02 · Recovery policy approved by Dan on 2026-10-04.** A write-ahead intent plus a marker carried into ClickHouse's mutation record can recover an accepted `UPDATE` after the client dies without reading its response. @@ -13,12 +13,17 @@ us whether the request never arrived, is still being submitted, or completed and was removed from the system table. Reissuing can apply `x = x + 1` twice; skipping can omit it. No command-normalization or time-window heuristic fixes that. -**Decision needed:** permit an explicit, nonzero, fail-closed result when durable -completion evidence is unavailable, with operator reconciliation rather than -automatic replay. That is the recommendation, not an accepted change to -[capabilities 4 and 5](2026-10-02-ch-migrate-1.0.md). Until that exception is -accepted, DRY-1399 and DRY-1400 must not implement a narrower guarantee and call it -complete. The current product definition remains authoritative. +**Approved recovery rule:** when an interrupted statement's outcome cannot be +proven, exit nonzero, do not reissue it, do not mark the migration complete, and +require operator reconciliation. Known unfinished work still reattaches +automatically. Dan approved this exception to +[capabilities 4 and 5](2026-10-02-ch-migrate-1.0.md) and explicitly resumed +local-only implementation on 2026-10-04. + +Approval revalidation on 2026-10-04: all 16 local experiments passed; the separate +three-trial kill/re-run pass passed. The local CI gate passed 641 tests on each of +Python 3.10, 3.11, 3.12, 3.13 and 3.14. The implementation rows below remain the +acceptance contract for DRY-1399/1400, not claims that those features already exist. ## Reproduce the evidence @@ -140,7 +145,7 @@ allocations in `ProfileEvents`. The marker is not free. It also requires DELETE privilege. Predicate markers avoid that extra command for suitable UPDATE/DELETE statements, but require syntax-aware insertion and their own corpus proof. -### Smallest proposed state +### Minimal runtime state One append-only server-side journal, deployed with the same replication policy as the version table, stores: @@ -165,7 +170,7 @@ one owner on both. Deduplication has a retention window. Do not turn this result into an unbounded exactly-once or concurrent-run claim. Concurrent migrators need serialization; this spike has not proved a portable fencing lock. -### The unresolved boundary +### Missing-evidence recovery If the process dies after preparing intent but before receiving a durable receipt, look up the marker. If it is present, attach without submitting again. If it is @@ -188,11 +193,13 @@ A resumed journal reader has no receipt to distinguish that history from an unsubmitted operation. Unfinished records surviving a normal mid-wait SIGKILL do not establish that completed records survive forever. -**Recommended exception:** fail with `outcome unknown; statement not reissued; -revision unchanged`, include the intent token and target, and require the operator -to reconcile the server outcome. No automatic stamp, retry flag, blind journal -deletion, or background KILL. Choosing the supported reconciliation interface is -part of accepting this exception. Until then, this is a blocker, not a fallback. +**Approved exception:** fail with `outcome unknown; statement not reissued; +migration incomplete`, including the intent token, revision, statement position, +target UUID, and read-only diagnostic queries. The operator must stop concurrent +runners, establish the actual outcome from independent evidence, and reconcile +the data and that specific journal record in a reviewed manual operation before +resuming. An absent server record alone never authorizes replay or completion. +There is no automatic stamp, retry override, blind journal deletion, or KILL. ## Q6. Foreign mutations ahead of ours @@ -385,7 +392,7 @@ finishes. This is not a promise of transactional rollback of the inserted head. ### Progress and exits -Proposed stable messages (data substituted, not a literal output fixture): +Stable message semantics (data substituted, not literal output fixtures): - Progress: `Waiting: db.table mutation ID host HOST; N parts; Ns elapsed`. - Failure, exit 1: `Mutation failed: ... REASON`, followed by an operator-only @@ -394,8 +401,8 @@ Proposed stable messages (data substituted, not a literal output fixture): - Timeout, exit 1: `Timed out; server work continues; revision not completed`, followed by every pending mutation/host/DDL entry and its last known state. - Foreign failure, exit 1: `Blocked by foreign mutation ...`, reason and command. -- Unknown outcome, proposed exit 1: `Outcome unknown; statement not reissued; - revision unchanged`, intent token, target UUID and reconciliation requirement. +- Unknown outcome, exit 1: `Outcome unknown; statement not reissued; + migration incomplete`, intent token, target UUID and reconciliation requirement. - Configuration/permission/topology error: exit 1 with the actual error; never a zero-work success. Existing CLI usage errors remain Click's exit 2. @@ -406,9 +413,9 @@ budget for each host. No default timeout; no automatic mutation/DDL cancellation ## Numbered implementation test matrix -These are required future checks, **not claimed implementation results**. Rows -involving unknown outcomes depend on the decision above. The experiment names -make the subset already demonstrated reproducible. +These are required implementation checks, **not claimed implementation results**. +Unknown-outcome rows use the recovery rule approved on 2026-10-04. The experiment +names make the subset already demonstrated reproducible. | Row | Owner | Required observable result | |---|---|---| @@ -421,7 +428,7 @@ make the subset already demonstrated reproducible. | W07 | DRY-1399 | Missing required host/record is not success, including a query using skip-unavailable behavior. | | W08 | DRY-1399 | Healthy foreign predecessor is reported and waited; failed predecessor is named; trailing foreign work is not adopted as ours. | | W09 | DRY-1399 | Same SQL from another writer is not owned; table recreation and changed migration digest refuse before replay. | -| W10 | DRY-1399 | Expired/missing ownership evidence never causes silent success or duplicate execution; recovery behavior requires the explicit decision. | +| W10 | DRY-1399 | Expired/missing ownership evidence exits nonzero with diagnostics and operator-reconciliation guidance; never silently completes or resubmits the statement. | | W11 | DRY-1399 | Polls longer than a short session timeout preserve a prior SET; an actually expired session fails loudly. | | W12 | DRY-1399 | Python op.execute, op.get_bind().execute/exec_driver_sql and run_sql all use the hook; polling does not recurse. | | W13 | DRY-1399 | Interrupted/failed version DELETE uses the same diagnostics, preserves insert-first safety, and never re-executes an applied revision. | @@ -448,5 +455,6 @@ make the subset already demonstrated reproducible. linked above. These probes are not the `up` implementation and do not imply that its acceptance tests pass yet. -This note does not authorize Cloud use, release a version, change accepted product -behavior, or claim an unbounded exactly-once protocol from bounded server records. +This note does not authorize Cloud use or release a version. Its approved +unknown-outcome rule does not claim an unbounded exactly-once protocol from +bounded server records. From 6d6f4feba6b272da0420afa3f5244124173fbe3d Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 10:52:56 -0400 Subject: [PATCH 03/33] [port] Bring the statement classifier waiting uses Waiting decides which statements leave a background mutation by classifying them, so the classifier ships in 0.7 as an internal module. The plan command, its README section and CHANGELOG entry stay on main under DRY-1542. This is the original classifier, without the rebuild case main added later in 780dd4c. (cherry picked from commit 7bd2fd3a, classifier and its tests only) Solves: DRY-1600 Tests: tests/test_classify.py, tests/integration/test_classification.py --- ch_migrate/classify.py | 302 +++++++++++++++++++++++ tests/corpus/classification.yaml | 166 +++++++++++++ tests/integration/test_classification.py | 69 ++++++ tests/test_classify.py | 77 ++++++ 4 files changed, 614 insertions(+) create mode 100644 ch_migrate/classify.py create mode 100644 tests/corpus/classification.yaml create mode 100644 tests/integration/test_classification.py create mode 100644 tests/test_classify.py diff --git a/ch_migrate/classify.py b/ch_migrate/classify.py new file mode 100644 index 0000000..6a19c3b --- /dev/null +++ b/ch_migrate/classify.py @@ -0,0 +1,302 @@ +"""Classify ClickHouse work without executing it; the server corpus is authoritative.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Literal + +from ch_migrate.idempotency import _ACTIONS, _action_clauses +from ch_migrate.introspect import Schema, TableDefinition +from ch_migrate.sql import _skip_quoted_or_comment +from ch_migrate.statements import MigrationStatement + +_WORD_OR_NUMBER = re.compile(r"[A-Za-z_][A-Za-z_0-9]*|\d+(?:\.\d+)?") +_TYPE_MODIFIERS = {"DEFAULT", "MATERIALIZED", "ALIAS", "CODEC", "COMMENT", "REMOVE", "TTL"} + + +@dataclass(frozen=True) +class Classification: + kind: Literal["metadata", "mutation", "rebuild", "other"] + table: str | None + detail: str + + +def classify( + statement: str | MigrationStatement, live_schema: Schema | None = None +) -> Classification: + """Classify metadata, background mutations, impossible ALTERs, or other work. + + The contract is tests/corpus/classification.yaml. Its real-server proof in + tests/integration/test_classification.py defines correctness, not a guessed + list of SQL verbs. With no live type to compare, MODIFY COLUMN is conservatively + a mutation *if the type changes*. Unsupported actions remain explicitly other. + """ + tokens = _tokens(statement if isinstance(statement, str) else statement.sql) + if not tokens: + return Classification("other", None, "No SQL statement") + words = tuple(token.upper() for token in tokens) + if words[:2] == ("ALTER", "TABLE"): + return _classify_alter(tokens, live_schema) + table = _statement_target(tokens) + if words[0] in ("CREATE", "DROP", "RENAME"): + return Classification("metadata", table, "Object metadata operation") + if words[:2] == ("DELETE", "FROM"): + return Classification( + "mutation", table, "Lightweight DELETE rewrites the row-existence mask" + ) + if words[0] == "UPDATE": + return Classification( + "other", + table, + "Synchronous lightweight UPDATE writes patch parts; no background mutation record", + ) + return Classification("other", table, "Not a classified schema operation") + + +@dataclass(frozen=True) +class _Alter: + table: str + definition: TableDefinition | None + actions: tuple[tuple[str, ...], ...] + settings: tuple[str, ...] + + +def _classify_alter(tokens: tuple[str, ...], schema: Schema | None) -> Classification: + index = 2 + if tuple(token.upper() for token in tokens[index : index + 2]) == ("IF", "EXISTS"): + index += 2 + table, index = _name(tokens, index) + if tuple(token.upper() for token in tokens[index : index + 2]) == ("ON", "CLUSTER"): + index += 3 + actions, settings = _split_settings(tokens[index:]) + definition = _table_definition(schema, table) + alter = _Alter(table, definition, _alter_actions(actions), settings) + classifications = [_classify_action(action, alter) for action in alter.actions] + priority = {"metadata": 0, "mutation": 1, "other": 2, "rebuild": 3} + kind = max(classifications, key=lambda item: priority[item.kind]).kind + return Classification( + kind, table, "; ".join(dict.fromkeys(item.detail for item in classifications)) + ) + + +def _alter_actions(tokens: tuple[str, ...]) -> tuple[tuple[str, ...], ...]: + actions: list[tuple[str, ...]] = [] + for clause in _action_clauses(tokens): + starts_action = clause and clause[0].upper() in _ACTIONS | {"REMOVE"} + if actions and (not starts_action or clause[1:2] == ("=",)): + actions[-1] += (",",) + clause + else: + actions.append(clause) + return tuple(actions) + + +def _classify_action(action: tuple[str, ...], alter: _Alter) -> Classification: + words = tuple(token.upper() for token in action) + prefix = words[:2] + if prefix == ("MODIFY", "COLUMN"): + return _modify_column(action[2:], alter) + if words[:3] == ("MODIFY", "ORDER", "BY"): + return _modify_order(action[3:], alter) + if prefix in (("MODIFY", "PRIMARY"), ("MODIFY", "PARTITION"), ("MODIFY", "ENGINE")): + return Classification( + "rebuild", alter.table, "This key or engine cannot be changed by an in-place ALTER" + ) + if prefix == ("MODIFY", "TTL"): + disabled = _setting_is_zero(alter.settings, "materialize_ttl_after_modify") + return Classification( + "metadata" if disabled else "mutation", + alter.table, + "TTL metadata only" if disabled else "MODIFY TTL materializes existing parts", + ) + if words[:1] in (("UPDATE",), ("DELETE",), ("CLEAR",), ("MATERIALIZE",)): + return Classification("mutation", alter.table, "Background mutation of existing parts") + if prefix in ( + ("DROP", "COLUMN"), + ("RENAME", "COLUMN"), + ("DROP", "INDEX"), + ("DROP", "PROJECTION"), + ): + return Classification( + "mutation", + alter.table, + "Column, index, or projection change creates a background mutation", + ) + if words[:1] in (("ADD",), ("COMMENT",), ("RESET",)) or prefix in ( + ("MODIFY", "SETTING"), + ("REMOVE", "TTL"), + ): + return Classification("metadata", alter.table, "Metadata change; no background mutation") + return Classification( + "other", alter.table, "ALTER action is not covered by the server-backed classification" + ) + + +def _modify_column(tokens: tuple[str, ...], alter: _Alter) -> Classification: + if tuple(token.upper() for token in tokens[:2]) == ("IF", "EXISTS"): + tokens = tokens[2:] + if not tokens: + return Classification("other", alter.table, "Missing column definition") + column = _identifier(tokens[0]) + type_tokens = _column_type(tokens[1:]) + if not type_tokens: + return Classification( + "metadata", alter.table, "Column default, comment, or codec metadata only" + ) + current = ( + next((item for item in alter.definition.columns if item.name == column), None) + if alter.definition + else None + ) + if current is None: + return Classification( + "mutation", alter.table, "Rewrites existing parts if the type changes" + ) + if _normalize_type(type_tokens) == _normalize_type(_tokens(current.type)): + return Classification("metadata", alter.table, "Column type is unchanged") + return Classification( + "mutation", + alter.table, + f"Column {column} changes type from {current.type} to {''.join(type_tokens)}", + ) + + +def _modify_order(tokens: tuple[str, ...], alter: _Alter) -> Classification: + if not alter.definition: + return Classification( + "rebuild", + alter.table, + "Sorting-key change requires live schema to prove a new-column-only extension", + ) + requested = _order_expressions(tokens) + current = [_normalize_type(_tokens(item)) for item in alter.definition.order_by] + added = set() + for action in alter.actions: + if tuple(token.upper() for token in action[:2]) == ("ADD", "COLUMN"): + index = ( + 5 if tuple(token.upper() for token in action[2:5]) == ("IF", "NOT", "EXISTS") else 2 + ) + if len(action) > index: + added.add(_normalize_type((action[index],))) + existing = {_normalize_type((column.name,)) for column in alter.definition.columns} + suffix = requested[len(current) :] + if requested[: len(current)] == current and all(item in added - existing for item in suffix): + return Classification( + "metadata", + alter.table, + "Sorting key unchanged or extended only by columns added in this ALTER", + ) + return Classification( + "rebuild", alter.table, "Changing existing sorting-key expressions requires a table rebuild" + ) + + +def _statement_target(tokens: tuple[str, ...]) -> str | None: + words = tuple(token.upper() for token in tokens) + if words[0] == "UPDATE": + return _name(tokens, 1)[0] + if words[:2] in (("INSERT", "INTO"), ("DELETE", "FROM")): + return _name(tokens, 2)[0] + if words[0] not in ("CREATE", "DROP", "RENAME"): + return None + types = {"TABLE", "VIEW", "DICTIONARY", "DATABASE", "USER", "ROLE"} + index = next((index + 1 for index, token in enumerate(words) if token in types), len(tokens)) + while index < len(words) and words[index] in ("IF", "NOT", "EXISTS"): + index += 1 + return _name(tokens, index)[0] if index < len(tokens) else None + + +def _table_definition(schema: Schema | None, name: str) -> TableDefinition | None: + if schema is None: + return None + database, _, table = name.rpartition(".") + if database and schema.database and database != schema.database: + return None + return schema.tables.get(table or name) + + +def _split_settings(tokens: tuple[str, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]: + depth = 0 + for index, token in enumerate(tokens): + if token in ("(", "["): + depth += 1 + elif token in (")", "]"): + depth -= 1 + elif token.upper() == "SETTINGS" and depth == 0: + return tokens[:index], tokens[index + 1 :] + return tokens, () + + +def _setting_is_zero(tokens: tuple[str, ...], name: str) -> bool: + value = None + for index in range(len(tokens) - 2): + if tokens[index].lower() == name and tokens[index + 1] == "=": + value = tokens[index + 2] + return value == "0" + + +def _column_type(tokens: tuple[str, ...]) -> tuple[str, ...]: + depth = 0 + for index, token in enumerate(tokens): + if depth == 0 and token.upper() in _TYPE_MODIFIERS: + return tokens[:index] + if token in ("(", "["): + depth += 1 + elif token in (")", "]"): + depth -= 1 + return tokens + + +def _order_expressions(tokens: tuple[str, ...]) -> list[str]: + if tokens[:1] == ("(",) and tokens[-1:] == (")",): + tokens = tokens[1:-1] + elif tokens[:2] == ("tuple", "(") and tokens[-1:] == (")",): + tokens = tokens[2:-1] + return [_normalize_type(action) for action in _action_clauses(tokens)] if tokens else [] + + +def _normalize_type(tokens: tuple[str, ...]) -> str: + return "".join(_identifier(token) for token in tokens) + + +def _name(tokens: tuple[str, ...], index: int) -> tuple[str, int]: + if index >= len(tokens): + return "", index + names = [_identifier(tokens[index])] + index += 1 + while index + 1 < len(tokens) and tokens[index] == ".": + names.append(_identifier(tokens[index + 1])) + index += 2 + return ".".join(names), index + + +def _identifier(token: str) -> str: + if token[:1] in ('"', "`") and token[-1:] == token[:1]: + quote = token[0] + return token[1:-1].replace(quote * 2, quote).replace("\\" + quote, quote) + return token + + +def _tokens(sql: str) -> tuple[str, ...]: + tokens = [] + index = 0 + while index < len(sql): + end = _skip_quoted_or_comment(sql, index) + if end is not None: + if sql[index] in "'\"`$": + tokens.append(sql[index:end]) + index = end + continue + if sql[index] == "{" and (end := sql.find("}", index + 1)) != -1: + tokens.append(sql[index : end + 1]) + index = end + 1 + continue + match = _WORD_OR_NUMBER.match(sql, index) + if match: + tokens.append(match.group()) + index = match.end() + continue + if not sql[index].isspace() and sql[index] != ";": + tokens.append(sql[index]) + index += 1 + return tuple(tokens) diff --git a/tests/corpus/classification.yaml b/tests/corpus/classification.yaml new file mode 100644 index 0000000..798cd5c --- /dev/null +++ b/tests/corpus/classification.yaml @@ -0,0 +1,166 @@ +# Server-backed classification contract. Each ALTER gets a fresh seeded table. +table_ddl: >- + CREATE TABLE {table} ( + id UInt64, x UInt64, y UInt64 DEFAULT x + 1, s String, ts DateTime, + INDEX idx id TYPE minmax GRANULARITY 1, + PROJECTION pr (SELECT id ORDER BY id) + ) ENGINE = MergeTree PARTITION BY toYYYYMM(ts) ORDER BY id + TTL ts + INTERVAL 100 YEAR +seed: >- + INSERT INTO {table} (id, x, s, ts) VALUES + (1, 10, 'one', '2020-01-01 00:00:00'), (2, 20, 'two', '2020-02-01 00:00:00') +statements: + - id: create_table + sql: CREATE TABLE IF NOT EXISTS {table} (id UInt64) ENGINE = Memory + expected: metadata + - id: create_view + sql: CREATE VIEW {db}.sample_view AS SELECT id FROM {table} + expected: metadata + - id: drop_table + sql: DROP TABLE IF EXISTS {table} + expected: metadata + - id: rename_table + sql: RENAME TABLE {table} TO {db}.renamed_table + expected: metadata + - id: create_dictionary + sql: CREATE DICTIONARY {db}.lookup (id UInt64) PRIMARY KEY id LAYOUT(FLAT()) + expected: metadata + - id: drop_view + sql: DROP VIEW IF EXISTS {db}.sample_view + expected: metadata + - id: read_only_select + sql: SELECT id FROM {table} + expected: other + - id: insert_select + sql: INSERT INTO {table} SELECT * FROM {db}.source + expected: other + - id: session_setting + sql: SET max_threads = 1 + expected: other + - id: add_column + sql: ALTER TABLE {table} ADD COLUMN added UInt64 + expected: metadata + - id: add_existing_column + sql: ALTER TABLE {table} ADD COLUMN IF NOT EXISTS x UInt64 + expected: metadata + - id: add_default_column + sql: ALTER TABLE {table} ADD COLUMN added UInt64 DEFAULT x + 1 + expected: metadata + - id: add_array_column + sql: ALTER TABLE {table} ADD COLUMN added Array(Tuple(UInt64, String)) + expected: metadata + - id: default_only + sql: ALTER TABLE {table} MODIFY COLUMN y DEFAULT x + 2 + expected: metadata + - id: same_type_default + sql: ALTER TABLE {table} MODIFY COLUMN y UInt64 DEFAULT x + 2 + expected: metadata + without_schema: mutation + - id: comment_modifier + sql: ALTER TABLE {table} MODIFY COLUMN x COMMENT 'mutation UPDATE DELETE is only text' + expected: metadata + - id: comment_column + sql: ALTER TABLE {table} COMMENT COLUMN x 'description' + expected: metadata + - id: modify_setting + sql: ALTER TABLE {table} MODIFY SETTING parts_to_throw_insert = 3000 + expected: metadata + - id: add_index + sql: ALTER TABLE {table} ADD INDEX another x TYPE minmax GRANULARITY 1 + expected: metadata + - id: add_projection + sql: ALTER TABLE {table} ADD PROJECTION another (SELECT id, x ORDER BY x) + expected: metadata + - id: drop_index + sql: ALTER TABLE {table} DROP INDEX idx + expected: mutation + - id: drop_projection + sql: ALTER TABLE {table} DROP PROJECTION pr + expected: mutation + - id: narrow_column_type + sql: ALTER TABLE {table} MODIFY COLUMN x UInt32 + expected: mutation + - id: nullable_column_type + sql: ALTER TABLE {table} MODIFY COLUMN x Nullable(UInt64) + expected: mutation + - id: low_cardinality_column_type + sql: ALTER TABLE {table} MODIFY COLUMN s LowCardinality(String) + expected: mutation + - id: drop_column + sql: ALTER TABLE {table} DROP COLUMN y + expected: mutation + - id: clear_column + sql: ALTER TABLE {table} CLEAR COLUMN y + expected: mutation + - id: materialize_column + sql: ALTER TABLE {table} MATERIALIZE COLUMN y + expected: mutation + - id: rename_column + sql: ALTER TABLE {table} RENAME COLUMN y TO renamed + expected: mutation + - id: update + sql: ALTER TABLE {table} UPDATE x = x + 1 WHERE id = 1 + expected: mutation + - id: delete + sql: ALTER TABLE {table} DELETE WHERE id = 1 + expected: mutation + - id: materialize_index + sql: ALTER TABLE {table} MATERIALIZE INDEX idx + expected: mutation + - id: clear_index + sql: ALTER TABLE {table} CLEAR INDEX idx + expected: mutation + - id: materialize_projection + sql: ALTER TABLE {table} MATERIALIZE PROJECTION pr + expected: mutation + - id: clear_projection + sql: ALTER TABLE {table} CLEAR PROJECTION pr + expected: mutation + - id: modify_ttl + sql: ALTER TABLE {table} MODIFY TTL ts + INTERVAL 99 YEAR + expected: mutation + - id: modify_ttl_without_materialization + sql: ALTER TABLE {table} MODIFY TTL ts + INTERVAL 99 YEAR SETTINGS materialize_ttl_after_modify = 0 + expected: metadata + - id: materialize_ttl + sql: ALTER TABLE {table} MATERIALIZE TTL + expected: mutation + - id: remove_ttl + sql: ALTER TABLE {table} REMOVE TTL + expected: metadata + - id: append_new_sorting_column + sql: ALTER TABLE {table} ADD COLUMN added UInt64, MODIFY ORDER BY (id, added) + expected: metadata + without_schema: rebuild + - id: append_existing_sorting_column + sql: ALTER TABLE {table} MODIFY ORDER BY (id, x) + expected: rebuild + - id: replace_sorting_key + sql: ALTER TABLE {table} MODIFY ORDER BY x + expected: rebuild + - id: change_primary_key + sql: ALTER TABLE {table} MODIFY PRIMARY KEY x + expected: rebuild + - id: change_partition_key + sql: ALTER TABLE {table} MODIFY PARTITION BY toYYYYMM(ts + INTERVAL 1 MONTH) + expected: rebuild + - id: change_engine + sql: ALTER TABLE {table} MODIFY ENGINE = ReplacingMergeTree + expected: rebuild + - id: unchanged_type + sql: ALTER TABLE {table} MODIFY COLUMN x UInt64 + expected: metadata + without_schema: mutation + - id: mixed_metadata_mutation + sql: ALTER TABLE {table} ADD COLUMN added String, MATERIALIZE COLUMN y + expected: mutation + - id: lightweight_delete + sql: DELETE FROM {table} WHERE id = 1 + expected: mutation + - id: lightweight_update + sql: UPDATE {table} SET x = x + 1 WHERE id = 1 + expected: other + detail: lightweight + - id: update_multiple_assignments + sql: ALTER TABLE {table} UPDATE x = x + 1, s = concat(s, '!') WHERE id = 1 + expected: mutation diff --git a/tests/integration/test_classification.py b/tests/integration/test_classification.py new file mode 100644 index 0000000..a5d775c --- /dev/null +++ b/tests/integration/test_classification.py @@ -0,0 +1,69 @@ +"""Every corpus ALTER is classified against actual mutation or rejection evidence.""" + +from pathlib import Path + +import pytest +import yaml +from clickhouse_connect.driver.exceptions import DatabaseError + +from ch_migrate.classify import classify +from ch_migrate.introspect import get_live_schema + +CORPUS = yaml.safe_load((Path(__file__).parents[1] / "corpus/classification.yaml").read_text()) +ALTERS = [case for case in CORPUS["statements"] if case["sql"].startswith("ALTER")] +pytestmark = pytest.mark.integration + + +@pytest.mark.parametrize("case", ALTERS, ids=lambda case: case["id"]) +def test_classification_matches_real_alter_work(project, case): + values = {"db": project.database, "table": f"{project.database}.sample"} + project.client.command(CORPUS["table_ddl"].format(**values)) + project.client.command(CORPUS["seed"].format(**values)) + statement = case["sql"].format(**values) + result = classify(statement, get_live_schema(project.client, project.database)) + assert result.kind == case["expected"], (statement, result) + before = _mutation_ids(project) + if result.kind == "rebuild": + code = 36 if "sorting" in case["id"] else 62 + with pytest.raises(DatabaseError, match=f"code: {code},"): + project.client.command(statement) + assert _mutation_ids(project) == before + return + project.client.command(statement, settings={"mutations_sync": 0, "alter_sync": 0}) + created_mutations = _mutation_ids(project) - before + assert bool(created_mutations) == (result.kind == "mutation"), ( + statement, + result, + created_mutations, + ) + + +def test_classification_lightweight_update_is_synchronous_patch_work(project): + table = f"{project.database}.patches" + project.client.command( + f"CREATE TABLE {table} (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id " + "SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1" + ) + project.client.command(f"INSERT INTO {table} VALUES (1, 10), (2, 20)") + statement = f"UPDATE {table} SET x = x + 1 WHERE id = 1" + result = classify(statement) + project.client.command(statement, settings={"allow_experimental_lightweight_update": 1}) + assert project.client.query(f"SELECT id, x FROM {table} ORDER BY id").result_rows == [ + (1, 11), + (2, 20), + ] + rows = project.client.query( + "SELECT mutation_id FROM system.mutations WHERE database = {db:String} AND table = 'patches'", + parameters={"db": project.database}, + ).result_rows + assert result.kind == "other" and "lightweight" in result.detail + assert rows == [] + + +def _mutation_ids(project): + rows = project.client.query( + "SELECT DISTINCT mutation_id FROM system.mutations " + "WHERE database = {db:String} AND table = 'sample'", + parameters={"db": project.database}, + ).result_rows + return {row[0] for row in rows} diff --git a/tests/test_classify.py b/tests/test_classify.py new file mode 100644 index 0000000..8fcaf6b --- /dev/null +++ b/tests/test_classify.py @@ -0,0 +1,77 @@ +"""Corpus classifications and uncertain or quoted SQL boundaries.""" + +from pathlib import Path + +import pytest +import yaml + +from ch_migrate.classify import classify +from ch_migrate.introspect import Schema, parse_create_table +from ch_migrate.statements import MigrationStatement + +CORPUS = yaml.safe_load((Path(__file__).parent / "corpus/classification.yaml").read_text()) +VALUES = {"db": "example", "table": "example.sample"} + + +@pytest.fixture +def schema(): + table = parse_create_table(CORPUS["table_ddl"].format(**VALUES)) + return Schema(database="example", tables={"sample": table}) + + +@pytest.mark.parametrize("case", CORPUS["statements"], ids=lambda case: case["id"]) +def test_classification_corpus(case, schema): + sql = case["sql"].format(**VALUES) + result = classify(sql, schema) + assert result.kind == case["expected"], (sql, result) + if "detail" in case: + assert case["detail"] in result.detail + if sql.startswith("ALTER"): + assert result.table == "example.sample" + + +@pytest.mark.parametrize( + "case", + [case for case in CORPUS["statements"] if "without_schema" in case], + ids=lambda case: case["id"], +) +def test_classification_without_live_schema_is_conservative(case): + result = classify(case["sql"].format(**VALUES)) + assert result.kind == case["without_schema"] + if "MODIFY COLUMN" in case["sql"]: + assert "if the type changes" in result.detail + + +def test_classification_ignores_comments_and_literal_keywords(schema): + statement = MigrationStatement( + "/* UPDATE x */ ALTER TABLE \"example\".`sample` ON CLUSTER 'local' " + "MODIFY COLUMN x COMMENT 'DROP COLUMN y, MODIFY ENGINE = Memory'", + "migration.sql", + 9, + (), + "upgrade", + ) + result = classify(statement, schema) + assert result.kind == "metadata" + assert result.table == "example.sample" + + +def test_classification_different_database_cannot_supply_a_type(schema): + result = classify("ALTER TABLE elsewhere.sample MODIFY COLUMN x UInt64", schema) + assert result.kind == "mutation" + assert "if the type changes" in result.detail + + +def test_classification_unknown_action_is_not_metadata(schema): + result = classify("ALTER TABLE example.sample SOMETHING UNRECOGNIZED", schema) + assert result.kind == "other" + + +def test_classification_preserves_identifier_case(schema): + result = classify("ALTER TABLE example.sample MODIFY ORDER BY ID", schema) + assert result.kind == "rebuild" + + +def test_classification_empty_and_read_only_have_no_table(): + assert classify("/* nothing */").kind == "other" + assert classify("SELECT 'ALTER TABLE x DELETE WHERE 1'").table is None From 13224ff5508df6a97c6101e60799ac6f4c57fdb8 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Sun, 4 Oct 2026 13:49:31 -0400 Subject: [PATCH 04/33] [db] Wait for owned mutations before completion Persist intent before submission and resume only proven server work. Poll every required replica in the checked session, preserve completed writes, and fail closed when ownership evidence cannot prove an outcome. Keep the CLI as the owning process and journal version bookkeeping. Solves: DRY-1399 Tests: 655 tests on Python 3.10-3.14; 127 integration; 3 crash repeats (cherry picked from commit 1d52b89677e0d72517a78e255896f7732012e5fb) (cherry picked from commit 39092ef7ec55aa1542f3d5ed6188f021acc1e575) --- README.md | 99 ++- ch_migrate/alembic_env.py | 121 ++- ch_migrate/bootstrap.py | 2 + ch_migrate/cli.py | 20 +- ch_migrate/migration_runner.py | 27 + ch_migrate/templates/bootstrap/init_users.sql | 2 + ch_migrate/version_table.py | 29 +- ch_migrate/waiting.py | 396 ++++++++++ ch_migrate/waiting_mutations.py | 238 ++++++ ch_migrate/waiting_sql.py | 227 ++++++ ch_migrate/waiting_store.py | 179 +++++ ch_migrate/waiting_types.py | 73 ++ ch_migrate/waiting_versions.py | 132 ++++ docs/design/2026-10-02-ch-migrate-1.0.md | 13 +- docs/design/2026-10-02-waiting.md | 21 + tests/integration/test_mutation_waiting.py | 718 ++++++++++++++++++ tests/integration/test_version_table.py | 45 +- tests/test_waiting.py | 93 +++ 18 files changed, 2316 insertions(+), 119 deletions(-) create mode 100644 ch_migrate/migration_runner.py create mode 100644 ch_migrate/waiting.py create mode 100644 ch_migrate/waiting_mutations.py create mode 100644 ch_migrate/waiting_sql.py create mode 100644 ch_migrate/waiting_store.py create mode 100644 ch_migrate/waiting_types.py create mode 100644 ch_migrate/waiting_versions.py create mode 100644 tests/integration/test_mutation_waiting.py create mode 100644 tests/test_waiting.py diff --git a/README.md b/README.md index e541b2a..ffedb30 100644 --- a/README.md +++ b/README.md @@ -173,13 +173,14 @@ New `alembic_version` tables use `ORDER BY version_num`. The runtime inspects | Atomic database with `cluster` configured | `ReplicatedMergeTree` created `ON CLUSTER`, with a Keeper path containing the database name and shard macro | | Single server | `MergeTree` | -The official Alembic implementation advances a version by **inserting the new -row, then deleting the old row with `mutations_sync = 2`**. It does not issue an -asynchronous version UPDATE. If the process stops while the delete is pending, -both rows remain and Alembic refuses their overlapping history rather than -repeating the migration. Let the version mutation finish, then retry. A -server-reported failed version mutation stops the run with its reason; ch-migrate -never kills that mutation. +Version advancement remains **insert the new row, then delete the old row**; +there is no asynchronous version UPDATE. Online `up` journals that bookkeeping +and polls its owned DELETE rather than holding an HTTP request open with +`mutations_sync`. If interrupted, both rows can remain. A subsequent `up` +finishes the proven bookkeeping without running the revision again. Overlapping +heads without an owned journal receipt are still refused. Failed version +mutations report their reason and an operator-only KILL statement; ch-migrate +never kills them itself. Before reading replicated heads, the tool catches up replicated database metadata and inserted version parts. The table barrier uses @@ -227,6 +228,80 @@ information or the single-server/Cloud default for version-table DDL. Review that DDL before using it for a self-hosted Replicated database. Cloud qualification is separate from the local suite; the local replicated harness has one shard. +### Migrations that wait + +`up` waits for the mutations its statements create before completing a revision. +It uses the same checked ClickHouse session for submission and polling, so prior +`SET` statements stay effective. `op.execute`, raw `op.get_bind()` execution, and +`run_sql` all pass through the cursor boundary. Upgrades run in the CLI process: +killing that process cannot leave a child Alembic runner advancing revisions. + +```bash +ch-migrate up dev # No default waiting timeout +ch-migrate up dev --timeout 120 # One waiting budget across the invocation +``` + +Progress names the table, mutation ID, host, remaining parts and elapsed time. +A terminal uses a live line; CI gets plain state changes and periodic summaries +on stderr. Zero remaining parts alone does not count as completion: every +required replica must report its owned mutation finished. Configure `cluster` +for self-hosted replicated tables, or use a Replicated database's automatic +cluster. Missing replicas or missing evidence are never treated as success. + +An active foreign mutation ahead of ours is reported and waited before more work +is submitted. A failed predecessor stops the run. Foreign work queued after our +completion barrier is not claimed as ours and does not hold our revision open. + +Ownership is persisted in `_ch_migrate_journal` **before** SQL submission. The +journal follows the version table's deployment policy and uses the same session. +It records revision/statement identities, table UUIDs and progress receipts. +Run only one migration runner per database, using a CI concurrency key or an +external lock: this journal is not a distributed lock. Do not delete or restore +it independently of the database state it describes. + +After a timeout, dropped connection or killed process, ClickHouse may still be +working. Run the same `up` again: it skips proven completed statements and +reattaches to its known unfinished mutation instead of issuing it again. +Successful downgrade starts a fresh journal generation so a later upgrade +actually executes. A changed unresolved statement or replaced table is refused. +An explicitly rejected repeat-safe metadata statement can be repaired and retried; +transport errors do not establish rejection. + +**Unknown outcome:** if ownership/completion evidence is missing or expired, +`up` exits nonzero, does not reissue the statement and does not complete the +migration. The error names the journal key, token and recorded UUIDs and provides +read-only inspection queries. Operator recovery is deliberately not a retry flag: + +1. Stop migration runners and quiesce other writers as needed. Back up the + affected journal key and relevant data before making changes. +2. Establish whether the original statement took effect using independent data, + schema and server evidence on every required replica. An absent mutation row + is not proof either way. +3. Reconcile that **specific** statement and journal key in a reviewed manual + operation. For a proven applied operation, its completed receipt must carry + the actual post-operation UUID map (`after_target.uuids`, or null for a dropped + target). For a proven unapplied operation, remove only that key's attempt + records after restoring any partial effects, and wait for that journal + change on every replica. Preserve the other completed statement receipts. +4. Recheck the recorded heads and rerun `up`. Never blindly stamp the revision, + delete the whole journal, or replay a non-idempotent statement to guess its + outcome. + +**Failed mutation:** `latest_fail_reason` stops the run and prints a scoped +`KILL MUTATION` statement for the operator. The tool does not execute it. Fix the +cause and reattach, or review a cancellation and reconcile its effects; killing +a mutation is not proof that it completed. + +`MODIFY TTL` has a separate metadata/materialization boundary: when materialization +is enabled, the runtime journals the metadata change with automatic materialization +disabled, then submits a separately owned `MATERIALIZE TTL`. This avoids confusing +TTL's comma-separated rules with ALTER actions. Session and per-query +`materialize_ttl_after_modify = 0` are honored. Parameter batches are executed +sequentially through the same single-statement path, with a receipt and completion +barrier for each parameter set; intermediate TTL effects are not collapsed. +Lightweight DELETE keeps its row-mask semantics; lightweight UPDATE is synchronous +patch work, not a fabricated `system.mutations` record. + ### Re-runnable migrations `up` statically checks pending upgrade statements before Alembic executes any of @@ -393,7 +468,7 @@ Example: `ch-migrate new dev add_status --table logs` ### `up` -`ch-migrate up ENV [-r REV] [--skip-mv-check] [--verbose]` applies migrations to `head` by default after the idempotency gate passes, printing one line per migration. `-r/--revision` selects a target. `--skip-mv-check` skips nonblocking materialized-view declaration checks, not the idempotency gate; use it only after reviewing those findings. If a migration fails, `up` names it, the SQL file, the statement and its line, and ClickHouse's error; `--verbose` adds the Python traceback. +`ch-migrate up ENV [-r REV] [--timeout SECONDS] [--skip-mv-check] [--verbose]` applies migrations to `head` by default after the idempotency gate passes, printing one line per migration. `-r/--revision` selects a target. `--timeout` is a finite positive waiting limit; omitting it waits without a deadline. `--skip-mv-check` skips nonblocking materialized-view declaration checks, not the idempotency gate; use it only after reviewing those findings. If a migration fails, `up` names it, the SQL file, the statement and its line, and ClickHouse's error; `--verbose` adds the Python traceback. Example: `ch-migrate up dev --revision abc123` @@ -507,7 +582,13 @@ The legacy `CH__PASSWORD` remains supported. Never commit credentials or pa ### Hooks -Top-level hooks run SQL on the migration connection. `pre_migrate` runs before the migration batch; `post_migrate` runs after each revision. `{db}` is substituted. Hooks execute as SQLAlchemy text, not through the SQL-file splitter; supply one statement per entry. Hook SQL is logged, so do not put secrets in it. +Top-level hooks run SQL on the migration connection. `pre_migrate` runs before the +migration batch; `post_migrate` runs after each revision body **before its version +is completed**, so hook mutations cannot escape the waiting barrier. A resumed +upgrade reuses its unfinished batch's receipts instead of repeating completed +hook writes. `{db}` is substituted. Hooks execute as SQLAlchemy text, not through +the SQL-file splitter; supply one statement per entry. Hook SQL is logged, so do +not put secrets in it. ```yaml hooks: diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index baca369..f59145f 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -6,17 +6,15 @@ import os from logging.config import fileConfig from pathlib import Path -from types import SimpleNamespace from typing import Any -from uuid import uuid4 import clickhouse_connect.cc_sqlalchemy.alembic # Registers the official implementation. from alembic import context from clickhouse_connect.cc_sqlalchemy.alembic.impl import ClickHouseImpl from clickhouse_connect.driver.httputil import get_pool_manager from dotenv import load_dotenv -from sqlalchemy import URL, Column, MetaData, String, Table, create_engine, pool, text -from sqlalchemy.sql.dml import Insert +from sqlalchemy import URL, Column, MetaData, String, Table, create_engine, pool +from sqlalchemy.sql.dml import Delete, Insert, Update from ch_migrate.config import get_env_config from ch_migrate.hooks import HookRegistry, run_hooks @@ -28,6 +26,7 @@ inspect_version_table, sync_version_replica, ) +from ch_migrate.waiting import MigrationWaiter ENV_VERSION = 2 DEFAULT_SESSION_TIMEOUT = 1800 @@ -58,52 +57,36 @@ def version_table_impl(self, *, version_table, version_table_schema, version_tab ) def _exec(self, construct, execution_options=None, multiparams=None, params=None): - if isinstance(construct, Insert) and self._is_version_table_construct(construct): - self._insert_version_row(construct) - return SimpleNamespace(rowcount=1) + # The official implementation owns these SQLAlchemy version constructs; + # all user SQL, including raw bind execution, is covered by cursor events. + waiter = self.context_opts.get("ch_migrate_waiter") + if ( + waiter is not None + and waiter.revision is not None + and isinstance(construct, (Insert, Update, Delete)) + and self._is_version_table_construct(construct) + ): + return waiter.versions.execute(construct, self) return super()._exec( construct, execution_options=execution_options, multiparams=multiparams, params=params ) - def _exec_version_update(self, construct, execution_options=None): - # The official INSERT then waited DELETE, with the INSERT written by _insert_version_row. - self._insert_version_row(construct) - where = self._compile_version_where(construct) - super()._exec( - text( - f"ALTER TABLE {self._version_table_name} DELETE WHERE {where} " - "SETTINGS mutations_sync = 2" - ) - ) - return SimpleNamespace(rowcount=1) - - def _insert_version_row(self, construct) -> None: - # Replicated and Shared MergeTree silently drop a block identical to an earlier insert - # (DRY-1447), so re-inserting a previous head after `down` left no version row at all. - # A token unique to this write keeps every intentional version row. - value = self._compile_clause(next(iter(construct._values.values()))) - token = f"'chm_version_{uuid4().hex}'" - super()._exec( - text( - f"INSERT INTO {self._version_table_name} (version_num) " - f"SETTINGS insert_deduplication_token = {token}, async_insert = 0 " - f"VALUES ({value})" - ) - ) - def run() -> None: """Run the selected environment on one official-dialect session.""" root = Path.cwd() load_dotenv(root / ".env.local") - env_config = get_env_config(os.environ.get("CH_ENVIRONMENT", "dev"), root / "config.yaml") + environment = context.config.attributes.get( + "ch_migrate_environment", os.environ.get("CH_ENVIRONMENT", "dev") + ) + env_config = get_env_config(environment, root / "config.yaml") os.environ["CH_DATABASE"] = env_config["database"] if env_config.get("cluster"): os.environ["CH_CLUSTER"] = env_config["cluster"] else: os.environ.pop("CH_CLUSTER", None) if context.config.config_file_name is not None: - fileConfig(context.config.config_file_name) + fileConfig(context.config.config_file_name, disable_existing_loggers=False) if context.is_offline_mode(): _run_offline(env_config) else: @@ -153,29 +136,49 @@ def _run_online(env_config: dict[str, Any]) -> None: def _run_connection(connection, env_config: dict[str, Any]) -> None: - hooks = HookRegistry.from_config(env_config.get("hooks")) - database = env_config["database"] - # Establish the session before requiring its existence on every later request. + # The checked session and listener registration share this failure boundary. connection.exec_driver_sql("SELECT 1") client = connection.connection.dbapi_connection.client client.set_client_setting("session_check", 1) - state = inspect_version_table(client, database, env_config.get("cluster")) + state = inspect_version_table(client, env_config["database"], env_config.get("cluster")) assert_version_mutations_healthy(client, state) state = sync_version_replica(client, state) if warning := state.warning(): logger.warning(warning) connection.dialect.ddl_compiler = VersionTableDDLCompiler - context.configure( - connection=connection, - target_metadata=None, - version_table="alembic_version", - version_table_schema=database, - on_version_apply=_post_migrate(hooks, database), - ch_migrate_version_state=state, - ) + waiter = MigrationWaiter(connection, state, context.config.attributes.get("ch_migrate_timeout")) + try: + waiter.install(context.config) + context.configure( + connection=connection, + target_metadata=None, + version_table="alembic_version", + version_table_schema=state.database, + ch_migrate_waiter=waiter, + ch_migrate_version_state=state, + ) + _run_context(waiter, HookRegistry.from_config(env_config.get("hooks"))) + finally: + waiter.close() + + +def _run_context(waiter: MigrationWaiter, hooks: HookRegistry) -> None: + runtime = context.get_context() + runtime._migrations_fn = waiter.steps(runtime._migrations_fn, hooks) with context.begin_transaction(): - run_hooks(connection, hooks.pre_migrate, db=database, phase="pre_migrate", revision="all") + if _is_upgrade(): + waiter.versions.resume() + waiter.run_pre_hooks(hooks) + else: + run_hooks( + waiter.connection, + hooks.pre_migrate, + db=waiter.state.database, + phase="pre_migrate", + revision="all", + ) context.run_migrations() + waiter.finish_run() def _url(env_config: dict[str, Any]) -> URL: @@ -194,19 +197,9 @@ def _url(env_config: dict[str, Any]) -> URL: ) -def _post_migrate(hooks: HookRegistry, database: str): - def after_revision(ctx, step, heads, run_args): - revision = ( - step.up_revision - if step.is_upgrade - else (step.down_revisions[0] if step.down_revisions else "unknown") - ) - run_hooks( - ctx.connection, - hooks.post_migrate, - db=database, - phase="post_migrate", - revision=revision, - ) - - return after_revision +def _is_upgrade() -> bool: + explicit = context.config.attributes.get("ch_migrate_command") + if explicit is not None: + return explicit == "upgrade" + command = getattr(context.config.cmd_opts, "cmd", None) + return bool(command and command[0].__name__ == "upgrade") diff --git a/ch_migrate/bootstrap.py b/ch_migrate/bootstrap.py index 67fe52d..6d9ad50 100644 --- a/ch_migrate/bootstrap.py +++ b/ch_migrate/bootstrap.py @@ -147,6 +147,8 @@ def build_bootstrap_sql( f"GRANT SELECT ON system.databases TO {project}_migration_role;", f"GRANT SELECT ON system.tables TO {project}_migration_role;", f"GRANT SELECT ON system.mutations TO {project}_migration_role;", + f"GRANT SELECT ON system.columns TO {project}_migration_role;", + f"GRANT CURRENT GRANTS(SELECT ON system.replicas) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SYSTEM SYNC REPLICA ON {db}.*) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SYSTEM SYNC DATABASE REPLICA ON {db}.*) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(CLUSTER ON *.*) TO {project}_migration_role;", diff --git a/ch_migrate/cli.py b/ch_migrate/cli.py index 8866ce6..7abbafe 100644 --- a/ch_migrate/cli.py +++ b/ch_migrate/cli.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import os import re import shutil @@ -258,13 +259,19 @@ def bootstrap(environment: str, dry_run: bool, verbose: bool) -> None: @main.command() @click.argument("environment") @click.option("--revision", "-r", default="head", help="Revision to upgrade to (default: head)") +@click.option( + "--timeout", + type=click.FloatRange(min=0, min_open=True), + default=None, + help="Maximum total waiting seconds; default waits without a deadline", +) @click.option( "--skip-mv-check", is_flag=True, help="Skip materialized view declaration validation", ) @click.option("--verbose", is_flag=True, help="Show the full traceback if a migration fails") -def up(environment: str, revision: str, skip_mv_check: bool, verbose: bool) -> None: +def up(environment: str, revision: str, skip_mv_check: bool, timeout: float | None, verbose: bool) -> None: """Apply pending migrations. Runs all unapplied migrations to bring the database to the latest version. @@ -273,9 +280,18 @@ def up(environment: str, revision: str, skip_mv_check: bool, verbose: bool) -> N Idempotency and standalone-SET gate errors refuse the run before Alembic. Other findings are warnings; --skip-mv-check only skips MV declaration checks. """ + if timeout is not None and not math.isfinite(timeout): + raise click.BadParameter("must be finite", param_hint="--timeout") _require_current_env() _enforce_up_gate(environment, skip_mv_check) - sys.exit(run_migrations(environment, ["upgrade", revision], verbose=verbose)) + from ch_migrate.migration_runner import run_upgrade + + try: + run_upgrade(environment, revision, timeout) + except Exception as error: + if verbose: + raise + raise click.ClickException(str(error)) from error @main.command() diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py new file mode 100644 index 0000000..f1d9ee9 --- /dev/null +++ b/ch_migrate/migration_runner.py @@ -0,0 +1,27 @@ +"""Own the upgrade process so SIGKILL cannot leave a child advancing revisions.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from alembic import command +from alembic.config import Config + + +def run_upgrade(environment: str, revision: str, timeout: float | None) -> None: + """Run Alembic directly: progress streams live and the CLI owns the waiter.""" + config = Config(str(Path.cwd() / "alembic.ini")) + config.attributes["ch_migrate_environment"] = environment + config.attributes["ch_migrate_command"] = "upgrade" + config.attributes["ch_migrate_timeout"] = timeout + # Project helpers intentionally read these during migration execution only. + saved = {name: os.environ.get(name) for name in ("CH_DATABASE", "CH_CLUSTER")} + try: + command.upgrade(config, revision) + finally: + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value diff --git a/ch_migrate/templates/bootstrap/init_users.sql b/ch_migrate/templates/bootstrap/init_users.sql index 200c03f..da29cbe 100644 --- a/ch_migrate/templates/bootstrap/init_users.sql +++ b/ch_migrate/templates/bootstrap/init_users.sql @@ -25,6 +25,8 @@ GRANT SELECT ON system.grants TO {project}_migration_role; GRANT SELECT ON system.databases TO {project}_migration_role; GRANT SELECT ON system.tables TO {project}_migration_role; GRANT SELECT ON system.mutations TO {project}_migration_role; +GRANT SELECT ON system.columns TO {project}_migration_role; +GRANT CURRENT GRANTS(SELECT ON system.replicas) TO {project}_migration_role; GRANT CURRENT GRANTS(SYSTEM SYNC REPLICA ON {db}.*) TO {project}_migration_role; GRANT CURRENT GRANTS(SYSTEM SYNC DATABASE REPLICA ON {db}.*) TO {project}_migration_role; GRANT CURRENT GRANTS(CLUSTER ON *.*) TO {project}_migration_role; diff --git a/ch_migrate/version_table.py b/ch_migrate/version_table.py index 5a9b2dd..3f13b11 100644 --- a/ch_migrate/version_table.py +++ b/ch_migrate/version_table.py @@ -31,16 +31,7 @@ def health_cluster(self) -> str | None: return self.cluster or (self.database if self.database_engine == "Replicated" else None) def new_engine(self): - if self.database_engine.startswith("Shared"): - return MergeTree(order_by="version_num") - if self.database_engine == "Replicated": - return ReplicatedMergeTree(order_by="version_num") - if self.cluster: - path = ( - f"/clickhouse/ch_migrate/{quote(self.database, safe='')}/{{shard}}/alembic_version" - ) - return ReplicatedMergeTree(order_by="version_num", zk_path=path, replica="{replica}") - return MergeTree(order_by="version_num") + return migration_state_engine(self, "alembic_version", "version_num") def warning(self) -> str | None: replicated = self.database_engine == "Replicated" or bool(self.cluster) @@ -73,6 +64,18 @@ class VersionTableMutationError(RuntimeError): """An unfinished version-table mutation has a server-reported failure.""" +def migration_state_engine(state: VersionTableState, table: str, order_by: str): + """Apply one deployment policy to version state and durable migration journals.""" + if state.database_engine.startswith("Shared"): + return MergeTree(order_by=order_by) + if state.database_engine == "Replicated": + return ReplicatedMergeTree(order_by=order_by) + if state.cluster: + path = f"/clickhouse/ch_migrate/{quote(state.database, safe='')}/{{shard}}/{table}" + return ReplicatedMergeTree(order_by=order_by, zk_path=path, replica="{replica}") + return MergeTree(order_by=order_by) + + def inspect_version_table(client, database: str, cluster: str | None = None) -> VersionTableState: rows = client.query( "SELECT engine FROM system.databases WHERE name = {db:String}", @@ -117,8 +120,12 @@ def assert_version_mutations_healthy(client, state: VersionTableState) -> None: settings=settings, ).result_rows if failed: + from ch_migrate.waiting_sql import sql_string + host, mutation, reason = failed[0] raise VersionTableMutationError( f"Version-table mutation {mutation} on {host} failed: {reason}. " - "Resolve the failed mutation manually before retrying; ch-migrate did not kill it." + "ch-migrate did not kill it.\n" + f"Operator only on {host}: KILL MUTATION WHERE database = {sql_string(state.database)} " + f"AND table = 'alembic_version' AND mutation_id = {sql_string(mutation)}" ) diff --git a/ch_migrate/waiting.py b/ch_migrate/waiting.py new file mode 100644 index 0000000..cc4d6a6 --- /dev/null +++ b/ch_migrate/waiting.py @@ -0,0 +1,396 @@ +"""Journal actual cursor writes and wait before Alembic completes a revision.""" + +from __future__ import annotations + +import hashlib +import json +import sys +from contextlib import contextmanager +from functools import wraps +from pathlib import Path +from uuid import uuid4 + +from alembic.script import ScriptDirectory +from clickhouse_connect.driver.exceptions import DatabaseError +from sqlalchemy import event + +from ch_migrate.classify import classify +from ch_migrate.hooks import run_hooks +from ch_migrate.idempotency import classify_idempotency +from ch_migrate.introspect import ColumnDefinition, Schema, TableDefinition, _parse_order_by +from ch_migrate.statements import migration_statements +from ch_migrate.waiting_mutations import MutationWaiter +from ch_migrate.waiting_sql import ( + bind_statement, + is_session_or_read, + modifies_ttl, + mutation_sql, + qualified_table, + query_setting, + query_settings, + sql_string, + statement_cluster, + statement_digest, + statement_table, +) +from ch_migrate.waiting_store import StepKey, WaitingJournal +from ch_migrate.waiting_types import UnknownOutcome, WaitBudget, WaitingError +from ch_migrate.waiting_versions import VersionWrites + + +class MigrationWaiter: + """One connection, one invocation deadline, and ordered durable receipts.""" + + def __init__(self, connection, state, timeout=None): + self.connection, self.state = connection, state + self.client = connection.connection.dbapi_connection.client + self.budget = WaitBudget(timeout) + self.journal = WaitingJournal(connection, state, self.budget) + self.mutations = MutationWaiter(self.client, state, self.budget) + self.versions = VersionWrites(self) + self.progress_stream = sys.stderr + self.revision = None + self.generation = self.position = self._internal = 0 + self.fingerprint = "" + self._pending = {} + self._batch_key = None + self._listeners = [] + self._scripts = None + + def install(self, config) -> None: + self._scripts = ScriptDirectory.from_config(config) + for target, name, listener, options in ( + (self.connection, "before_cursor_execute", self._before, {"retval": True}), + (self.connection, "after_cursor_execute", self._after, {}), + (self.connection.engine, "handle_error", self._error, {}), + ): + event.listen(target, name, listener, **options) + self._listeners.append((target, name, listener)) + + def close(self) -> None: + self.budget.finish() + for target, name, listener in reversed(self._listeners): + event.remove(target, name, listener) + self._listeners.clear() + self.revision = None + + @contextmanager + def internal(self): + self._internal += 1 + try: + yield + finally: + self._internal -= 1 + + def reserve(self, payload: dict) -> tuple[StepKey, dict, bool]: + self.position += 1 + key = StepKey(self.revision, self.generation, self.position) + previous = self.journal.read(key) + payload["fingerprint"] = self.fingerprint + if previous is None: + payload["token"] = "chm_mutation_" + uuid4().hex + return key, payload, True + retryable = ( + previous.get("phase") == "rejected" + and previous.get("repeat_safe") + and payload.get("repeat_safe") + ) + if previous["digest"] != payload["digest"] and not retryable: + raise UnknownOutcome( + f"Revision {key.revision} statement {key.position} changed after execution began; " + "statement not reissued. Reconcile its journal and the original SQL." + ) + if retryable: + payload["token"] = "chm_mutation_" + uuid4().hex + return key, payload if retryable else previous, bool(retryable) + + def validate_revision(self, revision: str, expected: str) -> None: + script = self._scripts.get_revision(revision) + if script is None or _fingerprint(Path(script.path)) != expected: + raise UnknownOutcome( + f"Revision {revision} changed during incomplete version bookkeeping; reconcile before resuming" + ) + + def unknown_outcome(self, key: StepKey, record: dict, error: Exception) -> UnknownOutcome: + receipt = record.get("receipt") or record.get("before_target") or {} + table = record.get("table") + if record.get("kind") == "version": + table = (self.state.database, "alembic_version") + receipt = next((step["receipt"] for step in record["steps"] if "receipt" in step), {}) + message = ( + f"{error}\nRevision {key.revision}, generation {key.generation}, statement {key.position}; " + f"token {record.get('token', receipt.get('token', 'unrecorded'))}; " + f"recorded target UUIDs {json.dumps(receipt.get('uuids', {}), sort_keys=True)}.\n" + "Stop migration runners. Establish the actual outcome independently; reconcile only this " + "journal key before rerunning. Do not stamp the revision or blindly delete its journal.\n" + f"Read-only inspection: SELECT sequence, payload FROM {self.journal.table} " + f"WHERE revision = {sql_string(key.revision)} AND generation = {key.generation} " + f"AND position = {key.position} ORDER BY sequence DESC;\n" + ) + if table: + source = ( + f"clusterAllReplicas({sql_string(receipt['cluster'])}, system.mutations)" + if receipt.get("cluster") + else "system.mutations" + ) + message += ( + "Read-only inspection: SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason " + f"FROM {source} WHERE database = {sql_string(table[0])} AND table = {sql_string(table[1])};" + ) + return UnknownOutcome(message) + + def steps(self, original, hooks): + def iterate(heads, context): + for step in original(heads, context): + if not hasattr(step, "revision"): + yield step # Native Alembic stamp steps have no migration body to track. + continue + revision = step.revision.revision + fingerprint = _fingerprint(Path(step.revision.path)) + if step.is_upgrade: + self._begin(revision, fingerprint) + else: + self.journal.invalidate(revision, fingerprint) + execute = step.migration_fn + step.migration_fn = self._revision_body(execute, hooks, revision) + try: + yield step + finally: + self.revision = None + + return iterate + + def run_pre_hooks(self, hooks) -> None: + fingerprint = hashlib.sha256(json.dumps(hooks.pre_migrate).encode()).hexdigest() + self._begin("__batch__", fingerprint) + key = StepKey(self.revision, self.generation, 0) + if self.journal.read(key).get("finished"): + self.journal.invalidate(self.revision, fingerprint) + self._begin("__batch__", fingerprint) + key = StepKey(self.revision, self.generation, 0) + self._batch_key = key + run_hooks( + self.connection, + hooks.pre_migrate, + db=self.state.database, + phase="pre_migrate", + revision="all", + ) + boundary, record, _ = self.reserve( + {"kind": "boundary", "phase": "done", "digest": "pre-hooks-complete"} + ) + self.journal.write(boundary, record) + self.revision = None + + def finish_run(self) -> None: + if self._batch_key: + control = self.journal.read(self._batch_key) + control["finished"] = True + self.journal.write(self._batch_key, control) + + def _begin(self, revision: str, fingerprint: str) -> None: + self.revision, self.fingerprint, self.position = revision, fingerprint, 0 + self.generation = self.journal.begin(revision, fingerprint) + self._completed_targets = {} + for key, record in self.journal.records(revision, self.generation): + if ( + record["phase"] not in ("done", "rejected") + and record.get("fingerprint") != fingerprint + ): + raise UnknownOutcome( + f"Revision {revision} changed with unresolved work; statement not reissued" + ) + if record["phase"] == "accepted" and record["kind"] == "write": + self._complete(key, record) + if record["phase"] == "done" and record.get("table"): + self._completed_targets[tuple(record["table"])] = record.get("after_target") + + def _revision_body(self, execute, hooks, revision): + @wraps(execute) + def run(**kwargs): + execute(**kwargs) + run_hooks( + self.connection, + hooks.post_migrate, + db=self.state.database, + phase="post_migrate", + revision=revision, + ) + + return run + + def _before(self, connection, cursor, statement, parameters, context, executemany): + # SQLAlchemy's event signature is an external contract. + if self._internal or self.revision is None: + return statement, parameters + sample = parameters[0] if executemany and parameters else parameters + sql = bind_statement(statement, sample, self.client) + if is_session_or_read(sql): + return statement, parameters + if executemany: + return self._execute_many(statement, parameters, context.execution_options) + table = statement_table(sql, self.state.database) + payload = { + "kind": "write", + "phase": "intent", + "digest": statement_digest(statement, parameters), + "table": list(table) if table else None, + "repeat_safe": classify_idempotency(sql).status == "ok", + } + key, record, fresh = self.reserve(payload) + if not fresh and self._resume(key, record): + return "SELECT 1 WHERE 0", {} + target = self.mutations.target(table) if table else None + record["before_target"] = target + kind = self._work_kind(sql, target, sample) if target else None + if kind: + token = record["token"] + record["kind"] = kind + record["receipt"] = self.mutations.prepare(target, token) + record["receipt"]["expected_markers"] = 1 + if kind == "ttl": + record.update(kind="ttl", work_submitted=False, cluster=statement_cluster(sql)) + statement = query_settings( + statement, {"materialize_ttl_after_modify": "0", "alter_sync": "0"} + ) + else: + statement = mutation_sql(statement, token) + self.journal.write(key, record) # Acknowledged intent MUST precede submission. + self._pending[id(context)] = (key, record) + return statement, parameters + + def _execute_many(self, statement, parameters, execution_options): + # Re-enter the single-statement cursor path, preserving statement order + # and an independently durable intent for each parameter set. + for binding in parameters: + self.connection.exec_driver_sql( + statement, binding, execution_options=execution_options + ).close() + return "SELECT 1 WHERE 0", [{}] + + def _after(self, connection, cursor, statement, parameters, context, executemany): + pending = self._pending.pop(id(context), None) + if pending is None: + return + key, record = pending + record["phase"] = "accepted" + self.journal.write(key, record) + if record["kind"] in ("mutation", "ttl"): + self._wait_work(key, record) + self._complete(key, record) + + def _error(self, context): + if context.connection is not self.connection: + return + pending = self._pending.pop(id(context.execution_context), None) + if pending is None: + return + key, record = pending + # A server rejection of repeat-safe metadata can be repaired and retried. + # Transport loss and mutation submissions remain unknown, never replayed. + if ( + record["kind"] == "write" + and record["repeat_safe"] + and isinstance(context.original_exception, DatabaseError) + and context.original_exception.code is not None + ): + record["phase"] = "rejected" + self.journal.write(key, record) + + def _resume(self, key: StepKey, record: dict) -> bool: + if record["phase"] == "done": + self._check_completed_target(record) + return True + if record["kind"] == "mutation" or ( + record["kind"] == "ttl" and record["phase"] == "accepted" + ): + print( + f"Resuming {key.revision} statement {key.position}; not reissuing SQL", + file=self.progress_stream, + ) + self._wait_work(key, record) + self._complete(key, record) + return True + if record["phase"] == "accepted": + self._complete(key, record) + return True + error = UnknownOutcome("Outcome unknown; statement not reissued; migration incomplete") + raise self.unknown_outcome(key, record, error) from error + + def _wait_work(self, key: StepKey, record: dict) -> None: + if record["kind"] == "ttl" and not record["work_submitted"]: + record["work_submitted"] = True + self.journal.write(key, record) + cluster = ( + f" ON CLUSTER {sql_string(record['cluster'])}" if record.get("cluster") else "" + ) + sql = f"ALTER TABLE {qualified_table(tuple(record['table']))}{cluster} MATERIALIZE TTL" + with self.internal(): + self.connection.exec_driver_sql(mutation_sql(sql, record["receipt"]["token"])) + try: + self.mutations.wait(record["receipt"], lambda: self.journal.write(key, record)) + except UnknownOutcome as error: + raise self.unknown_outcome(key, record, error) from error + + def _complete(self, key: StepKey, record: dict) -> None: + if record.get("table"): + record["after_target"] = self.mutations.target(tuple(record["table"])) + self._completed_targets[tuple(record["table"])] = record["after_target"] + record["phase"] = "done" + self.journal.write(key, record) + + def _check_completed_target(self, record: dict) -> None: + if not record.get("table"): + return + expected = self._completed_targets.get(tuple(record["table"]), record.get("after_target")) + actual = self.mutations.target(tuple(record["table"])) + if ( + (expected is None) != (actual is None) + or expected is not None + and expected["uuids"] != actual["uuids"] + ): + raise UnknownOutcome( + "Completed statement's target was dropped or replaced; reconcile before skipping its SQL" + ) + + def _work_kind(self, sql: str, target: dict, parameters) -> str | None: + if not target["engine"].endswith("MergeTree"): + return None + rows = self.client.query( + "SELECT name, type FROM system.columns WHERE database = {db:String} AND table = {table:String}", + parameters={"db": target["database"], "table": target["table"]}, + ).result_rows + table = TableDefinition( + target["table"], + target["engine"], + columns=[ColumnDefinition(*row) for row in rows], + order_by=_parse_order_by(target["sorting_key"]) if target["sorting_key"] else [], + ) + schema = Schema(database=target["database"], tables={target["table"]: table}) + ttl = modifies_ttl(sql) + materialize = self._materializes_ttl(sql, parameters) if ttl else False + analyzed = ( + query_settings(sql, {"materialize_ttl_after_modify": "0"}) + if ttl and not materialize + else sql + ) + if classify(analyzed, schema).kind != "mutation": + return None + return "ttl" if ttl and materialize else "mutation" + + def _materializes_ttl(self, sql: str, parameters) -> bool: + value = query_setting(sql, "materialize_ttl_after_modify") + if value is None: + value = str(self.client.command("SELECT getSetting('materialize_ttl_after_modify')")) + if value.startswith("{") and isinstance(parameters, dict): + value = str(parameters[value[1:].split(":", 1)[0]]) + if value.lower() not in ("0", "1", "true", "false"): + raise WaitingError("materialize_ttl_after_modify must resolve to a boolean value") + return value.lower() in ("1", "true") + + +def _fingerprint(path: Path) -> str: + digest = hashlib.sha256(path.read_bytes()) + for statement in migration_statements(path): + digest.update(statement.sql.encode()) + return digest.hexdigest() diff --git a/ch_migrate/waiting_mutations.py b/ch_migrate/waiting_mutations.py new file mode 100644 index 0000000..829dbb6 --- /dev/null +++ b/ch_migrate/waiting_mutations.py @@ -0,0 +1,238 @@ +"""Mutation ownership, predecessor handling, and per-replica completion evidence.""" + +from __future__ import annotations + +import re + +from ch_migrate.waiting_sql import sql_string +from ch_migrate.waiting_types import UnknownOutcome, WaitingError + + +class MutationWaiter: + def __init__(self, client, state, budget): + self.client, self.state, self.budget = client, state, budget + + def target(self, table: tuple[str, str]) -> dict | None: + rows = self.client.query( + "SELECT materialize(hostName()), toString(uuid), engine, sorting_key FROM system.tables " + "WHERE database = {db:String} AND name = {table:String}", + parameters={"db": table[0], "table": table[1]}, + ).result_rows + if not rows: + return None + host, uuid, engine, sorting_key = rows[0] + target = { + "database": table[0], + "table": table[1], + "engine": engine, + "initiator": host, + "uuids": {host: uuid}, + "cluster": None, + "sorting_key": sorting_key, + } + if engine.startswith("Shared"): + target["initiator"], target["uuids"] = "shared", {"shared": uuid} + elif engine.startswith("Replicated"): + self._replicated_target(target) + return target + + def prepare(self, target: dict, token: str) -> dict: + while True: + rows = self._rows(target) + pending = [row for row in rows if not row[3]] + self._failures(target, pending, "Blocked by foreign mutation") + if not pending: + self.budget.complete(f"{target['database']}.{target['table']} foreign predecessors") + break + self.budget.pause("foreign predecessor " + self._progress(target, pending)) + target = dict(target) + target.update(token=token, ids={}, done_hosts=[], baseline={}) + for host in target["uuids"]: + target["baseline"][host] = sorted({row[1] for row in rows if row[0] == host}) + return target + + def wait(self, receipt: dict, checkpoint) -> None: + while True: + identities = self._identities(receipt) + self._visible_hosts = set(identities) + for host, uuid in identities.items(): + if host in receipt["uuids"] and uuid != receipt["uuids"][host]: + raise UnknownOutcome( + f"Table {receipt['database']}.{receipt['table']} was replaced on {host}; statement not reissued" + ) + if receipt["initiator"] not in identities and not receipt.get("cluster"): + raise UnknownOutcome( + f"Table {receipt['database']}.{receipt['table']} disappeared; statement not reissued" + ) + rows = self._rows(receipt) + self._bind_markers(receipt, rows, checkpoint) + pending = self._pending(receipt, rows, checkpoint) + if not pending: + self.budget.complete( + f"{receipt['database']}.{receipt['table']}; every required replica finished" + ) + return + self.budget.pause("; ".join(pending)) + + def _replicated_target(self, target: dict) -> None: + path, total = self.client.query( + "SELECT zookeeper_path, total_replicas FROM system.replicas " + "WHERE database = {db:String} AND table = {table:String}", + parameters=self._parameters(target), + ).result_rows[0] + cluster = self.state.health_cluster + if not cluster: + if total > 1: + raise WaitingError( + "Replicated mutation waiting requires an environment cluster covering every replica" + ) + return + target["cluster"] = cluster + while True: + replicas = self.client.query( + "SELECT materialize(hostName()), replica_name FROM " + "clusterAllReplicas({cluster:String}, system.replicas) " + "WHERE database = {db:String} AND table = {table:String} AND zookeeper_path = {path:String}", + parameters={**self._parameters(target), "cluster": cluster, "path": path}, + settings=self._settings(), + ).result_rows + if len({replica for _, replica in replicas}) == total: + hosts = {host for host, _ in replicas} + target["uuids"] = { + host: uuid for host, uuid in self._identities(target).items() if host in hosts + } + if len(target["uuids"]) == total: + return + self.budget.pause( + f"{target['database']}.{target['table']}: cluster {cluster} exposes {len(replicas)}/{total} required replicas" + ) + + def _bind_markers(self, receipt: dict, rows: list, checkpoint) -> None: + found = False + for host in receipt["uuids"]: + if host in receipt["done_hosts"] or host in receipt["ids"]: + found = True + continue + local = [row for row in rows if row[0] == host] + markers = {row[1] for row in local if receipt["token"] in row[2]} + if not markers: + continue + if len(markers) != receipt.get("expected_markers", 1): + raise UnknownOutcome( + f"Outcome unknown for partially submitted batch on {host}; statement not reissued" + ) + found = True + barrier = max(_order(value) for value in markers) + baseline = set(receipt["baseline"].get(host, [])) + receipt["ids"][host] = sorted( + {row[1] for row in local if row[1] not in baseline and _order(row[1]) <= barrier} + ) + checkpoint() + if not found: + raise UnknownOutcome( + f"Outcome unknown for {receipt['database']}.{receipt['table']} token {receipt['token']}: " + "ownership evidence is missing or expired; statement not reissued; migration incomplete. " + "Stop runners and reconcile the journal with independent server evidence." + ) + + def _pending(self, receipt: dict, rows: list, checkpoint) -> list[str]: + pending = [] + for host in receipt["uuids"]: + if host in receipt["done_hosts"]: + continue + ids = receipt["ids"].get(host) + if not ids: + pending.append( + f"{receipt['database']}.{receipt['table']}: host {host} has not supplied the owned mutation" + ) + continue + local = [row for row in rows if row[0] == host and row[1] in ids] + missing = set(ids) - {row[1] for row in local} + if missing: + if host not in self._visible_hosts: + pending.append( + f"{receipt['database']}.{receipt['table']}: host {host} missing mutation evidence {sorted(missing)}" + ) + continue + raise UnknownOutcome( + f"Outcome unknown: mutation evidence {sorted(missing)} disappeared on {host}; statement not reissued" + ) + self._failures(receipt, local, "Mutation or preceding work failed") + unfinished = [row for row in local if not row[3]] + if unfinished: + pending.append(self._progress(receipt, unfinished)) + else: + receipt["done_hosts"].append(host) + checkpoint() + return pending + + def _rows(self, target: dict) -> list: + source, parameters = self._source(target, "mutations") + rows = self.client.query( + f"SELECT materialize(hostName()), mutation_id, command, is_done, parts_to_do, latest_fail_reason " + f"FROM {source} WHERE database = {{db:String}} AND table = {{table:String}}", + parameters=parameters, + settings=self._settings() if target.get("cluster") else None, + ).result_rows + if target["engine"].startswith("Shared"): + return [("shared", *row[1:]) for row in rows] + return [row for row in rows if row[0] in target["uuids"]] + + def _identities(self, target: dict) -> dict[str, str]: + source, parameters = self._source(target, "tables") + rows = self.client.query( + f"SELECT materialize(hostName()), toString(uuid) FROM {source} " + "WHERE database = {db:String} AND name = {table:String}", + parameters=parameters, + settings=self._settings() if target.get("cluster") else None, + ).result_rows + if target["engine"].startswith("Shared"): + return {"shared": rows[0][1]} if rows else {} + return dict(rows) + + def _source(self, target: dict, table: str) -> tuple[str, dict]: + parameters = self._parameters(target) + if target.get("cluster"): + parameters["cluster"] = target["cluster"] + return f"clusterAllReplicas({{cluster:String}}, system.{table})", parameters + return f"system.{table}", parameters + + def _settings(self) -> dict: + return { + "skip_unavailable_shards": 1, + "connect_timeout_with_failover_ms": 250, + "receive_timeout": 2, + } + + @staticmethod + def _parameters(target: dict) -> dict: + return {"db": target["database"], "table": target["table"]} + + @staticmethod + def _failures(target: dict, rows: list, prefix: str) -> None: + for host, mutation, _, done, _, reason in rows: + if not done and reason: + kill = ( + f"KILL MUTATION WHERE database = {sql_string(target['database'])} " + f"AND table = {sql_string(target['table'])} AND mutation_id = {sql_string(mutation)}" + ) + raise WaitingError( + f"{prefix}: {target['database']}.{target['table']} mutation {mutation} on {host}: {reason}\nOperator only on {host}: {kill}" + ) + + @staticmethod + def _progress(target: dict, rows: list) -> str: + parts = {} + for host, mutation, _, _, count, _ in rows: + parts[(host, mutation)] = max(parts.get((host, mutation), 0), count) + return "; ".join( + f"{target['database']}.{target['table']} mutation {mutation} host {host}; parts_to_do={count}" + for (host, mutation), count in sorted(parts.items()) + ) + + +def _order(mutation: str) -> int: + numbers = re.findall(r"\d+", mutation) + if len(numbers) != 1: + raise WaitingError(f"Unrecognized mutation ordering identifier: {mutation}") + return int(numbers[0]) diff --git a/ch_migrate/waiting_sql.py b/ch_migrate/waiting_sql.py new file mode 100644 index 0000000..7ea1e83 --- /dev/null +++ b/ch_migrate/waiting_sql.py @@ -0,0 +1,227 @@ +"""SQL identity and ownership markers without matching another writer's command.""" + +from __future__ import annotations + +import hashlib +import json +import re + +from clickhouse_connect.driver.binding import bind_query, quote_identifier + +from ch_migrate.classify import _identifier, _tokens +from ch_migrate.sql import _skip_quoted_or_comment +from ch_migrate.waiting_types import WaitingError + +_WORD = re.compile(r"[A-Za-z_][A-Za-z_0-9]*") +_MANAGED = frozenset(("mutations_sync", "alter_sync", "lightweight_deletes_sync")) + + +def bind_statement(statement, parameters, client) -> str: + bound, _ = bind_query(statement, parameters or None, client.server_tz) + return bound.decode() if isinstance(bound, bytes) else bound + + +def statement_digest(statement: str, parameters) -> str: + serialized = json.dumps( + [statement, parameters], sort_keys=True, default=str, separators=(",", ":") + ) + return hashlib.sha256(serialized.encode()).hexdigest() + + +def statement_table(statement: str, database: str) -> tuple[str, str] | None: + tokens = _tokens(statement) + words = tuple(token.upper() for token in tokens) + if words[:2] in (("ALTER", "TABLE"), ("INSERT", "INTO"), ("DELETE", "FROM")): + index = 2 + elif words[:1] == ("UPDATE",): + index = 1 + elif words[:1] in (("CREATE",), ("DROP",), ("RENAME",), ("TRUNCATE",)): + index = next( + (i + 1 for i, token in enumerate(words) if token in ("TABLE", "VIEW", "DICTIONARY")), + len(tokens), + ) + else: + return None + while index < len(tokens) and words[index] in ("IF", "NOT", "EXISTS"): + index += 1 + if index >= len(tokens): + return None + components = [_identifier(tokens[index])] + while index + 2 < len(tokens) and tokens[index + 1] == ".": + components.append(_identifier(tokens[index + 2])) + index += 2 + if any(component.startswith("{") for component in components): + raise WaitingError("Mutation tracking requires resolved database/table identifiers") + return (components[-2] if len(components) > 1 else database, components[-1]) + + +def statement_cluster(statement: str) -> str | None: + tokens = _tokens(statement) + for index in range(len(tokens) - 2): + if tuple(token.upper() for token in tokens[index : index + 2]) == ("ON", "CLUSTER"): + return tokens[index + 2].strip("'`\"") + return None + + +def is_session_or_read(statement: str) -> bool: + tokens = _tokens(statement) + if tokens and tokens[0].upper() == "WITH": + operation = next( + ( + span[0].upper() + for span in _spans(statement) + if span[3] == 0 and span[0].upper() in ("SELECT", "INSERT", "UPDATE", "DELETE") + ), + None, + ) + return operation == "SELECT" + return not tokens or tokens[0].upper() in ( + "SELECT", + "SHOW", + "DESCRIBE", + "DESC", + "EXPLAIN", + "SET", + "USE", + ) + + +def modifies_ttl(statement: str) -> bool: + tokens = _tokens(statement) + return any( + tuple(word.upper() for word in tokens[index : index + 2]) == ("MODIFY", "TTL") + for index in range(len(tokens) - 1) + ) + + +def mutation_sql(statement: str, token: str) -> str: + """Carry ownership into system.mutations and make submission asynchronous.""" + body, settings = _query_parts(statement) + words = _tokens(body) + marker = f"{sql_string(token)} = {sql_string(token)}" + overrides = {name: "0" for name in _MANAGED} + if tuple(word.upper() for word in words[:2]) == ("DELETE", "FROM"): + where = next( + (span for span in _spans(body) if span[0].upper() == "WHERE" and span[3] == 0), None + ) + if where is None: + raise WaitingError("A lightweight DELETE needs a WHERE predicate to track ownership") + body = body[: where[2]] + f" ({body[where[2]:].strip()}) AND ({marker})" + elif tuple(word.upper() for word in words[:2]) == ("ALTER", "TABLE"): + if modifies_ttl(body): + spans = list(_spans(body)) + start = next( + span[1] + for index, span in enumerate(spans[:-1]) + if span[0].upper() == "MODIFY" + and spans[index + 1][0].upper() == "TTL" + and span[3] == 0 + ) + body = body[:start].rstrip() + f" DELETE WHERE 0 AND ({marker}), " + body[start:] + overrides["materialize_ttl_after_modify"] = "0" + else: + body += f", DELETE WHERE 0 AND ({marker})" + else: + raise WaitingError("Cannot attach a mutation ownership marker to this statement") + return _with_settings(body, settings, overrides) + + +def query_settings(statement: str, overrides: dict[str, str]) -> str: + body, settings = _query_parts(statement) + return _with_settings(body, settings, overrides) + + +def query_setting(statement: str, name: str) -> str | None: + _, settings = _query_parts(statement) + for setting in reversed(settings): + key, _, value = setting.partition("=") + if _identifier(key.strip()).lower() == name: + return value.strip().strip("'") + return None + + +def sql_string(value: str) -> str: + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + + +def qualified_table(table: tuple[str, str]) -> str: + return ".".join(quote_identifier(part) for part in table) + + +def _query_parts(statement: str) -> tuple[str, list[str]]: + statement = _without_comments(statement) + spans = list(_spans(statement)) + if not spans: + raise WaitingError("Empty statement cannot be tracked") + sql = statement[: spans[-1][2]].rstrip() + candidates = [ + span + for index, span in enumerate(spans[:-2]) + if span[0].upper() == "SETTINGS" + and span[3] == 0 + and _WORD.fullmatch(_identifier(spans[index + 1][0])) + and spans[index + 2][0] == "=" + ] + if not candidates: + return sql, [] + settings = candidates[-1] + return sql[: settings[1]].rstrip(), _split_settings(sql[settings[2] :]) + + +def _split_settings(sql: str) -> list[str]: + starts = [span[1] for span in _spans(sql) if span[0] == "," and span[3] == 0] + result, previous = [], 0 + for index in starts: + result.append(sql[previous:index].strip()) + previous = index + 1 + if sql[previous:].strip(): + result.append(sql[previous:].strip()) + return result + + +def _with_settings(body: str, settings: list[str], overrides: dict[str, str]) -> str: + retained = [ + item + for item in settings + if _identifier(item.split("=", 1)[0].strip()).lower() not in overrides + ] + retained.extend(f"{name} = {value}" for name, value in sorted(overrides.items())) + return body + " SETTINGS " + ", ".join(retained) + + +def _without_comments(sql: str) -> str: + chunks, index = [], 0 + while index < len(sql): + end = _skip_quoted_or_comment(sql, index) + if end is None: + chunks.append(sql[index]) + index += 1 + else: + chunks.append(sql[index:end] if sql[index] in "'\"`$" else " ") + index = end + return "".join(chunks) + + +def _spans(sql: str): + """Yield significant tokens with offsets/depth, retaining quoted SQL verbatim.""" + index = depth = 0 + while index < len(sql): + end = _skip_quoted_or_comment(sql, index) + if end is not None: + if sql[index] in "'\"`$": + yield sql[index:end], index, end, depth + index = end + continue + match = _WORD.match(sql, index) + if match: + yield match.group(), index, match.end(), depth + index = match.end() + continue + char = sql[index] + if char in ")]": + depth -= 1 + if not char.isspace() and char != ";": + yield char, index, index + 1, depth + if char in "([": + depth += 1 + index += 1 diff --git a/ch_migrate/waiting_store.py b/ch_migrate/waiting_store.py new file mode 100644 index 0000000..a0c0d15 --- /dev/null +++ b/ch_migrate/waiting_store.py @@ -0,0 +1,179 @@ +"""Append-only waiting receipts, with a fresh generation after each downgrade.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass + +from clickhouse_connect.cc_sqlalchemy.datatypes.sqltypes import UInt32, UInt64 +from sqlalchemy import Column, MetaData, String, Table +from sqlalchemy.schema import CreateTable + +from ch_migrate.version_table import migration_state_engine +from ch_migrate.waiting_sql import qualified_table, query_settings +from ch_migrate.waiting_types import UnknownOutcome + +JOURNAL_TABLE = "_ch_migrate_journal" + + +@dataclass(frozen=True) +class StepKey: + revision: str + generation: int + position: int + + +class WaitingJournal: + """One ordered writer per database; readers synchronize before resuming.""" + + def __init__(self, connection, state, budget): + self.connection, self.state, self.budget = connection, state, budget + self.client = connection.connection.dbapi_connection.client + self.table = qualified_table((state.database, JOURNAL_TABLE)) + self._sequences: dict[StepKey, int] = {} + self._ready = False + + def exists(self) -> bool: + return bool( + self.client.query( + "SELECT count() FROM system.tables WHERE database = {db:String} AND name = {table:String}", + parameters={"db": self.state.database, "table": JOURNAL_TABLE}, + ).result_rows[0][0] + ) + + def synchronize(self) -> None: + if not self.exists(): + return + engine = self.client.query( + "SELECT engine FROM system.tables WHERE database = {db:String} AND name = {table:String}", + parameters={"db": self.state.database, "table": JOURNAL_TABLE}, + ).result_rows[0][0] + if engine.startswith(("Replicated", "Shared")): + self.client.command(f"SYSTEM SYNC REPLICA {self.table} LIGHTWEIGHT") + + def begin(self, revision: str, fingerprint: str) -> int: + self.ensure() + generation = self._generation(revision) + if generation == 0: + generation = 1 + self.write(StepKey(revision, generation, 0), {"fingerprint": fingerprint}) + else: + control = self.read(StepKey(revision, generation, 0)) + if control is None: + raise UnknownOutcome( + f"Journal generation for {revision} has no control record; reconcile before resuming" + ) + return generation + + def invalidate(self, revision: str, fingerprint: str) -> None: + if self.exists(): + self.write( + StepKey(revision, self._generation(revision) + 1, 0), {"fingerprint": fingerprint} + ) + + def records(self, revision: str, generation: int) -> list[tuple[StepKey, dict]]: + rows = self.client.query( + f"SELECT position, max(sequence), argMax(payload, sequence) FROM {self.table} " + "WHERE revision = {revision:String} AND generation = {generation:UInt64} AND position > 0 " + "GROUP BY position ORDER BY position", + parameters={"revision": revision, "generation": generation}, + ).result_rows + result = [] + for position, sequence, payload in rows: + key = StepKey(revision, generation, position) + self._sequences[key] = sequence + result.append((key, json.loads(payload))) + return result + + def read(self, key: StepKey) -> dict | None: + rows = self.client.query( + f"SELECT sequence, payload FROM {self.table} WHERE revision = {{revision:String}} " + "AND generation = {generation:UInt64} AND position = {position:UInt32} " + "ORDER BY sequence DESC LIMIT 2", + parameters=self._parameters(key), + ).result_rows + if not rows: + self._sequences[key] = 0 + return None + if len(rows) == 2 and rows[0][0] == rows[1][0] and rows[0][1] != rows[1][1]: + raise UnknownOutcome( + f"Conflicting journal writers for {key.revision}:{key.position}; serialize runners and reconcile" + ) + self._sequences[key] = rows[0][0] + return json.loads(rows[0][1]) + + def write(self, key: StepKey, payload: dict) -> None: + if key not in self._sequences: + self.read(key) + sequence = self._sequences[key] + 1 + self.client.insert( + f"{self.state.database}.{JOURNAL_TABLE}", + [ + [ + key.revision, + key.generation, + key.position, + sequence, + json.dumps(payload, sort_keys=True), + ] + ], + column_names=["revision", "generation", "position", "sequence", "payload"], + settings={"async_insert": 0}, + ) + self._sequences[key] = sequence + + def unfinished_versions(self) -> list[tuple[StepKey, dict]]: + if not self.exists(): + return [] + self.synchronize() + rows = self.client.query( + f"SELECT revision, generation, position, max(sequence), argMax(payload, sequence) AS latest " + f"FROM {self.table} WHERE position > 0 AND (revision, generation) IN " + f"(SELECT revision, max(generation) FROM {self.table} GROUP BY revision) " + "GROUP BY revision, generation, position HAVING JSONExtractString(latest, 'kind') = 'version' " + "AND JSONExtractString(latest, 'phase') != 'done' ORDER BY revision, position" + ).result_rows + result = [] + for revision, generation, position, sequence, payload in rows: + key = StepKey(revision, generation, position) + self._sequences[key] = sequence + result.append((key, json.loads(payload))) + return result + + def ensure(self) -> None: + if self._ready: + return + if self.exists(): + self.synchronize() + self._ready = True + return + table = Table( + JOURNAL_TABLE, + MetaData(), + Column("revision", String, nullable=False), + Column("generation", UInt64, nullable=False), + Column("position", UInt32, nullable=False), + Column("sequence", UInt64, nullable=False), + Column("payload", String, nullable=False), + migration_state_engine( + self.state, JOURNAL_TABLE, "(revision, generation, position, sequence)" + ), + schema=self.state.database, + info={"ch_migrate_on_cluster": self.state.on_cluster}, + ) + ddl = str(CreateTable(table, if_not_exists=True).compile(dialect=self.connection.dialect)) + self.client.command(query_settings(ddl, {"distributed_ddl_task_timeout": "0"})) + while not self.exists(): + self.budget.pause(f"journal {self.table} metadata is not available on this host") + self.synchronize() + self._ready = True + + def _generation(self, revision: str) -> int: + return self.client.query( + f"SELECT max(generation) FROM {self.table} WHERE revision = {{revision:String}}", + parameters={"revision": revision}, + ).result_rows[0][0] + + @staticmethod + def _parameters(key: StepKey) -> dict: + return {"revision": key.revision, "generation": key.generation, "position": key.position} diff --git a/ch_migrate/waiting_types.py b/ch_migrate/waiting_types.py new file mode 100644 index 0000000..47491c6 --- /dev/null +++ b/ch_migrate/waiting_types.py @@ -0,0 +1,73 @@ +"""Waiting errors, invocation deadline, and bounded progress output.""" + +from __future__ import annotations + +import sys +import time +from dataclasses import dataclass, field + + +class WaitingError(RuntimeError): + """Server work is unfinished; the revision must not be completed.""" + + +class UnknownOutcome(WaitingError): + """Available evidence cannot justify replay or completion.""" + + +class WaitTimeout(WaitingError): + """The invocation's waiting deadline expired without cancelling server work.""" + + +@dataclass +class WaitBudget: + timeout: float | None = None + started: float = field(default_factory=time.monotonic) + poll_interval: float = 0.2 + _last_line: str = "" + _last_printed: float = 0 + _live: bool = False + + @property + def elapsed(self) -> float: + return time.monotonic() - self.started + + def check(self, pending: str) -> None: + if self.timeout is not None and self.elapsed >= self.timeout: + self.finish() + raise WaitTimeout( + f"Timed out after {self.elapsed:.1f}s; server work continues; " + f"migration incomplete. Pending: {pending}" + ) + + def pause(self, pending: str) -> None: + self.check(pending) + self.progress(pending) + remaining = self.poll_interval + if self.timeout is not None: + remaining = min(remaining, max(0, self.timeout - self.elapsed)) + time.sleep(remaining) + self.check(pending) + + def progress(self, pending: str) -> None: + now = time.monotonic() + tty = sys.stderr.isatty() + if pending != self._last_line or now - self._last_printed >= (0.5 if tty else 5): + line = f"Waiting: {pending}; {self.elapsed:.1f}s elapsed" + if tty: + print("\r\033[K" + line, end="", file=sys.stderr, flush=True) + self._live = True + else: + print(line, file=sys.stderr, flush=True) + self._last_printed, self._last_line = now, pending + + def complete(self, message: str) -> None: + self.finish() + if self._last_line: + print(f"Finished: {message}; {self.elapsed:.1f}s elapsed", file=sys.stderr, flush=True) + self._last_line = "" + + def finish(self) -> None: + if self._live: + print(file=sys.stderr, flush=True) + self._live = False diff --git a/ch_migrate/waiting_versions.py b/ch_migrate/waiting_versions.py new file mode 100644 index 0000000..dcf5fe7 --- /dev/null +++ b/ch_migrate/waiting_versions.py @@ -0,0 +1,132 @@ +"""Resume insert-first version bookkeeping without invoking a revision twice.""" + +from __future__ import annotations + +import ast +from types import SimpleNamespace +from uuid import uuid4 + +from sqlalchemy.sql.dml import Delete, Insert, Update + +from ch_migrate.waiting_sql import mutation_sql, qualified_table, statement_digest +from ch_migrate.waiting_types import UnknownOutcome, WaitingError + + +class VersionWrites: + def __init__(self, owner): + self.owner = owner + + def execute(self, construct, impl): + steps = _version_steps(construct, impl) + payload = { + "kind": "version", + "phase": "intent", + "steps": steps, + "digest": statement_digest(str(construct), [step["sql"] for step in steps]), + } + key, record, _ = self.owner.reserve(payload) + self.continue_record(key, record) + return SimpleNamespace(rowcount=1) + + def resume(self) -> None: + for key, record in self.owner.journal.unfinished_versions(): + self.owner.validate_revision(key.revision, record["fingerprint"]) + print( + f"Resuming version bookkeeping for {key.revision}", file=self.owner.progress_stream + ) + self.continue_record(key, record) + + def continue_record(self, key, record: dict) -> None: + if record["phase"] == "done": + return + try: + for step in record["steps"]: + if step.get("done"): + continue + checkpoint = lambda: self.owner.journal.write(key, record) + if step["kind"] == "insert": + self._insert(step, checkpoint) + else: + self._delete(step, checkpoint) + step["done"] = True + checkpoint() + record["phase"] = "done" + self.owner.journal.write(key, record) + except UnknownOutcome as error: + raise self.owner.unknown_outcome(key, record, error) from error + + def _insert(self, step: dict, checkpoint) -> None: + table = qualified_table((self.owner.state.database, "alembic_version")) + count = self.owner.client.query( + f"SELECT count() FROM {table} WHERE version_num = {{version:String}}", + parameters={"version": step["value"]}, + ).result_rows[0][0] + if step.get("prepared"): + if count == 1: + return # The exact version-row effect proves the internal INSERT succeeded. + raise UnknownOutcome( + f"Outcome unknown for version INSERT {step['value']}: found {count} rows; " + "statement not reissued; reconcile version state before resuming" + ) + if count: + raise WaitingError( + f"Version {step['value']} already exists before its INSERT; reconcile overlapping heads" + ) + step["prepared"] = True + checkpoint() + self.owner.client.command( + step["sql"], settings={"async_insert": 0, "wait_for_async_insert": 1} + ) + + def _delete(self, step: dict, checkpoint) -> None: + if not step.get("prepared"): + target = self.owner.mutations.target((self.owner.state.database, "alembic_version")) + if target is None: + raise UnknownOutcome("Version table disappeared before deleting the previous head") + token = "chm_version_" + uuid4().hex + step["receipt"] = self.owner.mutations.prepare(target, token) + step["prepared"] = True + checkpoint() + with self.owner.internal(): + self.owner.connection.exec_driver_sql(mutation_sql(step["sql"], token)) + self.owner.mutations.wait(step["receipt"], checkpoint) + + +def _version_steps(construct, impl) -> list[dict]: + if isinstance(construct, Insert): + value = _version_value(construct) + sql = str(construct.compile(dialect=impl.dialect, compile_kwargs={"literal_binds": True})) + return [{"kind": "insert", "value": value, "sql": sql}] + if isinstance(construct, Update): + value = _version_value(construct) + literal = impl._compile_clause(next(iter(construct._values.values()))) + return [ + { + "kind": "insert", + "value": value, + "sql": f"INSERT INTO {impl._version_table_name} (version_num) VALUES ({literal})", + }, + { + "kind": "delete", + "sql": f"ALTER TABLE {impl._version_table_name} DELETE WHERE {impl._compile_version_where(construct)}", + }, + ] + if isinstance(construct, Delete): + return [ + { + "kind": "delete", + "sql": f"ALTER TABLE {impl._version_table_name} DELETE WHERE {impl._compile_version_where(construct)}", + } + ] + raise WaitingError("Unsupported version-table transition") + + +def _version_value(construct) -> str: + values = construct._values + expression = next(iter(values.values())) if values else None + value = getattr(expression, "value", None) + if value is None and getattr(expression, "is_literal", False): + value = ast.literal_eval(str(expression)) + if not isinstance(value, str): + raise WaitingError("Version-table INSERT/UPDATE must contain a literal revision identifier") + return value diff --git a/docs/design/2026-10-02-ch-migrate-1.0.md b/docs/design/2026-10-02-ch-migrate-1.0.md index f658184..67ed6e2 100644 --- a/docs/design/2026-10-02-ch-migrate-1.0.md +++ b/docs/design/2026-10-02-ch-migrate-1.0.md @@ -99,9 +99,11 @@ both; the README quick start contains no Python. ([dialect spike](2026-10-02-dialect-spike.md)). A session that expires while idle must fail loudly, not quietly drop its settings. - The version table cannot cause a migration to run twice. The version advances by inserting - the new row and then deleting the old one, waiting for the delete. An interrupted run is - therefore refused, not repeated. Today's `ALTER … UPDATE` returns before it is applied, and - the spike reproduced a migration running twice because of it. + the new row and then deleting the old one, waiting for the delete. Interrupted + bookkeeping is never permission to repeat the migration: without ownership + evidence it is refused; with capability 5's journal it resumes the same + bookkeeping. The spike reproduced duplicate execution with the old asynchronous + `ALTER … UPDATE` approach. - The version table is created to suit the deployment: shared on ClickHouse Cloud, replicated in a replicated database or on a configured cluster, plain on a single server. - `ch-migrate upgrade-env` moves an existing project to the new `env.py`. Existing version tables @@ -110,8 +112,9 @@ both; the README quick start contains no Python. **Accept:** an integration test against a real ClickHouse server shows a `SET` in one statement taking effect in the next; an existing 0.x project upgrades with `upgrade-env` and -runs `up` without editing migration files; with the version table's delete held back, a second -`up` refuses instead of re-running a migration. +runs `up` without editing migration files; with the version table's delete held back, +a second `up` cannot re-run the migration. It refuses unowned overlapping state, +or reattaches to proven owned bookkeeping under capability 5. ### 4. Re-runnable migrations diff --git a/docs/design/2026-10-02-waiting.md b/docs/design/2026-10-02-waiting.md index 0632aa4..9735e52 100644 --- a/docs/design/2026-10-02-waiting.md +++ b/docs/design/2026-10-02-waiting.md @@ -369,6 +369,13 @@ than its idle timeout. This both preserves session `SET`s and keeps the session alive. Propagate DRY-1392's expired-session failure; never silently recreate it. A separate monitoring client alone does not keep the migration session alive. +Online `up` invokes Alembic in the CLI process, so SIGKILL cannot leave a child +Alembic runner advancing the version after its parent dies. A narrow revision-step +adapter establishes the journal scope; cursor events still cover every SQL path. +Pre-hooks share the resumable batch scope. Post-hooks run after the revision body +but before its version transition, so their mutations are covered by the same +completion barrier. + ### Do not hold a request open to wait The managed request uses asynchronous mutation settings and polls afterward. @@ -379,6 +386,20 @@ is a general SQL-rewriting strategy. Parse and replace managed settings instead of relying on duplicate-setting precedence. Do not permanently reset unrelated session settings. +`MODIFY TTL` cannot take an appended ALTER marker: its comma-separated TTL-rule +grammar consumes the apparent next action. The runtime therefore acknowledges +the metadata change with automatic materialization disabled, journals that +boundary, and submits `MATERIALIZE TTL` with its own marker. A resume does not +repeat either submitted operation. Existing session/per-query opt-out is honored. +An unacknowledged metadata change remains an unknown outcome under the approved +rule; it is not silently reissued. Executemany writes re-enter the single-statement +path for each parameter set, preserving an independent intent and completion +barrier rather than collapsing intermediate TTL effects into one materialization. + +The journal and version INSERTs explicitly disable asynchronous insertion. A +migration's session `async_insert=1, wait_for_async_insert=0` must not turn control +state into queued, unacknowledged data. + ### The version table's delete DRY-1412 inserts the new head before deleting the previous one. Its delete is a diff --git a/tests/integration/test_mutation_waiting.py b/tests/integration/test_mutation_waiting.py new file mode 100644 index 0000000..02b244b --- /dev/null +++ b/tests/integration/test_mutation_waiting.py @@ -0,0 +1,718 @@ +"""Real-server waiting, crash recovery, failure, and timeout invariants.""" + +import json +import os +import subprocess +import sys +import time +from contextlib import contextmanager +from pathlib import Path + +import pytest +import yaml + +pytestmark = pytest.mark.integration +CORPUS = yaml.safe_load((Path(__file__).parents[1] / "corpus/classification.yaml").read_text()) +MUTATIONS = [case for case in CORPUS["statements"] if case["expected"] == "mutation"] + + +@pytest.mark.parametrize("case", MUTATIONS, ids=lambda case: case["id"]) +def test_mutation_waits_for_server_corpus(project, case): + values = {"db": project.database, "table": f"{project.database}.sample"} + project.client.command(CORPUS["table_ddl"].format(**values)) + project.client.command(CORPUS["seed"].format(**values)) + if case["id"] == "lightweight_delete": + # Lightweight deletes reject projections under the server's default policy. + project.client.command( + f"ALTER TABLE {values['table']} DROP PROJECTION pr SETTINGS mutations_sync=2" + ) + sql = case["sql"].format(**values) + project.write_revision( + "aaaa", + { + "upgrade": "# ch-migrate: allow-non-idempotent Server-backed mutation corpus\n" + + f"op.execute({sql!r})" + }, + ) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + rows = _mutations(project, "sample") + assert all(row[2] for row in rows), rows + assert any("chm_mutation_" in row[1] for row in rows), rows + assert _heads(project) == ["aaaa"] + + +def test_mutation_failure_leaves_head_and_prints_operator_kill(project): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = throwIf(x = 5) WHERE 1") + result = project.run("up", "it") + assert result.exit_code == 1, result.output + rows = _mutations(project) + failed = next(row for row in rows if row[4]) + assert failed[0] in result.output and "throwIf" in result.output + assert "KILL MUTATION WHERE database =" in result.output + assert f"mutation_id = '{failed[0]}'" in result.output + assert _heads(project) == ["aaaa"] + assert any(not row[2] and row[0] == failed[0] for row in _mutations(project)) + + +def test_mutation_timeout_preserves_work_and_resumes(project): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + try: + result = project.run("up", "it", "--timeout", "0.5") + assert result.exit_code == 1, result.output + rows = _mutations(project) + ids = {row[0] for row in rows} + assert "Timed out" in result.output and "parts_to_do=" in result.output + assert all(mutation in result.output for mutation in ids) + assert _heads(project) == ["aaaa"] + assert any(not row[2] for row in rows) + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert {row[0] for row in _mutations(project)} == ids + assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 1 + assert _heads(project) == ["bbbb"] + + +@pytest.mark.parametrize("trial", range(3)) +def test_mutation_sigkill_rerun_attaches_without_second_increment(project, trial): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + try: + with _running(project, ("up", "it")) as (first, first_log): + rows = _until(lambda: _mutations(project), "mutation was not submitted") + mutation = rows[0][0] + _until( + lambda: mutation in first_log.read_text(), "first process did not report waiting" + ) + first.kill() + first.wait(timeout=10) + original = {row[0] for row in _mutations(project)} + assert _heads(project) == ["aaaa"] + with _running(project, ("up", "it")) as (resumed, log): + _until( + lambda: mutation in log.read_text(), "rerun did not report the original mutation" + ) + assert resumed.poll() is None + assert {row[0] for row in _mutations(project)} == original + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + assert resumed.wait(timeout=30) == 0, log.read_text() + assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 1 + assert _heads(project) == ["bbbb"] + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + + +def test_mutation_crash_before_durable_receipt_recovers_from_intent(project): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + # Fault injection kills the real CLI after server acceptance, before journal acknowledgement. + code = """import os, signal +from ch_migrate.waiting import MigrationWaiter +original = MigrationWaiter._after +def crash(self, connection, cursor, statement, parameters, context, executemany): + pending = self._pending.get(id(context)) + if pending and pending[1]['kind'] == 'mutation' and pending[1]['table'][1] == 'counter': + os.kill(os.getpid(), signal.SIGKILL) + return original(self, connection, cursor, statement, parameters, context, executemany) +MigrationWaiter._after = crash +from ch_migrate.cli import main +main() +""" + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + try: + with _running(project, ("up", "it"), code) as (crashed, log): + assert crashed.wait(timeout=30) == -9, log.read_text() + rows = _mutations(project) + original_ids = {row[0] for row in rows} + assert rows and any("chm_mutation_" in row[1] for row in rows) + payload = project.client.query( + f"SELECT argMax(payload, sequence) FROM {project.database}._ch_migrate_journal " + "WHERE revision = 'bbbb' AND position = 1" + ).result_rows[0][0] + assert json.loads(payload)["phase"] == "intent" + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert {row[0] for row in _mutations(project)} == original_ids + assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 1 + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + + +def test_mutation_foreign_failure_is_not_adopted_or_followed(project): + _seed(project) + project.client.command( + f"ALTER TABLE {project.database}.counter UPDATE x = throwIf(x = 5) WHERE 1" + ) + _until(lambda: any(row[4] for row in _mutations(project)), "foreign mutation did not fail") + original_ids = {row[0] for row in _mutations(project)} + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + result = project.run("up", "it") + assert result.exit_code == 1 and "foreign mutation" in result.output, result.output + assert {row[0] for row in _mutations(project)} == original_ids + assert _heads(project) == ["aaaa"] + + +def test_mutation_expired_ownership_refuses_instead_of_replaying(project): + _seed( + project, + " SETTINGS finished_mutations_to_keep=1, old_parts_lifetime=0, cleanup_delay_period=1, cleanup_delay_period_random_add=0", + ) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + try: + timed_out = project.run("up", "it", "--timeout", "0.4") + assert timed_out.exit_code == 1 + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + _until(lambda: all(row[2] for row in _mutations(project)), "owned mutation did not finish") + project.client.command( + f"ALTER TABLE {project.database}.counter DELETE WHERE 0 SETTINGS mutations_sync=2" + ) + _until( + lambda: not any("chm_mutation_" in row[1] for row in _mutations(project)), + "ownership did not expire", + ) + resumed = project.run("up", "it") + assert resumed.exit_code == 1 and "Outcome unknown" in resumed.output, resumed.output + assert "not reissued" in resumed.output + assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 1 + assert _heads(project) == ["aaaa"] + + +def test_mutation_changed_sql_refuses_replay(project): + _seed(project) + path = _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + try: + assert project.run("up", "it", "--timeout", "0.4").exit_code == 1 + before = {row[0] for row in _mutations(project)} + path.write_text(path.read_text().replace("x + 1", "x + 2")) + result = project.run("up", "it") + assert result.exit_code == 1 and "changed" in result.output, result.output + assert {row[0] for row in _mutations(project)} == before + assert _heads(project) == ["aaaa"] + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + + +def test_mutation_completed_prefix_is_not_repeated_and_downgrade_resets(project): + _seed(project) + project.client.command(f"CREATE TABLE {project.database}.audit (n UInt64) ENGINE=Memory") + path = _upgrade_sql( + project, + "INSERT INTO {db}.audit VALUES (1);\n-- ch-migrate: allow-non-idempotent increment once\nALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1", + ) + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + try: + assert project.run("up", "it", "--timeout", "0.4").exit_code == 1 + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert project.client.command(f"SELECT count() FROM {project.database}.audit") == 1 + reverted = project.run("down", "it") + assert reverted.exit_code == 0, reverted.output + reapplied = project.run("up", "it") + assert reapplied.exit_code == 0, reapplied.output + assert project.client.command(f"SELECT count() FROM {project.database}.audit") == 2 + assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 2 + + +def test_mutation_waits_for_every_replica(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _seed_cluster(project, cluster) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + table = f"{project.database}.counter" + for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): + cluster.clients[2].command(f"SYSTEM STOP {action} {table}") + try: + with _running(project, ("up", "it")) as (process, log): + _until(lambda: _mutations(project), "replicated mutation was not submitted") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table} PULL") + _until(lambda: all(row[2] for row in _mutations(project)), "initiator did not finish") + second = ( + cluster.clients[2] + .query( + "SELECT is_done FROM system.mutations WHERE database = {db:String} AND table = 'counter'", + parameters={"db": project.database}, + ) + .result_rows + ) + assert second and all(not row[0] for row in second) + _until(lambda: "parts_to_do=" in log.read_text(), "replica wait was not reported") + assert process.poll() is None + for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): + cluster.clients[2].command(f"SYSTEM START {action} {table}") + assert process.wait(timeout=30) == 0, log.read_text() + for client in cluster.clients.values(): + assert client.command(f"SELECT x FROM {table} WHERE id = 1") == 1 + finally: + for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): + cluster.clients[2].command(f"SYSTEM START {action} {table}") + + +def test_mutation_missing_replica_is_not_success(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _seed_cluster(project, cluster) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + table = f"{project.database}.counter" + for client in cluster.clients.values(): + client.command(f"SYSTEM STOP MERGES {table}") + try: + first = project.run("up", "it", "--timeout", "0.5") + assert first.exit_code == 1, first.output + original = {row[0] for row in _mutations(project)} + cluster.stop_node(2) + result = project.run("up", "it", "--timeout", "1") + assert result.exit_code == 1 and "Timed out" in result.output, result.output + assert "host" in result.output + assert _heads(project) == ["aaaa"] + assert {row[0] for row in _mutations(project)} == original + finally: + if 2 in cluster.stopped: + cluster.start_node(2) + for client in cluster.clients.values(): + client.command(f"SYSTEM START MERGES {table}") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert _heads(project) == ["bbbb"] + + +def test_mutation_polling_preserves_session_settings(project): + _seed(project) + project.client.command(f"CREATE TABLE {project.database}.observed (value UInt64) ENGINE=Memory") + config_path = project.root / "config.yaml" + config = yaml.safe_load(config_path.read_text()) + config["environments"]["it"]["session_timeout"] = 1 + config_path.write_text(yaml.safe_dump(config)) + _upgrade_sql( + project, + "SET max_threads=3;\n" + "-- ch-migrate: allow-non-idempotent increment once\n" + "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1;\n" + "-- ch-migrate: allow-non-idempotent record the preserved session setting\n" + "INSERT INTO {db}.observed SELECT getSetting('max_threads')", + ) + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + try: + with _running(project, ("up", "it")) as (process, log): + mutation = _until(lambda: _mutations(project), "mutation was not submitted")[0][0] + _until(lambda: mutation in log.read_text(), "wait was not reported") + time.sleep(2.2) # Longer than the server session timeout, while polling continues. + assert process.poll() is None, log.read_text() + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + assert process.wait(timeout=30) == 0, log.read_text() + assert project.client.query( + f"SELECT value FROM {project.database}.observed" + ).result_rows == [(3,)] + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + + +@pytest.mark.parametrize("mode", ["execute", "driver", "executemany"]) +def test_mutation_raw_bind_paths_are_waited(project, mode): + _seed(project) + if mode == "execute": + body = ( + "from sqlalchemy import text\n" + "# ch-migrate: allow-non-idempotent bound counter update\n" + "op.get_bind().execute(text(f'ALTER TABLE {db}.counter UPDATE x=x+:delta WHERE id=:key'), {'delta': 1, 'key': 1})" + ) + else: + values = ( + "[{'delta': 1, 'key': 1}, {'delta': 1, 'key': 5}]" + if mode == "executemany" + else "{'delta': 1, 'key': 1}" + ) + body = ( + "# ch-migrate: allow-non-idempotent bound counter update\n" + f"op.get_bind().exec_driver_sql(f'ALTER TABLE {{db}}.counter UPDATE x=x+%(delta)s WHERE id=%(key)s', {values})" + ) + project.write_revision("bbbb", {"upgrade": body}, "aaaa") + result = project.run("up", "it") + assert result.exit_code == 0, result.output + assert all(row[2] for row in _mutations(project)) + assert project.client.query( + f"SELECT id, x FROM {project.database}.counter ORDER BY id" + ).result_rows == [(1, 1), (5, 6 if mode == "executemany" else 5)] + assert _heads(project) == ["bbbb"] + + +def test_mutation_replaced_table_refuses_resume(project): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + result = project.run("up", "it", "--timeout", "0.4") + assert result.exit_code == 1 + project.client.command(f"DROP TABLE {project.database}.counter SYNC") + project.client.command( + f"CREATE TABLE {project.database}.counter (id UInt64, x UInt64) ENGINE=MergeTree ORDER BY id" + ) + project.client.command(f"INSERT INTO {project.database}.counter VALUES (1, 0)") + resumed = project.run("up", "it") + assert resumed.exit_code == 1 and "replaced" in resumed.output, resumed.output + assert project.client.command(f"SELECT x FROM {project.database}.counter") == 0 + assert _heads(project) == ["aaaa"] + + +def test_mutation_healthy_foreign_predecessor_is_waited_before_submission(project): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.client.command(f"ALTER TABLE {project.database}.counter UPDATE x=x+10 WHERE id=1") + original = {row[0] for row in _mutations(project)} + try: + blocked = project.run("up", "it", "--timeout", "0.4") + assert blocked.exit_code == 1, blocked.output + assert {row[0] for row in _mutations(project)} == original + assert all("chm_mutation_" not in row[1] for row in _mutations(project)) + assert _heads(project) == ["aaaa"] + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id=1") == 11 + + +@pytest.mark.parametrize("scope", ["session", "query"]) +def test_mutation_ttl_materialization_setting_is_respected(project, scope): + project.client.command( + f"CREATE TABLE {project.database}.expiry (ts DateTime) ENGINE=MergeTree ORDER BY ts" + ) + project.client.command(f"INSERT INTO {project.database}.expiry VALUES ('2020-01-01 00:00:00')") + statement = "ALTER TABLE {db}.expiry MODIFY TTL ts + INTERVAL 100 YEAR" + sql = ( + "SET materialize_ttl_after_modify=0;\n" + statement + if scope == "session" + else statement + " SETTINGS materialize_ttl_after_modify=0" + ) + project.write_revision( + "aaaa", {"upgrade": "from ch_migrate import run_sql\nrun_sql('ttl.sql')"} + ) + (project.sql_dir / "ttl.sql").write_text(sql) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + assert _mutations(project, "expiry") == [] + assert "TTL" in project.client.command(f"SHOW CREATE TABLE {project.database}.expiry") + + +def test_mutation_timeout_budget_is_not_reset_for_next_statement(project): + _seed(project) + project.client.command( + f"CREATE TABLE {project.database}.second (id UInt64, x UInt64) ENGINE=MergeTree ORDER BY id" + ) + project.client.command(f"INSERT INTO {project.database}.second VALUES (1, 0)") + _upgrade_sql( + project, + "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1;\n" + "-- ch-migrate: allow-non-idempotent second controlled update\n" + "ALTER TABLE {db}.second UPDATE x=x+1 WHERE id=1", + ) + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.client.command(f"SYSTEM STOP MERGES {project.database}.second") + try: + with _running(project, ("up", "it", "--timeout", "1.5")) as (process, log): + mutation = _until(lambda: _mutations(project), "first statement did not start")[0][0] + _until(lambda: mutation in log.read_text(), "first wait was not visible") + time.sleep(0.7) + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + _until(lambda: _mutations(project, "second"), "second statement did not start") + began_second = time.monotonic() + assert process.wait(timeout=10) == 1, log.read_text() + assert time.monotonic() - began_second < 1.3 + assert "second" in log.read_text() and "Timed out" in log.read_text() + assert _heads(project) == ["aaaa"] + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + project.client.command(f"SYSTEM START MERGES {project.database}.second") + + +def test_mutation_version_rows_ignore_fire_and_forget_insert_settings(project): + project.write_revision( + "aaaa", + { + "upgrade": 'op.execute("SET async_insert=1, wait_for_async_insert=0, async_insert_use_adaptive_busy_timeout=0, async_insert_busy_timeout_ms=60000")' + }, + ) + project.write_revision("bbbb", {"upgrade": 'op.execute("SELECT 1")'}, "aaaa") + result = project.run("up", "it") + assert result.exit_code == 0, result.output + assert _heads(project) == ["bbbb"] + queued = project.client.query( + "SELECT count() FROM system.asynchronous_inserts WHERE database = {db:String} " + "AND table IN ('alembic_version', '_ch_migrate_journal')", + parameters={"db": project.database}, + ).result_rows[0][0] + assert queued == 0 + + +def test_mutation_version_insert_lost_receipt_finishes_bookkeeping_only(project): + _seed(project) + project.client.command(f"CREATE TABLE {project.database}.audit (n UInt64) ENGINE=Memory") + _upgrade_sql(project, "INSERT INTO {db}.audit VALUES (1)") + code = """import os, signal +from ch_migrate.waiting_versions import VersionWrites +original = VersionWrites._insert +def crash(self, step, checkpoint): + original(self, step, checkpoint) + if step['value'] == 'bbbb': + os.kill(os.getpid(), signal.SIGKILL) +VersionWrites._insert = crash +from ch_migrate.cli import main +main() +""" + with _running(project, ("up", "it"), code) as (process, log): + assert process.wait(timeout=30) == -9, log.read_text() + assert _heads(project) == ["aaaa", "bbbb"] + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert _heads(project) == ["bbbb"] + assert project.client.command(f"SELECT count() FROM {project.database}.audit") == 1 + + +def test_mutation_transport_loss_is_not_treated_as_server_rejection(project): + _seed(project) + _upgrade_sql(project, "CREATE TABLE IF NOT EXISTS {db}.uncertain (id UInt64) ENGINE=Memory") + code = """from ch_migrate.waiting import MigrationWaiter +from clickhouse_connect.driver.exceptions import OperationalError +original = MigrationWaiter._after +def lose_reply(self, connection, cursor, statement, parameters, context, executemany): + pending = self._pending.get(id(context)) + if pending and pending[1].get('table', [None, None])[-1] == 'uncertain': + raise OperationalError('Connection lost after server accepted the statement') + return original(self, connection, cursor, statement, parameters, context, executemany) +MigrationWaiter._after = lose_reply +from ch_migrate.cli import main +main() +""" + with _running(project, ("up", "it"), code) as (process, log): + assert process.wait(timeout=30) == 1, log.read_text() + assert project.client.command(f"EXISTS TABLE {project.database}.uncertain") == 1 + resumed = project.run("up", "it") + assert resumed.exit_code == 1 and "Outcome unknown" in resumed.output, resumed.output + assert "Read-only inspection:" in resumed.output + assert _heads(project) == ["aaaa"] + project.client.command("SYSTEM FLUSH LOGS") + requests = project.client.query( + "SELECT count() FROM system.query_log WHERE type = 'QueryStart' " + "AND startsWith(query, {prefix:String})", + parameters={"prefix": f"CREATE TABLE IF NOT EXISTS {project.database}.uncertain"}, + ).result_rows[0][0] + assert requests == 1 + + +def test_mutation_trailing_foreign_failure_does_not_hold_owned_completion(project): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") + code = """from ch_migrate.waiting import MigrationWaiter +original = MigrationWaiter._after +def append_foreign(self, connection, cursor, statement, parameters, context, executemany): + pending = self._pending.get(id(context)) + if pending and pending[1]['kind'] == 'mutation' and pending[1]['table'][1] == 'counter': + self.client.command(f"ALTER TABLE {self.state.database}.counter UPDATE x=throwIf(x=5) WHERE 1") + return original(self, connection, cursor, statement, parameters, context, executemany) +MigrationWaiter._after = append_foreign +from ch_migrate.cli import main +main() +""" + with _running(project, ("up", "it"), code) as (process, log): + assert process.wait(timeout=30) == 0, log.read_text() + _until(lambda: any(row[4] for row in _mutations(project)), "trailing mutation did not fail") + assert _heads(project) == ["bbbb"] + assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id=1") == 1 + + +def test_mutation_ttl_resume_does_not_materialize_twice(project): + project.client.command( + f"CREATE TABLE {project.database}.expiry (ts DateTime) ENGINE=MergeTree ORDER BY ts" + ) + project.client.command(f"INSERT INTO {project.database}.expiry VALUES ('2020-01-01 00:00:00')") + project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) + assert project.run("up", "it").exit_code == 0 + _upgrade_sql(project, "ALTER TABLE {db}.expiry MODIFY TTL ts + INTERVAL 100 YEAR") + project.client.command(f"SYSTEM STOP MERGES {project.database}.expiry") + try: + timed_out = project.run("up", "it", "--timeout", "0.4") + assert timed_out.exit_code == 1, timed_out.output + original = {row[0] for row in _mutations(project, "expiry")} + assert any(not row[2] for row in _mutations(project, "expiry")) + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.expiry") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert {row[0] for row in _mutations(project, "expiry")} == original + assert _heads(project) == ["bbbb"] + + +@pytest.mark.skipif(os.name != "posix", reason="A real PTY is required for terminal progress") +def test_mutation_terminal_progress_uses_a_live_line(project): + import pty + import select + + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + master, slave = pty.openpty() + process = None + try: + process = subprocess.Popen( + [sys.executable, "-m", "ch_migrate.cli", "up", "it", "--timeout", "0.8"], + cwd=project.root, + stdout=subprocess.DEVNULL, + stderr=slave, + ) + # Keep the PTY alive and drain it while the child writes; do not depend + # on buffered output surviving the final slave close. + chunks = [] + deadline = time.monotonic() + 30 + while process.poll() is None or select.select([master], [], [], 0)[0]: + assert time.monotonic() < deadline, "terminal run did not finish" + if select.select([master], [], [], 0.2)[0]: + chunks.append(os.read(master, 4096)) + assert process.wait(timeout=5) == 1 + output = b"".join(chunks) + assert b"\x1b[K" in output and b"parts_to_do=" in output + assert _heads(project) == ["aaaa"] + finally: + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=10) + if slave is not None: + os.close(slave) + os.close(master) + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + + +def test_mutation_parameterized_ttl_batch_preserves_each_operation(project): + project.client.command( + f"CREATE TABLE {project.database}.expiry (ts DateTime) ENGINE=MergeTree ORDER BY ts" + ) + project.client.command(f"INSERT INTO {project.database}.expiry VALUES ('2020-01-01 00:00:00')") + project.write_revision( + "aaaa", + { + "upgrade": "op.get_bind().exec_driver_sql(" + "f'ALTER TABLE {db}.expiry MODIFY TTL ts + toIntervalYear(%(years)s)', " + "[{'years': 100}, {'years': 99}])" + }, + ) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + rows = _mutations(project, "expiry") + assert all(row[2] for row in rows) + assert len({row[0] for row in rows if "chm_mutation_" in row[1]}) == 2 + assert _heads(project) == ["aaaa"] + + +def test_mutation_lightweight_update_finishes_without_fabricating_a_mutation(project): + project.client.command( + f"CREATE TABLE {project.database}.patches (id UInt64, x UInt64) " + "ENGINE=MergeTree ORDER BY id SETTINGS enable_block_number_column=1, " + "enable_block_offset_column=1" + ) + project.client.command(f"INSERT INTO {project.database}.patches VALUES (1, 0)") + project.write_revision( + "aaaa", + { + "upgrade": 'op.execute("SET allow_experimental_lightweight_update=1")\n' + "# ch-migrate: allow-non-idempotent apply one synchronous patch\n" + 'op.get_bind().exec_driver_sql(f"UPDATE {db}.patches SET x=x+1 WHERE id=1")' + }, + ) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + assert project.client.command(f"SELECT x FROM {project.database}.patches") == 1 + assert _mutations(project, "patches") == [] + assert _heads(project) == ["aaaa"] + + +def _seed_cluster(project, cluster): + table = f"{project.database}.counter" + cluster.clients[1].command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64, x UInt64) " + f"ENGINE=ReplicatedMergeTree('/clickhouse/waiting/{project.database}/counter', '{{replica}}') ORDER BY id" + ) + cluster.clients[1].command(f"INSERT INTO {table} VALUES (1, 0), (5, 5)") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") + project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + + +def _seed(project, settings=""): + project.client.command( + f"CREATE TABLE {project.database}.counter (id UInt64, x UInt64) ENGINE=MergeTree ORDER BY id{settings}" + ) + project.client.command(f"INSERT INTO {project.database}.counter VALUES (1, 0), (5, 5)") + project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + + +def _upgrade_sql(project, sql): + path = project.sql_dir / "mutation.sql" + path.write_text( + "-- ch-migrate: allow-non-idempotent Controlled waiting acceptance scenario\n" + sql + ";\n" + ) + project.write_revision( + "bbbb", + {"upgrade": "from ch_migrate import run_sql\nrun_sql('mutation.sql')"}, + "aaaa", + ) + return path + + +def _mutations(project, table="counter"): + return project.client.query( + "SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason FROM system.mutations " + "WHERE database = {db:String} AND table = {table:String} ORDER BY mutation_id", + parameters={"db": project.database, "table": table}, + ).result_rows + + +def _heads(project): + return [ + row[0] + for row in project.client.query( + f"SELECT version_num FROM {project.database}.alembic_version ORDER BY version_num" + ).result_rows + ] + + +def _until(predicate, message): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(0.05) + pytest.fail(message) + + +@contextmanager +def _running(project, args, code=None): + path = project.root / f"run-{time.monotonic_ns()}.log" + command = ( + [sys.executable, "-c", code] if code else [sys.executable, "-m", "ch_migrate.cli"] + ) + with path.open("w") as log: + process = subprocess.Popen( + [*command, *args], + cwd=project.root, + stdout=log, + stderr=subprocess.STDOUT, + env=os.environ.copy(), + ) + try: + yield process, path + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) diff --git a/tests/integration/test_version_table.py b/tests/integration/test_version_table.py index dddb7da..7a80492 100644 --- a/tests/integration/test_version_table.py +++ b/tests/integration/test_version_table.py @@ -24,11 +24,20 @@ def test_version_table_interrupted_delete_does_not_replay(project): process = _start_up(project) try: _wait_for(lambda: set(_heads(project)) == {"aaaa", "bbbb"}) - _wait_for_version_delete(project, process) + original_ids = _wait_for_version_delete(project, process) assert process.poll() is None, "up must still be waiting for the version DELETE" _stop_process(process) - refused = _completed_up(project) - assert refused[0] != 0 and "overlap" in refused[1].lower(), refused + refused = _completed_up(project, ("--timeout", "0.4")) + assert refused[0] != 0, refused + current_ids = { + row[0] + for row in project.client.query( + "SELECT mutation_id FROM system.mutations WHERE database = {db:String} " + "AND table = 'alembic_version'", + parameters={"db": project.database}, + ).result_rows + } + assert current_ids == original_ids assert _probe_counts(project) == [("aaaa", 1), ("bbbb", 1)] finally: _stop_process(process) @@ -38,7 +47,6 @@ def test_version_table_interrupted_delete_does_not_replay(project): assert completed[0] == 0, completed[1] assert _probe_counts(project) == [("aaaa", 1), ("bbbb", 1), ("cccc", 1)] assert _heads(project) == ["cccc"] - _assert_version_sql(project) def test_version_table_failed_mutation_refuses_without_killing(project): @@ -203,9 +211,9 @@ def _probe_revision(project, revision, parent): ) -def _start_up(project): +def _start_up(project, extra=()): return subprocess.Popen( - [sys.executable, "-m", "ch_migrate.cli", "up", "it"], + [sys.executable, "-m", "ch_migrate.cli", "up", "it", *extra], cwd=project.root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -224,8 +232,8 @@ def _stop_process(process): return process.communicate(timeout=5)[0] -def _completed_up(project): - process = _start_up(project) +def _completed_up(project, extra=()): + process = _start_up(project, extra) try: output = process.communicate(timeout=20)[0] return process.returncode, output @@ -268,7 +276,7 @@ def _wait_for_version_delete(project, process): parameters={"db": project.database}, ).result_rows if any(not done and "DELETE" in command.upper() for _, command, done, _ in rows): - return + return {mutation for mutation, _, _, _ in rows} if process.poll() is not None: break time.sleep(0.05) @@ -286,25 +294,6 @@ def _failed_mutations(project, client=None): ).result_rows -def _assert_version_sql(project): - project.client.command("SYSTEM FLUSH LOGS") - queries = [ - row[0] - for row in project.client.query( - "SELECT DISTINCT replaceAll(query, '`', '') FROM system.query_log WHERE type = 'QueryStart' " - "AND position(query, {db:String}) > 0 AND position(query, 'alembic_version') > 0", - parameters={"db": project.database}, - ).result_rows - ] - prefix = f"{project.database}.alembic_version" - assert any(query.startswith(f"INSERT INTO {prefix}") for query in queries) - deletes = [query for query in queries if query.startswith(f"ALTER TABLE {prefix} DELETE")] - assert deletes and all( - re.search(r"mutations_sync\s*=\s*2", query) for query in deletes - ), queries - assert not any(query.startswith(f"ALTER TABLE {prefix} UPDATE") for query in queries) - - def _bootstrap_cluster_user(project, cluster, request): monkeypatch = request.getfixturevalue("monkeypatch") user = project.database + "_migration" diff --git a/tests/test_waiting.py b/tests/test_waiting.py new file mode 100644 index 0000000..1a51af4 --- /dev/null +++ b/tests/test_waiting.py @@ -0,0 +1,93 @@ +"""Ownership SQL and deadline/completion boundaries that could silently replay work.""" + +import pytest +from click.testing import CliRunner + +from ch_migrate.cli import main +from ch_migrate.waiting_mutations import MutationWaiter +from ch_migrate.waiting_sql import ( + is_session_or_read, + mutation_sql, + query_setting, + statement_table, +) +from ch_migrate.waiting_types import WaitBudget, WaitTimeout + + +@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf", "-inf"]) +def test_wait_timeout_rejects_invalid_values_before_project_access(tmp_path, monkeypatch, timeout): + # Click's public command signature, not a database mock, decides usage errors. + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(main, ["up", "not-configured", "--timeout", timeout]) + assert result.exit_code == 2 + assert "--timeout" in result.output + + +def test_mutation_settings_override_ignores_comment_and_literal_keywords(): + sql = ( + "ALTER TABLE db.t UPDATE value = 'SETTINGS -- not a comment', settings=settings+1 " + "WHERE id=1 -- trailing predicate comment\n" + "SETTINGS /* policy */ mutations_sync=2, max_threads=3 -- trailing setting comment\n;" + ) + rewritten = mutation_sql(sql, "owned-token") + assert query_setting(rewritten, "mutations_sync") == "0" + assert query_setting(rewritten, "max_threads") == "3" + assert "'SETTINGS -- not a comment'" in rewritten + assert "settings=settings+1 WHERE id=1" in rewritten + assert "DELETE WHERE 0" in rewritten + + +def test_settings_column_is_not_a_settings_clause(): + rewritten = mutation_sql("ALTER TABLE db.t UPDATE settings=2 WHERE settings=1", "owned-token") + assert "UPDATE settings=2 WHERE settings=1" in rewritten + assert query_setting(rewritten, "mutations_sync") == "0" + + +def test_qualified_identifiers_preserve_dots_inside_quoted_names(): + assert statement_table( + 'ALTER TABLE "data.base".`some.table` UPDATE x=1 WHERE 1', "default" + ) == ("data.base", "some.table") + + +@pytest.mark.parametrize( + "sql,readonly", + [ + ("WITH 1 AS x SELECT x", True), + ("WITH source AS (SELECT 1) INSERT INTO t SELECT * FROM source", False), + ("INSERT INTO t SELECT 'SELECT'", False), + ("SET max_threads=3", True), + ], +) +def test_statement_tracking_distinguishes_reads_from_writes(sql, readonly): + assert is_session_or_read(sql) is readonly + + +def test_deadline_is_shared_across_waits(monkeypatch): + now = [10.0] + monkeypatch.setattr("ch_migrate.waiting_types.time.monotonic", lambda: now[0]) + budget = WaitBudget(timeout=2, started=now[0]) + now[0] = 11.5 + assert budget.elapsed == 1.5 + now[0] = 12.0 + with pytest.raises(WaitTimeout, match="second mutation"): + budget.check("second mutation") + + +def test_zero_parts_is_not_mutation_completion(monkeypatch): + waiter = MutationWaiter(None, None, WaitBudget(timeout=0)) + receipt = { + "database": "db", + "table": "t", + "engine": "MergeTree", + "initiator": "host", + "token": "owned", + "uuids": {"host": "uuid"}, + "ids": {"host": ["mutation_1.txt"]}, + "done_hosts": [], + "baseline": {"host": []}, + } + monkeypatch.setattr(waiter, "_identities", lambda _: {"host": "uuid"}) + monkeypatch.setattr(waiter, "_rows", lambda _: [("host", "mutation_1.txt", "owned", 0, 0, "")]) + with pytest.raises(WaitTimeout, match="mutation_1.txt"): + waiter.wait(receipt, lambda: None) + assert receipt["done_hosts"] == [] From 2452f8e87d8a99b0cf92c7e9dbf0e61981301a5e Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 13 Oct 2026 11:35:36 -0400 Subject: [PATCH 05/33] [db] Keep re-inserted version rows on replicas Replicated and Shared MergeTree drop an INSERT block identical to an earlier one. After `down` re-inserted a previous head, or `up` re-inserted a revision applied before a downgrade, the row was silently deduplicated while the DELETE of the old head still ran, leaving alembic_version empty so the next `up` replays everything. Each version-row INSERT now carries its own deduplication token, and downgrade version writes take the same path. Cloud ignored insert_deduplicate = 0 in probes; only a unique token kept the row. Solves: DRY-1447 Tests: round-trip down/up on Docker replicas (failed before) and single node; irreversible + version-table suite 5x on Cloud; 715 unit (cherry picked from commit c9d19ebc36b3cc0d3f13241e39e29d4a55facab8) --- ch_migrate/alembic_env.py | 15 +++++++------- ch_migrate/waiting_versions.py | 38 +++++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index f59145f..4d2901a 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -27,6 +27,7 @@ sync_version_replica, ) from ch_migrate.waiting import MigrationWaiter +from ch_migrate.waiting_versions import write_unjournaled ENV_VERSION = 2 DEFAULT_SESSION_TIMEOUT = 1800 @@ -59,14 +60,14 @@ def version_table_impl(self, *, version_table, version_table_schema, version_tab def _exec(self, construct, execution_options=None, multiparams=None, params=None): # The official implementation owns these SQLAlchemy version constructs; # all user SQL, including raw bind execution, is covered by cursor events. - waiter = self.context_opts.get("ch_migrate_waiter") - if ( - waiter is not None - and waiter.revision is not None - and isinstance(construct, (Insert, Update, Delete)) - and self._is_version_table_construct(construct) + if isinstance(construct, (Insert, Update, Delete)) and self._is_version_table_construct( + construct ): - return waiter.versions.execute(construct, self) + waiter = self.context_opts.get("ch_migrate_waiter") + if waiter is not None and waiter.revision is not None: + return waiter.versions.execute(construct, self) + if not isinstance(construct, Delete): + return write_unjournaled(construct, self) return super()._exec( construct, execution_options=execution_options, multiparams=multiparams, params=params ) diff --git a/ch_migrate/waiting_versions.py b/ch_migrate/waiting_versions.py index dcf5fe7..52e1bb1 100644 --- a/ch_migrate/waiting_versions.py +++ b/ch_migrate/waiting_versions.py @@ -6,9 +6,16 @@ from types import SimpleNamespace from uuid import uuid4 +from sqlalchemy import text from sqlalchemy.sql.dml import Delete, Insert, Update -from ch_migrate.waiting_sql import mutation_sql, qualified_table, statement_digest +from ch_migrate.waiting_sql import ( + mutation_sql, + qualified_table, + query_settings, + sql_string, + statement_digest, +) from ch_migrate.waiting_types import UnknownOutcome, WaitingError @@ -92,19 +99,29 @@ def _delete(self, step: dict, checkpoint) -> None: self.owner.mutations.wait(step["receipt"], checkpoint) +def write_unjournaled(construct, impl): + """Apply a version INSERT/UPDATE outside a tracked upgrade, such as a downgrade.""" + for step in _version_steps(construct, impl): + sql = step["sql"] + if step["kind"] == "delete": + sql = query_settings(sql, {"mutations_sync": "2"}) + impl._exec(text(sql)) + return SimpleNamespace(rowcount=1) + + def _version_steps(construct, impl) -> list[dict]: if isinstance(construct, Insert): value = _version_value(construct) - sql = str(construct.compile(dialect=impl.dialect, compile_kwargs={"literal_binds": True})) - return [{"kind": "insert", "value": value, "sql": sql}] + return [ + {"kind": "insert", "value": value, "sql": _insert_sql(impl._version_table_name, value)} + ] if isinstance(construct, Update): value = _version_value(construct) - literal = impl._compile_clause(next(iter(construct._values.values()))) return [ { "kind": "insert", "value": value, - "sql": f"INSERT INTO {impl._version_table_name} (version_num) VALUES ({literal})", + "sql": _insert_sql(impl._version_table_name, value), }, { "kind": "delete", @@ -121,6 +138,17 @@ def _version_steps(construct, impl) -> list[dict]: raise WaitingError("Unsupported version-table transition") +def _insert_sql(table: str, value: str) -> str: + # Replicated and Shared MergeTree silently drop a block identical to an earlier insert + # (DRY-1447), so re-inserting a previous head after `down` left no version row at all. + # A token unique to this write keeps every intentional version row. + token = sql_string("chm_version_" + uuid4().hex) + return ( + f"INSERT INTO {table} (version_num) SETTINGS insert_deduplication_token = {token}, " + f"async_insert = 0 VALUES ({sql_string(value)})" + ) + + def _version_value(construct) -> str: values = construct._values expression = next(iter(values.values())) if values else None From 474b845803634d71eded210d7f7a64598ccf14c8 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Sun, 4 Oct 2026 14:47:40 -0400 Subject: [PATCH 06/33] [db] Wait for every distributed DDL host Own queue entries with durable markers and require successful results from every target host. Resume the existing entry after interruption, then wait for any mutations it created before completing the revision. Apply the same deadline to cold version-table creation on a cluster. Solves: DRY-1400 Tests: DDL host/crash/TTL cases; 655 tests on Python 3.10-3.14; 137 integration (cherry picked from commit 34d95ebe602ee853024a7a7870247329e38cff68) (cherry picked from commit 52852784452e19bf43324e2b49ffa1ef935a5cfa) --- README.md | 20 ++ ch_migrate/alembic_env.py | 1 + ch_migrate/bootstrap.py | 2 + ch_migrate/templates/bootstrap/init_users.sql | 2 + ch_migrate/waiting.py | 140 +++++++- ch_migrate/waiting_ddl.py | 112 ++++++ ch_migrate/waiting_mutations.py | 7 +- ch_migrate/waiting_sql.py | 8 + ch_migrate/waiting_store.py | 6 + tests/integration/test_ddl_queue.py | 328 ++++++++++++++++++ 10 files changed, 609 insertions(+), 17 deletions(-) create mode 100644 ch_migrate/waiting_ddl.py create mode 100644 tests/integration/test_ddl_queue.py diff --git a/README.md b/README.md index ffedb30..78b55a3 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,26 @@ barrier for each parameter set; intermediate TTL effects are not collapsed. Lightweight DELETE keeps its row-mask semantics; lightweight UPDATE is synchronous patch work, not a fabricated `system.mutations` record. +For `ON CLUSTER` statements, the same journal first records a unique distributed +DDL marker. `up` submits without a server-side synchronous DDL wait, then polls +`system.distributed_ddl_queue`. Every configured target host must report +`Finished` **and** exception code zero. A failed host is named with its error; +a down host remains pending until it returns or the shared `--timeout` expires. +The tool neither deletes the queue entry nor resubmits the DDL on rerun. + +The queue entry is found by its recorded `log_comment`, not by matching SQL text +(the server can insert UUIDs into that text). Caller log comments are preserved +in the journal. A lost acknowledgement can reattach to the existing entry; +missing/expired queue evidence follows the unknown-outcome procedure above. +Mutation-producing distributed ALTERs must satisfy **both** barriers: DDL execution +on all hosts and completion of their owned mutations. TTL metadata and explicit +materialization retain separate queue receipts. + +Single-node statements without `ON CLUSTER` do not consult that queue. Ordinary +Cloud DDL does not need `ON CLUSTER`; Cloud qualification remains separate from +the local harness. Re-run bootstrap when upgrading a restricted migration user +to obtain the available cluster and distributed-queue inspection grants. + ### Re-runnable migrations `up` statically checks pending upgrade statements before Alembic executes any of diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index 4d2901a..072699c 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -168,6 +168,7 @@ def _run_context(waiter: MigrationWaiter, hooks: HookRegistry) -> None: runtime._migrations_fn = waiter.steps(runtime._migrations_fn, hooks) with context.begin_transaction(): if _is_upgrade(): + waiter.ensure_version_table(runtime._version) waiter.versions.resume() waiter.run_pre_hooks(hooks) else: diff --git a/ch_migrate/bootstrap.py b/ch_migrate/bootstrap.py index 6d9ad50..8f73952 100644 --- a/ch_migrate/bootstrap.py +++ b/ch_migrate/bootstrap.py @@ -149,6 +149,8 @@ def build_bootstrap_sql( f"GRANT SELECT ON system.mutations TO {project}_migration_role;", f"GRANT SELECT ON system.columns TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SELECT ON system.replicas) TO {project}_migration_role;", + f"GRANT CURRENT GRANTS(SELECT ON system.clusters) TO {project}_migration_role;", + f"GRANT CURRENT GRANTS(SELECT ON system.distributed_ddl_queue) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SYSTEM SYNC REPLICA ON {db}.*) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SYSTEM SYNC DATABASE REPLICA ON {db}.*) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(CLUSTER ON *.*) TO {project}_migration_role;", diff --git a/ch_migrate/templates/bootstrap/init_users.sql b/ch_migrate/templates/bootstrap/init_users.sql index da29cbe..0f29a45 100644 --- a/ch_migrate/templates/bootstrap/init_users.sql +++ b/ch_migrate/templates/bootstrap/init_users.sql @@ -27,6 +27,8 @@ GRANT SELECT ON system.tables TO {project}_migration_role; GRANT SELECT ON system.mutations TO {project}_migration_role; GRANT SELECT ON system.columns TO {project}_migration_role; GRANT CURRENT GRANTS(SELECT ON system.replicas) TO {project}_migration_role; +GRANT CURRENT GRANTS(SELECT ON system.clusters) TO {project}_migration_role; +GRANT CURRENT GRANTS(SELECT ON system.distributed_ddl_queue) TO {project}_migration_role; GRANT CURRENT GRANTS(SYSTEM SYNC REPLICA ON {db}.*) TO {project}_migration_role; GRANT CURRENT GRANTS(SYSTEM SYNC DATABASE REPLICA ON {db}.*) TO {project}_migration_role; GRANT CURRENT GRANTS(CLUSTER ON *.*) TO {project}_migration_role; diff --git a/ch_migrate/waiting.py b/ch_migrate/waiting.py index cc4d6a6..a1bc001 100644 --- a/ch_migrate/waiting.py +++ b/ch_migrate/waiting.py @@ -13,12 +13,14 @@ from alembic.script import ScriptDirectory from clickhouse_connect.driver.exceptions import DatabaseError from sqlalchemy import event +from sqlalchemy.schema import CreateTable from ch_migrate.classify import classify from ch_migrate.hooks import run_hooks from ch_migrate.idempotency import classify_idempotency from ch_migrate.introspect import ColumnDefinition, Schema, TableDefinition, _parse_order_by from ch_migrate.statements import migration_statements +from ch_migrate.waiting_ddl import DistributedDDLWaiter from ch_migrate.waiting_mutations import MutationWaiter from ch_migrate.waiting_sql import ( bind_statement, @@ -28,6 +30,7 @@ qualified_table, query_setting, query_settings, + remove_query_settings, sql_string, statement_cluster, statement_digest, @@ -47,6 +50,7 @@ def __init__(self, connection, state, timeout=None): self.budget = WaitBudget(timeout) self.journal = WaitingJournal(connection, state, self.budget) self.mutations = MutationWaiter(self.client, state, self.budget) + self.ddl = DistributedDDLWaiter(self.client, self.budget) self.versions = VersionWrites(self) self.progress_stream = sys.stderr self.revision = None @@ -137,6 +141,13 @@ def unknown_outcome(self, key: StepKey, record: dict, error: Exception) -> Unkno "Read-only inspection: SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason " f"FROM {source} WHERE database = {sql_string(table[0])} AND table = {sql_string(table[1])};" ) + for name in ("ddl", "work_ddl"): + if record.get(name): + message += ( + "\nRead-only inspection: SELECT entry, host, port, status, exception_code, exception_text " + "FROM system.distributed_ddl_queue WHERE settings['log_comment'] = " + f"{sql_string(record[name]['token'])};" + ) return UnknownOutcome(message) def steps(self, original, hooks): @@ -160,6 +171,19 @@ def iterate(heads, context): return iterate + def ensure_version_table(self, table) -> None: + if not self.state.on_cluster: + return + scope = "__version_table__" + if self.state.table_engine and not self.journal.unfinished_scope(scope): + return + ddl = str(CreateTable(table, if_not_exists=True).compile(dialect=self.connection.dialect)) + self._begin(scope, statement_digest(ddl, None)) + try: + self.connection.exec_driver_sql(ddl) + finally: + self.revision = None + def run_pre_hooks(self, hooks) -> None: fingerprint = hashlib.sha256(json.dumps(hooks.pre_migrate).encode()).hexdigest() self._begin("__batch__", fingerprint) @@ -200,7 +224,11 @@ def _begin(self, revision: str, fingerprint: str) -> None: raise UnknownOutcome( f"Revision {revision} changed with unresolved work; statement not reissued" ) - if record["phase"] == "accepted" and record["kind"] == "write": + if ( + record["phase"] == "accepted" + and record["kind"] == "write" + and not record.get("ddl") + ): self._complete(key, record) if record["phase"] == "done" and record.get("table"): self._completed_targets[tuple(record["table"])] = record.get("after_target") @@ -240,7 +268,15 @@ def _before(self, connection, cursor, statement, parameters, context, executeman key, record, fresh = self.reserve(payload) if not fresh and self._resume(key, record): return "SELECT 1 WHERE 0", {} - target = self.mutations.target(table) if table else None + cluster = statement_cluster(sql) + if cluster: + record["ddl"] = self.ddl.prepare(cluster, record["token"] + "_ddl") + record["caller_log_comment"] = query_setting(sql, "log_comment") + target = self._target(table, record, allow_missing=True) if table else None + if cluster and table and target is None and classify(sql).kind == "mutation": + raise WaitingError( + f"No reachable mutation target {table} in cluster {cluster}; nothing was submitted" + ) record["before_target"] = target kind = self._work_kind(sql, target, sample) if target else None if kind: @@ -255,6 +291,11 @@ def _before(self, connection, cursor, statement, parameters, context, executeman ) else: statement = mutation_sql(statement, token) + if record.get("ddl"): + statement, settings = self._ddl_submission(statement, record["ddl"]) + previous = self._statement_settings(context, settings) + if record["caller_log_comment"] is None: + record["caller_log_comment"] = previous.get("log_comment") self.journal.write(key, record) # Acknowledged intent MUST precede submission. self._pending[id(context)] = (key, record) return statement, parameters @@ -275,7 +316,7 @@ def _after(self, connection, cursor, statement, parameters, context, executemany key, record = pending record["phase"] = "accepted" self.journal.write(key, record) - if record["kind"] in ("mutation", "ttl"): + if record.get("ddl") or record["kind"] in ("mutation", "ttl"): self._wait_work(key, record) self._complete(key, record) @@ -291,6 +332,7 @@ def _error(self, context): if ( record["kind"] == "write" and record["repeat_safe"] + and not record.get("ddl") and isinstance(context.original_exception, DatabaseError) and context.original_exception.code is not None ): @@ -301,8 +343,10 @@ def _resume(self, key: StepKey, record: dict) -> bool: if record["phase"] == "done": self._check_completed_target(record) return True - if record["kind"] == "mutation" or ( - record["kind"] == "ttl" and record["phase"] == "accepted" + if ( + record.get("ddl") + or record["kind"] == "mutation" + or (record["kind"] == "ttl" and record["phase"] == "accepted") ): print( f"Resuming {key.revision} statement {key.position}; not reissuing SQL", @@ -318,23 +362,83 @@ def _resume(self, key: StepKey, record: dict) -> bool: raise self.unknown_outcome(key, record, error) from error def _wait_work(self, key: StepKey, record: dict) -> None: - if record["kind"] == "ttl" and not record["work_submitted"]: - record["work_submitted"] = True - self.journal.write(key, record) + checkpoint = lambda: self.journal.write(key, record) + try: + if record.get("ddl"): + self.ddl.wait(record["ddl"], checkpoint) + record["phase"] = "accepted" + checkpoint() + if record["kind"] == "ttl": + self._materialize_ttl(record, checkpoint) + if record["kind"] in ("mutation", "ttl"): + self._refresh_distributed_target(record) + self.mutations.wait(record["receipt"], checkpoint) + except UnknownOutcome as error: + raise self.unknown_outcome(key, record, error) from error + + def _materialize_ttl(self, record: dict, checkpoint) -> None: + if not record["work_submitted"]: cluster = ( f" ON CLUSTER {sql_string(record['cluster'])}" if record.get("cluster") else "" ) sql = f"ALTER TABLE {qualified_table(tuple(record['table']))}{cluster} MATERIALIZE TTL" + sql = mutation_sql(sql, record["receipt"]["token"]) + settings = {} + if record.get("ddl"): + record["work_ddl"] = self.ddl.prepare(record["cluster"], record["token"] + "_ttl") + sql, settings = self._ddl_submission(sql, record["work_ddl"]) + record["work_submitted"] = True + checkpoint() with self.internal(): - self.connection.exec_driver_sql(mutation_sql(sql, record["receipt"]["token"])) - try: - self.mutations.wait(record["receipt"], lambda: self.journal.write(key, record)) - except UnknownOutcome as error: - raise self.unknown_outcome(key, record, error) from error + self.connection.exec_driver_sql(sql, execution_options={"settings": settings}) + if record.get("work_ddl"): + self.ddl.wait(record["work_ddl"], checkpoint) + + def _refresh_distributed_target(self, record: dict) -> None: + if not record.get("ddl"): + return + actual = self._target(tuple(record["table"]), record) + if actual is None: + raise UnknownOutcome("Distributed mutation target disappeared; statement not reissued") + receipt = record["receipt"] + for host, uuid in receipt["uuids"].items(): + if actual["uuids"].get(host) != uuid: + raise UnknownOutcome( + f"Distributed mutation target was replaced on {host}; statement not reissued" + ) + receipt["uuids"] = actual["uuids"] + + def _target(self, table, record, allow_missing=False): + if record.get("ddl"): + return self.ddl.tables(table, record["ddl"]["cluster"], allow_missing) + return self.mutations.target(table) + + @staticmethod + def _ddl_submission(statement: str, receipt: dict) -> tuple[str, dict]: + settings = { + "distributed_ddl_task_timeout": 0, + "distributed_ddl_output_mode": "none", + "log_comment": receipt["token"], + } + return remove_query_settings(statement, set(settings)), settings + + @staticmethod + def _statement_settings(context, overrides: dict) -> dict: + previous = dict(context.execution_options.get("settings") or {}) + statement = getattr(context, "invoked_statement", None) + if statement is not None: + previous.update(statement.get_execution_options().get("settings") or {}) + context.invoked_statement = statement.execution_options( + settings={**previous, **overrides} + ) + context.execution_options = context.execution_options.union( + {"settings": {**previous, **overrides}} + ) + return previous def _complete(self, key: StepKey, record: dict) -> None: if record.get("table"): - record["after_target"] = self.mutations.target(tuple(record["table"])) + record["after_target"] = self._target(tuple(record["table"]), record) self._completed_targets[tuple(record["table"])] = record["after_target"] record["phase"] = "done" self.journal.write(key, record) @@ -343,7 +447,7 @@ def _check_completed_target(self, record: dict) -> None: if not record.get("table"): return expected = self._completed_targets.get(tuple(record["table"]), record.get("after_target")) - actual = self.mutations.target(tuple(record["table"])) + actual = self._target(tuple(record["table"]), record) if ( (expected is None) != (actual is None) or expected is not None @@ -366,7 +470,11 @@ def _work_kind(self, sql: str, target: dict, parameters) -> str | None: columns=[ColumnDefinition(*row) for row in rows], order_by=_parse_order_by(target["sorting_key"]) if target["sorting_key"] else [], ) - schema = Schema(database=target["database"], tables={target["table"]: table}) + schema = ( + None + if target.get("ddl_scope") + else Schema(database=target["database"], tables={target["table"]: table}) + ) ttl = modifies_ttl(sql) materialize = self._materializes_ttl(sql, parameters) if ttl else False analyzed = ( diff --git a/ch_migrate/waiting_ddl.py b/ch_migrate/waiting_ddl.py new file mode 100644 index 0000000..ab6c2c9 --- /dev/null +++ b/ch_migrate/waiting_ddl.py @@ -0,0 +1,112 @@ +"""Distributed DDL receipts identified by an owned queue marker, never query text.""" + +from __future__ import annotations + +from ch_migrate.waiting_types import UnknownOutcome, WaitingError + + +class DistributedDDLWaiter: + def __init__(self, client, budget): + self.client, self.budget = client, budget + + def prepare(self, cluster: str, token: str) -> dict: + hosts = self.client.query( + "SELECT host_name, port FROM system.clusters WHERE cluster = {cluster:String} " + "ORDER BY shard_num, replica_num", + parameters={"cluster": cluster}, + ).result_rows + if not hosts: + raise WaitingError(f"Cluster {cluster!r} has no configured DDL hosts") + return { + "cluster": cluster, + "token": token, + "hosts": [list(host) for host in hosts], + "entry": None, + "done_hosts": [], + } + + def wait(self, receipt: dict, checkpoint) -> None: + expected = {_host_key(host, port) for host, port in receipt["hosts"]} + while not expected.issubset(receipt["done_hosts"]): + rows = self._rows(receipt) + entries = {row[0] for row in rows} + if not entries: + raise UnknownOutcome( + "Distributed DDL outcome unknown: queue evidence is missing or expired; statement not reissued" + ) + if len(entries) != 1: + raise UnknownOutcome( + "Distributed DDL ownership marker identifies multiple queue entries; reconcile without resubmitting" + ) + entry = next(iter(entries)) + if receipt["entry"] is not None and receipt["entry"] != entry: + raise UnknownOutcome( + "Distributed DDL queue entry changed for a recorded intent; statement not reissued" + ) + if receipt["entry"] is None: + receipt["entry"] = entry + checkpoint() + pending = self._host_states(receipt, rows, checkpoint) + if pending: + self.budget.pause(f"DDL {entry}: " + "; ".join(pending)) + self.budget.complete(f"DDL {receipt['entry']}; every required host finished") + + def tables( + self, table: tuple[str, str], cluster: str, allow_missing: bool = False + ) -> dict | None: + rows = self.client.query( + "SELECT materialize(hostName()), toString(uuid), engine, sorting_key FROM " + "clusterAllReplicas({cluster:String}, system.tables) " + "WHERE database = {db:String} AND name = {table:String}", + parameters={"cluster": cluster, "db": table[0], "table": table[1]}, + settings={ + "skip_unavailable_shards": int(allow_missing), + "connect_timeout_with_failover_ms": 250, + }, + ).result_rows + if not rows: + return None + if len({row[2] for row in rows}) != 1: + raise WaitingError( + "Distributed DDL target engines differ across hosts; reconcile them before tracked execution" + ) + return { + "database": table[0], + "table": table[1], + "cluster": cluster, + "ddl_scope": True, + "engine": rows[0][2], + "sorting_key": rows[0][3], + "initiator": rows[0][0], + "uuids": {host: uuid for host, uuid, _, _ in rows}, + } + + def _rows(self, receipt: dict) -> list: + return self.client.query( + "SELECT entry, host, port, status, exception_code, exception_text " + "FROM system.distributed_ddl_queue WHERE settings['log_comment'] = {token:String}", + parameters={"token": receipt["token"]}, + ).result_rows + + def _host_states(self, receipt: dict, rows: list, checkpoint) -> list[str]: + observed = {} + for _, host, port, status, code, message in rows: + key = _host_key(host, port) + if code: + raise WaitingError( + f"Distributed DDL {receipt['entry']} failed on {key}: code {code}: {message}" + ) + observed[key] = status + if status == "Finished" and code == 0 and key not in receipt["done_hosts"]: + receipt["done_hosts"].append(key) + checkpoint() + pending = [] + for host, port in receipt["hosts"]: + key = _host_key(host, port) + if key not in receipt["done_hosts"]: + pending.append(f"host {key}: {observed.get(key) or 'missing queue row'}") + return pending + + +def _host_key(host, port) -> str: + return f"{host}:{port}" diff --git a/ch_migrate/waiting_mutations.py b/ch_migrate/waiting_mutations.py index 829dbb6..e46cdea 100644 --- a/ch_migrate/waiting_mutations.py +++ b/ch_migrate/waiting_mutations.py @@ -125,7 +125,12 @@ def _bind_markers(self, receipt: dict, rows: list, checkpoint) -> None: barrier = max(_order(value) for value in markers) baseline = set(receipt["baseline"].get(host, [])) receipt["ids"][host] = sorted( - {row[1] for row in local if row[1] not in baseline and _order(row[1]) <= barrier} + markers + | { + row[1] + for row in local + if not row[3] and row[1] not in baseline and _order(row[1]) <= barrier + } ) checkpoint() if not found: diff --git a/ch_migrate/waiting_sql.py b/ch_migrate/waiting_sql.py index 7ea1e83..0ba70f4 100644 --- a/ch_migrate/waiting_sql.py +++ b/ch_migrate/waiting_sql.py @@ -131,6 +131,14 @@ def query_settings(statement: str, overrides: dict[str, str]) -> str: return _with_settings(body, settings, overrides) +def remove_query_settings(statement: str, names: set[str]) -> str: + body, settings = _query_parts(statement) + retained = [ + item for item in settings if _identifier(item.split("=", 1)[0].strip()).lower() not in names + ] + return body + (" SETTINGS " + ", ".join(retained) if retained else "") + + def query_setting(statement: str, name: str) -> str | None: _, settings = _query_parts(statement) for setting in reversed(settings): diff --git a/ch_migrate/waiting_store.py b/ch_migrate/waiting_store.py index a0c0d15..d746c0b 100644 --- a/ch_migrate/waiting_store.py +++ b/ch_migrate/waiting_store.py @@ -122,6 +122,12 @@ def write(self, key: StepKey, payload: dict) -> None: ) self._sequences[key] = sequence + def unfinished_scope(self, revision: str) -> bool: + if not self.exists(): + return False + generation = self._generation(revision) + return any(record["phase"] != "done" for _, record in self.records(revision, generation)) + def unfinished_versions(self) -> list[tuple[StepKey, dict]]: if not self.exists(): return [] diff --git a/tests/integration/test_ddl_queue.py b/tests/integration/test_ddl_queue.py new file mode 100644 index 0000000..ede4797 --- /dev/null +++ b/tests/integration/test_ddl_queue.py @@ -0,0 +1,328 @@ +"""Owned distributed DDL waits, failure propagation, and queue-identity recovery.""" + +import json +import os +import subprocess +import sys +import time +from contextlib import contextmanager + +import pytest + +pytestmark = pytest.mark.integration + + +def test_ddl_queue_create_then_alter_finishes_every_host(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + _revision( + project, + [ + f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory", + f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} ADD COLUMN IF NOT EXISTS value String", + ], + ) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + for client in cluster.clients.values(): + assert client.query( + "SELECT name FROM system.columns WHERE database={db:String} AND table='owned' ORDER BY position", + parameters={"db": project.database}, + ).result_rows == [("id",), ("value",)] + rows = _queue(project) + assert len({row[0] for row in rows}) == 2 + assert all(row[3] == "Finished" and row[4] == 0 for row in rows) + assert _heads(project) == ["bbbb"] + + +def test_ddl_queue_down_host_times_out_then_reattaches(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + _revision( + project, + [ + f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" + ], + ) + cluster.stop_node(2) + try: + result = project.run("up", "it", "--timeout", "1") + assert result.exit_code == 1 and "Timed out" in result.output, result.output + assert cluster.hostnames[2] in result.output + before = {row[0] for row in _queue(project)} + assert len(before) == 1 + assert _heads(project) == ["aaaa"] + finally: + cluster.start_node(2) + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert {row[0] for row in _queue(project)} == before + assert all(row[3] == "Finished" and row[4] == 0 for row in _queue(project)) + assert _heads(project) == ["bbbb"] + + +def test_ddl_queue_host_exception_is_not_finished_success(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + table = f"{project.database}.owned" + project.client.command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" + ) + cluster.clients[2].command(f"ALTER TABLE {table} ADD COLUMN conflict String") + _revision( + project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} ADD COLUMN conflict UInt64"] + ) + result = project.run("up", "it") + assert result.exit_code == 1, result.output + assert cluster.hostnames[2] in result.output and "DUPLICATE_COLUMN" in result.output + assert _heads(project) == ["aaaa"] + assert any(row[3] == "Finished" and row[4] == 15 for row in _queue(project)) + + +def test_ddl_queue_lost_acknowledgement_uses_pre_send_marker(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + _revision( + project, + [ + f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" + ], + ) + code = """import os, signal +from ch_migrate.waiting import MigrationWaiter +original = MigrationWaiter._after +def crash(self, connection, cursor, statement, parameters, context, executemany): + pending = self._pending.get(id(context)) + if pending and pending[1].get('ddl') and pending[1].get('table', [])[-1] == 'owned': + os.kill(os.getpid(), signal.SIGKILL) + return original(self, connection, cursor, statement, parameters, context, executemany) +MigrationWaiter._after = crash +from ch_migrate.cli import main +main() +""" + with _running(project, code) as (process, log): + assert process.wait(timeout=30) == -9, log.read_text() + entries = {row[0] for row in _queue(project)} + assert len(entries) == 1 + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert {row[0] for row in _queue(project)} == entries + assert _heads(project) == ["bbbb"] + + +def test_ddl_queue_and_replicated_mutation_both_must_finish(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + table = f"{project.database}.owned" + project.client.command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64, x UInt64) " + f"ENGINE=ReplicatedMergeTree('/clickhouse/ddl_wait/{project.database}', '{{replica}}') ORDER BY id" + ) + project.client.command(f"INSERT INTO {table} VALUES (1, 0)") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") + _revision( + project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} UPDATE x=x+1 WHERE id=1"] + ) + for client in cluster.clients.values(): + client.command(f"SYSTEM STOP MERGES {table}") + try: + result = project.run("up", "it", "--timeout", "1") + assert result.exit_code == 1, result.output + rows = _queue(project) + assert rows and all(row[3] == "Finished" and row[4] == 0 for row in rows) + assert "parts_to_do=" in result.output + assert _heads(project) == ["aaaa"] + entries = {row[0] for row in rows} + finally: + for client in cluster.clients.values(): + client.command(f"SYSTEM START MERGES {table}") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert {row[0] for row in _queue(project)} == entries + for client in cluster.clients.values(): + assert client.command(f"SELECT x FROM {table}") == 1 + assert _heads(project) == ["bbbb"] + + +def test_ddl_queue_missing_ownership_evidence_refuses_resubmission( + cluster_project, clickhouse_cluster +): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + _revision( + project, + [ + f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" + ], + ) + cluster.stop_node(2) + try: + assert project.run("up", "it", "--timeout", "1").exit_code == 1 + entries = {row[0] for row in _queue(project)} + row = project.client.query( + f"SELECT generation, position, sequence, payload FROM {project.database}._ch_migrate_journal " + "WHERE revision='bbbb' ORDER BY sequence DESC LIMIT 1" + ).result_rows[0] + payload = json.loads(row[3]) + assert "ddl" in payload + # An operator-restored receipt whose ownership token has no server evidence. + payload["ddl"]["token"] = "missing-ownership-evidence" + project.client.insert( + f"{project.database}._ch_migrate_journal", + [["bbbb", row[0], row[1], row[2] + 1, json.dumps(payload)]], + column_names=["revision", "generation", "position", "sequence", "payload"], + ) + result = project.run("up", "it") + assert result.exit_code == 1 and "unknown" in result.output.lower(), result.output + assert "not reissued" in result.output + assert {row[0] for row in _queue(project)} == entries + assert _heads(project) == ["aaaa"] + finally: + cluster.start_node(2) + + +def test_ddl_queue_is_not_used_for_single_node_migrations(project): + project.write_revision( + "aaaa", + { + "upgrade": 'op.execute(f"CREATE TABLE IF NOT EXISTS {db}.owned (id UInt64) ENGINE=Memory")' + }, + ) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + project.client.command("SYSTEM FLUSH LOGS") + reads = project.client.query( + "SELECT count() FROM system.query_log WHERE type='QueryStart' " + "AND position(query, 'FROM system.distributed_ddl_queue') > 0 " + "AND position(query, 'system.query_log') = 0" + ).result_rows[0][0] + assert reads == 0 + + +def test_ddl_queue_ttl_metadata_and_materialization_resume(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + table = f"{project.database}.expiry" + project.client.command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (ts DateTime) " + f"ENGINE=ReplicatedMergeTree('/clickhouse/ddl_ttl/{project.database}', '{{replica}}') ORDER BY ts" + ) + project.client.command(f"INSERT INTO {table} VALUES ('2020-01-01 00:00:00')") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") + _revision( + project, + [f"ALTER TABLE {{db}}.expiry ON CLUSTER {cluster.name} MODIFY TTL ts + INTERVAL 100 YEAR"], + ) + for client in cluster.clients.values(): + client.command(f"SYSTEM STOP MERGES {table}") + try: + result = project.run("up", "it", "--timeout", "1") + assert result.exit_code == 1 and "parts_to_do=" in result.output, result.output + entries = {row[0] for row in _queue(project)} + assert len(entries) == 2 # The metadata and materialization have separate receipts. + assert _heads(project) == ["aaaa"] + finally: + for client in cluster.clients.values(): + client.command(f"SYSTEM START MERGES {table}") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert {row[0] for row in _queue(project)} == entries + assert _heads(project) == ["bbbb"] + + +def test_ddl_queue_nonreplicated_mutations_wait_on_each_host(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + table = f"{project.database}.owned" + project.client.command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (x UInt64) ENGINE=MergeTree ORDER BY tuple()" + ) + for client in cluster.clients.values(): + client.command(f"INSERT INTO {table} VALUES (0)") + _revision(project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} UPDATE x=x+1 WHERE 1"]) + cluster.clients[2].command(f"SYSTEM STOP MERGES {table}") + try: + result = project.run("up", "it", "--timeout", "1") + assert result.exit_code == 1, result.output + assert cluster.hostnames[2] in result.output + assert _heads(project) == ["aaaa"] + assert cluster.clients[1].command(f"SELECT x FROM {table}") == 1 + assert cluster.clients[2].command(f"SELECT x FROM {table}") == 0 + finally: + cluster.clients[2].command(f"SYSTEM START MERGES {table}") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + for client in cluster.clients.values(): + assert client.command(f"SELECT x FROM {table}") == 1 + + +def test_ddl_queue_cold_version_table_creation_obeys_timeout(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) + cluster.stop_node(2) + try: + result = project.run("up", "it", "--timeout", "1") + assert result.exit_code == 1 and "Timed out" in result.output, result.output + assert cluster.hostnames[2] in result.output + assert _heads(project) == [] + finally: + cluster.start_node(2) + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert _heads(project) == ["aaaa"] + + +def _baseline(project): + project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + + +def _revision(project, statements): + path = project.sql_dir / "ddl.sql" + path.write_text( + "\n".join( + "-- ch-migrate: allow-non-idempotent Controlled distributed DDL scenario\n" + sql + ";" + for sql in statements + ) + ) + project.write_revision( + "bbbb", {"upgrade": "from ch_migrate import run_sql\nrun_sql('ddl.sql')"}, "aaaa" + ) + + +def _queue(project): + return project.client.query( + "SELECT entry, host, port, status, exception_code FROM system.distributed_ddl_queue " + "WHERE startsWith(settings['log_comment'], 'chm_mutation_') AND position(query, {db:String}) > 0 " + "AND position(query, 'alembic_version') = 0 AND position(query, '_ch_migrate_journal') = 0", + parameters={"db": project.database}, + ).result_rows + + +def _heads(project): + return [ + row[0] + for row in project.client.query( + f"SELECT version_num FROM {project.database}.alembic_version ORDER BY version_num" + ).result_rows + ] + + +@contextmanager +def _running(project, code): + path = project.root / f"ddl-{time.monotonic_ns()}.log" + with path.open("w") as log: + process = subprocess.Popen( + [sys.executable, "-c", code, "up", "it"], + cwd=project.root, + stdout=log, + stderr=subprocess.STDOUT, + env=os.environ.copy(), + ) + try: + yield process, path + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) From 34557fec80c111abb0a806ae7c7efdb79451e8e6 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Fri, 9 Oct 2026 17:03:58 -0400 Subject: [PATCH 07/33] [errors] Preserve waiter operator guidance Keep multi-line waiting and unknown-outcome instructions when SQL execution wraps a server error, while retaining concise database errors. Solves: Operator recovery detail lost in compact upgrade output Tests: Docker mutation failure and unknown-outcome integration cases (cherry picked from commit 9f3b047d0f9fdbd7e158fb5576c164ce6c9d26f5) --- ch_migrate/sql.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ch_migrate/sql.py b/ch_migrate/sql.py index 6d98ed0..c691c37 100644 --- a/ch_migrate/sql.py +++ b/ch_migrate/sql.py @@ -126,7 +126,16 @@ def clean_database_error(message: str) -> str: def _database_reason(exc: Exception) -> str: - """First line of a driver error, without the wrapper and server-version noise.""" + """Preserve operator instructions from wait failures; trim driver noise only.""" + from ch_migrate.waiting_types import WaitingError + + current: BaseException | None = exc + seen: set[int] = set() + while current is not None and id(current) not in seen: + if isinstance(current, WaitingError): + return str(current) + seen.add(id(current)) + current = getattr(current, "orig", None) or current.__cause__ lines = str(exc).strip().splitlines() return clean_database_error(lines[0]) if lines else type(exc).__name__ From c92e03a722be92ca607085e39826079e3f396651 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 13 Oct 2026 12:54:47 -0400 Subject: [PATCH 08/33] [test] Prove Cloud crash-resume on a running mutation Cloud cannot hold a mutation with SYSTEM STOP MERGES, so the self-hosted SIGKILL tests skip there. A deliberately slow UPDATE (sleepEachRow) gives a real window instead. `up` is killed while the mutation is running, and the rerun must resume from its journal ("not reissuing SQL"), produce exactly one increment and one owned mutation, and then advance the version. Solves: DRY-1401 (W03/W23 on Cloud) Tests: integration and cloud on the DRY Cloud service, 4 passed (cherry picked from commit c6870955acccdce6f09479291fd8233b4c5d1df6) --- tests/integration/test_cloud.py | 104 ++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/integration/test_cloud.py b/tests/integration/test_cloud.py index b5969be..ab712a7 100644 --- a/tests/integration/test_cloud.py +++ b/tests/integration/test_cloud.py @@ -1,5 +1,9 @@ """ClickHouse Cloud-only checks; they skip on self-hosted servers (DRY-1413).""" +import subprocess +import sys +import time + import pytest pytestmark = [pytest.mark.integration, pytest.mark.cloud] @@ -61,3 +65,103 @@ def test_cloud_version_table_is_shared_and_round_trips(project): parameters={"db": project.database}, ) assert engine == "SharedMergeTree" + + +def test_cloud_up_advances_version_only_after_mutation_finishes(project): + """Cloud cannot hold a mutation with STOP MERGES; a deliberately slow one shows the wait.""" + db = project.database + _slow_increment(project, seconds=2) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + assert ( + project.client.command( + f"SELECT x FROM {db}.counter", settings={"select_sequential_consistency": 1} + ) + == 1 + ) + mutated, versioned = _part_log_times(project) + print(f"\nCLOUD_WAIT mutation_done={mutated} version_row={versioned}") + assert mutated.year > 1970, "part_log shows no MutatePart for counter; waiting unproven" + assert versioned >= mutated + + +def test_cloud_killed_up_reattaches_to_its_running_mutation(project): + """SIGKILL mid-wait; the rerun must resume from its journal, never resubmit the UPDATE.""" + db = project.database + _slow_increment(project, seconds=3) + log_path = project.root / "first-up.log" + with log_path.open("w") as log: + first = subprocess.Popen( + [sys.executable, "-m", "ch_migrate.cli", "up", "it"], + cwd=project.root, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + _until_running_mutation(project) + finally: + first.kill() + first.wait(timeout=10) + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert "not reissuing SQL" in resumed.output, resumed.output + increments = project.client.command( + f"SELECT x FROM {db}.counter", settings={"select_sequential_consistency": 1} + ) + rewrites = project.client.command( + "SELECT uniqExact(mutation_id) FROM system.mutations " + "WHERE database = {db:String} AND table = {table:String} AND " + project.REWRITES_PARTS, + parameters={"db": db, "table": "counter"}, + ) + print(f"\nCLOUD_SIGKILL increments={increments} owned_mutations={rewrites}") + assert (increments, rewrites) == (1, 1) + assert project.client.query(f"SELECT version_num FROM {db}.alembic_version").result_rows == [ + ("aaaa",) + ] + + +def _slow_increment(project, seconds): + db = project.database + project.client.command( + f"CREATE TABLE {db}.counter (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id" + ) + project.client.command(f"INSERT INTO {db}.counter VALUES (1, 0)") + (project.sql_dir / "slow.sql").write_text( + "-- ch-migrate: allow-non-idempotent One controlled increment\n" + f"ALTER TABLE {{db}}.counter UPDATE x = x + 1 + sleepEachRow({seconds}) WHERE id = 1;\n" + ) + project.write_revision( + "aaaa", {"upgrade": "from ch_migrate import run_sql\nrun_sql('slow.sql')"} + ) + + +def _until_running_mutation(project): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + pending = project.client.command( + "SELECT count() FROM system.mutations WHERE database = {db:String} " + "AND table = 'counter' AND NOT is_done", + parameters={"db": project.database}, + ) + if pending: + return + time.sleep(0.05) + pytest.fail("The slow UPDATE was never seen running") + + +def _part_log_times(project): + """Latest mutated `counter` part and the version row's part, across every replica.""" + deadline = time.monotonic() + 60 + while True: + project.client.command("SYSTEM FLUSH LOGS") + mutated, versioned = project.client.query( + "SELECT maxIf(event_time_microseconds, table = 'counter' AND event_type = 'MutatePart'), " + "maxIf(event_time_microseconds, table = 'alembic_version' AND event_type = 'NewPart') " + "FROM clusterAllReplicas('default', merge('system', '^part_log')) " + "WHERE database = {db:String}", + parameters={"db": project.database}, + settings={"skip_unavailable_shards": 1}, + ).result_rows[0] + if (mutated.year > 1970 and versioned.year > 1970) or time.monotonic() > deadline: + return mutated, versioned + time.sleep(2) From 5b62a5d56d4f0bc727adc9d0a8af4295e03e0bfd Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 10:54:24 -0400 Subject: [PATCH 09/33] [style] Match main's formatting of the waiting modules main's lint pass (f7b0f6f) turned two checkpoint lambdas into nested functions; taking it keeps the release line's waiting code identical to main's. Solves: DRY-1600 Tests: --- ch_migrate/waiting.py | 4 +++- ch_migrate/waiting_versions.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ch_migrate/waiting.py b/ch_migrate/waiting.py index a1bc001..9fe40dd 100644 --- a/ch_migrate/waiting.py +++ b/ch_migrate/waiting.py @@ -362,7 +362,9 @@ def _resume(self, key: StepKey, record: dict) -> bool: raise self.unknown_outcome(key, record, error) from error def _wait_work(self, key: StepKey, record: dict) -> None: - checkpoint = lambda: self.journal.write(key, record) + def checkpoint() -> None: + self.journal.write(key, record) + try: if record.get("ddl"): self.ddl.wait(record["ddl"], checkpoint) diff --git a/ch_migrate/waiting_versions.py b/ch_migrate/waiting_versions.py index 52e1bb1..78fcca8 100644 --- a/ch_migrate/waiting_versions.py +++ b/ch_migrate/waiting_versions.py @@ -50,7 +50,10 @@ def continue_record(self, key, record: dict) -> None: for step in record["steps"]: if step.get("done"): continue - checkpoint = lambda: self.owner.journal.write(key, record) + + def checkpoint() -> None: + self.owner.journal.write(key, record) + if step["kind"] == "insert": self._insert(step, checkpoint) else: From c5eb5ef4a55f23f47018b4ed91796a0e51f24000 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 10:54:24 -0400 Subject: [PATCH 10/33] [test] Count only part-rewriting mutations on Cloud The Cloud crash-resume test filters system.mutations with main's REWRITES_PARTS predicate, because SharedMergeTree lists metadata-only ALTERs there too (DRY-1446). Solves: DRY-1600 Tests: tests/integration/test_cloud.py --- tests/integration/conftest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3fb011d..3f75f12 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -81,6 +81,14 @@ class MigrationProject: database: str cloud: bool = False + # SharedMergeTree lists metadata-only ALTERs in system.mutations with no block numbers + # (DRY-1446). They rewrite no parts, so tests count only part-rewriting mutations. + # Needs {db:String} and {table:String} parameters. + REWRITES_PARTS = ( + "NOT (empty(block_numbers.number) AND startsWith((SELECT engine FROM system.tables " + "WHERE database = {db:String} AND name = {table:String}), 'Shared'))" + ) + @property def sql_dir(self): return self.root / "migrations" / "sql" From 39c922223819c41d95d7b1775048007f7577e4bf Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 10:55:21 -0400 Subject: [PATCH 11/33] [ci] Release 0.7.x tags from dy/ch-migrate-0.7 0.7 ships from its own line, cut from 0.6, while main carries the held 0.8-1.0 work. As on 0.6 (e0e2c3f), this branch's release.yml checks that a tag is on dy/ch-migrate-0.7, and CI runs on pushes to it. main's copy and its guard are unchanged. Solves: DRY-1600 Tests: both workflow files parse --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f635f7b..927535d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ name: ci on: push: - branches: [main, dy/ch-migrate-0.6] + branches: [main, dy/ch-migrate-0.7] pull_request: workflow_call: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 70d3664..efff80a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,13 +8,13 @@ name: release # PyPI publisher for project ch-migrate-cli: owner DRYCodeWorks, repository # ch-migrate, workflow release.yml, no environment. # -# This copy of the workflow is on the 0.6.x release branch, dy/ch-migrate-0.6. -# GitHub runs the workflow file from the tagged commit, so 0.6.x tags are +# This copy of the workflow is on the 0.7.x release branch, dy/ch-migrate-0.7. +# GitHub runs the workflow file from the tagged commit, so 0.7.x tags are # checked against this branch while tags on main keep main's own check. # -# To release 0.6.x: set __version__ in ch_migrate/__init__.py, commit, push -# dy/ch-migrate-0.6, then `git tag vX.Y.Z && git push origin vX.Y.Z`. Tags on -# commits that are not on dy/ch-migrate-0.6 are refused before anything builds. +# To release 0.7.x: set __version__ in ch_migrate/__init__.py, commit, push +# dy/ch-migrate-0.7, then `git tag vX.Y.Z && git push origin vX.Y.Z`. Tags on +# commits that are not on dy/ch-migrate-0.7 are refused before anything builds. on: push: @@ -24,7 +24,7 @@ permissions: contents: read env: - RELEASE_BRANCH: dy/ch-migrate-0.6 + RELEASE_BRANCH: dy/ch-migrate-0.7 jobs: on-release-branch: From 040f769dcf0a3b6d639458a4652496eddd7dbbc4 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 11:40:25 -0400 Subject: [PATCH 12/33] [db] Show live progress while up waits Waiting printed a raw rewritten line (parts_to_do=N) with no sense of how much work was left. Waiters now hand the server's state to one display module: a live bar per mutation in a terminal, weighted by bytes from system.parts and labelled with parts and time left, one row per replica, and a host count for ON CLUSTER DDL. Piped output gets plain lines at the start, each 10% or 30 s, and the finish. The version table's own delete is waited for but not shown. The display never decides completion. Without SELECT on system.parts it counts parts only and says so once; bootstrap now grants it. Failed mutations raise MutationFailed with the reason and KILL statement as fields, and WaitTimeout carries what was pending. Solves: DRY-1601 Tests: tests/test_waiting.py; manual terminal, piped and --timeout runs --- README.md | 30 +- ch_migrate/bootstrap.py | 5 +- ch_migrate/templates/bootstrap/init_users.sql | 1 + ch_migrate/ui.py | 8 +- ch_migrate/waiting.py | 12 +- ch_migrate/waiting_ddl.py | 5 +- ch_migrate/waiting_display.py | 286 ++++++++++++++++++ ch_migrate/waiting_mutations.py | 92 +++++- ch_migrate/waiting_types.py | 61 ++-- ch_migrate/waiting_versions.py | 5 +- tests/test_waiting.py | 4 +- 11 files changed, 457 insertions(+), 52 deletions(-) create mode 100644 ch_migrate/waiting_display.py diff --git a/README.md b/README.md index 78b55a3..e61f9e3 100644 --- a/README.md +++ b/README.md @@ -241,9 +241,33 @@ ch-migrate up dev # No default waiting timeout ch-migrate up dev --timeout 120 # One waiting budget across the invocation ``` -Progress names the table, mutation ID, host, remaining parts and elapsed time. -A terminal uses a live line; CI gets plain state changes and periodic summaries -on stderr. Zero remaining parts alone does not count as completion: every +While it waits, `up` shows progress on stderr. In a terminal, each mutation gets a +live bar weighted by bytes, with its parts and an estimate of the time left: + +```text +→ Applying 3f9c2a1b backfill_digest + ━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━ 18.2 / 41.0 GiB · 23/41 parts · 1:12 · ~1:30 left +✓ 3f9c2a1b backfill_digest mutation_7.txt · 41 parts · 41.0 GiB · 2:03 +``` + +Self-hosted replicated tables get one row per replica, and `ON CLUSTER` DDL +counts the hosts that have finished. When output isn't a terminal (CI logs, +pipes, coding agents), `up` prints plain lines instead: one when a wait starts, +one at each 10% of the parts or every 30 seconds, and one when it finishes: + +```text +→ Applying 3f9c2a1b backfill_digest + waiting on mutation_7.txt (demo_dev.events): 0/41 parts + waiting on mutation_7.txt (demo_dev.events): 5/41 parts · 0:14 +✓ 3f9c2a1b backfill_digest mutation_7.txt · 41 parts · 2:03 +``` + +Part sizes come from `system.parts`. If the migration role can't read it, the bar +counts parts only and says so once; progress never changes whether `up` +succeeds. After a resume the bar counts the parts still left when it reattached. +A failed mutation shows ClickHouse's `latest_fail_reason` and the scoped +`KILL MUTATION` statement; a timeout names what is still pending and how to +resume. Zero remaining parts alone does not count as completion: every required replica must report its owned mutation finished. Configure `cluster` for self-hosted replicated tables, or use a Replicated database's automatic cluster. Missing replicas or missing evidence are never treated as success. diff --git a/ch_migrate/bootstrap.py b/ch_migrate/bootstrap.py index 8f73952..0ff8fb0 100644 --- a/ch_migrate/bootstrap.py +++ b/ch_migrate/bootstrap.py @@ -151,6 +151,7 @@ def build_bootstrap_sql( f"GRANT CURRENT GRANTS(SELECT ON system.replicas) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SELECT ON system.clusters) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SELECT ON system.distributed_ddl_queue) TO {project}_migration_role;", + f"GRANT CURRENT GRANTS(SELECT ON system.parts) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SYSTEM SYNC REPLICA ON {db}.*) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SYSTEM SYNC DATABASE REPLICA ON {db}.*) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(CLUSTER ON *.*) TO {project}_migration_role;", @@ -328,7 +329,9 @@ def run_bootstrap( secure = env_config.get("secure", True) port = env_config.get("port", 8443 if secure else 8123) - ui.step(f"Bootstrapping {env_name}: database {env_config['database']} on {host}, as {admin_user}") + ui.step( + f"Bootstrapping {env_name}: database {env_config['database']} on {host}, as {admin_user}" + ) ui.detail(f"Migration user: {migration_user}") if dict_reader_name: ui.detail(f"Dict reader: {dict_reader_name}") diff --git a/ch_migrate/templates/bootstrap/init_users.sql b/ch_migrate/templates/bootstrap/init_users.sql index 0f29a45..49ab9c4 100644 --- a/ch_migrate/templates/bootstrap/init_users.sql +++ b/ch_migrate/templates/bootstrap/init_users.sql @@ -29,6 +29,7 @@ GRANT SELECT ON system.columns TO {project}_migration_role; GRANT CURRENT GRANTS(SELECT ON system.replicas) TO {project}_migration_role; GRANT CURRENT GRANTS(SELECT ON system.clusters) TO {project}_migration_role; GRANT CURRENT GRANTS(SELECT ON system.distributed_ddl_queue) TO {project}_migration_role; +GRANT CURRENT GRANTS(SELECT ON system.parts) TO {project}_migration_role; GRANT CURRENT GRANTS(SYSTEM SYNC REPLICA ON {db}.*) TO {project}_migration_role; GRANT CURRENT GRANTS(SYSTEM SYNC DATABASE REPLICA ON {db}.*) TO {project}_migration_role; GRANT CURRENT GRANTS(CLUSTER ON *.*) TO {project}_migration_role; diff --git a/ch_migrate/ui.py b/ch_migrate/ui.py index e9809b4..803d98d 100644 --- a/ch_migrate/ui.py +++ b/ch_migrate/ui.py @@ -46,14 +46,14 @@ def show(self, file: IO[Any] | None = None) -> None: hint(line, stderr=True) -def step(message: str) -> None: +def step(message: str, *, stderr: bool = False) -> None: """Something is starting or in progress.""" - out.print(_marked("→ ", "cyan", message)) + (err if stderr else out).print(_marked("→ ", "cyan", message)) -def success(message: str) -> None: +def success(message: str, *, stderr: bool = False) -> None: """Something finished as intended.""" - out.print(_marked("✓ ", "green", message)) + (err if stderr else out).print(_marked("✓ ", "green", message)) def detail(message: str, *, stderr: bool = False) -> None: diff --git a/ch_migrate/waiting.py b/ch_migrate/waiting.py index 9fe40dd..390df19 100644 --- a/ch_migrate/waiting.py +++ b/ch_migrate/waiting.py @@ -4,7 +4,6 @@ import hashlib import json -import sys from contextlib import contextmanager from functools import wraps from pathlib import Path @@ -15,6 +14,7 @@ from sqlalchemy import event from sqlalchemy.schema import CreateTable +from ch_migrate import ui from ch_migrate.classify import classify from ch_migrate.hooks import run_hooks from ch_migrate.idempotency import classify_idempotency @@ -52,7 +52,6 @@ def __init__(self, connection, state, timeout=None): self.mutations = MutationWaiter(self.client, state, self.budget) self.ddl = DistributedDDLWaiter(self.client, self.budget) self.versions = VersionWrites(self) - self.progress_stream = sys.stderr self.revision = None self.generation = self.position = self._internal = 0 self.fingerprint = "" @@ -62,6 +61,8 @@ def __init__(self, connection, state, timeout=None): self._scripts = None def install(self, config) -> None: + # `ch-migrate up` passes the display it labels with each migration's name. + self.budget.display = config.attributes.get("ch_migrate_display", self.budget.display) self._scripts = ScriptDirectory.from_config(config) for target, name, listener, options in ( (self.connection, "before_cursor_execute", self._before, {"retval": True}), @@ -348,9 +349,10 @@ def _resume(self, key: StepKey, record: dict) -> bool: or record["kind"] == "mutation" or (record["kind"] == "ttl" and record["phase"] == "accepted") ): - print( - f"Resuming {key.revision} statement {key.position}; not reissuing SQL", - file=self.progress_stream, + ui.step( + f"Resuming {key.revision[:8]} statement {key.position}: waiting on the work it " + "started; not reissuing SQL", + stderr=True, ) self._wait_work(key, record) self._complete(key, record) diff --git a/ch_migrate/waiting_ddl.py b/ch_migrate/waiting_ddl.py index ab6c2c9..bbb22a0 100644 --- a/ch_migrate/waiting_ddl.py +++ b/ch_migrate/waiting_ddl.py @@ -2,6 +2,7 @@ from __future__ import annotations +from ch_migrate.waiting_display import DDLProgress from ch_migrate.waiting_types import UnknownOutcome, WaitingError @@ -48,7 +49,9 @@ def wait(self, receipt: dict, checkpoint) -> None: checkpoint() pending = self._host_states(receipt, rows, checkpoint) if pending: - self.budget.pause(f"DDL {entry}: " + "; ".join(pending)) + finished = len(expected.intersection(receipt["done_hosts"])) + progress = DDLProgress(entry, finished, len(expected)) + self.budget.pause(f"DDL {entry}: " + "; ".join(pending), progress) self.budget.complete(f"DDL {receipt['entry']}; every required host finished") def tables( diff --git a/ch_migrate/waiting_display.py b/ch_migrate/waiting_display.py new file mode 100644 index 0000000..fb09f46 --- /dev/null +++ b/ch_migrate/waiting_display.py @@ -0,0 +1,286 @@ +"""What `up` shows while it waits for mutations and ON CLUSTER DDL. + +Waiters hand over what the server reports; this module only draws it, and nothing +here decides whether work is done. On a terminal each wait is a live bar on +stderr, weighted by bytes when part sizes are known and labelled with parts. Piped +output, for CI logs and agents, gets plain lines instead: one when a wait starts, +one at each 10% step or every 30 seconds, and one when it finishes. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Union + +from rich.console import Console, Group +from rich.live import Live +from rich.progress_bar import ProgressBar +from rich.table import Table +from rich.text import Text + +from ch_migrate import ui + +PIPED_INTERVAL = 30.0 +PIPED_STEPS = 10 +BAR_WIDTH = 38 + + +@dataclass(frozen=True) +class HostParts: + """One host's remaining parts. `sizes` maps each remaining part to its bytes, if known.""" + + host: str + names: frozenset[str] + count: int + sizes: dict[str, int] | None = None + + @property + def bytes_left(self) -> int | None: + if self.sizes is None or not self.names.issubset(self.sizes): + return None + return sum(self.sizes[name] for name in self.names) + + +@dataclass(frozen=True) +class MutationProgress: + """The mutation a statement started, on every host that hasn't finished it.""" + + table: str + mutation: str + hosts: tuple[HostParts, ...] + bookkeeping: bool = False # the version table's own delete: waited for, not shown + + +@dataclass(frozen=True) +class DDLProgress: + """An ON CLUSTER statement's distributed DDL queue entry.""" + + entry: str + finished: int + total: int + + +Progress = Union[MutationProgress, DDLProgress, str] + + +class WaitDisplay: + """Draws one wait at a time; `complete` prints its result line, `stop` clears it.""" + + def __init__(self, console: Console | None = None) -> None: + self.console = console or ui.err + self.migration = "" + self._wait: _Wait | None = None + self._live: Live | None = None + self._parts_only = False + + @property + def wants_bytes(self) -> bool: + """Only the terminal bar shows bytes, so piped runs skip the size lookup.""" + return self._terminal and not self._parts_only + + @property + def _terminal(self) -> bool: + # A dumb terminal can't redraw in place, so it gets the piped lines. + return self.console.is_terminal and not self.console.is_dumb_terminal + + def start_migration(self, label: str) -> None: + self.migration = label + + def update(self, progress: Progress) -> None: + if isinstance(progress, MutationProgress) and progress.bookkeeping: + return + wait = self._wait_for(progress) + wait.observe(progress, time.monotonic()) + if self._terminal: + self._draw_live(wait) + else: + self._print_piped(wait) + + def parts_only(self, reason: str) -> None: + """Part sizes can't be read; say so once, then count parts.""" + if self._parts_only: + return + self._parts_only = True + self.console.print( + Text.assemble(("! ", "yellow"), (f"Progress counts parts only: {reason}", "yellow")) + ) + + def complete(self, fallback: str) -> None: + wait = self._wait + self.stop() + if wait is not None: + self.console.print( + Text.assemble(("✓ ", "green"), wait.summary(self.migration, fallback)) + ) + + def stop(self) -> None: + if self._live is not None: + self._live.stop() + self._live = None + self._wait = None + + def _wait_for(self, progress: Progress) -> _Wait: + key = _key(progress) + if self._wait is None or self._wait.key != key: + self.stop() + self._wait = _Wait(key, started=time.monotonic()) + return self._wait + + def _draw_live(self, wait: _Wait) -> None: + if self._live is None: + self._live = Live(console=self.console, auto_refresh=False, transient=True) + self._live.start() + self._live.update(wait.renderable(), refresh=True) + + def _print_piped(self, wait: _Wait) -> None: + line = wait.piped_line(time.monotonic()) + if line is not None: + self.console.print(Text(" " + line)) + + +@dataclass +class _Wait: + key: str + started: float + latest: Progress = "" + parts_total: dict[str, int] = field(default_factory=dict) + bytes_total: dict[str, int] = field(default_factory=dict) + printed_at: float | None = None + printed_step: int = -1 + + def observe(self, progress: Progress, now: float) -> None: + self.latest = progress + if isinstance(progress, MutationProgress): + for host in progress.hosts: + self.parts_total[host.host] = max(self.parts_total.get(host.host, 0), host.count) + if host.host not in self.bytes_total and host.bytes_left is not None: + self.bytes_total[host.host] = host.bytes_left + + def renderable(self): + elapsed = _duration(time.monotonic() - self.started) + progress = self.latest + if isinstance(progress, DDLProgress): + stats = f"{progress.finished}/{progress.total} hosts · {elapsed}" + return _bar_row("", progress.finished, progress.total, stats) + if isinstance(progress, str): + return Text(f" waiting: {progress} · {elapsed}", style="dim") + rows = [self._host_row(host, len(progress.hosts) > 1) for host in progress.hosts] + return Group(*rows) + + def piped_line(self, now: float) -> str | None: + step = int(self._fraction() * PIPED_STEPS) + due = self.printed_at is None or now - self.printed_at >= PIPED_INTERVAL + if not due and step <= self.printed_step: + return None + first = self.printed_at is None + self.printed_at, self.printed_step = now, step + elapsed = "" if first else f" · {_duration(now - self.started)}" + return self._piped_text() + elapsed + + def summary(self, migration: str, fallback: str) -> str: + elapsed = _duration(time.monotonic() - self.started) + progress = self.latest + if isinstance(progress, DDLProgress): + label = migration or f"ON CLUSTER DDL {progress.entry}" + return f"{label} ON CLUSTER DDL on {progress.total} hosts · {elapsed}" + if isinstance(progress, str): + return f"{migration or fallback} waited for {progress} · {elapsed}" + return f"{migration or progress.table} {self._mutation_totals(progress)} · {elapsed}" + + def _host_row(self, host: HostParts, named: bool): + total = self.parts_total.get(host.host, host.count) + parts = f"{total - host.count}/{total} parts" + left, size_total = host.bytes_left, self.bytes_total.get(host.host) + if left is None or not size_total: + return _bar_row(host.host if named else "", total - host.count, total, parts) + done = size_total - left + stats = ( + f"{_sizes(done, size_total)} · {parts} · {_duration(time.monotonic() - self.started)}" + ) + if eta := self._eta(done, left): + stats += f" · ~{eta} left" + return _bar_row(host.host if named else "", done, size_total, stats) + + def _eta(self, done: int, left: int) -> str: + elapsed = time.monotonic() - self.started + if done <= 0 or elapsed < 2: + return "" + return _duration(left / (done / elapsed)) + + def _fraction(self) -> float: + progress = self.latest + if isinstance(progress, DDLProgress): + return progress.finished / progress.total if progress.total else 0.0 + if isinstance(progress, str): + return 0.0 + total = sum(self.parts_total.values()) + left = sum(host.count for host in progress.hosts) + return (total - left) / total if total else 0.0 + + def _piped_text(self) -> str: + progress = self.latest + if isinstance(progress, DDLProgress): + return f"waiting on ON CLUSTER DDL {progress.entry}: {progress.finished}/{progress.total} hosts" + if isinstance(progress, str): + return f"waiting: {progress}" + total = sum(self.parts_total.values()) + done = total - sum(host.count for host in progress.hosts) + replicas = f" on {len(progress.hosts)} replicas" if len(progress.hosts) > 1 else "" + return f"waiting on {_named(progress.mutation)} ({progress.table}): {done}/{total} parts{replicas}" + + def _mutation_totals(self, progress: MutationProgress) -> str: + parts = sum(self.parts_total.values()) + text = f"{_named(progress.mutation)} · {parts} {'part' if parts == 1 else 'parts'}" + if self.bytes_total and len(self.bytes_total) == len(self.parts_total): + text += f" · {_size(sum(self.bytes_total.values()))}" + if len(self.parts_total) > 1: + text += f" · {len(self.parts_total)} replicas" + return text + + +def _named(mutation: str) -> str: + # MergeTree ids already read "mutation_7.txt"; replicated ones are bare numbers. + return mutation if mutation.startswith("mutation") else f"mutation {mutation}" + + +def _key(progress: Progress) -> str: + if isinstance(progress, MutationProgress): + return f"mutation {progress.table} {progress.mutation}" + if isinstance(progress, DDLProgress): + return f"ddl {progress.entry}" + return "status" + + +def _bar_row(label: str, completed: float, total: float, stats: str) -> Table: + row = Table.grid(padding=(0, 1)) + row.add_column(no_wrap=True) + row.add_column(width=BAR_WIDTH) + row.add_column(no_wrap=True) + bar = ProgressBar(total=max(total, 1), completed=min(completed, total), width=BAR_WIDTH) + row.add_row(Text(f" {label}".rstrip() if label else " "), bar, Text(stats)) + return row + + +def _duration(seconds: float) -> str: + seconds = max(0, int(seconds)) + hours, rest = divmod(seconds, 3600) + minutes, secs = divmod(rest, 60) + return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" + + +def _sizes(done: int, total: int) -> str: + unit, scale = _unit(total) + return f"{done / scale:.1f} / {total / scale:.1f} {unit}" + + +def _size(total: int) -> str: + unit, scale = _unit(total) + return f"{total / scale:.1f} {unit}" + + +def _unit(total: int) -> tuple[str, int]: + for unit, scale in (("GiB", 1 << 30), ("MiB", 1 << 20), ("KiB", 1 << 10)): + if total >= scale: + return unit, scale + return "B", 1 diff --git a/ch_migrate/waiting_mutations.py b/ch_migrate/waiting_mutations.py index e46cdea..f2b9016 100644 --- a/ch_migrate/waiting_mutations.py +++ b/ch_migrate/waiting_mutations.py @@ -3,14 +3,21 @@ from __future__ import annotations import re +from dataclasses import replace +from clickhouse_connect.driver.exceptions import DatabaseError + +from ch_migrate.waiting_display import HostParts, MutationProgress from ch_migrate.waiting_sql import sql_string -from ch_migrate.waiting_types import UnknownOutcome, WaitingError +from ch_migrate.waiting_types import FailedMutation, MutationFailed, UnknownOutcome, WaitingError + +VERSION_TABLE = "alembic_version" class MutationWaiter: def __init__(self, client, state, budget): self.client, self.state, self.budget = client, state, budget + self._part_sizes: dict[tuple[str, str], int] = {} def target(self, table: tuple[str, str]) -> dict | None: rows = self.client.query( @@ -72,7 +79,7 @@ def wait(self, receipt: dict, checkpoint) -> None: f"{receipt['database']}.{receipt['table']}; every required replica finished" ) return - self.budget.pause("; ".join(pending)) + self.budget.pause("; ".join(pending), self._snapshot(receipt, rows)) def _replicated_target(self, target: dict) -> None: path, total = self.client.query( @@ -171,11 +178,59 @@ def _pending(self, receipt: dict, rows: list, checkpoint) -> list[str]: checkpoint() return pending + def _snapshot(self, receipt: dict, rows: list) -> MutationProgress | None: + """What the display shows about this statement's unfinished mutations; never a verdict.""" + hosts = [ + parts + for host in receipt["uuids"] + if host not in receipt["done_hosts"] + and (parts := _host_parts(host, receipt["ids"].get(host) or [], rows)) is not None + ] + owned = [mutation for ids in receipt["ids"].values() for mutation in ids] + if not hosts or not owned: + return None + if self.budget.display.wants_bytes: + hosts = self._with_sizes(receipt, hosts) + table = f"{receipt['database']}.{receipt['table']}" + bookkeeping = receipt["table"] == VERSION_TABLE + return MutationProgress(table, min(owned, key=_order), tuple(hosts), bookkeeping) + + def _with_sizes(self, receipt: dict, hosts: list[HostParts]) -> list[HostParts]: + # Parts never change size, so each is read once; only new names cost a query. + wanted = {(host.host, name) for host in hosts for name in host.names} + unknown = {name for _, name in wanted - self._part_sizes.keys()} + if unknown: + try: + self._part_sizes.update(self._read_sizes(receipt, unknown)) + except DatabaseError as error: + self.budget.display.parts_only(_sizes_unreadable(error)) + return hosts + sizes = self._part_sizes + return [ + replace( + host, + sizes={n: sizes[(host.host, n)] for n in host.names if (host.host, n) in sizes}, + ) + for host in hosts + ] + + def _read_sizes(self, receipt: dict, names: set[str]) -> dict[tuple[str, str], int]: + source, parameters = self._source(receipt, "parts") + rows = self.client.query( + "SELECT materialize(hostName()), name, bytes_on_disk " + f"FROM {source} WHERE database = {{db:String}} AND table = {{table:String}} " + "AND active AND has({names:Array(String)}, name)", + parameters={**parameters, "names": sorted(names)}, + settings=self._settings() if receipt.get("cluster") else None, + ).result_rows + shared = receipt["engine"].startswith("Shared") + return {("shared" if shared else host, name): size for host, name, size in rows} + def _rows(self, target: dict) -> list: source, parameters = self._source(target, "mutations") rows = self.client.query( - f"SELECT materialize(hostName()), mutation_id, command, is_done, parts_to_do, latest_fail_reason " - f"FROM {source} WHERE database = {{db:String}} AND table = {{table:String}}", + "SELECT materialize(hostName()), mutation_id, command, is_done, parts_to_do, " + f"latest_fail_reason, parts_to_do_names FROM {source} WHERE database = {{db:String}} AND table = {{table:String}}", parameters=parameters, settings=self._settings() if target.get("cluster") else None, ).result_rows @@ -215,20 +270,24 @@ def _parameters(target: dict) -> dict: @staticmethod def _failures(target: dict, rows: list, prefix: str) -> None: - for host, mutation, _, done, _, reason in rows: + for host, mutation, _, done, _, reason, *_ in rows: if not done and reason: kill = ( f"KILL MUTATION WHERE database = {sql_string(target['database'])} " f"AND table = {sql_string(target['table'])} AND mutation_id = {sql_string(mutation)}" ) - raise WaitingError( - f"{prefix}: {target['database']}.{target['table']} mutation {mutation} on {host}: {reason}\nOperator only on {host}: {kill}" + table = f"{target['database']}.{target['table']}" + raise MutationFailed( + f"{prefix}: {table} mutation {mutation} on {host}: {reason}\nOperator only on {host}: {kill}", + FailedMutation( + table, mutation, host, reason, kill, prefix.startswith("Blocked") + ), ) @staticmethod def _progress(target: dict, rows: list) -> str: parts = {} - for host, mutation, _, _, count, _ in rows: + for host, mutation, _, _, count, *_ in rows: parts[(host, mutation)] = max(parts.get((host, mutation), 0), count) return "; ".join( f"{target['database']}.{target['table']} mutation {mutation} host {host}; parts_to_do={count}" @@ -241,3 +300,20 @@ def _order(mutation: str) -> int: if len(numbers) != 1: raise WaitingError(f"Unrecognized mutation ordering identifier: {mutation}") return int(numbers[0]) + + +def _host_parts(host: str, ids: list, rows: list) -> HostParts | None: + unfinished = [row for row in rows if row[0] == host and row[1] in ids and not row[3]] + if not unfinished: + return None + names = frozenset(name for row in unfinished for name in row[6]) + return HostParts(host, names, len(names) or max(row[4] for row in unfinished)) + + +def _sizes_unreadable(error: Exception) -> str: + code = re.search(r"\(([A-Z][A-Z_]+)\)", str(error)) + why = f" ({code.group(1)})" if code else "" + return ( + f"can't read system.parts{why}. Grant SELECT ON system.parts to the migration role " + "to see bytes and a time estimate." + ) diff --git a/ch_migrate/waiting_types.py b/ch_migrate/waiting_types.py index 47491c6..776b944 100644 --- a/ch_migrate/waiting_types.py +++ b/ch_migrate/waiting_types.py @@ -2,10 +2,11 @@ from __future__ import annotations -import sys import time from dataclasses import dataclass, field +from ch_migrate.waiting_display import Progress, WaitDisplay + class WaitingError(RuntimeError): """Server work is unfinished; the revision must not be completed.""" @@ -18,15 +19,37 @@ class UnknownOutcome(WaitingError): class WaitTimeout(WaitingError): """The invocation's waiting deadline expired without cancelling server work.""" + def __init__(self, message: str, pending: str, elapsed: float) -> None: + super().__init__(message) + self.pending, self.elapsed = pending, elapsed + + +@dataclass(frozen=True) +class FailedMutation: + """What the operator needs: where it failed, why, and the scoped KILL statement.""" + + table: str + mutation: str + host: str + reason: str + kill: str + foreign: bool = False + + +class MutationFailed(WaitingError): + """ClickHouse reports `latest_fail_reason` for a mutation this run owns or waits behind.""" + + def __init__(self, message: str, failure: FailedMutation) -> None: + super().__init__(message) + self.failure = failure + @dataclass class WaitBudget: timeout: float | None = None started: float = field(default_factory=time.monotonic) poll_interval: float = 0.2 - _last_line: str = "" - _last_printed: float = 0 - _live: bool = False + display: WaitDisplay = field(default_factory=WaitDisplay) @property def elapsed(self) -> float: @@ -37,37 +60,23 @@ def check(self, pending: str) -> None: self.finish() raise WaitTimeout( f"Timed out after {self.elapsed:.1f}s; server work continues; " - f"migration incomplete. Pending: {pending}" + f"migration incomplete. Pending: {pending}", + pending, + self.elapsed, ) - def pause(self, pending: str) -> None: + def pause(self, pending: str, progress: Progress | None = None) -> None: + """Show `progress` (or the pending text), then sleep one poll within the deadline.""" self.check(pending) - self.progress(pending) + self.display.update(progress if progress is not None else pending) remaining = self.poll_interval if self.timeout is not None: remaining = min(remaining, max(0, self.timeout - self.elapsed)) time.sleep(remaining) self.check(pending) - def progress(self, pending: str) -> None: - now = time.monotonic() - tty = sys.stderr.isatty() - if pending != self._last_line or now - self._last_printed >= (0.5 if tty else 5): - line = f"Waiting: {pending}; {self.elapsed:.1f}s elapsed" - if tty: - print("\r\033[K" + line, end="", file=sys.stderr, flush=True) - self._live = True - else: - print(line, file=sys.stderr, flush=True) - self._last_printed, self._last_line = now, pending - def complete(self, message: str) -> None: - self.finish() - if self._last_line: - print(f"Finished: {message}; {self.elapsed:.1f}s elapsed", file=sys.stderr, flush=True) - self._last_line = "" + self.display.complete(message) def finish(self) -> None: - if self._live: - print(file=sys.stderr, flush=True) - self._live = False + self.display.stop() diff --git a/ch_migrate/waiting_versions.py b/ch_migrate/waiting_versions.py index 78fcca8..890d621 100644 --- a/ch_migrate/waiting_versions.py +++ b/ch_migrate/waiting_versions.py @@ -9,6 +9,7 @@ from sqlalchemy import text from sqlalchemy.sql.dml import Delete, Insert, Update +from ch_migrate import ui from ch_migrate.waiting_sql import ( mutation_sql, qualified_table, @@ -38,9 +39,7 @@ def execute(self, construct, impl): def resume(self) -> None: for key, record in self.owner.journal.unfinished_versions(): self.owner.validate_revision(key.revision, record["fingerprint"]) - print( - f"Resuming version bookkeeping for {key.revision}", file=self.owner.progress_stream - ) + ui.step(f"Resuming version bookkeeping for {key.revision[:8]}", stderr=True) self.continue_record(key, record) def continue_record(self, key, record: dict) -> None: diff --git a/tests/test_waiting.py b/tests/test_waiting.py index 1a51af4..f5396a6 100644 --- a/tests/test_waiting.py +++ b/tests/test_waiting.py @@ -87,7 +87,9 @@ def test_zero_parts_is_not_mutation_completion(monkeypatch): "baseline": {"host": []}, } monkeypatch.setattr(waiter, "_identities", lambda _: {"host": "uuid"}) - monkeypatch.setattr(waiter, "_rows", lambda _: [("host", "mutation_1.txt", "owned", 0, 0, "")]) + monkeypatch.setattr( + waiter, "_rows", lambda _: [("host", "mutation_1.txt", "owned", 0, 0, "", [])] + ) with pytest.raises(WaitTimeout, match="mutation_1.txt"): waiter.wait(receipt, lambda: None) assert receipt["done_hosts"] == [] From 013b33acfaec24db6cc1bb751773ebd9ea571f6f Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 11:40:26 -0400 Subject: [PATCH 13/33] [cli] Report up one line per migration again Running Alembic in-process (39092ef) dropped 0.6's translated output: up printed Alembic's INFO lines and wrapped every failure in one plain error. The runner now routes Alembic's log records to the same translation runner.py uses for down, labels the waiting display with the migration it belongs to, and reports a failed mutation, a timeout or an unknown outcome as an error with what to do next. Solves: DRY-1601 Tests: tests/test_waiting_display.py; piped and terminal integration tests --- ch_migrate/alembic_env.py | 13 ++ ch_migrate/cli.py | 35 ++-- ch_migrate/migration_runner.py | 205 +++++++++++++++++++-- tests/integration/test_mutation_waiting.py | 37 +++- tests/test_waiting_display.py | 160 ++++++++++++++++ 5 files changed, 420 insertions(+), 30 deletions(-) create mode 100644 tests/test_waiting_display.py diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index 072699c..1508dec 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -88,6 +88,7 @@ def run() -> None: os.environ.pop("CH_CLUSTER", None) if context.config.config_file_name is not None: fileConfig(context.config.config_file_name, disable_existing_loggers=False) + _route_logs(context.config.attributes.get("ch_migrate_log_handler")) if context.is_offline_mode(): _run_offline(env_config) else: @@ -183,6 +184,18 @@ def _run_context(waiter: MigrationWaiter, hooks: HookRegistry) -> None: waiter.finish_run() +def _route_logs(handler: logging.Handler | None) -> None: + # `ch-migrate up` reports each migration in its own words, so Alembic's and + # ch-migrate's records go to its handler instead of alembic.ini's console handler. + if handler is None: + return + for name in ("alembic", "ch_migrate"): + named = logging.getLogger(name) + named.handlers = [handler] + named.propagate = False + named.setLevel(logging.INFO) + + def _url(env_config: dict[str, Any]) -> URL: secure = env_config.get("secure", True) return URL.create( diff --git a/ch_migrate/cli.py b/ch_migrate/cli.py index 7abbafe..88b7e87 100644 --- a/ch_migrate/cli.py +++ b/ch_migrate/cli.py @@ -123,11 +123,15 @@ def _report_irreversible_range( summary = "The migration it would revert is irreversible:" else: verb = "is" if len(blocked) == 1 else "are" - summary = f"Of the {len(reasons)} migrations it would revert, {len(blocked)} {verb} irreversible:" + summary = ( + f"Of the {len(reasons)} migrations it would revert, {len(blocked)} {verb} irreversible:" + ) ui.error(f"Downgrade refused; nothing was run. {summary}") for rev, reason in reasons.items(): name = graph.migrations[rev].description or "" - marker, status = ("✗", f"irreversible: {reason}") if reason is not None else (" ", "reversible") + marker, status = ( + ("✗", f"irreversible: {reason}") if reason is not None else (" ", "reversible") + ) ui.detail(f"{marker} {rev[:8]} {name} ({status})", stderr=True) newest_blocked = blocked[0] above = list(reasons).index(newest_blocked) @@ -271,7 +275,9 @@ def bootstrap(environment: str, dry_run: bool, verbose: bool) -> None: help="Skip materialized view declaration validation", ) @click.option("--verbose", is_flag=True, help="Show the full traceback if a migration fails") -def up(environment: str, revision: str, skip_mv_check: bool, timeout: float | None, verbose: bool) -> None: +def up( + environment: str, revision: str, skip_mv_check: bool, timeout: float | None, verbose: bool +) -> None: """Apply pending migrations. Runs all unapplied migrations to bring the database to the latest version. @@ -284,14 +290,9 @@ def up(environment: str, revision: str, skip_mv_check: bool, timeout: float | No raise click.BadParameter("must be finite", param_hint="--timeout") _require_current_env() _enforce_up_gate(environment, skip_mv_check) - from ch_migrate.migration_runner import run_upgrade + from ch_migrate.migration_runner import Upgrade, run_upgrade - try: - run_upgrade(environment, revision, timeout) - except Exception as error: - if verbose: - raise - raise click.ClickException(str(error)) from error + sys.exit(run_upgrade(Upgrade(environment, revision, timeout, verbose))) @main.command() @@ -321,7 +322,9 @@ def status(environment: str) -> None: from ch_migrate.display import render_status state = _load_migration_state(environment) - render_status(environment, state.env_config, state.graph, state.applied, db_error=state.db_error) + render_status( + environment, state.env_config, state.graph, state.applied, db_error=state.db_error + ) if state.db_error: ui.warn(f"Could not reach the database: {state.db_error.strip().splitlines()[0]}") return @@ -832,8 +835,13 @@ def lint(environment: str | None) -> None: @main.command() @click.argument("environment") -@click.option("--validate", "-v", "validate_sql", type=click.Path(exists=True), - help="Validate a SQL file against the dependency graph") +@click.option( + "--validate", + "-v", + "validate_sql", + type=click.Path(exists=True), + help="Validate a SQL file against the dependency graph", +) def deps(environment: str, validate_sql: str | None) -> None: """Show materialized view and dictionary dependency graph. @@ -957,6 +965,7 @@ def diff_cmd(environment: str, snapshot_dir: str | None) -> None: TableDefinition, ViewDefinition, ) + fallback_types = { "table": lambda: TableDefinition(name=name, engine="", raw_ddl=ddl), "view": lambda: ViewDefinition(name=name, select_query="", raw_ddl=ddl), diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py index f1d9ee9..04fd702 100644 --- a/ch_migrate/migration_runner.py +++ b/ch_migrate/migration_runner.py @@ -1,27 +1,208 @@ -"""Own the upgrade process so SIGKILL cannot leave a child advancing revisions.""" +"""Own the upgrade process so SIGKILL cannot leave a child advancing revisions. + +`ch-migrate up` runs Alembic in this process and reports in ch-migrate's own words, +as runner.py does for `down`: one line per migration, setup chatter dropped, and a +failure reduced to the migration, what went wrong and what to do next. Waiting +progress is drawn by a display the waiter receives through the Alembic config. +""" from __future__ import annotations +import logging import os +import re +import traceback +from contextlib import contextmanager +from dataclasses import dataclass from pathlib import Path from alembic import command from alembic.config import Config +from ch_migrate import ui +from ch_migrate.runner import ( + _RUNNING, + _SELF_DESCRIBING, + _SETUP_CHATTER, + _Progress, + _report_success, + _start_migration, +) +from ch_migrate.sql import clean_database_error +from ch_migrate.waiting_display import WaitDisplay +from ch_migrate.waiting_types import ( + FailedMutation, + MutationFailed, + UnknownOutcome, + WaitingError, + WaitTimeout, +) + +_LOCATION = re.compile( + r"^(?Pmigrations/sql/\S+) \(statement \d+ of \d+, line (?P\d+)\): " +) + + +@dataclass(frozen=True) +class Upgrade: + """One `ch-migrate up`: where, how far, how long to wait, and how much to show.""" -def run_upgrade(environment: str, revision: str, timeout: float | None) -> None: - """Run Alembic directly: progress streams live and the CLI owns the waiter.""" + environment: str + revision: str = "head" + timeout: float | None = None + verbose: bool = False + + +def run_upgrade(upgrade: Upgrade) -> int: + """Apply migrations in this process, one reported line each; return the exit status.""" + progress, display = _Progress(), WaitDisplay() config = Config(str(Path.cwd() / "alembic.ini")) - config.attributes["ch_migrate_environment"] = environment - config.attributes["ch_migrate_command"] = "upgrade" - config.attributes["ch_migrate_timeout"] = timeout + config.attributes.update( + ch_migrate_environment=upgrade.environment, + ch_migrate_command="upgrade", + ch_migrate_timeout=upgrade.timeout, + ch_migrate_display=display, + ch_migrate_log_handler=_MigrationLog(progress, display), + ) # Project helpers intentionally read these during migration execution only. saved = {name: os.environ.get(name) for name in ("CH_DATABASE", "CH_CLUSTER")} try: - command.upgrade(config, revision) + with _restored_loggers("alembic", "ch_migrate"): + command.upgrade(config, upgrade.revision) + except Exception as error: + display.stop() + _report_failure(upgrade, progress, error) + return 1 + finally: + _restore_environment(saved) + _report_success(upgrade.environment, upgrade.revision, progress) + return 0 + + +class _MigrationLog(logging.Handler): + """Turns Alembic's log records into ch-migrate's lines, as runner.py does for its output.""" + + def __init__(self, progress: _Progress, display: WaitDisplay) -> None: + super().__init__(logging.INFO) + self.progress, self.display = progress, display + + def emit(self, record: logging.LogRecord) -> None: + message = record.getMessage() + running = _RUNNING.match(message) + if running: + _start_migration(running, self.progress) + self.display.start_migration(self.progress.current) + elif message.startswith(_SETUP_CHATTER): + return + elif record.levelno >= logging.WARNING: + ui.warn(message) + else: + ui.detail(message) + + +def _report_failure(upgrade: Upgrade, progress: _Progress, error: Exception) -> None: + where = progress.current or "Upgrade" + waiting = _waiting_error(error) + if waiting is not None and (location := _LOCATION.match(str(error))): + where = f"{where} ({location['path']}:{location['line']})" + if isinstance(waiting, MutationFailed): + _report_mutation_failure(upgrade.environment, where, waiting.failure) + elif isinstance(waiting, WaitTimeout): + _report_timeout(upgrade.environment, where, waiting) + elif isinstance(waiting, UnknownOutcome): + _report_unknown_outcome(where, waiting) + else: + _report_statement_failure(upgrade, where, error) + + +def _report_mutation_failure(environment: str, where: str, failure: FailedMutation) -> None: + on_host = "" if failure.host == "shared" else f" on {failure.host}" + if failure.foreign: + ui.error( + f"{where} is blocked: foreign mutation {failure.mutation} on {failure.table} " + f"failed{on_host}" + ) + else: + ui.error(f"{where} failed: mutation {failure.mutation} on {failure.table} failed{on_host}") + ui.detail(failure.reason, stderr=True) + ui.hint( + "It was not recorded as applied. ClickHouse keeps retrying a failed mutation, and later " + f"mutations on {failure.table} wait behind it.", + stderr=True, + ) + ui.hint( + f"Fix the cause, then run `ch-migrate up {environment}` to wait for the retry. " + f"To stop it instead (operator only{on_host}): {failure.kill}", + stderr=True, + ) + + +def _report_timeout(environment: str, where: str, timeout: WaitTimeout) -> None: + ui.error(f"Timed out after {timeout.elapsed:.1f}s waiting on {where}") + ui.detail(f"Pending: {timeout.pending}", stderr=True) + ui.hint( + "ClickHouse keeps doing the work, and the migration is not recorded as applied. " + f"Run `ch-migrate up {environment}` again to keep waiting; it won't reissue the statement.", + stderr=True, + ) + + +def _report_unknown_outcome(where: str, unknown: UnknownOutcome) -> None: + ui.error(f"{where}: outcome unknown, so the statement was not reissued") + for line in str(unknown).strip().splitlines(): + ui.detail(line, stderr=True) + ui.hint("Follow the README's Unknown outcome steps before you run `up` again.", stderr=True) + + +def _report_statement_failure(upgrade: Upgrade, where: str, error: Exception) -> None: + ui.error(f"{where} failed") + ui.detail(_reason(error), stderr=True) + ui.hint( + "It was not recorded as applied. Anything it ran before the failure stays in place, " + f"so make the SQL safe to re-run, then run `ch-migrate up {upgrade.environment}` again.", + stderr=True, + ) + if upgrade.verbose: + traceback.print_exception(error) + else: + ui.hint("Add `--verbose` to see the full traceback.", stderr=True) + + +def _reason(error: Exception) -> str: + lines = str(error).strip().splitlines() + message = clean_database_error(lines[0]) if lines else "" + kind = type(error).__name__ + if not message: + return kind + return message if kind in _SELF_DESCRIBING else f"{kind}: {message}" + + +def _waiting_error(error: BaseException) -> WaitingError | None: + current: BaseException | None = error + seen: set[int] = set() + while current is not None and id(current) not in seen: + if isinstance(current, WaitingError): + return current + seen.add(id(current)) + current = getattr(current, "orig", None) or current.__cause__ or current.__context__ + return None + + +@contextmanager +def _restored_loggers(*names: str): + saved = [(logging.getLogger(n), logging.getLogger(n).handlers[:]) for n in names] + states = [(logger.propagate, logger.level) for logger, _ in saved] + try: + yield finally: - for name, value in saved.items(): - if value is None: - os.environ.pop(name, None) - else: - os.environ[name] = value + for (logger, handlers), (propagate, level) in zip(saved, states): + logger.handlers, logger.propagate = handlers, propagate + logger.setLevel(level) + + +def _restore_environment(saved: dict[str, str | None]) -> None: + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value diff --git a/tests/integration/test_mutation_waiting.py b/tests/integration/test_mutation_waiting.py index 02b244b..97e1cb5 100644 --- a/tests/integration/test_mutation_waiting.py +++ b/tests/integration/test_mutation_waiting.py @@ -246,7 +246,9 @@ def test_mutation_waits_for_every_replica(cluster_project, clickhouse_cluster): .result_rows ) assert second and all(not row[0] for row in second) - _until(lambda: "parts_to_do=" in log.read_text(), "replica wait was not reported") + _until( + lambda: "waiting on mutation" in log.read_text(), "replica wait was not reported" + ) assert process.poll() is None for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): cluster.clients[2].command(f"SYSTEM START {action} {table}") @@ -551,6 +553,30 @@ def test_mutation_ttl_resume_does_not_materialize_twice(project): assert _heads(project) == ["bbbb"] +def test_mutation_piped_progress_prints_plain_lines(project): + """CI logs and agents get a start line naming the mutation, never a redrawn bar.""" + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") + project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + try: + result = subprocess.run( + [sys.executable, "-m", "ch_migrate.cli", "up", "it", "--timeout", "1"], + cwd=project.root, + env={**os.environ, "TERM": "xterm-256color"}, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 1, result.stderr + start = f" waiting on mutation_2.txt ({project.database}.counter): 0/1 parts" + assert start in result.stderr.splitlines(), result.stderr + assert "\x1b" not in result.stderr and "\r" not in result.stderr + assert "Run ch-migrate up it again to keep waiting" in result.stderr + finally: + project.client.command(f"SYSTEM START MERGES {project.database}.counter") + assert _heads(project) == ["aaaa"] + + @pytest.mark.skipif(os.name != "posix", reason="A real PTY is required for terminal progress") def test_mutation_terminal_progress_uses_a_live_line(project): import pty @@ -565,6 +591,7 @@ def test_mutation_terminal_progress_uses_a_live_line(project): process = subprocess.Popen( [sys.executable, "-m", "ch_migrate.cli", "up", "it", "--timeout", "0.8"], cwd=project.root, + env={**os.environ, "TERM": "xterm-256color"}, stdout=subprocess.DEVNULL, stderr=slave, ) @@ -578,7 +605,9 @@ def test_mutation_terminal_progress_uses_a_live_line(project): chunks.append(os.read(master, 4096)) assert process.wait(timeout=5) == 1 output = b"".join(chunks) - assert b"\x1b[K" in output and b"parts_to_do=" in output + # The live bar redraws in place; the timeout still names what was pending. + assert "━".encode() in output and b" parts" in output + assert b"Timed out" in output and b"parts_to_do=" in output assert _heads(project) == ["aaaa"] finally: if process is not None and process.poll() is None: @@ -699,9 +728,7 @@ def _until(predicate, message): @contextmanager def _running(project, args, code=None): path = project.root / f"run-{time.monotonic_ns()}.log" - command = ( - [sys.executable, "-c", code] if code else [sys.executable, "-m", "ch_migrate.cli"] - ) + command = [sys.executable, "-c", code] if code else [sys.executable, "-m", "ch_migrate.cli"] with path.open("w") as log: process = subprocess.Popen( [*command, *args], diff --git a/tests/test_waiting_display.py b/tests/test_waiting_display.py new file mode 100644 index 0000000..078aef4 --- /dev/null +++ b/tests/test_waiting_display.py @@ -0,0 +1,160 @@ +"""What `up` shows while it waits: the terminal bar, piped lines and failure reports.""" + +import io + +import pytest +from rich.console import Console + +from ch_migrate.migration_runner import Upgrade, _report_failure +from ch_migrate.runner import _Progress +from ch_migrate.sql import SqlStatementError +from ch_migrate.waiting_display import DDLProgress, HostParts, MutationProgress, WaitDisplay +from ch_migrate.waiting_types import FailedMutation, MutationFailed, UnknownOutcome, WaitTimeout + +GIB = 1 << 30 + + +@pytest.fixture +def clock(monkeypatch): + now = [100.0] + monkeypatch.setattr("ch_migrate.waiting_display.time.monotonic", lambda: now[0]) + return now + + +def _display(terminal): + buffer = io.StringIO() + console = Console(file=buffer, force_terminal=terminal, width=160, color_system=None) + return WaitDisplay(console), buffer + + +def _mutation(left, host="h1", sizes=True): + names = frozenset(f"all_{i}_{i}_0" for i in range(left)) + return HostParts(host, names, left, {name: GIB for name in names} if sizes else None) + + +def _progress(*hosts): + return MutationProgress("demo_dev.events", "7", hosts) + + +def test_terminal_bar_weights_by_bytes_and_estimates_time(clock): + display, buffer = _display(terminal=True) + display.start_migration("3f9c2a1b backfill_status") + display.update(_progress(_mutation(41))) + clock[0] = 172.0 + display.update(_progress(_mutation(18))) + assert "23.0 / 41.0 GiB · 23/41 parts · 1:12 · ~0:56 left" in buffer.getvalue() + assert "━" in buffer.getvalue() + clock[0] = 223.0 + display.complete("demo_dev.events") + assert ( + buffer.getvalue() + .rstrip() + .endswith("✓ 3f9c2a1b backfill_status mutation 7 · 41 parts · 41.0 GiB · 2:03") + ) + + +def test_piped_lines_start_then_every_tenth_or_thirty_seconds(clock): + display, buffer = _display(terminal=False) + display.start_migration("3f9c2a1b backfill_status") + for at, left in ((100, 40), (101, 38), (102, 36), (140, 35)): + clock[0] = float(at) + display.update(_progress(_mutation(left, sizes=False))) + clock[0] = 141.0 + display.complete("demo_dev.events") + assert buffer.getvalue().splitlines() == [ + " waiting on mutation 7 (demo_dev.events): 0/40 parts", + " waiting on mutation 7 (demo_dev.events): 4/40 parts · 0:02", + " waiting on mutation 7 (demo_dev.events): 5/40 parts · 0:40", + "✓ 3f9c2a1b backfill_status mutation 7 · 40 parts · 0:41", + ] + assert "\x1b" not in buffer.getvalue() and "\r" not in buffer.getvalue() + + +def test_unreadable_part_sizes_fall_back_to_parts_once(): + display, buffer = _display(terminal=True) + assert display.wants_bytes + display.parts_only("can't read system.parts (ACCESS_DENIED)") + display.parts_only("can't read system.parts (ACCESS_DENIED)") + assert buffer.getvalue().count("! Progress counts parts only") == 1 + assert not display.wants_bytes + + +def test_parts_only_bar_has_no_time_estimate(clock): + display, buffer = _display(terminal=True) + display.update(_progress(_mutation(10, sizes=False))) + clock[0] = 110.0 + display.update(_progress(_mutation(4, sizes=False))) + assert "6/10 parts" in buffer.getvalue() and "left" not in buffer.getvalue() + + +def test_each_replica_gets_its_own_row(clock): + display, buffer = _display(terminal=True) + display.update(_progress(_mutation(5, "ch-1"), _mutation(5, "ch-2"))) + clock[0] = 105.0 + display.update(_progress(_mutation(2, "ch-2"))) + text = buffer.getvalue() + assert "ch-1" in text and "ch-2" in text + display.complete("demo_dev.events") + assert "· 2 replicas" in buffer.getvalue().splitlines()[-1] + + +def test_on_cluster_ddl_counts_hosts(clock): + display, buffer = _display(terminal=True) + display.update(DDLProgress("query-0000000012", 2, 3)) + assert "2/3 hosts" in buffer.getvalue() + piped, lines = _display(terminal=False) + piped.update(DDLProgress("query-0000000012", 2, 3)) + assert lines.getvalue() == " waiting on ON CLUSTER DDL query-0000000012: 2/3 hosts\n" + + +def test_failed_mutation_names_reason_retry_and_kill(capsys): + failure = FailedMutation( + "demo_dev.events", + "mutation_7.txt", + "h1", + "Cannot parse string 'n/a' as UInt32", + "KILL MUTATION WHERE database = 'demo_dev' AND table = 'events' AND mutation_id = 'mutation_7.txt'", + ) + error = _statement_error(MutationFailed("failed", failure)) + _report_failure(Upgrade("dev"), _current("3f9c2a1b widen_amount"), error) + err = capsys.readouterr().err + assert err.startswith( + "✗ 3f9c2a1b widen_amount (migrations/sql/x.up.sql:2) failed: mutation mutation_7.txt " + "on demo_dev.events failed on h1" + ) + assert "Cannot parse string 'n/a' as UInt32" in err + assert "keeps retrying a failed mutation" in err and failure.kill in err + + +def test_timeout_says_the_work_continues_and_how_to_resume(capsys): + timeout = WaitTimeout( + "Timed out", "demo_dev.events mutation mutation_7.txt; parts_to_do=3", 3.0 + ) + _report_failure(Upgrade("dev"), _current("3f9c2a1b backfill"), _statement_error(timeout)) + err = capsys.readouterr().err + assert err.startswith("✗ Timed out after 3.0s waiting on 3f9c2a1b backfill") + assert "Pending: demo_dev.events mutation mutation_7.txt; parts_to_do=3" in err + assert "ch-migrate up dev again to keep waiting; it won't reissue the statement" in err + + +def test_unknown_outcome_shows_the_operator_guidance(capsys): + unknown = UnknownOutcome("Outcome unknown for demo_dev.events\nRead-only inspection: SELECT 1") + _report_failure(Upgrade("dev"), _current("3f9c2a1b backfill"), _statement_error(unknown)) + err = capsys.readouterr().err + assert "outcome unknown, so the statement was not reissued" in err + assert " Read-only inspection: SELECT 1" in err + + +def _current(label): + progress = _Progress() + progress.current, progress.count = label, 1 + return progress + + +def _statement_error(cause): + try: + raise SqlStatementError( + f"migrations/sql/x.up.sql (statement 1 of 1, line 2): {cause}" + ) from cause + except SqlStatementError as error: + return error From 5c6612eaf882cef5da071e85acf2d8b66ff3217b Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 11:42:45 -0400 Subject: [PATCH 14/33] [release] Prepare 0.7.0 0.7.0 is 0.6.0 plus waiting on mutations and ON CLUSTER DDL, with a progress display. The CHANGELOG leads with what changes for a 0.6 project: up now blocks with no default timeout, and roles bootstrapped by 0.6 on Cloud or a cluster need five more grants. No upgrade-env is needed. The README's Cloud retry note now says up waits for the column mutation itself. Solves: DRY-1602 Tests: unit suite; integration suites at 013b33a --- CHANGELOG.md | 33 +++++++++++++++++++++++++++++++++ README.md | 20 ++++++++++++++------ ch_migrate/__init__.py | 2 +- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee91b0b..f96783a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,39 @@ Changes are recorded in [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ## [Unreleased] +## [0.7.0] + +0.7 waits for the work a migration starts. `up` doesn't record a migration as applied until the mutations and `ON CLUSTER` DDL its statements started have finished, and it shows their progress while it waits. A normal `pip install -U` reaches it from 0.6, so read this before upgrading: + +1. **`up` now blocks until that work finishes, with no default timeout.** If a CI job's time limit stops it mid-wait, ClickHouse keeps working and the next `up` reattaches to the same mutation instead of sending the statement again. `--timeout SECONDS` sets a limit of your own. +2. **Migration roles that 0.6's `bootstrap` created on ClickHouse Cloud or a cluster need five more grants.** Self-hosted single servers already give the role `SELECT ON system.*`. As the admin user, replacing `` with your project name: + ```sql + GRANT SELECT ON system.columns TO _migration_role; + GRANT CURRENT GRANTS(SELECT ON system.replicas) TO _migration_role; + GRANT CURRENT GRANTS(SELECT ON system.clusters) TO _migration_role; + GRANT CURRENT GRANTS(SELECT ON system.distributed_ddl_queue) TO _migration_role; + GRANT CURRENT GRANTS(SELECT ON system.parts) TO _migration_role; + ``` +3. **No `upgrade-env` this time.** The environment 0.6 wrote is current. + +To stay on 0.6 until then, pin `ch-migrate-cli==0.6.0`. + +### Changed + +- `up` waits before it records a migration as applied: for the mutations its statements start, on every required replica, and for `ON CLUSTER` DDL, on every host. It polls `system.mutations` and `system.distributed_ddl_queue` on the migration session instead of holding a request open with `mutations_sync`, so a dropped connection doesn't lose the outcome. +- A failed mutation fails the migration with ClickHouse's `latest_fail_reason` and prints the scoped `KILL MUTATION` statement for an operator. `ch-migrate` never kills a mutation itself. +- `up` runs Alembic in its own process, so killing `up` can't leave a child process advancing the version table. Running `alembic upgrade` directly goes through the same environment and waits too; `--timeout` exists only on `ch-migrate up`. +- The version table's delete goes through the same waiting, keeping 0.6's insert-first bookkeeping. +- If ClickHouse rejects a statement that would start a mutation, for example `MODIFY COLUMN` on a sorting-key column, the migration can't be re-run or edited until that statement's journal entry is reconciled by hand. The README's "Unknown outcome" section has the steps. + +### Added + +- `up --timeout SECONDS`: one waiting budget for the whole run. When it runs out, `up` exits 1, names what is still pending and leaves the work running; the next `up` waits on it again. +- `_ch_migrate_journal`, created by `up` in the project database with the version table's engine. It records which work each statement started before the statement is sent, so a rerun reattaches instead of reissuing. When the evidence can't prove what happened, `up` stops without reissuing the statement or completing the migration, and prints read-only inspection queries. Don't delete the journal or restore it separately from the database. +- Progress while `up` waits: in a terminal, a live bar per mutation weighted by bytes from `system.parts`, with parts and time left, one row per replica, and a host count for `ON CLUSTER` DDL. Piped output gets plain lines when a wait starts, at each 10% or every 30 seconds, and when it finishes. Without access to `system.parts` the bar counts parts only. +- Foreign mutations already queued on the same table are reported and waited for first; a failed one stops the run. +- `bootstrap` grants `SELECT` on `system.columns`, `system.replicas`, `system.clusters`, `system.distributed_ddl_queue` and `system.parts`. + ## [0.6.0] 0.6 runs migrations on ClickHouse's official clickhouse-connect dialect in one session-safe connection, and stops a migration from running twice. A normal `pip install -U` reaches it from 0.5.x, so read this before upgrading: diff --git a/README.md b/README.md index e61f9e3..29bedfd 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,15 @@ This README describes the source checkout, which may be ahead of PyPI. To try an ## Upgrade from 0.x -0.6 requires Python 3.10+; 0.5.x is the last line supporting Python 3.9. -A loose requirement such as `ch-migrate-cli` or `~=0.5` upgrades to 0.6; pin +**From 0.6:** 0.7 needs no `upgrade-env`. `up` now waits for the mutations and +`ON CLUSTER` DDL its statements start (see [Migrations that wait](#migrations-that-wait)), +so give CI jobs time to finish or pass `--timeout`. Migration roles that 0.6's +`bootstrap` created on ClickHouse Cloud or a cluster need the five grants listed +in the [CHANGELOG](CHANGELOG.md). A loose requirement such as `ch-migrate-cli` +upgrades to 0.7; pin `ch-migrate-cli==0.6.0` to stay on 0.6. + +**From 0.5:** 0.6 and later require Python 3.10+; 0.5.x is the last line supporting Python 3.9. +A loose requirement such as `ch-migrate-cli` or `~=0.5` upgrades past 0.5; pin `ch-migrate-cli==0.5.1` until each project has run the command below. After upgrading the package, run this in each migration project: @@ -395,10 +402,11 @@ The reviewable classification corpus is `tests/corpus/idempotency.yaml`. **Recovery:** fix the cause of a partial failure, then run `ch-migrate up ENV` again. Earlier idempotent statements can run again; the version advances only -when the revision completes. On ClickHouse Cloud, a column that the failed attempt -dropped or renamed stays locked until that change's mutation finishes: a retry -before then fails with "affected by mutation ... which is not finished yet", even -with `IF EXISTS`. Wait for it in `system.mutations`, then run `up` again. Before retrying waived operations, review what the +when the revision completes. A dropped or renamed column stays locked until that +change's mutation finishes, and `up` waits for it before running the next +statement, so a retry after a later failure doesn't hit "affected by mutation ... +which is not finished yet". If `up` was stopped while it waited, run it again: it +reattaches to that mutation. Before retrying waived operations, review what the earlier attempt actually changed. **Existing history:** `upgrade-env` records the current local script head(s) in diff --git a/ch_migrate/__init__.py b/ch_migrate/__init__.py index d1c9f6c..e01ef59 100644 --- a/ch_migrate/__init__.py +++ b/ch_migrate/__init__.py @@ -5,7 +5,7 @@ from ch_migrate import read_sql, get_db, get_env_config, create_dictionary """ -__version__ = "0.6.0" +__version__ = "0.7.0" from typing import Any From 284e8fdd92ffef7943c053815f4f481a14060bcb Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 11:52:03 -0400 Subject: [PATCH 15/33] [cli] Say when a failed mutation needs an operator "Fix the cause, then run up" only holds for a temporary cause. When the statement itself is wrong (a value that can't be converted, say), any fix is a mutation that queues behind the failed one, and killing it leaves an unknown outcome: re-running or editing the revision refuses until the journal entry is reconciled by hand. The hint now says both, the reason line drops the server version, and the CHANGELOG states the limitation once. Solves: DRY-1601, DRY-1604 (documents current behaviour) Tests: tests/test_waiting_display.py; reproduction on a local 26.3 server --- CHANGELOG.md | 3 +-- ch_migrate/migration_runner.py | 11 ++++++++--- tests/test_waiting_display.py | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f96783a..fac0d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,10 +24,9 @@ To stay on 0.6 until then, pin `ch-migrate-cli==0.6.0`. ### Changed - `up` waits before it records a migration as applied: for the mutations its statements start, on every required replica, and for `ON CLUSTER` DDL, on every host. It polls `system.mutations` and `system.distributed_ddl_queue` on the migration session instead of holding a request open with `mutations_sync`, so a dropped connection doesn't lose the outcome. -- A failed mutation fails the migration with ClickHouse's `latest_fail_reason` and prints the scoped `KILL MUTATION` statement for an operator. `ch-migrate` never kills a mutation itself. +- A failed mutation fails the migration with ClickHouse's `latest_fail_reason` and prints the scoped `KILL MUTATION` statement for an operator. `ch-migrate` never kills a mutation itself. Once a mutation is killed, or ClickHouse rejects a statement that would start one (for example `MODIFY COLUMN` on a sorting-key column), the migration can't be re-run or edited until that statement's journal entry is reconciled by hand; the README's "Unknown outcome" section has the steps. - `up` runs Alembic in its own process, so killing `up` can't leave a child process advancing the version table. Running `alembic upgrade` directly goes through the same environment and waits too; `--timeout` exists only on `ch-migrate up`. - The version table's delete goes through the same waiting, keeping 0.6's insert-first bookkeeping. -- If ClickHouse rejects a statement that would start a mutation, for example `MODIFY COLUMN` on a sorting-key column, the migration can't be re-run or edited until that statement's journal entry is reconciled by hand. The README's "Unknown outcome" section has the steps. ### Added diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py index 04fd702..9ac25f5 100644 --- a/ch_migrate/migration_runner.py +++ b/ch_migrate/migration_runner.py @@ -124,15 +124,20 @@ def _report_mutation_failure(environment: str, where: str, failure: FailedMutati ) else: ui.error(f"{where} failed: mutation {failure.mutation} on {failure.table} failed{on_host}") - ui.detail(failure.reason, stderr=True) + ui.detail(clean_database_error(failure.reason), stderr=True) ui.hint( "It was not recorded as applied. ClickHouse keeps retrying a failed mutation, and later " f"mutations on {failure.table} wait behind it.", stderr=True, ) ui.hint( - f"Fix the cause, then run `ch-migrate up {environment}` to wait for the retry. " - f"To stop it instead (operator only{on_host}): {failure.kill}", + "If the cause is temporary, such as memory or disk, fix it and run " + f"`ch-migrate up {environment}` to wait for the retry.", + stderr=True, + ) + ui.hint( + f"If the statement itself is wrong, an operator stops it{on_host} with {failure.kill}, " + "then reconciles the journal as the README's Unknown outcome section describes.", stderr=True, ) diff --git a/tests/test_waiting_display.py b/tests/test_waiting_display.py index 078aef4..7f84726 100644 --- a/tests/test_waiting_display.py +++ b/tests/test_waiting_display.py @@ -124,6 +124,7 @@ def test_failed_mutation_names_reason_retry_and_kill(capsys): ) assert "Cannot parse string 'n/a' as UInt32" in err assert "keeps retrying a failed mutation" in err and failure.kill in err + assert "If the cause is temporary" in err and "reconciles the journal" in err def test_timeout_says_the_work_continues_and_how_to_resume(capsys): From d80857a6a41a236a5db05db4116b27eb904a4468 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 11:53:30 -0400 Subject: [PATCH 16/33] [cli] Report Ctrl-C during up like a timeout KeyboardInterrupt skipped the runner, so a user who stopped a long wait got click's bare "Aborted!" and a terminal with the bar's cursor still hidden. up now clears the display and says the same as a timeout: the work keeps running, nothing is recorded, and the next up reattaches. Exit status 130. Solves: DRY-1601 Tests: tests/test_waiting_display.py; SIGINT mid-wait then rerun on a local server --- ch_migrate/migration_runner.py | 14 ++++++++++++++ tests/test_waiting_display.py | 9 ++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py index 9ac25f5..9335cef 100644 --- a/ch_migrate/migration_runner.py +++ b/ch_migrate/migration_runner.py @@ -69,6 +69,10 @@ def run_upgrade(upgrade: Upgrade) -> int: try: with _restored_loggers("alembic", "ch_migrate"): command.upgrade(config, upgrade.revision) + except KeyboardInterrupt: + display.stop() + _report_interrupt(upgrade.environment, progress) + return 130 except Exception as error: display.stop() _report_failure(upgrade, progress, error) @@ -100,6 +104,16 @@ def emit(self, record: logging.LogRecord) -> None: ui.detail(message) +def _report_interrupt(environment: str, progress: _Progress) -> None: + ui.error(f"Interrupted{' during ' + progress.current if progress.current else ''}") + ui.hint( + "Work ClickHouse already started keeps running, and the migration is not recorded as " + f"applied. Run `ch-migrate up {environment}` to pick up where it stopped; it won't " + "reissue a statement it already sent.", + stderr=True, + ) + + def _report_failure(upgrade: Upgrade, progress: _Progress, error: Exception) -> None: where = progress.current or "Upgrade" waiting = _waiting_error(error) diff --git a/tests/test_waiting_display.py b/tests/test_waiting_display.py index 7f84726..a66d77a 100644 --- a/tests/test_waiting_display.py +++ b/tests/test_waiting_display.py @@ -5,7 +5,7 @@ import pytest from rich.console import Console -from ch_migrate.migration_runner import Upgrade, _report_failure +from ch_migrate.migration_runner import Upgrade, _report_failure, _report_interrupt from ch_migrate.runner import _Progress from ch_migrate.sql import SqlStatementError from ch_migrate.waiting_display import DDLProgress, HostParts, MutationProgress, WaitDisplay @@ -146,6 +146,13 @@ def test_unknown_outcome_shows_the_operator_guidance(capsys): assert " Read-only inspection: SELECT 1" in err +def test_interrupt_says_the_work_continues(capsys): + _report_interrupt("dev", _current("3f9c2a1b backfill")) + err = capsys.readouterr().err + assert err.startswith("✗ Interrupted during 3f9c2a1b backfill") + assert "Run ch-migrate up dev to pick up where it stopped" in err + + def _current(label): progress = _Progress() progress.current, progress.count = label, 1 From 86751baeab681e355d69ba13e7e9211c8d996478 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 12:05:53 -0400 Subject: [PATCH 17/33] [test] Port main's Cloud changes to the waiting tests The first Cloud run on this branch failed 30 tests: the classifier and waiting tests came over from their original commits, before fad701e taught them about Cloud. SharedMergeTree lists metadata-only ALTERs in system.mutations (DRY-1446), Memory rows and query logs live on one replica, and Cloud refuses SYSTEM STOP MERGES. These are main's copies of conftest.py and both test files, with this branch's progress-output assertions and piped-progress test on top. (cherry picked from commit fad701e5, test files only) Solves: DRY-1602 Tests: Docker integration for both files --- tests/integration/conftest.py | 29 +++++++++- tests/integration/test_classification.py | 4 +- tests/integration/test_mutation_waiting.py | 62 ++++++++++++++-------- 3 files changed, 68 insertions(+), 27 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3f75f12..a30d5a7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -39,7 +39,8 @@ def connect(self): password=self.password, secure=self.secure, connect_timeout=5, - send_receive_timeout=15, + # Cloud DDL such as DROP DATABASE SYNC can take longer than a local container. + send_receive_timeout=60 if self.secure else 15, ) @@ -134,11 +135,35 @@ def query_log(self): return "system.query_log" def everywhere(self, table): - """FROM source for a Memory table: on Cloud its rows live only on the writing replica.""" + """FROM source for a per-node table (Memory, most system tables) on every Cloud replica.""" if self.cloud: return f"clusterAllReplicas('default', {table})" return table + def flush_logs(self): + """Flush query logs on every replica; Cloud keeps them per node.""" + on = " ON CLUSTER 'default'" if self.cloud else "" + self.client.command(f"SYSTEM FLUSH LOGS{on}") + + def drain_async_queue(self, table): + """Flush `table`'s async queue; Cloud forbids the flush, so wait for it to empty.""" + if not self.cloud: + self.client.command(f"SYSTEM FLUSH ASYNC INSERT QUEUE {self.database}.{table}") + return + deadline = time.monotonic() + 60 + while self.client.command( + f"SELECT count() FROM {self.everywhere('system.asynchronous_inserts')} " + "WHERE database = {db:String} AND table = {table:String}", + parameters={"db": self.database, "table": table}, + ): + if time.monotonic() > deadline: + pytest.fail(f"Async insert queue for {table} did not drain within 60 seconds") + time.sleep(0.2) + + def budget(self, seconds): + """An `up --timeout` value; Cloud's all-replica checks and grace need at least a minute.""" + return str(max(seconds, 60) if self.cloud else seconds) + @staticmethod def new_password(): # Cloud's password policy needs an uppercase letter and a special character. diff --git a/tests/integration/test_classification.py b/tests/integration/test_classification.py index a5d775c..29805eb 100644 --- a/tests/integration/test_classification.py +++ b/tests/integration/test_classification.py @@ -63,7 +63,7 @@ def test_classification_lightweight_update_is_synchronous_patch_work(project): def _mutation_ids(project): rows = project.client.query( "SELECT DISTINCT mutation_id FROM system.mutations " - "WHERE database = {db:String} AND table = 'sample'", - parameters={"db": project.database}, + "WHERE database = {db:String} AND table = {table:String} AND " + project.REWRITES_PARTS, + parameters={"db": project.database, "table": "sample"}, ).result_rows return {row[0] for row in rows} diff --git a/tests/integration/test_mutation_waiting.py b/tests/integration/test_mutation_waiting.py index 97e1cb5..ed5ac6a 100644 --- a/tests/integration/test_mutation_waiting.py +++ b/tests/integration/test_mutation_waiting.py @@ -59,7 +59,7 @@ def test_mutation_failure_leaves_head_and_prints_operator_kill(project): def test_mutation_timeout_preserves_work_and_resumes(project): _seed(project) _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") try: result = project.run("up", "it", "--timeout", "0.5") assert result.exit_code == 1, result.output @@ -82,7 +82,7 @@ def test_mutation_timeout_preserves_work_and_resumes(project): def test_mutation_sigkill_rerun_attaches_without_second_increment(project, trial): _seed(project) _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") try: with _running(project, ("up", "it")) as (first, first_log): rows = _until(lambda: _mutations(project), "mutation was not submitted") @@ -124,7 +124,7 @@ def crash(self, connection, cursor, statement, parameters, context, executemany) from ch_migrate.cli import main main() """ - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") try: with _running(project, ("up", "it"), code) as (crashed, log): assert crashed.wait(timeout=30) == -9, log.read_text() @@ -165,7 +165,7 @@ def test_mutation_expired_ownership_refuses_instead_of_replaying(project): " SETTINGS finished_mutations_to_keep=1, old_parts_lifetime=0, cleanup_delay_period=1, cleanup_delay_period_random_add=0", ) _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") try: timed_out = project.run("up", "it", "--timeout", "0.4") assert timed_out.exit_code == 1 @@ -189,7 +189,7 @@ def test_mutation_expired_ownership_refuses_instead_of_replaying(project): def test_mutation_changed_sql_refuses_replay(project): _seed(project) path = _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") try: assert project.run("up", "it", "--timeout", "0.4").exit_code == 1 before = {row[0] for row in _mutations(project)} @@ -205,11 +205,11 @@ def test_mutation_changed_sql_refuses_replay(project): def test_mutation_completed_prefix_is_not_repeated_and_downgrade_resets(project): _seed(project) project.client.command(f"CREATE TABLE {project.database}.audit (n UInt64) ENGINE=Memory") - path = _upgrade_sql( + _upgrade_sql( project, "INSERT INTO {db}.audit VALUES (1);\n-- ch-migrate: allow-non-idempotent increment once\nALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1", ) - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") try: assert project.run("up", "it", "--timeout", "0.4").exit_code == 1 finally: @@ -302,7 +302,7 @@ def test_mutation_polling_preserves_session_settings(project): "-- ch-migrate: allow-non-idempotent record the preserved session setting\n" "INSERT INTO {db}.observed SELECT getSetting('max_threads')", ) - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") try: with _running(project, ("up", "it")) as (process, log): mutation = _until(lambda: _mutations(project), "mutation was not submitted")[0][0] @@ -350,7 +350,7 @@ def test_mutation_raw_bind_paths_are_waited(project, mode): def test_mutation_replaced_table_refuses_resume(project): _seed(project) _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") result = project.run("up", "it", "--timeout", "0.4") assert result.exit_code == 1 project.client.command(f"DROP TABLE {project.database}.counter SYNC") @@ -367,7 +367,7 @@ def test_mutation_replaced_table_refuses_resume(project): def test_mutation_healthy_foreign_predecessor_is_waited_before_submission(project): _seed(project) _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") project.client.command(f"ALTER TABLE {project.database}.counter UPDATE x=x+10 WHERE id=1") original = {row[0] for row in _mutations(project)} try: @@ -417,8 +417,8 @@ def test_mutation_timeout_budget_is_not_reset_for_next_statement(project): "-- ch-migrate: allow-non-idempotent second controlled update\n" "ALTER TABLE {db}.second UPDATE x=x+1 WHERE id=1", ) - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") - project.client.command(f"SYSTEM STOP MERGES {project.database}.second") + project.stop_merges(f"{project.database}.counter") + project.stop_merges(f"{project.database}.second") try: with _running(project, ("up", "it", "--timeout", "1.5")) as (process, log): mutation = _until(lambda: _mutations(project), "first statement did not start")[0][0] @@ -476,7 +476,8 @@ def crash(self, step, checkpoint): resumed = project.run("up", "it") assert resumed.exit_code == 0, resumed.output assert _heads(project) == ["bbbb"] - assert project.client.command(f"SELECT count() FROM {project.database}.audit") == 1 + audit = project.everywhere(f"{project.database}.audit") + assert project.client.command(f"SELECT count() FROM {audit}") == 1 def test_mutation_transport_loss_is_not_treated_as_server_rejection(project): @@ -502,12 +503,10 @@ def lose_reply(self, connection, cursor, statement, parameters, context, execute assert "Read-only inspection:" in resumed.output assert _heads(project) == ["aaaa"] project.client.command("SYSTEM FLUSH LOGS") - requests = project.client.query( - "SELECT count() FROM system.query_log WHERE type = 'QueryStart' " - "AND startsWith(query, {prefix:String})", - parameters={"prefix": f"CREATE TABLE IF NOT EXISTS {project.database}.uncertain"}, - ).result_rows[0][0] - assert requests == 1 + # The client's text continues with the column list; a Shared database also logs its own + # re-execution of the CREATE, under another query_id, with `UUID '...'` inserted there. + prefix = f"CREATE TABLE IF NOT EXISTS {project.database}.uncertain (" + assert _requests_started(project, prefix) == 1 def test_mutation_trailing_foreign_failure_does_not_hold_owned_completion(project): @@ -539,7 +538,7 @@ def test_mutation_ttl_resume_does_not_materialize_twice(project): project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) assert project.run("up", "it").exit_code == 0 _upgrade_sql(project, "ALTER TABLE {db}.expiry MODIFY TTL ts + INTERVAL 100 YEAR") - project.client.command(f"SYSTEM STOP MERGES {project.database}.expiry") + project.stop_merges(f"{project.database}.expiry") try: timed_out = project.run("up", "it", "--timeout", "0.4") assert timed_out.exit_code == 1, timed_out.output @@ -557,7 +556,7 @@ def test_mutation_piped_progress_prints_plain_lines(project): """CI logs and agents get a start line naming the mutation, never a redrawn bar.""" _seed(project) _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") try: result = subprocess.run( [sys.executable, "-m", "ch_migrate.cli", "up", "it", "--timeout", "1"], @@ -584,7 +583,7 @@ def test_mutation_terminal_progress_uses_a_live_line(project): _seed(project) _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") - project.client.command(f"SYSTEM STOP MERGES {project.database}.counter") + project.stop_merges(f"{project.database}.counter") master, slave = pty.openpty() process = None try: @@ -701,11 +700,28 @@ def _upgrade_sql(project, sql): def _mutations(project, table="counter"): return project.client.query( "SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason FROM system.mutations " - "WHERE database = {db:String} AND table = {table:String} ORDER BY mutation_id", + "WHERE database = {db:String} AND table = {table:String} AND " + + project.REWRITES_PARTS + + " ORDER BY mutation_id", parameters={"db": project.database, "table": table}, ).result_rows +def _requests_started(project, prefix): + """Requests whose logged text starts with `prefix`, counted on every replica.""" + deadline = time.monotonic() + 30 + while True: + count = project.client.query( + f"SELECT uniqExact(query_id) FROM {project.query_log()} WHERE type = 'QueryStart' " + "AND startsWith(query, {prefix:String})", + parameters={"prefix": prefix}, + settings={"skip_unavailable_shards": 1}, + ).result_rows[0][0] + if count or time.monotonic() > deadline: + return count + time.sleep(1) + + def _heads(project): return [ row[0] From e1b7c02d760e42fb67103000fe4cf5cf8e892340 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 12:11:11 -0400 Subject: [PATCH 18/33] [docs] Scope the new grants to ClickHouse Cloud Self-hosted migration roles already read every system table through bootstrap's CURRENT GRANTS(SELECT ON system.*), whether single server or cluster. The individual grants matter where that wildcard can't be granted, which is ClickHouse Cloud. Solves: DRY-1602 Tests: --- CHANGELOG.md | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fac0d13..f358b9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Changes are recorded in [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) 0.7 waits for the work a migration starts. `up` doesn't record a migration as applied until the mutations and `ON CLUSTER` DDL its statements started have finished, and it shows their progress while it waits. A normal `pip install -U` reaches it from 0.6, so read this before upgrading: 1. **`up` now blocks until that work finishes, with no default timeout.** If a CI job's time limit stops it mid-wait, ClickHouse keeps working and the next `up` reattaches to the same mutation instead of sending the statement again. `--timeout SECONDS` sets a limit of your own. -2. **Migration roles that 0.6's `bootstrap` created on ClickHouse Cloud or a cluster need five more grants.** Self-hosted single servers already give the role `SELECT ON system.*`. As the admin user, replacing `` with your project name: +2. **Migration roles that 0.6's `bootstrap` created on ClickHouse Cloud need five more grants.** On self-hosted servers the role usually has them already, because `bootstrap` grants `SELECT ON system.*` when the admin user can; Cloud's admin can't. As the admin user, replacing `` with your project name: ```sql GRANT SELECT ON system.columns TO _migration_role; GRANT CURRENT GRANTS(SELECT ON system.replicas) TO _migration_role; diff --git a/README.md b/README.md index 29bedfd..0c16169 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ This README describes the source checkout, which may be ahead of PyPI. To try an **From 0.6:** 0.7 needs no `upgrade-env`. `up` now waits for the mutations and `ON CLUSTER` DDL its statements start (see [Migrations that wait](#migrations-that-wait)), so give CI jobs time to finish or pass `--timeout`. Migration roles that 0.6's -`bootstrap` created on ClickHouse Cloud or a cluster need the five grants listed -in the [CHANGELOG](CHANGELOG.md). A loose requirement such as `ch-migrate-cli` +`bootstrap` created on ClickHouse Cloud need the five grants listed in the +[CHANGELOG](CHANGELOG.md); self-hosted roles usually have them through `SELECT ON system.*`. A loose requirement such as `ch-migrate-cli` upgrades to 0.7; pin `ch-migrate-cli==0.6.0` to stay on 0.6. **From 0.5:** 0.6 and later require Python 3.10+; 0.5.x is the last line supporting Python 3.9. From f71a8a3e98fa933fcfa3f757ec59bdbb5d592673 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 12:25:34 -0400 Subject: [PATCH 19/33] [test] Count only this test's DDL-queue reads On a shared server such as ClickHouse Cloud, earlier tests' ON CLUSTER commands also read system.distributed_ddl_queue, so the single-node check failed on Cloud without ch-migrate reading the queue at all. It now counts reads after the test starts, on every replica. (cherry picked from commit 41b69ce4, this test only) Solves: DRY-1602 Tests: test_ddl_queue.py on Docker and Cloud --- tests/integration/test_ddl_queue.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_ddl_queue.py b/tests/integration/test_ddl_queue.py index ede4797..6aa40fa 100644 --- a/tests/integration/test_ddl_queue.py +++ b/tests/integration/test_ddl_queue.py @@ -188,13 +188,17 @@ def test_ddl_queue_is_not_used_for_single_node_migrations(project): "upgrade": 'op.execute(f"CREATE TABLE IF NOT EXISTS {db}.owned (id UInt64) ENGINE=Memory")' }, ) + started = project.client.command("SELECT toString(now64(6))") result = project.run("up", "it") assert result.exit_code == 0, result.output - project.client.command("SYSTEM FLUSH LOGS") + project.flush_logs() + # Earlier tests on a shared server (Cloud's ON CLUSTER commands) also read the queue. reads = project.client.query( - "SELECT count() FROM system.query_log WHERE type='QueryStart' " + f"SELECT count() FROM {project.query_log()} WHERE type='QueryStart' " + "AND event_time_microseconds >= toDateTime64({started:String}, 6) " "AND position(query, 'FROM system.distributed_ddl_queue') > 0 " - "AND position(query, 'system.query_log') = 0" + "AND position(query, 'system.query_log') = 0", + parameters={"started": started}, ).result_rows[0][0] assert reads == 0 From ff0026e8c54eddbbc227166a03c7fda9e5f1653f Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 12:35:48 -0400 Subject: [PATCH 20/33] [style] Keep the progress helpers to three parameters _bar_row took a label, two numbers and the stats text; the numbers now arrive as the ProgressBar that _bar builds, so each helper does one thing. _Wait.observe dropped a timestamp it never read. run_upgrade both configured and ran the upgrade; the config is now _alembic_config. The Progress alias names its types as strings so it can sit with the module constants. No behaviour change. Solves: DRY-1601 (python-style pass) Tests: unit suite --- ch_migrate/migration_runner.py | 22 ++++++++++++++-------- ch_migrate/waiting_display.py | 23 +++++++++++++---------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py index 9335cef..5a381d6 100644 --- a/ch_migrate/migration_runner.py +++ b/ch_migrate/migration_runner.py @@ -56,14 +56,7 @@ class Upgrade: def run_upgrade(upgrade: Upgrade) -> int: """Apply migrations in this process, one reported line each; return the exit status.""" progress, display = _Progress(), WaitDisplay() - config = Config(str(Path.cwd() / "alembic.ini")) - config.attributes.update( - ch_migrate_environment=upgrade.environment, - ch_migrate_command="upgrade", - ch_migrate_timeout=upgrade.timeout, - ch_migrate_display=display, - ch_migrate_log_handler=_MigrationLog(progress, display), - ) + config = _alembic_config(upgrade, progress, display) # Project helpers intentionally read these during migration execution only. saved = {name: os.environ.get(name) for name in ("CH_DATABASE", "CH_CLUSTER")} try: @@ -104,6 +97,19 @@ def emit(self, record: logging.LogRecord) -> None: ui.detail(message) +def _alembic_config(upgrade: Upgrade, progress: _Progress, display: WaitDisplay) -> Config: + """The project's Alembic config, carrying what env.py needs from this invocation.""" + config = Config(str(Path.cwd() / "alembic.ini")) + config.attributes.update( + ch_migrate_environment=upgrade.environment, + ch_migrate_command="upgrade", + ch_migrate_timeout=upgrade.timeout, + ch_migrate_display=display, + ch_migrate_log_handler=_MigrationLog(progress, display), + ) + return config + + def _report_interrupt(environment: str, progress: _Progress) -> None: ui.error(f"Interrupted{' during ' + progress.current if progress.current else ''}") ui.hint( diff --git a/ch_migrate/waiting_display.py b/ch_migrate/waiting_display.py index fb09f46..43afe56 100644 --- a/ch_migrate/waiting_display.py +++ b/ch_migrate/waiting_display.py @@ -25,6 +25,9 @@ PIPED_STEPS = 10 BAR_WIDTH = 38 +# What a waiter reports: a mutation, an ON CLUSTER queue entry, or a plain status line. +Progress = Union["MutationProgress", "DDLProgress", str] + @dataclass(frozen=True) class HostParts: @@ -61,9 +64,6 @@ class DDLProgress: total: int -Progress = Union[MutationProgress, DDLProgress, str] - - class WaitDisplay: """Draws one wait at a time; `complete` prints its result line, `stop` clears it.""" @@ -91,7 +91,7 @@ def update(self, progress: Progress) -> None: if isinstance(progress, MutationProgress) and progress.bookkeeping: return wait = self._wait_for(progress) - wait.observe(progress, time.monotonic()) + wait.observe(progress) if self._terminal: self._draw_live(wait) else: @@ -149,7 +149,7 @@ class _Wait: printed_at: float | None = None printed_step: int = -1 - def observe(self, progress: Progress, now: float) -> None: + def observe(self, progress: Progress) -> None: self.latest = progress if isinstance(progress, MutationProgress): for host in progress.hosts: @@ -162,7 +162,7 @@ def renderable(self): progress = self.latest if isinstance(progress, DDLProgress): stats = f"{progress.finished}/{progress.total} hosts · {elapsed}" - return _bar_row("", progress.finished, progress.total, stats) + return _bar_row("", _bar(progress.finished, progress.total), stats) if isinstance(progress, str): return Text(f" waiting: {progress} · {elapsed}", style="dim") rows = [self._host_row(host, len(progress.hosts) > 1) for host in progress.hosts] @@ -193,14 +193,14 @@ def _host_row(self, host: HostParts, named: bool): parts = f"{total - host.count}/{total} parts" left, size_total = host.bytes_left, self.bytes_total.get(host.host) if left is None or not size_total: - return _bar_row(host.host if named else "", total - host.count, total, parts) + return _bar_row(host.host if named else "", _bar(total - host.count, total), parts) done = size_total - left stats = ( f"{_sizes(done, size_total)} · {parts} · {_duration(time.monotonic() - self.started)}" ) if eta := self._eta(done, left): stats += f" · ~{eta} left" - return _bar_row(host.host if named else "", done, size_total, stats) + return _bar_row(host.host if named else "", _bar(done, size_total), stats) def _eta(self, done: int, left: int) -> str: elapsed = time.monotonic() - self.started @@ -252,16 +252,19 @@ def _key(progress: Progress) -> str: return "status" -def _bar_row(label: str, completed: float, total: float, stats: str) -> Table: +def _bar_row(label: str, bar: ProgressBar, stats: str) -> Table: row = Table.grid(padding=(0, 1)) row.add_column(no_wrap=True) row.add_column(width=BAR_WIDTH) row.add_column(no_wrap=True) - bar = ProgressBar(total=max(total, 1), completed=min(completed, total), width=BAR_WIDTH) row.add_row(Text(f" {label}".rstrip() if label else " "), bar, Text(stats)) return row +def _bar(completed: float, total: float) -> ProgressBar: + return ProgressBar(total=max(total, 1), completed=min(completed, total), width=BAR_WIDTH) + + def _duration(seconds: float) -> str: seconds = max(0, int(seconds)) hours, rest = divmod(seconds, 3600) From ce85baa1167bc6bfdeff14bccaf772ae0a0288b0 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 12:36:30 -0400 Subject: [PATCH 21/33] [style] Order the progress modules top-down Pure reordering, no other change, so blame on the moved lines stays readable. WaitDisplay now leads waiting_display.py ahead of the progress records it draws, with its private helpers below its public methods; private functions follow the order their callers use them. MutationFailed sits above the FailedMutation it carries, _route_logs above the _run_* helpers that run() calls after it, and the display tests' private helpers below the tests. Solves: DRY-1601 (python-style pass) Tests: unit suite; sorted-line comparison shows only moved lines --- ch_migrate/alembic_env.py | 24 ++++----- ch_migrate/waiting_display.py | 96 +++++++++++++++++------------------ ch_migrate/waiting_types.py | 16 +++--- tests/test_waiting_display.py | 30 +++++------ 4 files changed, 83 insertions(+), 83 deletions(-) diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index 1508dec..8e2306d 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -103,6 +103,18 @@ def has_current_env(path: Path) -> bool: ) +def _route_logs(handler: logging.Handler | None) -> None: + # `ch-migrate up` reports each migration in its own words, so Alembic's and + # ch-migrate's records go to its handler instead of alembic.ini's console handler. + if handler is None: + return + for name in ("alembic", "ch_migrate"): + named = logging.getLogger(name) + named.handlers = [handler] + named.propagate = False + named.setLevel(logging.INFO) + + def _run_offline(env_config: dict[str, Any]) -> None: context.configure( url=_url(env_config), @@ -184,18 +196,6 @@ def _run_context(waiter: MigrationWaiter, hooks: HookRegistry) -> None: waiter.finish_run() -def _route_logs(handler: logging.Handler | None) -> None: - # `ch-migrate up` reports each migration in its own words, so Alembic's and - # ch-migrate's records go to its handler instead of alembic.ini's console handler. - if handler is None: - return - for name in ("alembic", "ch_migrate"): - named = logging.getLogger(name) - named.handlers = [handler] - named.propagate = False - named.setLevel(logging.INFO) - - def _url(env_config: dict[str, Any]) -> URL: secure = env_config.get("secure", True) return URL.create( diff --git a/ch_migrate/waiting_display.py b/ch_migrate/waiting_display.py index 43afe56..8708b2c 100644 --- a/ch_migrate/waiting_display.py +++ b/ch_migrate/waiting_display.py @@ -29,41 +29,6 @@ Progress = Union["MutationProgress", "DDLProgress", str] -@dataclass(frozen=True) -class HostParts: - """One host's remaining parts. `sizes` maps each remaining part to its bytes, if known.""" - - host: str - names: frozenset[str] - count: int - sizes: dict[str, int] | None = None - - @property - def bytes_left(self) -> int | None: - if self.sizes is None or not self.names.issubset(self.sizes): - return None - return sum(self.sizes[name] for name in self.names) - - -@dataclass(frozen=True) -class MutationProgress: - """The mutation a statement started, on every host that hasn't finished it.""" - - table: str - mutation: str - hosts: tuple[HostParts, ...] - bookkeeping: bool = False # the version table's own delete: waited for, not shown - - -@dataclass(frozen=True) -class DDLProgress: - """An ON CLUSTER statement's distributed DDL queue entry.""" - - entry: str - finished: int - total: int - - class WaitDisplay: """Draws one wait at a time; `complete` prints its result line, `stop` clears it.""" @@ -74,19 +39,14 @@ def __init__(self, console: Console | None = None) -> None: self._live: Live | None = None self._parts_only = False + def start_migration(self, label: str) -> None: + self.migration = label + @property def wants_bytes(self) -> bool: """Only the terminal bar shows bytes, so piped runs skip the size lookup.""" return self._terminal and not self._parts_only - @property - def _terminal(self) -> bool: - # A dumb terminal can't redraw in place, so it gets the piped lines. - return self.console.is_terminal and not self.console.is_dumb_terminal - - def start_migration(self, label: str) -> None: - self.migration = label - def update(self, progress: Progress) -> None: if isinstance(progress, MutationProgress) and progress.bookkeeping: return @@ -127,6 +87,11 @@ def _wait_for(self, progress: Progress) -> _Wait: self._wait = _Wait(key, started=time.monotonic()) return self._wait + @property + def _terminal(self) -> bool: + # A dumb terminal can't redraw in place, so it gets the piped lines. + return self.console.is_terminal and not self.console.is_dumb_terminal + def _draw_live(self, wait: _Wait) -> None: if self._live is None: self._live = Live(console=self.console, auto_refresh=False, transient=True) @@ -139,6 +104,41 @@ def _print_piped(self, wait: _Wait) -> None: self.console.print(Text(" " + line)) +@dataclass(frozen=True) +class MutationProgress: + """The mutation a statement started, on every host that hasn't finished it.""" + + table: str + mutation: str + hosts: tuple[HostParts, ...] + bookkeeping: bool = False # the version table's own delete: waited for, not shown + + +@dataclass(frozen=True) +class HostParts: + """One host's remaining parts. `sizes` maps each remaining part to its bytes, if known.""" + + host: str + names: frozenset[str] + count: int + sizes: dict[str, int] | None = None + + @property + def bytes_left(self) -> int | None: + if self.sizes is None or not self.names.issubset(self.sizes): + return None + return sum(self.sizes[name] for name in self.names) + + +@dataclass(frozen=True) +class DDLProgress: + """An ON CLUSTER statement's distributed DDL queue entry.""" + + entry: str + finished: int + total: int + + @dataclass class _Wait: key: str @@ -239,11 +239,6 @@ def _mutation_totals(self, progress: MutationProgress) -> str: return text -def _named(mutation: str) -> str: - # MergeTree ids already read "mutation_7.txt"; replicated ones are bare numbers. - return mutation if mutation.startswith("mutation") else f"mutation {mutation}" - - def _key(progress: Progress) -> str: if isinstance(progress, MutationProgress): return f"mutation {progress.table} {progress.mutation}" @@ -265,6 +260,11 @@ def _bar(completed: float, total: float) -> ProgressBar: return ProgressBar(total=max(total, 1), completed=min(completed, total), width=BAR_WIDTH) +def _named(mutation: str) -> str: + # MergeTree ids already read "mutation_7.txt"; replicated ones are bare numbers. + return mutation if mutation.startswith("mutation") else f"mutation {mutation}" + + def _duration(seconds: float) -> str: seconds = max(0, int(seconds)) hours, rest = divmod(seconds, 3600) diff --git a/ch_migrate/waiting_types.py b/ch_migrate/waiting_types.py index 776b944..6afafc7 100644 --- a/ch_migrate/waiting_types.py +++ b/ch_migrate/waiting_types.py @@ -24,6 +24,14 @@ def __init__(self, message: str, pending: str, elapsed: float) -> None: self.pending, self.elapsed = pending, elapsed +class MutationFailed(WaitingError): + """ClickHouse reports `latest_fail_reason` for a mutation this run owns or waits behind.""" + + def __init__(self, message: str, failure: FailedMutation) -> None: + super().__init__(message) + self.failure = failure + + @dataclass(frozen=True) class FailedMutation: """What the operator needs: where it failed, why, and the scoped KILL statement.""" @@ -36,14 +44,6 @@ class FailedMutation: foreign: bool = False -class MutationFailed(WaitingError): - """ClickHouse reports `latest_fail_reason` for a mutation this run owns or waits behind.""" - - def __init__(self, message: str, failure: FailedMutation) -> None: - super().__init__(message) - self.failure = failure - - @dataclass class WaitBudget: timeout: float | None = None diff --git a/tests/test_waiting_display.py b/tests/test_waiting_display.py index a66d77a..2228e51 100644 --- a/tests/test_waiting_display.py +++ b/tests/test_waiting_display.py @@ -21,21 +21,6 @@ def clock(monkeypatch): return now -def _display(terminal): - buffer = io.StringIO() - console = Console(file=buffer, force_terminal=terminal, width=160, color_system=None) - return WaitDisplay(console), buffer - - -def _mutation(left, host="h1", sizes=True): - names = frozenset(f"all_{i}_{i}_0" for i in range(left)) - return HostParts(host, names, left, {name: GIB for name in names} if sizes else None) - - -def _progress(*hosts): - return MutationProgress("demo_dev.events", "7", hosts) - - def test_terminal_bar_weights_by_bytes_and_estimates_time(clock): display, buffer = _display(terminal=True) display.start_migration("3f9c2a1b backfill_status") @@ -153,6 +138,21 @@ def test_interrupt_says_the_work_continues(capsys): assert "Run ch-migrate up dev to pick up where it stopped" in err +def _display(terminal): + buffer = io.StringIO() + console = Console(file=buffer, force_terminal=terminal, width=160, color_system=None) + return WaitDisplay(console), buffer + + +def _mutation(left, host="h1", sizes=True): + names = frozenset(f"all_{i}_{i}_0" for i in range(left)) + return HostParts(host, names, left, {name: GIB for name in names} if sizes else None) + + +def _progress(*hosts): + return MutationProgress("demo_dev.events", "7", hosts) + + def _current(label): progress = _Progress() progress.current, progress.count = label, 1 From 73a00d1c747ac5d40449b2de24f3b8c449948ba9 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 12:42:50 -0400 Subject: [PATCH 22/33] [test] Give the DDL-and-mutation wait test room on slow CI With --timeout 1, a slow runner could run out of time while the ON CLUSTER queue entry was still finishing on the second host, so the timeout named the DDL wait instead of the mutation that STOP MERGES holds open, and the parts_to_do assertion failed (CI run 40543073241). Five seconds lets the queue finish first, so the test checks what it means to. main has the same test and the same race. Solves: DRY-1602 Tests: test_ddl_queue.py both_must_finish on the local cluster harness --- tests/integration/test_ddl_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_ddl_queue.py b/tests/integration/test_ddl_queue.py index 6aa40fa..725cc18 100644 --- a/tests/integration/test_ddl_queue.py +++ b/tests/integration/test_ddl_queue.py @@ -126,7 +126,9 @@ def test_ddl_queue_and_replicated_mutation_both_must_finish(cluster_project, cli for client in cluster.clients.values(): client.command(f"SYSTEM STOP MERGES {table}") try: - result = project.run("up", "it", "--timeout", "1") + # Long enough for the queue entry to finish on both hosts on a slow CI runner, so the + # deadline falls in the mutation wait that STOP MERGES holds open. + result = project.run("up", "it", "--timeout", "5") assert result.exit_code == 1, result.output rows = _queue(project) assert rows and all(row[3] == "Finished" and row[4] == 0 for row in rows) From 4722f5187e3933b7b6b6bb68f8778e96cf2d2886 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 12:52:21 -0400 Subject: [PATCH 23/33] [refactor] Move the waiting modules into ch_migrate.waiting Eight waiting_*.py modules sat in the package root beside unrelated commands; they are one feature, so they become a subpackage (Dan, 2026-10-27). waiting.py is now waiting/waiter.py, waiting_store.py is waiting/journal.py after the class it holds, and waiting_types.py is split into errors.py and budget.py so no module shadows the standard library's types. The package exports MigrationWaiter, so `from ch_migrate.waiting import MigrationWaiter` still works. Only imports change; no code moves between functions. main still has the flat modules, so the forward-port to main makes the same move. Solves: DRY-1601 (Dan's review) Tests: unit suite; imports resolve from the CLI and env.py --- ch_migrate/alembic_env.py | 2 +- ch_migrate/migration_runner.py | 4 +- ch_migrate/sql.py | 3 +- ch_migrate/version_table.py | 2 +- ch_migrate/waiting/__init__.py | 12 ++++++ .../{waiting_types.py => waiting/budget.py} | 41 ++----------------- ch_migrate/{waiting_ddl.py => waiting/ddl.py} | 4 +- .../display.py} | 0 ch_migrate/waiting/errors.py | 41 +++++++++++++++++++ .../{waiting_store.py => waiting/journal.py} | 4 +- .../mutations.py} | 6 +-- ch_migrate/{waiting_sql.py => waiting/sql.py} | 2 +- .../versions.py} | 4 +- ch_migrate/{waiting.py => waiting/waiter.py} | 13 +++--- tests/integration/test_mutation_waiting.py | 2 +- tests/test_waiting.py | 9 ++-- tests/test_waiting_display.py | 6 +-- 17 files changed, 87 insertions(+), 68 deletions(-) create mode 100644 ch_migrate/waiting/__init__.py rename ch_migrate/{waiting_types.py => waiting/budget.py} (55%) rename ch_migrate/{waiting_ddl.py => waiting/ddl.py} (97%) rename ch_migrate/{waiting_display.py => waiting/display.py} (100%) create mode 100644 ch_migrate/waiting/errors.py rename ch_migrate/{waiting_store.py => waiting/journal.py} (98%) rename ch_migrate/{waiting_mutations.py => waiting/mutations.py} (98%) rename ch_migrate/{waiting_sql.py => waiting/sql.py} (99%) rename ch_migrate/{waiting_versions.py => waiting/versions.py} (98%) rename ch_migrate/{waiting.py => waiting/waiter.py} (98%) diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index 8e2306d..9ef552b 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -27,7 +27,7 @@ sync_version_replica, ) from ch_migrate.waiting import MigrationWaiter -from ch_migrate.waiting_versions import write_unjournaled +from ch_migrate.waiting.versions import write_unjournaled ENV_VERSION = 2 DEFAULT_SESSION_TIMEOUT = 1800 diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py index 5a381d6..8c2e2cf 100644 --- a/ch_migrate/migration_runner.py +++ b/ch_migrate/migration_runner.py @@ -29,8 +29,8 @@ _start_migration, ) from ch_migrate.sql import clean_database_error -from ch_migrate.waiting_display import WaitDisplay -from ch_migrate.waiting_types import ( +from ch_migrate.waiting.display import WaitDisplay +from ch_migrate.waiting.errors import ( FailedMutation, MutationFailed, UnknownOutcome, diff --git a/ch_migrate/sql.py b/ch_migrate/sql.py index c691c37..1c2f84e 100644 --- a/ch_migrate/sql.py +++ b/ch_migrate/sql.py @@ -41,7 +41,6 @@ class SqlStatementError(RuntimeError): """ - def run_sql(path: str, **values: Any) -> None: """Run every statement in a SQL file under migrations/sql/, one at a time. @@ -127,7 +126,7 @@ def clean_database_error(message: str) -> str: def _database_reason(exc: Exception) -> str: """Preserve operator instructions from wait failures; trim driver noise only.""" - from ch_migrate.waiting_types import WaitingError + from ch_migrate.waiting.errors import WaitingError current: BaseException | None = exc seen: set[int] = set() diff --git a/ch_migrate/version_table.py b/ch_migrate/version_table.py index 3f13b11..50a0830 100644 --- a/ch_migrate/version_table.py +++ b/ch_migrate/version_table.py @@ -120,7 +120,7 @@ def assert_version_mutations_healthy(client, state: VersionTableState) -> None: settings=settings, ).result_rows if failed: - from ch_migrate.waiting_sql import sql_string + from ch_migrate.waiting.sql import sql_string host, mutation, reason = failed[0] raise VersionTableMutationError( diff --git a/ch_migrate/waiting/__init__.py b/ch_migrate/waiting/__init__.py new file mode 100644 index 0000000..47fd138 --- /dev/null +++ b/ch_migrate/waiting/__init__.py @@ -0,0 +1,12 @@ +"""Waiting for the work migrations start: mutations and ON CLUSTER DDL. + +`up` doesn't record a migration until the mutations and distributed DDL its +statements started have finished. `MigrationWaiter` (waiter.py) drives that from +Alembic's cursor events; mutations.py and ddl.py wait on the server, journal.py +records what each statement started, versions.py writes the version table, and +display.py shows progress. The design is docs/design/2026-10-02-waiting.md. +""" + +from ch_migrate.waiting.waiter import MigrationWaiter + +__all__ = ["MigrationWaiter"] diff --git a/ch_migrate/waiting_types.py b/ch_migrate/waiting/budget.py similarity index 55% rename from ch_migrate/waiting_types.py rename to ch_migrate/waiting/budget.py index 6afafc7..6da2baa 100644 --- a/ch_migrate/waiting_types.py +++ b/ch_migrate/waiting/budget.py @@ -1,47 +1,12 @@ -"""Waiting errors, invocation deadline, and bounded progress output.""" +"""One waiting deadline per invocation, and the poll between checks.""" from __future__ import annotations import time from dataclasses import dataclass, field -from ch_migrate.waiting_display import Progress, WaitDisplay - - -class WaitingError(RuntimeError): - """Server work is unfinished; the revision must not be completed.""" - - -class UnknownOutcome(WaitingError): - """Available evidence cannot justify replay or completion.""" - - -class WaitTimeout(WaitingError): - """The invocation's waiting deadline expired without cancelling server work.""" - - def __init__(self, message: str, pending: str, elapsed: float) -> None: - super().__init__(message) - self.pending, self.elapsed = pending, elapsed - - -class MutationFailed(WaitingError): - """ClickHouse reports `latest_fail_reason` for a mutation this run owns or waits behind.""" - - def __init__(self, message: str, failure: FailedMutation) -> None: - super().__init__(message) - self.failure = failure - - -@dataclass(frozen=True) -class FailedMutation: - """What the operator needs: where it failed, why, and the scoped KILL statement.""" - - table: str - mutation: str - host: str - reason: str - kill: str - foreign: bool = False +from ch_migrate.waiting.display import Progress, WaitDisplay +from ch_migrate.waiting.errors import WaitTimeout @dataclass diff --git a/ch_migrate/waiting_ddl.py b/ch_migrate/waiting/ddl.py similarity index 97% rename from ch_migrate/waiting_ddl.py rename to ch_migrate/waiting/ddl.py index bbb22a0..75cc934 100644 --- a/ch_migrate/waiting_ddl.py +++ b/ch_migrate/waiting/ddl.py @@ -2,8 +2,8 @@ from __future__ import annotations -from ch_migrate.waiting_display import DDLProgress -from ch_migrate.waiting_types import UnknownOutcome, WaitingError +from ch_migrate.waiting.display import DDLProgress +from ch_migrate.waiting.errors import UnknownOutcome, WaitingError class DistributedDDLWaiter: diff --git a/ch_migrate/waiting_display.py b/ch_migrate/waiting/display.py similarity index 100% rename from ch_migrate/waiting_display.py rename to ch_migrate/waiting/display.py diff --git a/ch_migrate/waiting/errors.py b/ch_migrate/waiting/errors.py new file mode 100644 index 0000000..499dc7c --- /dev/null +++ b/ch_migrate/waiting/errors.py @@ -0,0 +1,41 @@ +"""Why waiting stopped short: unfinished work, an unknown outcome, a deadline, a failed mutation.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +class WaitingError(RuntimeError): + """Server work is unfinished; the revision must not be completed.""" + + +class UnknownOutcome(WaitingError): + """Available evidence cannot justify replay or completion.""" + + +class WaitTimeout(WaitingError): + """The invocation's waiting deadline expired without cancelling server work.""" + + def __init__(self, message: str, pending: str, elapsed: float) -> None: + super().__init__(message) + self.pending, self.elapsed = pending, elapsed + + +class MutationFailed(WaitingError): + """ClickHouse reports `latest_fail_reason` for a mutation this run owns or waits behind.""" + + def __init__(self, message: str, failure: FailedMutation) -> None: + super().__init__(message) + self.failure = failure + + +@dataclass(frozen=True) +class FailedMutation: + """What the operator needs: where it failed, why, and the scoped KILL statement.""" + + table: str + mutation: str + host: str + reason: str + kill: str + foreign: bool = False diff --git a/ch_migrate/waiting_store.py b/ch_migrate/waiting/journal.py similarity index 98% rename from ch_migrate/waiting_store.py rename to ch_migrate/waiting/journal.py index d746c0b..6a9952b 100644 --- a/ch_migrate/waiting_store.py +++ b/ch_migrate/waiting/journal.py @@ -10,8 +10,8 @@ from sqlalchemy.schema import CreateTable from ch_migrate.version_table import migration_state_engine -from ch_migrate.waiting_sql import qualified_table, query_settings -from ch_migrate.waiting_types import UnknownOutcome +from ch_migrate.waiting.errors import UnknownOutcome +from ch_migrate.waiting.sql import qualified_table, query_settings JOURNAL_TABLE = "_ch_migrate_journal" diff --git a/ch_migrate/waiting_mutations.py b/ch_migrate/waiting/mutations.py similarity index 98% rename from ch_migrate/waiting_mutations.py rename to ch_migrate/waiting/mutations.py index f2b9016..7068e53 100644 --- a/ch_migrate/waiting_mutations.py +++ b/ch_migrate/waiting/mutations.py @@ -7,9 +7,9 @@ from clickhouse_connect.driver.exceptions import DatabaseError -from ch_migrate.waiting_display import HostParts, MutationProgress -from ch_migrate.waiting_sql import sql_string -from ch_migrate.waiting_types import FailedMutation, MutationFailed, UnknownOutcome, WaitingError +from ch_migrate.waiting.display import HostParts, MutationProgress +from ch_migrate.waiting.errors import FailedMutation, MutationFailed, UnknownOutcome, WaitingError +from ch_migrate.waiting.sql import sql_string VERSION_TABLE = "alembic_version" diff --git a/ch_migrate/waiting_sql.py b/ch_migrate/waiting/sql.py similarity index 99% rename from ch_migrate/waiting_sql.py rename to ch_migrate/waiting/sql.py index 0ba70f4..c865d35 100644 --- a/ch_migrate/waiting_sql.py +++ b/ch_migrate/waiting/sql.py @@ -10,7 +10,7 @@ from ch_migrate.classify import _identifier, _tokens from ch_migrate.sql import _skip_quoted_or_comment -from ch_migrate.waiting_types import WaitingError +from ch_migrate.waiting.errors import WaitingError _WORD = re.compile(r"[A-Za-z_][A-Za-z_0-9]*") _MANAGED = frozenset(("mutations_sync", "alter_sync", "lightweight_deletes_sync")) diff --git a/ch_migrate/waiting_versions.py b/ch_migrate/waiting/versions.py similarity index 98% rename from ch_migrate/waiting_versions.py rename to ch_migrate/waiting/versions.py index 890d621..f50dc0a 100644 --- a/ch_migrate/waiting_versions.py +++ b/ch_migrate/waiting/versions.py @@ -10,14 +10,14 @@ from sqlalchemy.sql.dml import Delete, Insert, Update from ch_migrate import ui -from ch_migrate.waiting_sql import ( +from ch_migrate.waiting.errors import UnknownOutcome, WaitingError +from ch_migrate.waiting.sql import ( mutation_sql, qualified_table, query_settings, sql_string, statement_digest, ) -from ch_migrate.waiting_types import UnknownOutcome, WaitingError class VersionWrites: diff --git a/ch_migrate/waiting.py b/ch_migrate/waiting/waiter.py similarity index 98% rename from ch_migrate/waiting.py rename to ch_migrate/waiting/waiter.py index 390df19..b4f1523 100644 --- a/ch_migrate/waiting.py +++ b/ch_migrate/waiting/waiter.py @@ -20,9 +20,12 @@ from ch_migrate.idempotency import classify_idempotency from ch_migrate.introspect import ColumnDefinition, Schema, TableDefinition, _parse_order_by from ch_migrate.statements import migration_statements -from ch_migrate.waiting_ddl import DistributedDDLWaiter -from ch_migrate.waiting_mutations import MutationWaiter -from ch_migrate.waiting_sql import ( +from ch_migrate.waiting.budget import WaitBudget +from ch_migrate.waiting.ddl import DistributedDDLWaiter +from ch_migrate.waiting.errors import UnknownOutcome, WaitingError +from ch_migrate.waiting.journal import StepKey, WaitingJournal +from ch_migrate.waiting.mutations import MutationWaiter +from ch_migrate.waiting.sql import ( bind_statement, is_session_or_read, modifies_ttl, @@ -36,9 +39,7 @@ statement_digest, statement_table, ) -from ch_migrate.waiting_store import StepKey, WaitingJournal -from ch_migrate.waiting_types import UnknownOutcome, WaitBudget, WaitingError -from ch_migrate.waiting_versions import VersionWrites +from ch_migrate.waiting.versions import VersionWrites class MigrationWaiter: diff --git a/tests/integration/test_mutation_waiting.py b/tests/integration/test_mutation_waiting.py index ed5ac6a..96257ad 100644 --- a/tests/integration/test_mutation_waiting.py +++ b/tests/integration/test_mutation_waiting.py @@ -460,7 +460,7 @@ def test_mutation_version_insert_lost_receipt_finishes_bookkeeping_only(project) project.client.command(f"CREATE TABLE {project.database}.audit (n UInt64) ENGINE=Memory") _upgrade_sql(project, "INSERT INTO {db}.audit VALUES (1)") code = """import os, signal -from ch_migrate.waiting_versions import VersionWrites +from ch_migrate.waiting.versions import VersionWrites original = VersionWrites._insert def crash(self, step, checkpoint): original(self, step, checkpoint) diff --git a/tests/test_waiting.py b/tests/test_waiting.py index f5396a6..a4c25e6 100644 --- a/tests/test_waiting.py +++ b/tests/test_waiting.py @@ -4,14 +4,15 @@ from click.testing import CliRunner from ch_migrate.cli import main -from ch_migrate.waiting_mutations import MutationWaiter -from ch_migrate.waiting_sql import ( +from ch_migrate.waiting.budget import WaitBudget +from ch_migrate.waiting.errors import WaitTimeout +from ch_migrate.waiting.mutations import MutationWaiter +from ch_migrate.waiting.sql import ( is_session_or_read, mutation_sql, query_setting, statement_table, ) -from ch_migrate.waiting_types import WaitBudget, WaitTimeout @pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf", "-inf"]) @@ -64,7 +65,7 @@ def test_statement_tracking_distinguishes_reads_from_writes(sql, readonly): def test_deadline_is_shared_across_waits(monkeypatch): now = [10.0] - monkeypatch.setattr("ch_migrate.waiting_types.time.monotonic", lambda: now[0]) + monkeypatch.setattr("ch_migrate.waiting.budget.time.monotonic", lambda: now[0]) budget = WaitBudget(timeout=2, started=now[0]) now[0] = 11.5 assert budget.elapsed == 1.5 diff --git a/tests/test_waiting_display.py b/tests/test_waiting_display.py index 2228e51..e438457 100644 --- a/tests/test_waiting_display.py +++ b/tests/test_waiting_display.py @@ -8,8 +8,8 @@ from ch_migrate.migration_runner import Upgrade, _report_failure, _report_interrupt from ch_migrate.runner import _Progress from ch_migrate.sql import SqlStatementError -from ch_migrate.waiting_display import DDLProgress, HostParts, MutationProgress, WaitDisplay -from ch_migrate.waiting_types import FailedMutation, MutationFailed, UnknownOutcome, WaitTimeout +from ch_migrate.waiting.display import DDLProgress, HostParts, MutationProgress, WaitDisplay +from ch_migrate.waiting.errors import FailedMutation, MutationFailed, UnknownOutcome, WaitTimeout GIB = 1 << 30 @@ -17,7 +17,7 @@ @pytest.fixture def clock(monkeypatch): now = [100.0] - monkeypatch.setattr("ch_migrate.waiting_display.time.monotonic", lambda: now[0]) + monkeypatch.setattr("ch_migrate.waiting.display.time.monotonic", lambda: now[0]) return now From 6201e7ff4d6420db97f7b9d44b46e55c28058de5 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 13:33:59 -0400 Subject: [PATCH 24/33] [waiting] Wait by observation; drop the journal 0.7 waited through a journal that recorded which work each statement started, so a rerun never sent a statement twice. That was about 1,300 of the feature's 2,100 lines, duplicated what the idempotency gate already guarantees, and left a killed or rejected mutation unrecoverable without editing the journal by hand (DRY-1604). Dan chose the simpler design on 2026-10-27. After each statement that can write, up now polls system.mutations on the migration session until the project database has no unfinished mutation on any replica that has to apply it. The ids the connected server reports are expected on every replica of a ReplicatedMergeTree table, so a replica that hasn't received one yet is pending, not finished. ON CLUSTER statements wait inside ClickHouse in throw mode, bounded by 270 seconds and what's left of --timeout; wait_end_of_query makes a timeout arrive as an error rather than a truncated response. Work already queued is waited for before the first migration runs, and a rerun waits for unfinished work and then runs the migration again. Removed: the journal and version-write modules, the classifier and its corpus, statement markers, the TTL split and the DDL-queue poller. version_table.py and its tests are 0.6's again. bootstrap grants only system.replicas and system.parts. The README, CHANGELOG and capability 5 describe the new behaviour; the waiting design note is marked superseded. Solves: DRY-1600, DRY-1604 Tests: unit 672; Docker single 53 passed 4 skipped; cluster 16, incl. killed-then-fixed and rejected-then-fixed reruns (DRY-1604) --- CHANGELOG.md | 31 +- README.md | 172 ++--- ch_migrate/alembic_env.py | 135 ++-- ch_migrate/bootstrap.py | 3 - ch_migrate/classify.py | 302 -------- ch_migrate/migration_runner.py | 25 +- ch_migrate/templates/bootstrap/init_users.sql | 3 - ch_migrate/version_table.py | 29 +- ch_migrate/waiting/__init__.py | 11 +- ch_migrate/waiting/ddl.py | 115 --- ch_migrate/waiting/display.py | 30 +- ch_migrate/waiting/errors.py | 6 +- ch_migrate/waiting/journal.py | 185 ----- ch_migrate/waiting/mutations.py | 428 +++++------- ch_migrate/waiting/sql.py | 239 +------ ch_migrate/waiting/versions.py | 162 ----- ch_migrate/waiting/waiter.py | 533 ++------------ docs/design/2026-10-02-ch-migrate-1.0.md | 34 +- docs/design/2026-10-02-waiting.md | 8 + tests/corpus/classification.yaml | 166 ----- tests/integration/test_classification.py | 69 -- tests/integration/test_cloud.py | 31 +- tests/integration/test_ddl_queue.py | 334 --------- tests/integration/test_mutation_waiting.py | 654 +++++------------- tests/integration/test_on_cluster.py | 200 ++++++ tests/integration/test_version_table.py | 45 +- tests/integration/waiting_spike_probe.py | 558 --------------- tests/test_classify.py | 77 --- tests/test_waiting.py | 135 ++-- tests/test_waiting_display.py | 30 +- 30 files changed, 946 insertions(+), 3804 deletions(-) delete mode 100644 ch_migrate/classify.py delete mode 100644 ch_migrate/waiting/ddl.py delete mode 100644 ch_migrate/waiting/journal.py delete mode 100644 ch_migrate/waiting/versions.py delete mode 100644 tests/corpus/classification.yaml delete mode 100644 tests/integration/test_classification.py delete mode 100644 tests/integration/test_ddl_queue.py create mode 100644 tests/integration/test_on_cluster.py delete mode 100644 tests/integration/waiting_spike_probe.py delete mode 100644 tests/test_classify.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f358b9b..61a6b96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,33 +8,30 @@ Changes are recorded in [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) 0.7 waits for the work a migration starts. `up` doesn't record a migration as applied until the mutations and `ON CLUSTER` DDL its statements started have finished, and it shows their progress while it waits. A normal `pip install -U` reaches it from 0.6, so read this before upgrading: -1. **`up` now blocks until that work finishes, with no default timeout.** If a CI job's time limit stops it mid-wait, ClickHouse keeps working and the next `up` reattaches to the same mutation instead of sending the statement again. `--timeout SECONDS` sets a limit of your own. -2. **Migration roles that 0.6's `bootstrap` created on ClickHouse Cloud need five more grants.** On self-hosted servers the role usually has them already, because `bootstrap` grants `SELECT ON system.*` when the admin user can; Cloud's admin can't. As the admin user, replacing `` with your project name: +1. **`up` now blocks until that work finishes, with no default timeout.** If a CI job's time limit stops it mid-wait, ClickHouse keeps working, and the next `up` waits for that work before it runs the migration again. `--timeout SECONDS` sets a limit of your own. +2. **A rerun runs the stopped migration again from its first statement.** 0.6's idempotency gate already requires statements that are safe to repeat, or a written waiver; check what a waived statement changed before you rerun it. +3. **Two grants, only where `bootstrap` couldn't grant `SELECT ON system.*`**, such as on ClickHouse Cloud. As the admin user, replacing `` with your project name: ```sql - GRANT SELECT ON system.columns TO _migration_role; - GRANT CURRENT GRANTS(SELECT ON system.replicas) TO _migration_role; - GRANT CURRENT GRANTS(SELECT ON system.clusters) TO _migration_role; - GRANT CURRENT GRANTS(SELECT ON system.distributed_ddl_queue) TO _migration_role; GRANT CURRENT GRANTS(SELECT ON system.parts) TO _migration_role; + GRANT CURRENT GRANTS(SELECT ON system.replicas) TO _migration_role; ``` -3. **No `upgrade-env` this time.** The environment 0.6 wrote is current. + `system.parts` gives the progress bar its bytes and time estimate; without it the bar counts parts. `system.replicas` is read only for self-hosted replicated tables, so Cloud needs just the first. +4. **No `upgrade-env` this time.** The environment 0.6 wrote is current. -To stay on 0.6 until then, pin `ch-migrate-cli==0.6.0`. +To stay on 0.6, pin `ch-migrate-cli==0.6.0`. ### Changed -- `up` waits before it records a migration as applied: for the mutations its statements start, on every required replica, and for `ON CLUSTER` DDL, on every host. It polls `system.mutations` and `system.distributed_ddl_queue` on the migration session instead of holding a request open with `mutations_sync`, so a dropped connection doesn't lose the outcome. -- A failed mutation fails the migration with ClickHouse's `latest_fail_reason` and prints the scoped `KILL MUTATION` statement for an operator. `ch-migrate` never kills a mutation itself. Once a mutation is killed, or ClickHouse rejects a statement that would start one (for example `MODIFY COLUMN` on a sorting-key column), the migration can't be re-run or edited until that statement's journal entry is reconciled by hand; the README's "Unknown outcome" section has the steps. -- `up` runs Alembic in its own process, so killing `up` can't leave a child process advancing the version table. Running `alembic upgrade` directly goes through the same environment and waits too; `--timeout` exists only on `ch-migrate up`. -- The version table's delete goes through the same waiting, keeping 0.6's insert-first bookkeeping. +- `up` waits before it records a migration as applied. After each statement that can write, it polls `system.mutations` on the migration session until the project database has no unfinished mutation on any replica that has to apply it, instead of holding a request open with `mutations_sync`. Mutations another writer started count too, and ones already queued are waited for before the first migration runs. The version table's own mutations stay with its 0.6 bookkeeping. +- `ON CLUSTER` statements wait inside ClickHouse: `up` sends them with `distributed_ddl_output_mode = 'throw'` and a limit of 270 seconds, or what's left of `--timeout` if that's less, so a host that fails or doesn't finish in time fails the migration with ClickHouse's message. +- A failed mutation fails the migration with ClickHouse's `latest_fail_reason` and prints the scoped `KILL MUTATION` statement for an operator; `ch-migrate` never kills a mutation itself. A failed mutation from before the run blocks it the same way. After a kill, fix the SQL and run `up` again. +- `up` runs Alembic in its own process, so killing `up` can't leave a child process advancing the version table. `down` and a direct `alembic upgrade` go through the same environment and wait too; `--timeout` exists only on `ch-migrate up`. ### Added -- `up --timeout SECONDS`: one waiting budget for the whole run. When it runs out, `up` exits 1, names what is still pending and leaves the work running; the next `up` waits on it again. -- `_ch_migrate_journal`, created by `up` in the project database with the version table's engine. It records which work each statement started before the statement is sent, so a rerun reattaches instead of reissuing. When the evidence can't prove what happened, `up` stops without reissuing the statement or completing the migration, and prints read-only inspection queries. Don't delete the journal or restore it separately from the database. -- Progress while `up` waits: in a terminal, a live bar per mutation weighted by bytes from `system.parts`, with parts and time left, one row per replica, and a host count for `ON CLUSTER` DDL. Piped output gets plain lines when a wait starts, at each 10% or every 30 seconds, and when it finishes. Without access to `system.parts` the bar counts parts only. -- Foreign mutations already queued on the same table are reported and waited for first; a failed one stops the run. -- `bootstrap` grants `SELECT` on `system.columns`, `system.replicas`, `system.clusters`, `system.distributed_ddl_queue` and `system.parts`. +- `up --timeout SECONDS`: one waiting budget for the whole run. When it runs out, `up` exits 1, names what is still pending and leaves the work running; the next `up` waits on it again, then runs the migration. +- Progress while `up` waits: in a terminal, a live bar per mutation weighted by bytes from `system.parts`, with parts and time left, and one row per replica. Piped output gets plain lines when a wait starts, at each 10% or every 30 seconds, and when it finishes. Without access to `system.parts` the bar counts parts only. +- `bootstrap` grants `SELECT` on `system.replicas` and `system.parts`. ## [0.6.0] diff --git a/README.md b/README.md index 0c16169..7fb5efd 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ This README describes the source checkout, which may be ahead of PyPI. To try an **From 0.6:** 0.7 needs no `upgrade-env`. `up` now waits for the mutations and `ON CLUSTER` DDL its statements start (see [Migrations that wait](#migrations-that-wait)), -so give CI jobs time to finish or pass `--timeout`. Migration roles that 0.6's -`bootstrap` created on ClickHouse Cloud need the five grants listed in the -[CHANGELOG](CHANGELOG.md); self-hosted roles usually have them through `SELECT ON system.*`. A loose requirement such as `ch-migrate-cli` -upgrades to 0.7; pin `ch-migrate-cli==0.6.0` to stay on 0.6. +so give CI jobs time to finish or pass `--timeout`. Where `bootstrap` couldn't grant +`SELECT ON system.*`, such as on ClickHouse Cloud, add the grants listed in the +[CHANGELOG](CHANGELOG.md). A loose requirement such as `ch-migrate-cli` upgrades to +0.7; pin `ch-migrate-cli==0.6.0` to stay on 0.6. **From 0.5:** 0.6 and later require Python 3.10+; 0.5.x is the last line supporting Python 3.9. A loose requirement such as `ch-migrate-cli` or `~=0.5` upgrades past 0.5; pin @@ -180,14 +180,13 @@ New `alembic_version` tables use `ORDER BY version_num`. The runtime inspects | Atomic database with `cluster` configured | `ReplicatedMergeTree` created `ON CLUSTER`, with a Keeper path containing the database name and shard macro | | Single server | `MergeTree` | -Version advancement remains **insert the new row, then delete the old row**; -there is no asynchronous version UPDATE. Online `up` journals that bookkeeping -and polls its owned DELETE rather than holding an HTTP request open with -`mutations_sync`. If interrupted, both rows can remain. A subsequent `up` -finishes the proven bookkeeping without running the revision again. Overlapping -heads without an owned journal receipt are still refused. Failed version -mutations report their reason and an operator-only KILL statement; ch-migrate -never kills them itself. +The official Alembic implementation advances a version by **inserting the new +row, then deleting the old row with `mutations_sync = 2`**. It does not issue an +asynchronous version UPDATE. If the process stops while the delete is pending, +both rows remain and Alembic refuses their overlapping history rather than +repeating the migration. Let the version mutation finish, then retry. A +server-reported failed version mutation stops the run with its reason; ch-migrate +never kills that mutation. Before reading replicated heads, the tool catches up replicated database metadata and inserted version parts. The table barrier uses @@ -237,17 +236,42 @@ is separate from the local suite; the local replicated harness has one shard. ### Migrations that wait -`up` waits for the mutations its statements create before completing a revision. -It uses the same checked ClickHouse session for submission and polling, so prior -`SET` statements stay effective. `op.execute`, raw `op.get_bind()` execution, and -`run_sql` all pass through the cursor boundary. Upgrades run in the CLI process: -killing that process cannot leave a child Alembic runner advancing revisions. +`up` doesn't record a migration as applied until the work its statements started +has finished. After each statement that can write, it polls `system.mutations` on +the same checked ClickHouse session, so earlier `SET` statements stay in effect. +`op.execute`, raw `op.get_bind()` execution and `run_sql` all pass through that +check. Upgrades run in the CLI process, so killing it can't leave a child Alembic +runner advancing revisions. `down` waits the same way. ```bash ch-migrate up dev # No default waiting timeout ch-migrate up dev --timeout 120 # One waiting budget across the invocation ``` +What `up` waits for: + +- **Every unfinished mutation in the project database,** including one another + writer started. Before the first migration runs, `up` waits for any already + queued, and a failed one blocks the run. The version table's own mutations are + left to [its bookkeeping](#the-version-table). +- **Each replica that has to apply it.** Zero parts left isn't completion on its + own: the mutation must report `is_done`. A ReplicatedMergeTree mutation must be + finished on every replica, and a replica that hasn't received it yet is still + pending. Configure `cluster` for self-hosted replicated tables, or use a + Replicated database's automatic cluster; without either, `up` refuses a table + with more than one replica. SharedMergeTree on ClickHouse Cloud reports one + shared mutation. +- **`ON CLUSTER` DDL on every host.** `up` sends these statements with + `distributed_ddl_output_mode = 'throw'`, so ClickHouse answers once every host + has run the statement, and a host that fails or doesn't finish in time fails the + migration with ClickHouse's message. The limit is 270 seconds, below the + client's read timeout, or what's left of `--timeout` if that's less. Mutations + the statement starts are then waited for as above. + +Statements ClickHouse already holds until they finish, such as a type-changing +`MODIFY COLUMN`, a `MODIFY TTL` that materializes, or a lightweight `DELETE`, +still hold their request as they did in 0.6. + While it waits, `up` shows progress on stderr. In a terminal, each mutation gets a live bar weighted by bytes, with its parts and an estimate of the time left: @@ -257,10 +281,10 @@ live bar weighted by bytes, with its parts and an estimate of the time left: ✓ 3f9c2a1b backfill_digest mutation_7.txt · 41 parts · 41.0 GiB · 2:03 ``` -Self-hosted replicated tables get one row per replica, and `ON CLUSTER` DDL -counts the hosts that have finished. When output isn't a terminal (CI logs, -pipes, coding agents), `up` prints plain lines instead: one when a wait starts, -one at each 10% of the parts or every 30 seconds, and one when it finishes: +Self-hosted replicated tables get one row per replica. When output isn't a +terminal (CI logs, pipes, coding agents), `up` prints plain lines instead: one +when a wait starts, one at each 10% of the parts or every 30 seconds, and one +when it finishes: ```text → Applying 3f9c2a1b backfill_digest @@ -270,88 +294,26 @@ one at each 10% of the parts or every 30 seconds, and one when it finishes: ``` Part sizes come from `system.parts`. If the migration role can't read it, the bar -counts parts only and says so once; progress never changes whether `up` -succeeds. After a resume the bar counts the parts still left when it reattached. -A failed mutation shows ClickHouse's `latest_fail_reason` and the scoped -`KILL MUTATION` statement; a timeout names what is still pending and how to -resume. Zero remaining parts alone does not count as completion: every -required replica must report its owned mutation finished. Configure `cluster` -for self-hosted replicated tables, or use a Replicated database's automatic -cluster. Missing replicas or missing evidence are never treated as success. - -An active foreign mutation ahead of ours is reported and waited before more work -is submitted. A failed predecessor stops the run. Foreign work queued after our -completion barrier is not claimed as ours and does not hold our revision open. - -Ownership is persisted in `_ch_migrate_journal` **before** SQL submission. The -journal follows the version table's deployment policy and uses the same session. -It records revision/statement identities, table UUIDs and progress receipts. -Run only one migration runner per database, using a CI concurrency key or an -external lock: this journal is not a distributed lock. Do not delete or restore -it independently of the database state it describes. - -After a timeout, dropped connection or killed process, ClickHouse may still be -working. Run the same `up` again: it skips proven completed statements and -reattaches to its known unfinished mutation instead of issuing it again. -Successful downgrade starts a fresh journal generation so a later upgrade -actually executes. A changed unresolved statement or replaced table is refused. -An explicitly rejected repeat-safe metadata statement can be repaired and retried; -transport errors do not establish rejection. - -**Unknown outcome:** if ownership/completion evidence is missing or expired, -`up` exits nonzero, does not reissue the statement and does not complete the -migration. The error names the journal key, token and recorded UUIDs and provides -read-only inspection queries. Operator recovery is deliberately not a retry flag: - -1. Stop migration runners and quiesce other writers as needed. Back up the - affected journal key and relevant data before making changes. -2. Establish whether the original statement took effect using independent data, - schema and server evidence on every required replica. An absent mutation row - is not proof either way. -3. Reconcile that **specific** statement and journal key in a reviewed manual - operation. For a proven applied operation, its completed receipt must carry - the actual post-operation UUID map (`after_target.uuids`, or null for a dropped - target). For a proven unapplied operation, remove only that key's attempt - records after restoring any partial effects, and wait for that journal - change on every replica. Preserve the other completed statement receipts. -4. Recheck the recorded heads and rerun `up`. Never blindly stamp the revision, - delete the whole journal, or replay a non-idempotent statement to guess its - outcome. - -**Failed mutation:** `latest_fail_reason` stops the run and prints a scoped -`KILL MUTATION` statement for the operator. The tool does not execute it. Fix the -cause and reattach, or review a cancellation and reconcile its effects; killing -a mutation is not proof that it completed. - -`MODIFY TTL` has a separate metadata/materialization boundary: when materialization -is enabled, the runtime journals the metadata change with automatic materialization -disabled, then submits a separately owned `MATERIALIZE TTL`. This avoids confusing -TTL's comma-separated rules with ALTER actions. Session and per-query -`materialize_ttl_after_modify = 0` are honored. Parameter batches are executed -sequentially through the same single-statement path, with a receipt and completion -barrier for each parameter set; intermediate TTL effects are not collapsed. -Lightweight DELETE keeps its row-mask semantics; lightweight UPDATE is synchronous -patch work, not a fabricated `system.mutations` record. - -For `ON CLUSTER` statements, the same journal first records a unique distributed -DDL marker. `up` submits without a server-side synchronous DDL wait, then polls -`system.distributed_ddl_queue`. Every configured target host must report -`Finished` **and** exception code zero. A failed host is named with its error; -a down host remains pending until it returns or the shared `--timeout` expires. -The tool neither deletes the queue entry nor resubmits the DDL on rerun. - -The queue entry is found by its recorded `log_comment`, not by matching SQL text -(the server can insert UUIDs into that text). Caller log comments are preserved -in the journal. A lost acknowledgement can reattach to the existing entry; -missing/expired queue evidence follows the unknown-outcome procedure above. -Mutation-producing distributed ALTERs must satisfy **both** barriers: DDL execution -on all hosts and completion of their owned mutations. TTL metadata and explicit -materialization retain separate queue receipts. - -Single-node statements without `ON CLUSTER` do not consult that queue. Ordinary -Cloud DDL does not need `ON CLUSTER`; Cloud qualification remains separate from -the local harness. Re-run bootstrap when upgrading a restricted migration user -to obtain the available cluster and distributed-queue inspection grants. +counts parts only and says so once; progress never changes whether `up` succeeds. + +**When `up` stops short.** After a timeout, Ctrl-C, dropped connection or killed +process, ClickHouse keeps working and the migration isn't recorded. Run `up` +again: it waits for that work to finish, then runs the migration again from its +first statement. That's safe because the idempotency gate below requires every +statement to be safe to repeat, or to carry a written waiver; before rerunning a +waived statement, check what the first attempt changed. + +**Failed mutation.** ClickHouse's `latest_fail_reason` stops the run, and `up` +prints the scoped `KILL MUTATION` statement for an operator; ch-migrate never +runs it. ClickHouse keeps retrying a failed mutation, and later mutations on that +table queue behind it. If the cause is temporary, such as memory or disk, fix it +and run `up` again. If the statement itself is wrong, kill the mutation, fix the +SQL and run `up` again. Killing a mutation doesn't undo the parts it already +rewrote. + +Run one migration runner per database, using a CI concurrency key or an external +lock. Re-run `bootstrap` when upgrading a restricted migration user to obtain the +`system.replicas` and `system.parts` grants. ### Re-runnable migrations @@ -406,8 +368,8 @@ when the revision completes. A dropped or renamed column stays locked until that change's mutation finishes, and `up` waits for it before running the next statement, so a retry after a later failure doesn't hit "affected by mutation ... which is not finished yet". If `up` was stopped while it waited, run it again: it -reattaches to that mutation. Before retrying waived operations, review what the -earlier attempt actually changed. +waits for that mutation, then runs the revision again. Before retrying waived +operations, review what the earlier attempt actually changed. **Existing history:** `upgrade-env` records the current local script head(s) in `config.yaml` as `lint.gate_baseline`, preserving YAML comments. It records diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index 9ef552b..8a9e330 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -6,15 +6,17 @@ import os from logging.config import fileConfig from pathlib import Path +from types import SimpleNamespace from typing import Any +from uuid import uuid4 import clickhouse_connect.cc_sqlalchemy.alembic # Registers the official implementation. from alembic import context from clickhouse_connect.cc_sqlalchemy.alembic.impl import ClickHouseImpl from clickhouse_connect.driver.httputil import get_pool_manager from dotenv import load_dotenv -from sqlalchemy import URL, Column, MetaData, String, Table, create_engine, pool -from sqlalchemy.sql.dml import Delete, Insert, Update +from sqlalchemy import URL, Column, MetaData, String, Table, create_engine, pool, text +from sqlalchemy.sql.dml import Insert from ch_migrate.config import get_env_config from ch_migrate.hooks import HookRegistry, run_hooks @@ -27,7 +29,6 @@ sync_version_replica, ) from ch_migrate.waiting import MigrationWaiter -from ch_migrate.waiting.versions import write_unjournaled ENV_VERSION = 2 DEFAULT_SESSION_TIMEOUT = 1800 @@ -58,25 +59,45 @@ def version_table_impl(self, *, version_table, version_table_schema, version_tab ) def _exec(self, construct, execution_options=None, multiparams=None, params=None): - # The official implementation owns these SQLAlchemy version constructs; - # all user SQL, including raw bind execution, is covered by cursor events. - if isinstance(construct, (Insert, Update, Delete)) and self._is_version_table_construct( - construct - ): - waiter = self.context_opts.get("ch_migrate_waiter") - if waiter is not None and waiter.revision is not None: - return waiter.versions.execute(construct, self) - if not isinstance(construct, Delete): - return write_unjournaled(construct, self) + if isinstance(construct, Insert) and self._is_version_table_construct(construct): + self._insert_version_row(construct) + return SimpleNamespace(rowcount=1) return super()._exec( construct, execution_options=execution_options, multiparams=multiparams, params=params ) + def _exec_version_update(self, construct, execution_options=None): + # The official INSERT then waited DELETE, with the INSERT written by _insert_version_row. + self._insert_version_row(construct) + where = self._compile_version_where(construct) + super()._exec( + text( + f"ALTER TABLE {self._version_table_name} DELETE WHERE {where} " + "SETTINGS mutations_sync = 2" + ) + ) + return SimpleNamespace(rowcount=1) + + def _insert_version_row(self, construct) -> None: + # Replicated and Shared MergeTree silently drop a block identical to an earlier insert + # (DRY-1447), so re-inserting a previous head after `down` left no version row at all. + # A token unique to this write keeps every intentional version row. + value = self._compile_clause(next(iter(construct._values.values()))) + token = f"'chm_version_{uuid4().hex}'" + super()._exec( + text( + f"INSERT INTO {self._version_table_name} (version_num) " + f"SETTINGS insert_deduplication_token = {token}, async_insert = 0 " + f"VALUES ({value})" + ) + ) + def run() -> None: """Run the selected environment on one official-dialect session.""" root = Path.cwd() load_dotenv(root / ".env.local") + # `ch-migrate up` runs Alembic in its own process and names the environment directly. environment = context.config.attributes.get( "ch_migrate_environment", os.environ.get("CH_ENVIRONMENT", "dev") ) @@ -150,7 +171,36 @@ def _run_online(env_config: dict[str, Any]) -> None: def _run_connection(connection, env_config: dict[str, Any]) -> None: - # The checked session and listener registration share this failure boundary. + hooks = HookRegistry.from_config(env_config.get("hooks")) + state = _checked_session(connection, env_config) + context.configure( + connection=connection, + target_metadata=None, + version_table="alembic_version", + version_table_schema=state.database, + on_version_apply=_post_migrate(hooks, state.database), + ch_migrate_version_state=state, + ) + waiter = MigrationWaiter(connection, state, context.config.attributes.get("ch_migrate_timeout")) + try: + waiter.install(context.config) + with context.begin_transaction(): + # Work an earlier run left unfinished finishes before anything is sent again. + waiter.wait(before_run=True) + run_hooks( + connection, + hooks.pre_migrate, + db=state.database, + phase="pre_migrate", + revision="all", + ) + context.run_migrations() + finally: + waiter.close() + + +def _checked_session(connection, env_config: dict[str, Any]) -> VersionTableState: + # Establish the session before requiring its existence on every later request. connection.exec_driver_sql("SELECT 1") client = connection.connection.dbapi_connection.client client.set_client_setting("session_check", 1) @@ -160,40 +210,7 @@ def _run_connection(connection, env_config: dict[str, Any]) -> None: if warning := state.warning(): logger.warning(warning) connection.dialect.ddl_compiler = VersionTableDDLCompiler - waiter = MigrationWaiter(connection, state, context.config.attributes.get("ch_migrate_timeout")) - try: - waiter.install(context.config) - context.configure( - connection=connection, - target_metadata=None, - version_table="alembic_version", - version_table_schema=state.database, - ch_migrate_waiter=waiter, - ch_migrate_version_state=state, - ) - _run_context(waiter, HookRegistry.from_config(env_config.get("hooks"))) - finally: - waiter.close() - - -def _run_context(waiter: MigrationWaiter, hooks: HookRegistry) -> None: - runtime = context.get_context() - runtime._migrations_fn = waiter.steps(runtime._migrations_fn, hooks) - with context.begin_transaction(): - if _is_upgrade(): - waiter.ensure_version_table(runtime._version) - waiter.versions.resume() - waiter.run_pre_hooks(hooks) - else: - run_hooks( - waiter.connection, - hooks.pre_migrate, - db=waiter.state.database, - phase="pre_migrate", - revision="all", - ) - context.run_migrations() - waiter.finish_run() + return state def _url(env_config: dict[str, Any]) -> URL: @@ -212,9 +229,19 @@ def _url(env_config: dict[str, Any]) -> URL: ) -def _is_upgrade() -> bool: - explicit = context.config.attributes.get("ch_migrate_command") - if explicit is not None: - return explicit == "upgrade" - command = getattr(context.config.cmd_opts, "cmd", None) - return bool(command and command[0].__name__ == "upgrade") +def _post_migrate(hooks: HookRegistry, database: str): + def after_revision(ctx, step, heads, run_args): + revision = ( + step.up_revision + if step.is_upgrade + else (step.down_revisions[0] if step.down_revisions else "unknown") + ) + run_hooks( + ctx.connection, + hooks.post_migrate, + db=database, + phase="post_migrate", + revision=revision, + ) + + return after_revision diff --git a/ch_migrate/bootstrap.py b/ch_migrate/bootstrap.py index 0ff8fb0..5954db7 100644 --- a/ch_migrate/bootstrap.py +++ b/ch_migrate/bootstrap.py @@ -147,10 +147,7 @@ def build_bootstrap_sql( f"GRANT SELECT ON system.databases TO {project}_migration_role;", f"GRANT SELECT ON system.tables TO {project}_migration_role;", f"GRANT SELECT ON system.mutations TO {project}_migration_role;", - f"GRANT SELECT ON system.columns TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SELECT ON system.replicas) TO {project}_migration_role;", - f"GRANT CURRENT GRANTS(SELECT ON system.clusters) TO {project}_migration_role;", - f"GRANT CURRENT GRANTS(SELECT ON system.distributed_ddl_queue) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SELECT ON system.parts) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SYSTEM SYNC REPLICA ON {db}.*) TO {project}_migration_role;", f"GRANT CURRENT GRANTS(SYSTEM SYNC DATABASE REPLICA ON {db}.*) TO {project}_migration_role;", diff --git a/ch_migrate/classify.py b/ch_migrate/classify.py deleted file mode 100644 index 6a19c3b..0000000 --- a/ch_migrate/classify.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Classify ClickHouse work without executing it; the server corpus is authoritative.""" - -from __future__ import annotations - -import re -from dataclasses import dataclass -from typing import Literal - -from ch_migrate.idempotency import _ACTIONS, _action_clauses -from ch_migrate.introspect import Schema, TableDefinition -from ch_migrate.sql import _skip_quoted_or_comment -from ch_migrate.statements import MigrationStatement - -_WORD_OR_NUMBER = re.compile(r"[A-Za-z_][A-Za-z_0-9]*|\d+(?:\.\d+)?") -_TYPE_MODIFIERS = {"DEFAULT", "MATERIALIZED", "ALIAS", "CODEC", "COMMENT", "REMOVE", "TTL"} - - -@dataclass(frozen=True) -class Classification: - kind: Literal["metadata", "mutation", "rebuild", "other"] - table: str | None - detail: str - - -def classify( - statement: str | MigrationStatement, live_schema: Schema | None = None -) -> Classification: - """Classify metadata, background mutations, impossible ALTERs, or other work. - - The contract is tests/corpus/classification.yaml. Its real-server proof in - tests/integration/test_classification.py defines correctness, not a guessed - list of SQL verbs. With no live type to compare, MODIFY COLUMN is conservatively - a mutation *if the type changes*. Unsupported actions remain explicitly other. - """ - tokens = _tokens(statement if isinstance(statement, str) else statement.sql) - if not tokens: - return Classification("other", None, "No SQL statement") - words = tuple(token.upper() for token in tokens) - if words[:2] == ("ALTER", "TABLE"): - return _classify_alter(tokens, live_schema) - table = _statement_target(tokens) - if words[0] in ("CREATE", "DROP", "RENAME"): - return Classification("metadata", table, "Object metadata operation") - if words[:2] == ("DELETE", "FROM"): - return Classification( - "mutation", table, "Lightweight DELETE rewrites the row-existence mask" - ) - if words[0] == "UPDATE": - return Classification( - "other", - table, - "Synchronous lightweight UPDATE writes patch parts; no background mutation record", - ) - return Classification("other", table, "Not a classified schema operation") - - -@dataclass(frozen=True) -class _Alter: - table: str - definition: TableDefinition | None - actions: tuple[tuple[str, ...], ...] - settings: tuple[str, ...] - - -def _classify_alter(tokens: tuple[str, ...], schema: Schema | None) -> Classification: - index = 2 - if tuple(token.upper() for token in tokens[index : index + 2]) == ("IF", "EXISTS"): - index += 2 - table, index = _name(tokens, index) - if tuple(token.upper() for token in tokens[index : index + 2]) == ("ON", "CLUSTER"): - index += 3 - actions, settings = _split_settings(tokens[index:]) - definition = _table_definition(schema, table) - alter = _Alter(table, definition, _alter_actions(actions), settings) - classifications = [_classify_action(action, alter) for action in alter.actions] - priority = {"metadata": 0, "mutation": 1, "other": 2, "rebuild": 3} - kind = max(classifications, key=lambda item: priority[item.kind]).kind - return Classification( - kind, table, "; ".join(dict.fromkeys(item.detail for item in classifications)) - ) - - -def _alter_actions(tokens: tuple[str, ...]) -> tuple[tuple[str, ...], ...]: - actions: list[tuple[str, ...]] = [] - for clause in _action_clauses(tokens): - starts_action = clause and clause[0].upper() in _ACTIONS | {"REMOVE"} - if actions and (not starts_action or clause[1:2] == ("=",)): - actions[-1] += (",",) + clause - else: - actions.append(clause) - return tuple(actions) - - -def _classify_action(action: tuple[str, ...], alter: _Alter) -> Classification: - words = tuple(token.upper() for token in action) - prefix = words[:2] - if prefix == ("MODIFY", "COLUMN"): - return _modify_column(action[2:], alter) - if words[:3] == ("MODIFY", "ORDER", "BY"): - return _modify_order(action[3:], alter) - if prefix in (("MODIFY", "PRIMARY"), ("MODIFY", "PARTITION"), ("MODIFY", "ENGINE")): - return Classification( - "rebuild", alter.table, "This key or engine cannot be changed by an in-place ALTER" - ) - if prefix == ("MODIFY", "TTL"): - disabled = _setting_is_zero(alter.settings, "materialize_ttl_after_modify") - return Classification( - "metadata" if disabled else "mutation", - alter.table, - "TTL metadata only" if disabled else "MODIFY TTL materializes existing parts", - ) - if words[:1] in (("UPDATE",), ("DELETE",), ("CLEAR",), ("MATERIALIZE",)): - return Classification("mutation", alter.table, "Background mutation of existing parts") - if prefix in ( - ("DROP", "COLUMN"), - ("RENAME", "COLUMN"), - ("DROP", "INDEX"), - ("DROP", "PROJECTION"), - ): - return Classification( - "mutation", - alter.table, - "Column, index, or projection change creates a background mutation", - ) - if words[:1] in (("ADD",), ("COMMENT",), ("RESET",)) or prefix in ( - ("MODIFY", "SETTING"), - ("REMOVE", "TTL"), - ): - return Classification("metadata", alter.table, "Metadata change; no background mutation") - return Classification( - "other", alter.table, "ALTER action is not covered by the server-backed classification" - ) - - -def _modify_column(tokens: tuple[str, ...], alter: _Alter) -> Classification: - if tuple(token.upper() for token in tokens[:2]) == ("IF", "EXISTS"): - tokens = tokens[2:] - if not tokens: - return Classification("other", alter.table, "Missing column definition") - column = _identifier(tokens[0]) - type_tokens = _column_type(tokens[1:]) - if not type_tokens: - return Classification( - "metadata", alter.table, "Column default, comment, or codec metadata only" - ) - current = ( - next((item for item in alter.definition.columns if item.name == column), None) - if alter.definition - else None - ) - if current is None: - return Classification( - "mutation", alter.table, "Rewrites existing parts if the type changes" - ) - if _normalize_type(type_tokens) == _normalize_type(_tokens(current.type)): - return Classification("metadata", alter.table, "Column type is unchanged") - return Classification( - "mutation", - alter.table, - f"Column {column} changes type from {current.type} to {''.join(type_tokens)}", - ) - - -def _modify_order(tokens: tuple[str, ...], alter: _Alter) -> Classification: - if not alter.definition: - return Classification( - "rebuild", - alter.table, - "Sorting-key change requires live schema to prove a new-column-only extension", - ) - requested = _order_expressions(tokens) - current = [_normalize_type(_tokens(item)) for item in alter.definition.order_by] - added = set() - for action in alter.actions: - if tuple(token.upper() for token in action[:2]) == ("ADD", "COLUMN"): - index = ( - 5 if tuple(token.upper() for token in action[2:5]) == ("IF", "NOT", "EXISTS") else 2 - ) - if len(action) > index: - added.add(_normalize_type((action[index],))) - existing = {_normalize_type((column.name,)) for column in alter.definition.columns} - suffix = requested[len(current) :] - if requested[: len(current)] == current and all(item in added - existing for item in suffix): - return Classification( - "metadata", - alter.table, - "Sorting key unchanged or extended only by columns added in this ALTER", - ) - return Classification( - "rebuild", alter.table, "Changing existing sorting-key expressions requires a table rebuild" - ) - - -def _statement_target(tokens: tuple[str, ...]) -> str | None: - words = tuple(token.upper() for token in tokens) - if words[0] == "UPDATE": - return _name(tokens, 1)[0] - if words[:2] in (("INSERT", "INTO"), ("DELETE", "FROM")): - return _name(tokens, 2)[0] - if words[0] not in ("CREATE", "DROP", "RENAME"): - return None - types = {"TABLE", "VIEW", "DICTIONARY", "DATABASE", "USER", "ROLE"} - index = next((index + 1 for index, token in enumerate(words) if token in types), len(tokens)) - while index < len(words) and words[index] in ("IF", "NOT", "EXISTS"): - index += 1 - return _name(tokens, index)[0] if index < len(tokens) else None - - -def _table_definition(schema: Schema | None, name: str) -> TableDefinition | None: - if schema is None: - return None - database, _, table = name.rpartition(".") - if database and schema.database and database != schema.database: - return None - return schema.tables.get(table or name) - - -def _split_settings(tokens: tuple[str, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]: - depth = 0 - for index, token in enumerate(tokens): - if token in ("(", "["): - depth += 1 - elif token in (")", "]"): - depth -= 1 - elif token.upper() == "SETTINGS" and depth == 0: - return tokens[:index], tokens[index + 1 :] - return tokens, () - - -def _setting_is_zero(tokens: tuple[str, ...], name: str) -> bool: - value = None - for index in range(len(tokens) - 2): - if tokens[index].lower() == name and tokens[index + 1] == "=": - value = tokens[index + 2] - return value == "0" - - -def _column_type(tokens: tuple[str, ...]) -> tuple[str, ...]: - depth = 0 - for index, token in enumerate(tokens): - if depth == 0 and token.upper() in _TYPE_MODIFIERS: - return tokens[:index] - if token in ("(", "["): - depth += 1 - elif token in (")", "]"): - depth -= 1 - return tokens - - -def _order_expressions(tokens: tuple[str, ...]) -> list[str]: - if tokens[:1] == ("(",) and tokens[-1:] == (")",): - tokens = tokens[1:-1] - elif tokens[:2] == ("tuple", "(") and tokens[-1:] == (")",): - tokens = tokens[2:-1] - return [_normalize_type(action) for action in _action_clauses(tokens)] if tokens else [] - - -def _normalize_type(tokens: tuple[str, ...]) -> str: - return "".join(_identifier(token) for token in tokens) - - -def _name(tokens: tuple[str, ...], index: int) -> tuple[str, int]: - if index >= len(tokens): - return "", index - names = [_identifier(tokens[index])] - index += 1 - while index + 1 < len(tokens) and tokens[index] == ".": - names.append(_identifier(tokens[index + 1])) - index += 2 - return ".".join(names), index - - -def _identifier(token: str) -> str: - if token[:1] in ('"', "`") and token[-1:] == token[:1]: - quote = token[0] - return token[1:-1].replace(quote * 2, quote).replace("\\" + quote, quote) - return token - - -def _tokens(sql: str) -> tuple[str, ...]: - tokens = [] - index = 0 - while index < len(sql): - end = _skip_quoted_or_comment(sql, index) - if end is not None: - if sql[index] in "'\"`$": - tokens.append(sql[index:end]) - index = end - continue - if sql[index] == "{" and (end := sql.find("}", index + 1)) != -1: - tokens.append(sql[index : end + 1]) - index = end + 1 - continue - match = _WORD_OR_NUMBER.match(sql, index) - if match: - tokens.append(match.group()) - index = match.end() - continue - if not sql[index].isspace() and sql[index] != ";": - tokens.append(sql[index]) - index += 1 - return tuple(tokens) diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py index 8c2e2cf..c535716 100644 --- a/ch_migrate/migration_runner.py +++ b/ch_migrate/migration_runner.py @@ -33,7 +33,6 @@ from ch_migrate.waiting.errors import ( FailedMutation, MutationFailed, - UnknownOutcome, WaitingError, WaitTimeout, ) @@ -114,8 +113,8 @@ def _report_interrupt(environment: str, progress: _Progress) -> None: ui.error(f"Interrupted{' during ' + progress.current if progress.current else ''}") ui.hint( "Work ClickHouse already started keeps running, and the migration is not recorded as " - f"applied. Run `ch-migrate up {environment}` to pick up where it stopped; it won't " - "reissue a statement it already sent.", + f"applied. Run `ch-migrate up {environment}` again: it waits for that work to finish, " + "then runs the migration again.", stderr=True, ) @@ -129,8 +128,8 @@ def _report_failure(upgrade: Upgrade, progress: _Progress, error: Exception) -> _report_mutation_failure(upgrade.environment, where, waiting.failure) elif isinstance(waiting, WaitTimeout): _report_timeout(upgrade.environment, where, waiting) - elif isinstance(waiting, UnknownOutcome): - _report_unknown_outcome(where, waiting) + elif waiting is not None: + ui.error(f"{where}: {waiting}") else: _report_statement_failure(upgrade, where, error) @@ -139,8 +138,8 @@ def _report_mutation_failure(environment: str, where: str, failure: FailedMutati on_host = "" if failure.host == "shared" else f" on {failure.host}" if failure.foreign: ui.error( - f"{where} is blocked: foreign mutation {failure.mutation} on {failure.table} " - f"failed{on_host}" + f"{where} is blocked: mutation {failure.mutation} on {failure.table}, started " + f"before this run, failed{on_host}" ) else: ui.error(f"{where} failed: mutation {failure.mutation} on {failure.table} failed{on_host}") @@ -157,7 +156,7 @@ def _report_mutation_failure(environment: str, where: str, failure: FailedMutati ) ui.hint( f"If the statement itself is wrong, an operator stops it{on_host} with {failure.kill}, " - "then reconciles the journal as the README's Unknown outcome section describes.", + f"then fixes the SQL and runs `ch-migrate up {environment}` again.", stderr=True, ) @@ -167,18 +166,12 @@ def _report_timeout(environment: str, where: str, timeout: WaitTimeout) -> None: ui.detail(f"Pending: {timeout.pending}", stderr=True) ui.hint( "ClickHouse keeps doing the work, and the migration is not recorded as applied. " - f"Run `ch-migrate up {environment}` again to keep waiting; it won't reissue the statement.", + f"Run `ch-migrate up {environment}` again: it waits for that work to finish, then runs " + "the migration again.", stderr=True, ) -def _report_unknown_outcome(where: str, unknown: UnknownOutcome) -> None: - ui.error(f"{where}: outcome unknown, so the statement was not reissued") - for line in str(unknown).strip().splitlines(): - ui.detail(line, stderr=True) - ui.hint("Follow the README's Unknown outcome steps before you run `up` again.", stderr=True) - - def _report_statement_failure(upgrade: Upgrade, where: str, error: Exception) -> None: ui.error(f"{where} failed") ui.detail(_reason(error), stderr=True) diff --git a/ch_migrate/templates/bootstrap/init_users.sql b/ch_migrate/templates/bootstrap/init_users.sql index 49ab9c4..d62fa6e 100644 --- a/ch_migrate/templates/bootstrap/init_users.sql +++ b/ch_migrate/templates/bootstrap/init_users.sql @@ -25,10 +25,7 @@ GRANT SELECT ON system.grants TO {project}_migration_role; GRANT SELECT ON system.databases TO {project}_migration_role; GRANT SELECT ON system.tables TO {project}_migration_role; GRANT SELECT ON system.mutations TO {project}_migration_role; -GRANT SELECT ON system.columns TO {project}_migration_role; GRANT CURRENT GRANTS(SELECT ON system.replicas) TO {project}_migration_role; -GRANT CURRENT GRANTS(SELECT ON system.clusters) TO {project}_migration_role; -GRANT CURRENT GRANTS(SELECT ON system.distributed_ddl_queue) TO {project}_migration_role; GRANT CURRENT GRANTS(SELECT ON system.parts) TO {project}_migration_role; GRANT CURRENT GRANTS(SYSTEM SYNC REPLICA ON {db}.*) TO {project}_migration_role; GRANT CURRENT GRANTS(SYSTEM SYNC DATABASE REPLICA ON {db}.*) TO {project}_migration_role; diff --git a/ch_migrate/version_table.py b/ch_migrate/version_table.py index 50a0830..5a9b2dd 100644 --- a/ch_migrate/version_table.py +++ b/ch_migrate/version_table.py @@ -31,7 +31,16 @@ def health_cluster(self) -> str | None: return self.cluster or (self.database if self.database_engine == "Replicated" else None) def new_engine(self): - return migration_state_engine(self, "alembic_version", "version_num") + if self.database_engine.startswith("Shared"): + return MergeTree(order_by="version_num") + if self.database_engine == "Replicated": + return ReplicatedMergeTree(order_by="version_num") + if self.cluster: + path = ( + f"/clickhouse/ch_migrate/{quote(self.database, safe='')}/{{shard}}/alembic_version" + ) + return ReplicatedMergeTree(order_by="version_num", zk_path=path, replica="{replica}") + return MergeTree(order_by="version_num") def warning(self) -> str | None: replicated = self.database_engine == "Replicated" or bool(self.cluster) @@ -64,18 +73,6 @@ class VersionTableMutationError(RuntimeError): """An unfinished version-table mutation has a server-reported failure.""" -def migration_state_engine(state: VersionTableState, table: str, order_by: str): - """Apply one deployment policy to version state and durable migration journals.""" - if state.database_engine.startswith("Shared"): - return MergeTree(order_by=order_by) - if state.database_engine == "Replicated": - return ReplicatedMergeTree(order_by=order_by) - if state.cluster: - path = f"/clickhouse/ch_migrate/{quote(state.database, safe='')}/{{shard}}/{table}" - return ReplicatedMergeTree(order_by=order_by, zk_path=path, replica="{replica}") - return MergeTree(order_by=order_by) - - def inspect_version_table(client, database: str, cluster: str | None = None) -> VersionTableState: rows = client.query( "SELECT engine FROM system.databases WHERE name = {db:String}", @@ -120,12 +117,8 @@ def assert_version_mutations_healthy(client, state: VersionTableState) -> None: settings=settings, ).result_rows if failed: - from ch_migrate.waiting.sql import sql_string - host, mutation, reason = failed[0] raise VersionTableMutationError( f"Version-table mutation {mutation} on {host} failed: {reason}. " - "ch-migrate did not kill it.\n" - f"Operator only on {host}: KILL MUTATION WHERE database = {sql_string(state.database)} " - f"AND table = 'alembic_version' AND mutation_id = {sql_string(mutation)}" + "Resolve the failed mutation manually before retrying; ch-migrate did not kill it." ) diff --git a/ch_migrate/waiting/__init__.py b/ch_migrate/waiting/__init__.py index 47fd138..cbf3221 100644 --- a/ch_migrate/waiting/__init__.py +++ b/ch_migrate/waiting/__init__.py @@ -1,10 +1,11 @@ """Waiting for the work migrations start: mutations and ON CLUSTER DDL. -`up` doesn't record a migration until the mutations and distributed DDL its -statements started have finished. `MigrationWaiter` (waiter.py) drives that from -Alembic's cursor events; mutations.py and ddl.py wait on the server, journal.py -records what each statement started, versions.py writes the version table, and -display.py shows progress. The design is docs/design/2026-10-02-waiting.md. +`up` doesn't record a migration until the mutations and distributed DDL its statements +started have finished. `MigrationWaiter` (waiter.py) hooks the migration connection's +cursor events: before each write it bounds ClickHouse's own distributed DDL wait, and +after it, mutations.py waits until the database has no unfinished mutations on any +replica that must apply them. display.py shows the progress; budget.py holds the +invocation's `--timeout`. """ from ch_migrate.waiting.waiter import MigrationWaiter diff --git a/ch_migrate/waiting/ddl.py b/ch_migrate/waiting/ddl.py deleted file mode 100644 index 75cc934..0000000 --- a/ch_migrate/waiting/ddl.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Distributed DDL receipts identified by an owned queue marker, never query text.""" - -from __future__ import annotations - -from ch_migrate.waiting.display import DDLProgress -from ch_migrate.waiting.errors import UnknownOutcome, WaitingError - - -class DistributedDDLWaiter: - def __init__(self, client, budget): - self.client, self.budget = client, budget - - def prepare(self, cluster: str, token: str) -> dict: - hosts = self.client.query( - "SELECT host_name, port FROM system.clusters WHERE cluster = {cluster:String} " - "ORDER BY shard_num, replica_num", - parameters={"cluster": cluster}, - ).result_rows - if not hosts: - raise WaitingError(f"Cluster {cluster!r} has no configured DDL hosts") - return { - "cluster": cluster, - "token": token, - "hosts": [list(host) for host in hosts], - "entry": None, - "done_hosts": [], - } - - def wait(self, receipt: dict, checkpoint) -> None: - expected = {_host_key(host, port) for host, port in receipt["hosts"]} - while not expected.issubset(receipt["done_hosts"]): - rows = self._rows(receipt) - entries = {row[0] for row in rows} - if not entries: - raise UnknownOutcome( - "Distributed DDL outcome unknown: queue evidence is missing or expired; statement not reissued" - ) - if len(entries) != 1: - raise UnknownOutcome( - "Distributed DDL ownership marker identifies multiple queue entries; reconcile without resubmitting" - ) - entry = next(iter(entries)) - if receipt["entry"] is not None and receipt["entry"] != entry: - raise UnknownOutcome( - "Distributed DDL queue entry changed for a recorded intent; statement not reissued" - ) - if receipt["entry"] is None: - receipt["entry"] = entry - checkpoint() - pending = self._host_states(receipt, rows, checkpoint) - if pending: - finished = len(expected.intersection(receipt["done_hosts"])) - progress = DDLProgress(entry, finished, len(expected)) - self.budget.pause(f"DDL {entry}: " + "; ".join(pending), progress) - self.budget.complete(f"DDL {receipt['entry']}; every required host finished") - - def tables( - self, table: tuple[str, str], cluster: str, allow_missing: bool = False - ) -> dict | None: - rows = self.client.query( - "SELECT materialize(hostName()), toString(uuid), engine, sorting_key FROM " - "clusterAllReplicas({cluster:String}, system.tables) " - "WHERE database = {db:String} AND name = {table:String}", - parameters={"cluster": cluster, "db": table[0], "table": table[1]}, - settings={ - "skip_unavailable_shards": int(allow_missing), - "connect_timeout_with_failover_ms": 250, - }, - ).result_rows - if not rows: - return None - if len({row[2] for row in rows}) != 1: - raise WaitingError( - "Distributed DDL target engines differ across hosts; reconcile them before tracked execution" - ) - return { - "database": table[0], - "table": table[1], - "cluster": cluster, - "ddl_scope": True, - "engine": rows[0][2], - "sorting_key": rows[0][3], - "initiator": rows[0][0], - "uuids": {host: uuid for host, uuid, _, _ in rows}, - } - - def _rows(self, receipt: dict) -> list: - return self.client.query( - "SELECT entry, host, port, status, exception_code, exception_text " - "FROM system.distributed_ddl_queue WHERE settings['log_comment'] = {token:String}", - parameters={"token": receipt["token"]}, - ).result_rows - - def _host_states(self, receipt: dict, rows: list, checkpoint) -> list[str]: - observed = {} - for _, host, port, status, code, message in rows: - key = _host_key(host, port) - if code: - raise WaitingError( - f"Distributed DDL {receipt['entry']} failed on {key}: code {code}: {message}" - ) - observed[key] = status - if status == "Finished" and code == 0 and key not in receipt["done_hosts"]: - receipt["done_hosts"].append(key) - checkpoint() - pending = [] - for host, port in receipt["hosts"]: - key = _host_key(host, port) - if key not in receipt["done_hosts"]: - pending.append(f"host {key}: {observed.get(key) or 'missing queue row'}") - return pending - - -def _host_key(host, port) -> str: - return f"{host}:{port}" diff --git a/ch_migrate/waiting/display.py b/ch_migrate/waiting/display.py index 8708b2c..7dcbbf1 100644 --- a/ch_migrate/waiting/display.py +++ b/ch_migrate/waiting/display.py @@ -1,4 +1,4 @@ -"""What `up` shows while it waits for mutations and ON CLUSTER DDL. +"""What `up` shows while it waits for mutations. Waiters hand over what the server reports; this module only draws it, and nothing here decides whether work is done. On a terminal each wait is a live bar on @@ -25,8 +25,8 @@ PIPED_STEPS = 10 BAR_WIDTH = 38 -# What a waiter reports: a mutation, an ON CLUSTER queue entry, or a plain status line. -Progress = Union["MutationProgress", "DDLProgress", str] +# What a waiter reports: a mutation, or a plain status line. +Progress = Union["MutationProgress", str] class WaitDisplay: @@ -48,8 +48,6 @@ def wants_bytes(self) -> bool: return self._terminal and not self._parts_only def update(self, progress: Progress) -> None: - if isinstance(progress, MutationProgress) and progress.bookkeeping: - return wait = self._wait_for(progress) wait.observe(progress) if self._terminal: @@ -111,7 +109,6 @@ class MutationProgress: table: str mutation: str hosts: tuple[HostParts, ...] - bookkeeping: bool = False # the version table's own delete: waited for, not shown @dataclass(frozen=True) @@ -130,15 +127,6 @@ def bytes_left(self) -> int | None: return sum(self.sizes[name] for name in self.names) -@dataclass(frozen=True) -class DDLProgress: - """An ON CLUSTER statement's distributed DDL queue entry.""" - - entry: str - finished: int - total: int - - @dataclass class _Wait: key: str @@ -160,9 +148,6 @@ def observe(self, progress: Progress) -> None: def renderable(self): elapsed = _duration(time.monotonic() - self.started) progress = self.latest - if isinstance(progress, DDLProgress): - stats = f"{progress.finished}/{progress.total} hosts · {elapsed}" - return _bar_row("", _bar(progress.finished, progress.total), stats) if isinstance(progress, str): return Text(f" waiting: {progress} · {elapsed}", style="dim") rows = [self._host_row(host, len(progress.hosts) > 1) for host in progress.hosts] @@ -181,9 +166,6 @@ def piped_line(self, now: float) -> str | None: def summary(self, migration: str, fallback: str) -> str: elapsed = _duration(time.monotonic() - self.started) progress = self.latest - if isinstance(progress, DDLProgress): - label = migration or f"ON CLUSTER DDL {progress.entry}" - return f"{label} ON CLUSTER DDL on {progress.total} hosts · {elapsed}" if isinstance(progress, str): return f"{migration or fallback} waited for {progress} · {elapsed}" return f"{migration or progress.table} {self._mutation_totals(progress)} · {elapsed}" @@ -210,8 +192,6 @@ def _eta(self, done: int, left: int) -> str: def _fraction(self) -> float: progress = self.latest - if isinstance(progress, DDLProgress): - return progress.finished / progress.total if progress.total else 0.0 if isinstance(progress, str): return 0.0 total = sum(self.parts_total.values()) @@ -220,8 +200,6 @@ def _fraction(self) -> float: def _piped_text(self) -> str: progress = self.latest - if isinstance(progress, DDLProgress): - return f"waiting on ON CLUSTER DDL {progress.entry}: {progress.finished}/{progress.total} hosts" if isinstance(progress, str): return f"waiting: {progress}" total = sum(self.parts_total.values()) @@ -242,8 +220,6 @@ def _mutation_totals(self, progress: MutationProgress) -> str: def _key(progress: Progress) -> str: if isinstance(progress, MutationProgress): return f"mutation {progress.table} {progress.mutation}" - if isinstance(progress, DDLProgress): - return f"ddl {progress.entry}" return "status" diff --git a/ch_migrate/waiting/errors.py b/ch_migrate/waiting/errors.py index 499dc7c..311798a 100644 --- a/ch_migrate/waiting/errors.py +++ b/ch_migrate/waiting/errors.py @@ -1,4 +1,4 @@ -"""Why waiting stopped short: unfinished work, an unknown outcome, a deadline, a failed mutation.""" +"""Why waiting stopped short: a deadline, a failed mutation, or a deployment it can't check.""" from __future__ import annotations @@ -9,10 +9,6 @@ class WaitingError(RuntimeError): """Server work is unfinished; the revision must not be completed.""" -class UnknownOutcome(WaitingError): - """Available evidence cannot justify replay or completion.""" - - class WaitTimeout(WaitingError): """The invocation's waiting deadline expired without cancelling server work.""" diff --git a/ch_migrate/waiting/journal.py b/ch_migrate/waiting/journal.py deleted file mode 100644 index 6a9952b..0000000 --- a/ch_migrate/waiting/journal.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Append-only waiting receipts, with a fresh generation after each downgrade.""" - -from __future__ import annotations - -import json -from dataclasses import dataclass - -from clickhouse_connect.cc_sqlalchemy.datatypes.sqltypes import UInt32, UInt64 -from sqlalchemy import Column, MetaData, String, Table -from sqlalchemy.schema import CreateTable - -from ch_migrate.version_table import migration_state_engine -from ch_migrate.waiting.errors import UnknownOutcome -from ch_migrate.waiting.sql import qualified_table, query_settings - -JOURNAL_TABLE = "_ch_migrate_journal" - - -@dataclass(frozen=True) -class StepKey: - revision: str - generation: int - position: int - - -class WaitingJournal: - """One ordered writer per database; readers synchronize before resuming.""" - - def __init__(self, connection, state, budget): - self.connection, self.state, self.budget = connection, state, budget - self.client = connection.connection.dbapi_connection.client - self.table = qualified_table((state.database, JOURNAL_TABLE)) - self._sequences: dict[StepKey, int] = {} - self._ready = False - - def exists(self) -> bool: - return bool( - self.client.query( - "SELECT count() FROM system.tables WHERE database = {db:String} AND name = {table:String}", - parameters={"db": self.state.database, "table": JOURNAL_TABLE}, - ).result_rows[0][0] - ) - - def synchronize(self) -> None: - if not self.exists(): - return - engine = self.client.query( - "SELECT engine FROM system.tables WHERE database = {db:String} AND name = {table:String}", - parameters={"db": self.state.database, "table": JOURNAL_TABLE}, - ).result_rows[0][0] - if engine.startswith(("Replicated", "Shared")): - self.client.command(f"SYSTEM SYNC REPLICA {self.table} LIGHTWEIGHT") - - def begin(self, revision: str, fingerprint: str) -> int: - self.ensure() - generation = self._generation(revision) - if generation == 0: - generation = 1 - self.write(StepKey(revision, generation, 0), {"fingerprint": fingerprint}) - else: - control = self.read(StepKey(revision, generation, 0)) - if control is None: - raise UnknownOutcome( - f"Journal generation for {revision} has no control record; reconcile before resuming" - ) - return generation - - def invalidate(self, revision: str, fingerprint: str) -> None: - if self.exists(): - self.write( - StepKey(revision, self._generation(revision) + 1, 0), {"fingerprint": fingerprint} - ) - - def records(self, revision: str, generation: int) -> list[tuple[StepKey, dict]]: - rows = self.client.query( - f"SELECT position, max(sequence), argMax(payload, sequence) FROM {self.table} " - "WHERE revision = {revision:String} AND generation = {generation:UInt64} AND position > 0 " - "GROUP BY position ORDER BY position", - parameters={"revision": revision, "generation": generation}, - ).result_rows - result = [] - for position, sequence, payload in rows: - key = StepKey(revision, generation, position) - self._sequences[key] = sequence - result.append((key, json.loads(payload))) - return result - - def read(self, key: StepKey) -> dict | None: - rows = self.client.query( - f"SELECT sequence, payload FROM {self.table} WHERE revision = {{revision:String}} " - "AND generation = {generation:UInt64} AND position = {position:UInt32} " - "ORDER BY sequence DESC LIMIT 2", - parameters=self._parameters(key), - ).result_rows - if not rows: - self._sequences[key] = 0 - return None - if len(rows) == 2 and rows[0][0] == rows[1][0] and rows[0][1] != rows[1][1]: - raise UnknownOutcome( - f"Conflicting journal writers for {key.revision}:{key.position}; serialize runners and reconcile" - ) - self._sequences[key] = rows[0][0] - return json.loads(rows[0][1]) - - def write(self, key: StepKey, payload: dict) -> None: - if key not in self._sequences: - self.read(key) - sequence = self._sequences[key] + 1 - self.client.insert( - f"{self.state.database}.{JOURNAL_TABLE}", - [ - [ - key.revision, - key.generation, - key.position, - sequence, - json.dumps(payload, sort_keys=True), - ] - ], - column_names=["revision", "generation", "position", "sequence", "payload"], - settings={"async_insert": 0}, - ) - self._sequences[key] = sequence - - def unfinished_scope(self, revision: str) -> bool: - if not self.exists(): - return False - generation = self._generation(revision) - return any(record["phase"] != "done" for _, record in self.records(revision, generation)) - - def unfinished_versions(self) -> list[tuple[StepKey, dict]]: - if not self.exists(): - return [] - self.synchronize() - rows = self.client.query( - f"SELECT revision, generation, position, max(sequence), argMax(payload, sequence) AS latest " - f"FROM {self.table} WHERE position > 0 AND (revision, generation) IN " - f"(SELECT revision, max(generation) FROM {self.table} GROUP BY revision) " - "GROUP BY revision, generation, position HAVING JSONExtractString(latest, 'kind') = 'version' " - "AND JSONExtractString(latest, 'phase') != 'done' ORDER BY revision, position" - ).result_rows - result = [] - for revision, generation, position, sequence, payload in rows: - key = StepKey(revision, generation, position) - self._sequences[key] = sequence - result.append((key, json.loads(payload))) - return result - - def ensure(self) -> None: - if self._ready: - return - if self.exists(): - self.synchronize() - self._ready = True - return - table = Table( - JOURNAL_TABLE, - MetaData(), - Column("revision", String, nullable=False), - Column("generation", UInt64, nullable=False), - Column("position", UInt32, nullable=False), - Column("sequence", UInt64, nullable=False), - Column("payload", String, nullable=False), - migration_state_engine( - self.state, JOURNAL_TABLE, "(revision, generation, position, sequence)" - ), - schema=self.state.database, - info={"ch_migrate_on_cluster": self.state.on_cluster}, - ) - ddl = str(CreateTable(table, if_not_exists=True).compile(dialect=self.connection.dialect)) - self.client.command(query_settings(ddl, {"distributed_ddl_task_timeout": "0"})) - while not self.exists(): - self.budget.pause(f"journal {self.table} metadata is not available on this host") - self.synchronize() - self._ready = True - - def _generation(self, revision: str) -> int: - return self.client.query( - f"SELECT max(generation) FROM {self.table} WHERE revision = {{revision:String}}", - parameters={"revision": revision}, - ).result_rows[0][0] - - @staticmethod - def _parameters(key: StepKey) -> dict: - return {"revision": key.revision, "generation": key.generation, "position": key.position} diff --git a/ch_migrate/waiting/mutations.py b/ch_migrate/waiting/mutations.py index 7068e53..83b78f8 100644 --- a/ch_migrate/waiting/mutations.py +++ b/ch_migrate/waiting/mutations.py @@ -1,207 +1,169 @@ -"""Mutation ownership, predecessor handling, and per-replica completion evidence.""" +"""Wait until a database has no unfinished mutations on any replica that has to apply them. + +The waiter doesn't track which statement started which mutation. After each write it +records the unfinished mutations the connected server reports, then polls until each one +is done everywhere it applies: on the service for SharedMergeTree, on every replica for +ReplicatedMergeTree (a replica that hasn't pulled the mutation yet is still pending), and +where it runs for anything else. Every unfinished mutation in the database counts, so a +migration also waits for, and is stopped by, another writer's work. +""" from __future__ import annotations import re -from dataclasses import replace +from dataclasses import dataclass, replace from clickhouse_connect.driver.exceptions import DatabaseError from ch_migrate.waiting.display import HostParts, MutationProgress -from ch_migrate.waiting.errors import FailedMutation, MutationFailed, UnknownOutcome, WaitingError +from ch_migrate.waiting.errors import FailedMutation, MutationFailed, WaitingError from ch_migrate.waiting.sql import sql_string +SHARED = "shared" +# version_table.py owns the version table's mutations: its DELETE waits on its own request, +# and a stuck one is refused before the run starts. VERSION_TABLE = "alembic_version" class MutationWaiter: + """Polls `system.mutations` on the migration session; never kills or resends anything.""" + def __init__(self, client, state, budget): self.client, self.state, self.budget = client, state, budget + self._hosts: dict[str, tuple[str, ...] | None] = {} self._part_sizes: dict[tuple[str, str], int] = {} - def target(self, table: tuple[str, str]) -> dict | None: + def wait(self, before_run: bool = False) -> None: + """Return once every unfinished mutation seen now has finished where it applies.""" + self._hosts.clear() + expected = {(row.table, row.mutation) for row in self._rows(set())} + while expected: + rows = self._rows(expected) + expected |= {(row.table, row.mutation) for row in rows if not row.done} + _raise_failures(self.state.database, rows, before_run) + pending = self._pending(expected, rows) + if not pending: + self.budget.complete(f"{self.state.database} mutations") + return + self.budget.pause(_pending_text(self.state.database, pending), self._snapshot(pending)) + + def _rows(self, expected: set[tuple[str, str]]) -> list[_Row]: + """Unfinished mutations in the database, and every row for the ones already expected.""" + cluster = self.state.health_cluster + source = ( + "clusterAllReplicas({cluster:String}, system.mutations)" + if cluster + else "system.mutations" + ) rows = self.client.query( - "SELECT materialize(hostName()), toString(uuid), engine, sorting_key FROM system.tables " - "WHERE database = {db:String} AND name = {table:String}", - parameters={"db": table[0], "table": table[1]}, + "SELECT materialize(hostName()), table, mutation_id, is_done, parts_to_do, " + f"latest_fail_reason, parts_to_do_names FROM {source} WHERE database = {{db:String}} " + "AND table != {version:String} " + "AND (NOT is_done OR has({expected:Array(String)}, concat(table, '/', mutation_id)))", + parameters={ + "db": self.state.database, + "version": VERSION_TABLE, + "cluster": cluster, + "expected": sorted(f"{table}/{mutation}" for table, mutation in expected), + }, + settings=_cluster_settings() if cluster else None, ).result_rows - if not rows: - return None - host, uuid, engine, sorting_key = rows[0] - target = { - "database": table[0], - "table": table[1], - "engine": engine, - "initiator": host, - "uuids": {host: uuid}, - "cluster": None, - "sorting_key": sorting_key, - } - if engine.startswith("Shared"): - target["initiator"], target["uuids"] = "shared", {"shared": uuid} - elif engine.startswith("Replicated"): - self._replicated_target(target) - return target - - def prepare(self, target: dict, token: str) -> dict: - while True: - rows = self._rows(target) - pending = [row for row in rows if not row[3]] - self._failures(target, pending, "Blocked by foreign mutation") - if not pending: - self.budget.complete(f"{target['database']}.{target['table']} foreign predecessors") - break - self.budget.pause("foreign predecessor " + self._progress(target, pending)) - target = dict(target) - target.update(token=token, ids={}, done_hosts=[], baseline={}) - for host in target["uuids"]: - target["baseline"][host] = sorted({row[1] for row in rows if row[0] == host}) - return target - - def wait(self, receipt: dict, checkpoint) -> None: - while True: - identities = self._identities(receipt) - self._visible_hosts = set(identities) - for host, uuid in identities.items(): - if host in receipt["uuids"] and uuid != receipt["uuids"][host]: - raise UnknownOutcome( - f"Table {receipt['database']}.{receipt['table']} was replaced on {host}; statement not reissued" + return [self._row(values) for values in rows] + + def _row(self, values: tuple) -> _Row: + host, table, mutation, done, parts, reason, names = values + # Every replica of a SharedMergeTree table reports the same shared mutation. + if self._required_hosts(table) == (SHARED,): + host = SHARED + return _Row(host, table, mutation, bool(done), parts, reason, frozenset(names)) + + def _pending(self, expected: set[tuple[str, str]], rows: list[_Row]) -> list[_Row]: + """Unfinished rows, plus a placeholder for each replica that hasn't reported one yet.""" + pending = [row for row in rows if not row.done] + seen = {(row.host, row.table, row.mutation) for row in rows} + for table, mutation in sorted(expected): + for host in self._required_hosts(table) or (): + if host != SHARED and (host, table, mutation) not in seen: + pending.append( + _Row(host, table, mutation, False, 0, "", frozenset(), missing=True) ) - if receipt["initiator"] not in identities and not receipt.get("cluster"): - raise UnknownOutcome( - f"Table {receipt['database']}.{receipt['table']} disappeared; statement not reissued" - ) - rows = self._rows(receipt) - self._bind_markers(receipt, rows, checkpoint) - pending = self._pending(receipt, rows, checkpoint) - if not pending: - self.budget.complete( - f"{receipt['database']}.{receipt['table']}; every required replica finished" - ) - return - self.budget.pause("; ".join(pending), self._snapshot(receipt, rows)) + return pending + + def _required_hosts(self, table: str) -> tuple[str, ...] | None: + """Hosts that must each report a mutation done; None means wherever it appears.""" + if table not in self._hosts: + self._hosts[table] = self._resolve_hosts(table) + return self._hosts[table] + + def _resolve_hosts(self, table: str) -> tuple[str, ...] | None: + rows = self.client.query( + "SELECT engine FROM system.tables WHERE database = {db:String} AND name = {table:String}", + parameters={"db": self.state.database, "table": table}, + ).result_rows + engine = rows[0][0] if rows else "" + if engine.startswith("Shared"): + return (SHARED,) + if engine.startswith("Replicated"): + return self._replicas(table) + return None - def _replicated_target(self, target: dict) -> None: + def _replicas(self, table: str) -> tuple[str, ...]: path, total = self.client.query( "SELECT zookeeper_path, total_replicas FROM system.replicas " "WHERE database = {db:String} AND table = {table:String}", - parameters=self._parameters(target), + parameters={"db": self.state.database, "table": table}, ).result_rows[0] cluster = self.state.health_cluster if not cluster: if total > 1: raise WaitingError( - "Replicated mutation waiting requires an environment cluster covering every replica" + f"{self.state.database}.{table} has {total} replicas; set `cluster` in " + "config.yaml so `up` can wait for the mutation on every one" ) - return - target["cluster"] = cluster - while True: - replicas = self.client.query( - "SELECT materialize(hostName()), replica_name FROM " - "clusterAllReplicas({cluster:String}, system.replicas) " - "WHERE database = {db:String} AND table = {table:String} AND zookeeper_path = {path:String}", - parameters={**self._parameters(target), "cluster": cluster, "path": path}, - settings=self._settings(), - ).result_rows - if len({replica for _, replica in replicas}) == total: - hosts = {host for host, _ in replicas} - target["uuids"] = { - host: uuid for host, uuid in self._identities(target).items() if host in hosts - } - if len(target["uuids"]) == total: - return + return (self.client.command("SELECT hostName()"),) + while len(hosts := self._cluster_replicas(table, path)) < total: self.budget.pause( - f"{target['database']}.{target['table']}: cluster {cluster} exposes {len(replicas)}/{total} required replicas" - ) - - def _bind_markers(self, receipt: dict, rows: list, checkpoint) -> None: - found = False - for host in receipt["uuids"]: - if host in receipt["done_hosts"] or host in receipt["ids"]: - found = True - continue - local = [row for row in rows if row[0] == host] - markers = {row[1] for row in local if receipt["token"] in row[2]} - if not markers: - continue - if len(markers) != receipt.get("expected_markers", 1): - raise UnknownOutcome( - f"Outcome unknown for partially submitted batch on {host}; statement not reissued" - ) - found = True - barrier = max(_order(value) for value in markers) - baseline = set(receipt["baseline"].get(host, [])) - receipt["ids"][host] = sorted( - markers - | { - row[1] - for row in local - if not row[3] and row[1] not in baseline and _order(row[1]) <= barrier - } - ) - checkpoint() - if not found: - raise UnknownOutcome( - f"Outcome unknown for {receipt['database']}.{receipt['table']} token {receipt['token']}: " - "ownership evidence is missing or expired; statement not reissued; migration incomplete. " - "Stop runners and reconcile the journal with independent server evidence." + f"{self.state.database}.{table}: cluster {cluster} shows {len(hosts)}/{total} replicas" ) + return hosts - def _pending(self, receipt: dict, rows: list, checkpoint) -> list[str]: - pending = [] - for host in receipt["uuids"]: - if host in receipt["done_hosts"]: - continue - ids = receipt["ids"].get(host) - if not ids: - pending.append( - f"{receipt['database']}.{receipt['table']}: host {host} has not supplied the owned mutation" - ) - continue - local = [row for row in rows if row[0] == host and row[1] in ids] - missing = set(ids) - {row[1] for row in local} - if missing: - if host not in self._visible_hosts: - pending.append( - f"{receipt['database']}.{receipt['table']}: host {host} missing mutation evidence {sorted(missing)}" - ) - continue - raise UnknownOutcome( - f"Outcome unknown: mutation evidence {sorted(missing)} disappeared on {host}; statement not reissued" - ) - self._failures(receipt, local, "Mutation or preceding work failed") - unfinished = [row for row in local if not row[3]] - if unfinished: - pending.append(self._progress(receipt, unfinished)) - else: - receipt["done_hosts"].append(host) - checkpoint() - return pending + def _cluster_replicas(self, table: str, path: str) -> tuple[str, ...]: + """Hosts the cluster can reach that hold a replica of this table's Keeper path.""" + rows = self.client.query( + "SELECT DISTINCT materialize(hostName()) FROM clusterAllReplicas({cluster:String}, " + "system.replicas) WHERE database = {db:String} AND table = {table:String} " + "AND zookeeper_path = {path:String}", + parameters={ + "cluster": self.state.health_cluster, + "db": self.state.database, + "table": table, + "path": path, + }, + settings=_cluster_settings(), + ).result_rows + return tuple(sorted(host for (host,) in rows)) - def _snapshot(self, receipt: dict, rows: list) -> MutationProgress | None: - """What the display shows about this statement's unfinished mutations; never a verdict.""" + def _snapshot(self, pending: list[_Row]) -> MutationProgress: + """The oldest pending mutation, as the display shows it.""" + table, mutation = min(((row.table, row.mutation) for row in pending), key=_mutation_order) + rows = [row for row in pending if (row.table, row.mutation) == (table, mutation)] + # A replica that hasn't received the mutation has every part its peers report. + most = max((len(row.names) or row.parts for row in rows if not row.missing), default=0) hosts = [ - parts - for host in receipt["uuids"] - if host not in receipt["done_hosts"] - and (parts := _host_parts(host, receipt["ids"].get(host) or [], rows)) is not None + HostParts(row.host, row.names, most if row.missing else len(row.names) or row.parts) + for row in rows ] - owned = [mutation for ids in receipt["ids"].values() for mutation in ids] - if not hosts or not owned: - return None if self.budget.display.wants_bytes: - hosts = self._with_sizes(receipt, hosts) - table = f"{receipt['database']}.{receipt['table']}" - bookkeeping = receipt["table"] == VERSION_TABLE - return MutationProgress(table, min(owned, key=_order), tuple(hosts), bookkeeping) + hosts = self._with_sizes(table, hosts) + return MutationProgress(f"{self.state.database}.{table}", mutation, tuple(hosts)) - def _with_sizes(self, receipt: dict, hosts: list[HostParts]) -> list[HostParts]: + def _with_sizes(self, table: str, hosts: list[HostParts]) -> list[HostParts]: # Parts never change size, so each is read once; only new names cost a query. wanted = {(host.host, name) for host in hosts for name in host.names} unknown = {name for _, name in wanted - self._part_sizes.keys()} if unknown: try: - self._part_sizes.update(self._read_sizes(receipt, unknown)) + self._part_sizes.update(self._read_sizes(table, unknown)) except DatabaseError as error: self.budget.display.parts_only(_sizes_unreadable(error)) return hosts @@ -214,100 +176,82 @@ def _with_sizes(self, receipt: dict, hosts: list[HostParts]) -> list[HostParts]: for host in hosts ] - def _read_sizes(self, receipt: dict, names: set[str]) -> dict[tuple[str, str], int]: - source, parameters = self._source(receipt, "parts") + def _read_sizes(self, table: str, names: set[str]) -> dict[tuple[str, str], int]: + cluster = self.state.health_cluster + source = "clusterAllReplicas({cluster:String}, system.parts)" if cluster else "system.parts" rows = self.client.query( - "SELECT materialize(hostName()), name, bytes_on_disk " - f"FROM {source} WHERE database = {{db:String}} AND table = {{table:String}} " - "AND active AND has({names:Array(String)}, name)", - parameters={**parameters, "names": sorted(names)}, - settings=self._settings() if receipt.get("cluster") else None, + f"SELECT materialize(hostName()), name, bytes_on_disk FROM {source} " + "WHERE database = {db:String} AND table = {table:String} AND active " + "AND has({names:Array(String)}, name)", + parameters={ + "cluster": cluster, + "db": self.state.database, + "table": table, + "names": sorted(names), + }, + settings=_cluster_settings() if cluster else None, ).result_rows - shared = receipt["engine"].startswith("Shared") - return {("shared" if shared else host, name): size for host, name, size in rows} + shared = self._required_hosts(table) == (SHARED,) + return {(SHARED if shared else host, name): size for host, name, size in rows} - def _rows(self, target: dict) -> list: - source, parameters = self._source(target, "mutations") - rows = self.client.query( - "SELECT materialize(hostName()), mutation_id, command, is_done, parts_to_do, " - f"latest_fail_reason, parts_to_do_names FROM {source} WHERE database = {{db:String}} AND table = {{table:String}}", - parameters=parameters, - settings=self._settings() if target.get("cluster") else None, - ).result_rows - if target["engine"].startswith("Shared"): - return [("shared", *row[1:]) for row in rows] - return [row for row in rows if row[0] in target["uuids"]] - def _identities(self, target: dict) -> dict[str, str]: - source, parameters = self._source(target, "tables") - rows = self.client.query( - f"SELECT materialize(hostName()), toString(uuid) FROM {source} " - "WHERE database = {db:String} AND name = {table:String}", - parameters=parameters, - settings=self._settings() if target.get("cluster") else None, - ).result_rows - if target["engine"].startswith("Shared"): - return {"shared": rows[0][1]} if rows else {} - return dict(rows) - - def _source(self, target: dict, table: str) -> tuple[str, dict]: - parameters = self._parameters(target) - if target.get("cluster"): - parameters["cluster"] = target["cluster"] - return f"clusterAllReplicas({{cluster:String}}, system.{table})", parameters - return f"system.{table}", parameters - - def _settings(self) -> dict: - return { - "skip_unavailable_shards": 1, - "connect_timeout_with_failover_ms": 250, - "receive_timeout": 2, - } - - @staticmethod - def _parameters(target: dict) -> dict: - return {"db": target["database"], "table": target["table"]} - - @staticmethod - def _failures(target: dict, rows: list, prefix: str) -> None: - for host, mutation, _, done, _, reason, *_ in rows: - if not done and reason: - kill = ( - f"KILL MUTATION WHERE database = {sql_string(target['database'])} " - f"AND table = {sql_string(target['table'])} AND mutation_id = {sql_string(mutation)}" - ) - table = f"{target['database']}.{target['table']}" - raise MutationFailed( - f"{prefix}: {table} mutation {mutation} on {host}: {reason}\nOperator only on {host}: {kill}", - FailedMutation( - table, mutation, host, reason, kill, prefix.startswith("Blocked") - ), - ) +@dataclass(frozen=True) +class _Row: + """One host's view of one mutation, or a placeholder for a replica that hasn't got it.""" + + host: str + table: str + mutation: str + done: bool + parts: int + reason: str + names: frozenset[str] + missing: bool = False + + +def _raise_failures(database: str, rows: list[_Row], before_run: bool) -> None: + for row in rows: + if row.done or not row.reason: + continue + kill = ( + f"KILL MUTATION WHERE database = {sql_string(database)} " + f"AND table = {sql_string(row.table)} AND mutation_id = {sql_string(row.mutation)}" + ) + table = f"{database}.{row.table}" + prefix = "Blocked by a failed mutation" if before_run else "Mutation failed" + raise MutationFailed( + f"{prefix}: {table} mutation {row.mutation} on {row.host}: {row.reason}\n" + f"Operator only on {row.host}: {kill}", + FailedMutation(table, row.mutation, row.host, row.reason, kill, before_run), + ) + - @staticmethod - def _progress(target: dict, rows: list) -> str: - parts = {} - for host, mutation, _, _, count, *_ in rows: - parts[(host, mutation)] = max(parts.get((host, mutation), 0), count) - return "; ".join( - f"{target['database']}.{target['table']} mutation {mutation} host {host}; parts_to_do={count}" - for (host, mutation), count in sorted(parts.items()) +def _pending_text(database: str, pending: list[_Row]) -> str: + parts = [] + for row in sorted(pending, key=lambda row: (row.table, _order(row.mutation), row.host)): + where = f"{database}.{row.table} mutation {row.mutation} host {row.host}" + parts.append( + f"{where}; not received yet" if row.missing else f"{where}; parts_to_do={row.parts}" ) + return "; ".join(parts) + + +def _mutation_order(key: tuple[str, str]) -> tuple[int, str]: + return _order(key[1]), key[0] def _order(mutation: str) -> int: numbers = re.findall(r"\d+", mutation) - if len(numbers) != 1: - raise WaitingError(f"Unrecognized mutation ordering identifier: {mutation}") - return int(numbers[0]) + return int(numbers[-1]) if numbers else 0 -def _host_parts(host: str, ids: list, rows: list) -> HostParts | None: - unfinished = [row for row in rows if row[0] == host and row[1] in ids and not row[3]] - if not unfinished: - return None - names = frozenset(name for row in unfinished for name in row[6]) - return HostParts(host, names, len(names) or max(row[4] for row in unfinished)) +def _cluster_settings() -> dict: + # A replica that's down is skipped, so its missing rows read as pending, not as done. + return { + "skip_unavailable_shards": 1, + "connect_timeout_with_failover_ms": 250, + "receive_timeout": 2, + } def _sizes_unreadable(error: Exception) -> str: diff --git a/ch_migrate/waiting/sql.py b/ch_migrate/waiting/sql.py index c865d35..bb28485 100644 --- a/ch_migrate/waiting/sql.py +++ b/ch_migrate/waiting/sql.py @@ -1,235 +1,36 @@ -"""SQL identity and ownership markers without matching another writer's command.""" +"""The little SQL the waiter needs: which statements can start work, and string literals.""" from __future__ import annotations -import hashlib -import json import re -from clickhouse_connect.driver.binding import bind_query, quote_identifier - -from ch_migrate.classify import _identifier, _tokens from ch_migrate.sql import _skip_quoted_or_comment -from ch_migrate.waiting.errors import WaitingError - -_WORD = re.compile(r"[A-Za-z_][A-Za-z_0-9]*") -_MANAGED = frozenset(("mutations_sync", "alter_sync", "lightweight_deletes_sync")) - - -def bind_statement(statement, parameters, client) -> str: - bound, _ = bind_query(statement, parameters or None, client.server_tz) - return bound.decode() if isinstance(bound, bytes) else bound - - -def statement_digest(statement: str, parameters) -> str: - serialized = json.dumps( - [statement, parameters], sort_keys=True, default=str, separators=(",", ":") - ) - return hashlib.sha256(serialized.encode()).hexdigest() - - -def statement_table(statement: str, database: str) -> tuple[str, str] | None: - tokens = _tokens(statement) - words = tuple(token.upper() for token in tokens) - if words[:2] in (("ALTER", "TABLE"), ("INSERT", "INTO"), ("DELETE", "FROM")): - index = 2 - elif words[:1] == ("UPDATE",): - index = 1 - elif words[:1] in (("CREATE",), ("DROP",), ("RENAME",), ("TRUNCATE",)): - index = next( - (i + 1 for i, token in enumerate(words) if token in ("TABLE", "VIEW", "DICTIONARY")), - len(tokens), - ) - else: - return None - while index < len(tokens) and words[index] in ("IF", "NOT", "EXISTS"): - index += 1 - if index >= len(tokens): - return None - components = [_identifier(tokens[index])] - while index + 2 < len(tokens) and tokens[index + 1] == ".": - components.append(_identifier(tokens[index + 2])) - index += 2 - if any(component.startswith("{") for component in components): - raise WaitingError("Mutation tracking requires resolved database/table identifiers") - return (components[-2] if len(components) > 1 else database, components[-1]) - - -def statement_cluster(statement: str) -> str | None: - tokens = _tokens(statement) - for index in range(len(tokens) - 2): - if tuple(token.upper() for token in tokens[index : index + 2]) == ("ON", "CLUSTER"): - return tokens[index + 2].strip("'`\"") - return None - - -def is_session_or_read(statement: str) -> bool: - tokens = _tokens(statement) - if tokens and tokens[0].upper() == "WITH": - operation = next( - ( - span[0].upper() - for span in _spans(statement) - if span[3] == 0 and span[0].upper() in ("SELECT", "INSERT", "UPDATE", "DELETE") - ), - None, - ) - return operation == "SELECT" - return not tokens or tokens[0].upper() in ( - "SELECT", - "SHOW", - "DESCRIBE", - "DESC", - "EXPLAIN", - "SET", - "USE", - ) - - -def modifies_ttl(statement: str) -> bool: - tokens = _tokens(statement) - return any( - tuple(word.upper() for word in tokens[index : index + 2]) == ("MODIFY", "TTL") - for index in range(len(tokens) - 1) - ) - -def mutation_sql(statement: str, token: str) -> str: - """Carry ownership into system.mutations and make submission asynchronous.""" - body, settings = _query_parts(statement) - words = _tokens(body) - marker = f"{sql_string(token)} = {sql_string(token)}" - overrides = {name: "0" for name in _MANAGED} - if tuple(word.upper() for word in words[:2]) == ("DELETE", "FROM"): - where = next( - (span for span in _spans(body) if span[0].upper() == "WHERE" and span[3] == 0), None - ) - if where is None: - raise WaitingError("A lightweight DELETE needs a WHERE predicate to track ownership") - body = body[: where[2]] + f" ({body[where[2]:].strip()}) AND ({marker})" - elif tuple(word.upper() for word in words[:2]) == ("ALTER", "TABLE"): - if modifies_ttl(body): - spans = list(_spans(body)) - start = next( - span[1] - for index, span in enumerate(spans[:-1]) - if span[0].upper() == "MODIFY" - and spans[index + 1][0].upper() == "TTL" - and span[3] == 0 - ) - body = body[:start].rstrip() + f" DELETE WHERE 0 AND ({marker}), " + body[start:] - overrides["materialize_ttl_after_modify"] = "0" - else: - body += f", DELETE WHERE 0 AND ({marker})" - else: - raise WaitingError("Cannot attach a mutation ownership marker to this statement") - return _with_settings(body, settings, overrides) +# Statements that only read or change the session; everything else may start a mutation +# or distributed DDL. Waiting after a read costs one poll, so unknown forms count as writes. +_READS = frozenset(("SELECT", "SHOW", "DESCRIBE", "DESC", "EXPLAIN", "EXISTS", "SET", "USE")) +_WORD = re.compile(r"[A-Za-z]+") -def query_settings(statement: str, overrides: dict[str, str]) -> str: - body, settings = _query_parts(statement) - return _with_settings(body, settings, overrides) - - -def remove_query_settings(statement: str, names: set[str]) -> str: - body, settings = _query_parts(statement) - retained = [ - item for item in settings if _identifier(item.split("=", 1)[0].strip()).lower() not in names - ] - return body + (" SETTINGS " + ", ".join(retained) if retained else "") - - -def query_setting(statement: str, name: str) -> str | None: - _, settings = _query_parts(statement) - for setting in reversed(settings): - key, _, value = setting.partition("=") - if _identifier(key.strip()).lower() == name: - return value.strip().strip("'") - return None +def is_write(statement: str) -> bool: + """True unless the statement's first keyword only reads or sets session state.""" + word = _WORD.match(_first_code(statement)) + return word is None or word.group(0).upper() not in _READS def sql_string(value: str) -> str: return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" -def qualified_table(table: tuple[str, str]) -> str: - return ".".join(quote_identifier(part) for part in table) - - -def _query_parts(statement: str) -> tuple[str, list[str]]: - statement = _without_comments(statement) - spans = list(_spans(statement)) - if not spans: - raise WaitingError("Empty statement cannot be tracked") - sql = statement[: spans[-1][2]].rstrip() - candidates = [ - span - for index, span in enumerate(spans[:-2]) - if span[0].upper() == "SETTINGS" - and span[3] == 0 - and _WORD.fullmatch(_identifier(spans[index + 1][0])) - and spans[index + 2][0] == "=" - ] - if not candidates: - return sql, [] - settings = candidates[-1] - return sql[: settings[1]].rstrip(), _split_settings(sql[settings[2] :]) - - -def _split_settings(sql: str) -> list[str]: - starts = [span[1] for span in _spans(sql) if span[0] == "," and span[3] == 0] - result, previous = [], 0 - for index in starts: - result.append(sql[previous:index].strip()) - previous = index + 1 - if sql[previous:].strip(): - result.append(sql[previous:].strip()) - return result - - -def _with_settings(body: str, settings: list[str], overrides: dict[str, str]) -> str: - retained = [ - item - for item in settings - if _identifier(item.split("=", 1)[0].strip()).lower() not in overrides - ] - retained.extend(f"{name} = {value}" for name, value in sorted(overrides.items())) - return body + " SETTINGS " + ", ".join(retained) - - -def _without_comments(sql: str) -> str: - chunks, index = [], 0 - while index < len(sql): - end = _skip_quoted_or_comment(sql, index) - if end is None: - chunks.append(sql[index]) - index += 1 - else: - chunks.append(sql[index:end] if sql[index] in "'\"`$" else " ") - index = end - return "".join(chunks) - - -def _spans(sql: str): - """Yield significant tokens with offsets/depth, retaining quoted SQL verbatim.""" - index = depth = 0 - while index < len(sql): - end = _skip_quoted_or_comment(sql, index) - if end is not None: - if sql[index] in "'\"`$": - yield sql[index:end], index, end, depth - index = end - continue - match = _WORD.match(sql, index) - if match: - yield match.group(), index, match.end(), depth - index = match.end() +def _first_code(sql: str) -> str: + """The statement from its first character that isn't whitespace or a comment.""" + i = 0 + while i < len(sql): + if sql[i].isspace(): + i += 1 continue - char = sql[index] - if char in ")]": - depth -= 1 - if not char.isspace() and char != ";": - yield char, index, index + 1, depth - if char in "([": - depth += 1 - index += 1 + skip_to = _skip_quoted_or_comment(sql, i) + if skip_to is None or sql[i] in "'\"`": + return sql[i:] + i = skip_to + return "" diff --git a/ch_migrate/waiting/versions.py b/ch_migrate/waiting/versions.py deleted file mode 100644 index f50dc0a..0000000 --- a/ch_migrate/waiting/versions.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Resume insert-first version bookkeeping without invoking a revision twice.""" - -from __future__ import annotations - -import ast -from types import SimpleNamespace -from uuid import uuid4 - -from sqlalchemy import text -from sqlalchemy.sql.dml import Delete, Insert, Update - -from ch_migrate import ui -from ch_migrate.waiting.errors import UnknownOutcome, WaitingError -from ch_migrate.waiting.sql import ( - mutation_sql, - qualified_table, - query_settings, - sql_string, - statement_digest, -) - - -class VersionWrites: - def __init__(self, owner): - self.owner = owner - - def execute(self, construct, impl): - steps = _version_steps(construct, impl) - payload = { - "kind": "version", - "phase": "intent", - "steps": steps, - "digest": statement_digest(str(construct), [step["sql"] for step in steps]), - } - key, record, _ = self.owner.reserve(payload) - self.continue_record(key, record) - return SimpleNamespace(rowcount=1) - - def resume(self) -> None: - for key, record in self.owner.journal.unfinished_versions(): - self.owner.validate_revision(key.revision, record["fingerprint"]) - ui.step(f"Resuming version bookkeeping for {key.revision[:8]}", stderr=True) - self.continue_record(key, record) - - def continue_record(self, key, record: dict) -> None: - if record["phase"] == "done": - return - try: - for step in record["steps"]: - if step.get("done"): - continue - - def checkpoint() -> None: - self.owner.journal.write(key, record) - - if step["kind"] == "insert": - self._insert(step, checkpoint) - else: - self._delete(step, checkpoint) - step["done"] = True - checkpoint() - record["phase"] = "done" - self.owner.journal.write(key, record) - except UnknownOutcome as error: - raise self.owner.unknown_outcome(key, record, error) from error - - def _insert(self, step: dict, checkpoint) -> None: - table = qualified_table((self.owner.state.database, "alembic_version")) - count = self.owner.client.query( - f"SELECT count() FROM {table} WHERE version_num = {{version:String}}", - parameters={"version": step["value"]}, - ).result_rows[0][0] - if step.get("prepared"): - if count == 1: - return # The exact version-row effect proves the internal INSERT succeeded. - raise UnknownOutcome( - f"Outcome unknown for version INSERT {step['value']}: found {count} rows; " - "statement not reissued; reconcile version state before resuming" - ) - if count: - raise WaitingError( - f"Version {step['value']} already exists before its INSERT; reconcile overlapping heads" - ) - step["prepared"] = True - checkpoint() - self.owner.client.command( - step["sql"], settings={"async_insert": 0, "wait_for_async_insert": 1} - ) - - def _delete(self, step: dict, checkpoint) -> None: - if not step.get("prepared"): - target = self.owner.mutations.target((self.owner.state.database, "alembic_version")) - if target is None: - raise UnknownOutcome("Version table disappeared before deleting the previous head") - token = "chm_version_" + uuid4().hex - step["receipt"] = self.owner.mutations.prepare(target, token) - step["prepared"] = True - checkpoint() - with self.owner.internal(): - self.owner.connection.exec_driver_sql(mutation_sql(step["sql"], token)) - self.owner.mutations.wait(step["receipt"], checkpoint) - - -def write_unjournaled(construct, impl): - """Apply a version INSERT/UPDATE outside a tracked upgrade, such as a downgrade.""" - for step in _version_steps(construct, impl): - sql = step["sql"] - if step["kind"] == "delete": - sql = query_settings(sql, {"mutations_sync": "2"}) - impl._exec(text(sql)) - return SimpleNamespace(rowcount=1) - - -def _version_steps(construct, impl) -> list[dict]: - if isinstance(construct, Insert): - value = _version_value(construct) - return [ - {"kind": "insert", "value": value, "sql": _insert_sql(impl._version_table_name, value)} - ] - if isinstance(construct, Update): - value = _version_value(construct) - return [ - { - "kind": "insert", - "value": value, - "sql": _insert_sql(impl._version_table_name, value), - }, - { - "kind": "delete", - "sql": f"ALTER TABLE {impl._version_table_name} DELETE WHERE {impl._compile_version_where(construct)}", - }, - ] - if isinstance(construct, Delete): - return [ - { - "kind": "delete", - "sql": f"ALTER TABLE {impl._version_table_name} DELETE WHERE {impl._compile_version_where(construct)}", - } - ] - raise WaitingError("Unsupported version-table transition") - - -def _insert_sql(table: str, value: str) -> str: - # Replicated and Shared MergeTree silently drop a block identical to an earlier insert - # (DRY-1447), so re-inserting a previous head after `down` left no version row at all. - # A token unique to this write keeps every intentional version row. - token = sql_string("chm_version_" + uuid4().hex) - return ( - f"INSERT INTO {table} (version_num) SETTINGS insert_deduplication_token = {token}, " - f"async_insert = 0 VALUES ({sql_string(value)})" - ) - - -def _version_value(construct) -> str: - values = construct._values - expression = next(iter(values.values())) if values else None - value = getattr(expression, "value", None) - if value is None and getattr(expression, "is_literal", False): - value = ast.literal_eval(str(expression)) - if not isinstance(value, str): - raise WaitingError("Version-table INSERT/UPDATE must contain a literal revision identifier") - return value diff --git a/ch_migrate/waiting/waiter.py b/ch_migrate/waiting/waiter.py index b4f1523..bc75c2e 100644 --- a/ch_migrate/waiting/waiter.py +++ b/ch_migrate/waiting/waiter.py @@ -1,509 +1,68 @@ -"""Journal actual cursor writes and wait before Alembic completes a revision.""" +"""Hold each migration until ClickHouse has finished the work its statements started.""" from __future__ import annotations -import hashlib -import json -from contextlib import contextmanager -from functools import wraps -from pathlib import Path -from uuid import uuid4 - -from alembic.script import ScriptDirectory -from clickhouse_connect.driver.exceptions import DatabaseError from sqlalchemy import event -from sqlalchemy.schema import CreateTable -from ch_migrate import ui -from ch_migrate.classify import classify -from ch_migrate.hooks import run_hooks -from ch_migrate.idempotency import classify_idempotency -from ch_migrate.introspect import ColumnDefinition, Schema, TableDefinition, _parse_order_by -from ch_migrate.statements import migration_statements from ch_migrate.waiting.budget import WaitBudget -from ch_migrate.waiting.ddl import DistributedDDLWaiter -from ch_migrate.waiting.errors import UnknownOutcome, WaitingError -from ch_migrate.waiting.journal import StepKey, WaitingJournal from ch_migrate.waiting.mutations import MutationWaiter -from ch_migrate.waiting.sql import ( - bind_statement, - is_session_or_read, - modifies_ttl, - mutation_sql, - qualified_table, - query_setting, - query_settings, - remove_query_settings, - sql_string, - statement_cluster, - statement_digest, - statement_table, -) -from ch_migrate.waiting.versions import VersionWrites +from ch_migrate.waiting.sql import is_write + +# ON CLUSTER statements wait inside ClickHouse. The limit stays below clickhouse-connect's +# default 300-second read timeout, so ClickHouse answers before the client gives up. +DDL_WAIT_LIMIT = 270 class MigrationWaiter: - """One connection, one invocation deadline, and ordered durable receipts.""" + """Waits after every write on the migration connection; reads and `SET`s pass through. + + ON CLUSTER statements return only once every host has run them; a host that fails or + runs out of time is an error. After each write it waits for the database's unfinished + mutations. + """ - def __init__(self, connection, state, timeout=None): - self.connection, self.state = connection, state + def __init__(self, connection, state, timeout: float | None = None): + self.connection = connection self.client = connection.connection.dbapi_connection.client self.budget = WaitBudget(timeout) - self.journal = WaitingJournal(connection, state, self.budget) self.mutations = MutationWaiter(self.client, state, self.budget) - self.ddl = DistributedDDLWaiter(self.client, self.budget) - self.versions = VersionWrites(self) - self.revision = None - self.generation = self.position = self._internal = 0 - self.fingerprint = "" - self._pending = {} - self._batch_key = None - self._listeners = [] - self._scripts = None + self._installed = False def install(self, config) -> None: # `ch-migrate up` passes the display it labels with each migration's name. self.budget.display = config.attributes.get("ch_migrate_display", self.budget.display) - self._scripts = ScriptDirectory.from_config(config) - for target, name, listener, options in ( - (self.connection, "before_cursor_execute", self._before, {"retval": True}), - (self.connection, "after_cursor_execute", self._after, {}), - (self.connection.engine, "handle_error", self._error, {}), - ): - event.listen(target, name, listener, **options) - self._listeners.append((target, name, listener)) + # Request settings, not a session SET, so a migration's own SETs can't undo them. + # wait_end_of_query makes a DDL timeout arrive as an error, not a truncated response. + self.client.set_client_setting("distributed_ddl_output_mode", "throw") + self.client.set_client_setting("distributed_ddl_task_timeout", DDL_WAIT_LIMIT) + self.client.set_client_setting("wait_end_of_query", 1) + event.listen(self.connection, "before_cursor_execute", self._before) + event.listen(self.connection, "after_cursor_execute", self._after) + self._installed = True + + def wait(self, before_run: bool = False) -> None: + """Wait for the database's unfinished mutations, including another writer's.""" + self.mutations.wait(before_run) def close(self) -> None: self.budget.finish() - for target, name, listener in reversed(self._listeners): - event.remove(target, name, listener) - self._listeners.clear() - self.revision = None - - @contextmanager - def internal(self): - self._internal += 1 - try: - yield - finally: - self._internal -= 1 - - def reserve(self, payload: dict) -> tuple[StepKey, dict, bool]: - self.position += 1 - key = StepKey(self.revision, self.generation, self.position) - previous = self.journal.read(key) - payload["fingerprint"] = self.fingerprint - if previous is None: - payload["token"] = "chm_mutation_" + uuid4().hex - return key, payload, True - retryable = ( - previous.get("phase") == "rejected" - and previous.get("repeat_safe") - and payload.get("repeat_safe") - ) - if previous["digest"] != payload["digest"] and not retryable: - raise UnknownOutcome( - f"Revision {key.revision} statement {key.position} changed after execution began; " - "statement not reissued. Reconcile its journal and the original SQL." - ) - if retryable: - payload["token"] = "chm_mutation_" + uuid4().hex - return key, payload if retryable else previous, bool(retryable) - - def validate_revision(self, revision: str, expected: str) -> None: - script = self._scripts.get_revision(revision) - if script is None or _fingerprint(Path(script.path)) != expected: - raise UnknownOutcome( - f"Revision {revision} changed during incomplete version bookkeeping; reconcile before resuming" - ) - - def unknown_outcome(self, key: StepKey, record: dict, error: Exception) -> UnknownOutcome: - receipt = record.get("receipt") or record.get("before_target") or {} - table = record.get("table") - if record.get("kind") == "version": - table = (self.state.database, "alembic_version") - receipt = next((step["receipt"] for step in record["steps"] if "receipt" in step), {}) - message = ( - f"{error}\nRevision {key.revision}, generation {key.generation}, statement {key.position}; " - f"token {record.get('token', receipt.get('token', 'unrecorded'))}; " - f"recorded target UUIDs {json.dumps(receipt.get('uuids', {}), sort_keys=True)}.\n" - "Stop migration runners. Establish the actual outcome independently; reconcile only this " - "journal key before rerunning. Do not stamp the revision or blindly delete its journal.\n" - f"Read-only inspection: SELECT sequence, payload FROM {self.journal.table} " - f"WHERE revision = {sql_string(key.revision)} AND generation = {key.generation} " - f"AND position = {key.position} ORDER BY sequence DESC;\n" - ) - if table: - source = ( - f"clusterAllReplicas({sql_string(receipt['cluster'])}, system.mutations)" - if receipt.get("cluster") - else "system.mutations" - ) - message += ( - "Read-only inspection: SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason " - f"FROM {source} WHERE database = {sql_string(table[0])} AND table = {sql_string(table[1])};" - ) - for name in ("ddl", "work_ddl"): - if record.get(name): - message += ( - "\nRead-only inspection: SELECT entry, host, port, status, exception_code, exception_text " - "FROM system.distributed_ddl_queue WHERE settings['log_comment'] = " - f"{sql_string(record[name]['token'])};" - ) - return UnknownOutcome(message) - - def steps(self, original, hooks): - def iterate(heads, context): - for step in original(heads, context): - if not hasattr(step, "revision"): - yield step # Native Alembic stamp steps have no migration body to track. - continue - revision = step.revision.revision - fingerprint = _fingerprint(Path(step.revision.path)) - if step.is_upgrade: - self._begin(revision, fingerprint) - else: - self.journal.invalidate(revision, fingerprint) - execute = step.migration_fn - step.migration_fn = self._revision_body(execute, hooks, revision) - try: - yield step - finally: - self.revision = None - - return iterate - - def ensure_version_table(self, table) -> None: - if not self.state.on_cluster: - return - scope = "__version_table__" - if self.state.table_engine and not self.journal.unfinished_scope(scope): - return - ddl = str(CreateTable(table, if_not_exists=True).compile(dialect=self.connection.dialect)) - self._begin(scope, statement_digest(ddl, None)) - try: - self.connection.exec_driver_sql(ddl) - finally: - self.revision = None - - def run_pre_hooks(self, hooks) -> None: - fingerprint = hashlib.sha256(json.dumps(hooks.pre_migrate).encode()).hexdigest() - self._begin("__batch__", fingerprint) - key = StepKey(self.revision, self.generation, 0) - if self.journal.read(key).get("finished"): - self.journal.invalidate(self.revision, fingerprint) - self._begin("__batch__", fingerprint) - key = StepKey(self.revision, self.generation, 0) - self._batch_key = key - run_hooks( - self.connection, - hooks.pre_migrate, - db=self.state.database, - phase="pre_migrate", - revision="all", - ) - boundary, record, _ = self.reserve( - {"kind": "boundary", "phase": "done", "digest": "pre-hooks-complete"} - ) - self.journal.write(boundary, record) - self.revision = None - - def finish_run(self) -> None: - if self._batch_key: - control = self.journal.read(self._batch_key) - control["finished"] = True - self.journal.write(self._batch_key, control) - - def _begin(self, revision: str, fingerprint: str) -> None: - self.revision, self.fingerprint, self.position = revision, fingerprint, 0 - self.generation = self.journal.begin(revision, fingerprint) - self._completed_targets = {} - for key, record in self.journal.records(revision, self.generation): - if ( - record["phase"] not in ("done", "rejected") - and record.get("fingerprint") != fingerprint - ): - raise UnknownOutcome( - f"Revision {revision} changed with unresolved work; statement not reissued" - ) - if ( - record["phase"] == "accepted" - and record["kind"] == "write" - and not record.get("ddl") - ): - self._complete(key, record) - if record["phase"] == "done" and record.get("table"): - self._completed_targets[tuple(record["table"])] = record.get("after_target") - - def _revision_body(self, execute, hooks, revision): - @wraps(execute) - def run(**kwargs): - execute(**kwargs) - run_hooks( - self.connection, - hooks.post_migrate, - db=self.state.database, - phase="post_migrate", - revision=revision, - ) - - return run - - def _before(self, connection, cursor, statement, parameters, context, executemany): - # SQLAlchemy's event signature is an external contract. - if self._internal or self.revision is None: - return statement, parameters - sample = parameters[0] if executemany and parameters else parameters - sql = bind_statement(statement, sample, self.client) - if is_session_or_read(sql): - return statement, parameters - if executemany: - return self._execute_many(statement, parameters, context.execution_options) - table = statement_table(sql, self.state.database) - payload = { - "kind": "write", - "phase": "intent", - "digest": statement_digest(statement, parameters), - "table": list(table) if table else None, - "repeat_safe": classify_idempotency(sql).status == "ok", - } - key, record, fresh = self.reserve(payload) - if not fresh and self._resume(key, record): - return "SELECT 1 WHERE 0", {} - cluster = statement_cluster(sql) - if cluster: - record["ddl"] = self.ddl.prepare(cluster, record["token"] + "_ddl") - record["caller_log_comment"] = query_setting(sql, "log_comment") - target = self._target(table, record, allow_missing=True) if table else None - if cluster and table and target is None and classify(sql).kind == "mutation": - raise WaitingError( - f"No reachable mutation target {table} in cluster {cluster}; nothing was submitted" - ) - record["before_target"] = target - kind = self._work_kind(sql, target, sample) if target else None - if kind: - token = record["token"] - record["kind"] = kind - record["receipt"] = self.mutations.prepare(target, token) - record["receipt"]["expected_markers"] = 1 - if kind == "ttl": - record.update(kind="ttl", work_submitted=False, cluster=statement_cluster(sql)) - statement = query_settings( - statement, {"materialize_ttl_after_modify": "0", "alter_sync": "0"} - ) - else: - statement = mutation_sql(statement, token) - if record.get("ddl"): - statement, settings = self._ddl_submission(statement, record["ddl"]) - previous = self._statement_settings(context, settings) - if record["caller_log_comment"] is None: - record["caller_log_comment"] = previous.get("log_comment") - self.journal.write(key, record) # Acknowledged intent MUST precede submission. - self._pending[id(context)] = (key, record) - return statement, parameters - - def _execute_many(self, statement, parameters, execution_options): - # Re-enter the single-statement cursor path, preserving statement order - # and an independently durable intent for each parameter set. - for binding in parameters: - self.connection.exec_driver_sql( - statement, binding, execution_options=execution_options - ).close() - return "SELECT 1 WHERE 0", [{}] - - def _after(self, connection, cursor, statement, parameters, context, executemany): - pending = self._pending.pop(id(context), None) - if pending is None: - return - key, record = pending - record["phase"] = "accepted" - self.journal.write(key, record) - if record.get("ddl") or record["kind"] in ("mutation", "ttl"): - self._wait_work(key, record) - self._complete(key, record) - - def _error(self, context): - if context.connection is not self.connection: - return - pending = self._pending.pop(id(context.execution_context), None) - if pending is None: - return - key, record = pending - # A server rejection of repeat-safe metadata can be repaired and retried. - # Transport loss and mutation submissions remain unknown, never replayed. - if ( - record["kind"] == "write" - and record["repeat_safe"] - and not record.get("ddl") - and isinstance(context.original_exception, DatabaseError) - and context.original_exception.code is not None - ): - record["phase"] = "rejected" - self.journal.write(key, record) - - def _resume(self, key: StepKey, record: dict) -> bool: - if record["phase"] == "done": - self._check_completed_target(record) - return True - if ( - record.get("ddl") - or record["kind"] == "mutation" - or (record["kind"] == "ttl" and record["phase"] == "accepted") - ): - ui.step( - f"Resuming {key.revision[:8]} statement {key.position}: waiting on the work it " - "started; not reissuing SQL", - stderr=True, - ) - self._wait_work(key, record) - self._complete(key, record) - return True - if record["phase"] == "accepted": - self._complete(key, record) - return True - error = UnknownOutcome("Outcome unknown; statement not reissued; migration incomplete") - raise self.unknown_outcome(key, record, error) from error - - def _wait_work(self, key: StepKey, record: dict) -> None: - def checkpoint() -> None: - self.journal.write(key, record) - - try: - if record.get("ddl"): - self.ddl.wait(record["ddl"], checkpoint) - record["phase"] = "accepted" - checkpoint() - if record["kind"] == "ttl": - self._materialize_ttl(record, checkpoint) - if record["kind"] in ("mutation", "ttl"): - self._refresh_distributed_target(record) - self.mutations.wait(record["receipt"], checkpoint) - except UnknownOutcome as error: - raise self.unknown_outcome(key, record, error) from error - - def _materialize_ttl(self, record: dict, checkpoint) -> None: - if not record["work_submitted"]: - cluster = ( - f" ON CLUSTER {sql_string(record['cluster'])}" if record.get("cluster") else "" - ) - sql = f"ALTER TABLE {qualified_table(tuple(record['table']))}{cluster} MATERIALIZE TTL" - sql = mutation_sql(sql, record["receipt"]["token"]) - settings = {} - if record.get("ddl"): - record["work_ddl"] = self.ddl.prepare(record["cluster"], record["token"] + "_ttl") - sql, settings = self._ddl_submission(sql, record["work_ddl"]) - record["work_submitted"] = True - checkpoint() - with self.internal(): - self.connection.exec_driver_sql(sql, execution_options={"settings": settings}) - if record.get("work_ddl"): - self.ddl.wait(record["work_ddl"], checkpoint) - - def _refresh_distributed_target(self, record: dict) -> None: - if not record.get("ddl"): - return - actual = self._target(tuple(record["table"]), record) - if actual is None: - raise UnknownOutcome("Distributed mutation target disappeared; statement not reissued") - receipt = record["receipt"] - for host, uuid in receipt["uuids"].items(): - if actual["uuids"].get(host) != uuid: - raise UnknownOutcome( - f"Distributed mutation target was replaced on {host}; statement not reissued" - ) - receipt["uuids"] = actual["uuids"] - - def _target(self, table, record, allow_missing=False): - if record.get("ddl"): - return self.ddl.tables(table, record["ddl"]["cluster"], allow_missing) - return self.mutations.target(table) - - @staticmethod - def _ddl_submission(statement: str, receipt: dict) -> tuple[str, dict]: - settings = { - "distributed_ddl_task_timeout": 0, - "distributed_ddl_output_mode": "none", - "log_comment": receipt["token"], - } - return remove_query_settings(statement, set(settings)), settings - - @staticmethod - def _statement_settings(context, overrides: dict) -> dict: - previous = dict(context.execution_options.get("settings") or {}) - statement = getattr(context, "invoked_statement", None) - if statement is not None: - previous.update(statement.get_execution_options().get("settings") or {}) - context.invoked_statement = statement.execution_options( - settings={**previous, **overrides} - ) - context.execution_options = context.execution_options.union( - {"settings": {**previous, **overrides}} - ) - return previous - - def _complete(self, key: StepKey, record: dict) -> None: - if record.get("table"): - record["after_target"] = self._target(tuple(record["table"]), record) - self._completed_targets[tuple(record["table"])] = record["after_target"] - record["phase"] = "done" - self.journal.write(key, record) - - def _check_completed_target(self, record: dict) -> None: - if not record.get("table"): - return - expected = self._completed_targets.get(tuple(record["table"]), record.get("after_target")) - actual = self._target(tuple(record["table"]), record) - if ( - (expected is None) != (actual is None) - or expected is not None - and expected["uuids"] != actual["uuids"] - ): - raise UnknownOutcome( - "Completed statement's target was dropped or replaced; reconcile before skipping its SQL" - ) - - def _work_kind(self, sql: str, target: dict, parameters) -> str | None: - if not target["engine"].endswith("MergeTree"): - return None - rows = self.client.query( - "SELECT name, type FROM system.columns WHERE database = {db:String} AND table = {table:String}", - parameters={"db": target["database"], "table": target["table"]}, - ).result_rows - table = TableDefinition( - target["table"], - target["engine"], - columns=[ColumnDefinition(*row) for row in rows], - order_by=_parse_order_by(target["sorting_key"]) if target["sorting_key"] else [], + if self._installed: + event.remove(self.connection, "before_cursor_execute", self._before) + event.remove(self.connection, "after_cursor_execute", self._after) + self._installed = False + + def _before(self, conn, cursor, statement, parameters, context, executemany) -> None: + if is_write(statement) and self.budget.timeout is not None: + self._limit_distributed_ddl() + + def _after(self, conn, cursor, statement, parameters, context, executemany) -> None: + if is_write(statement): + self.wait() + + def _limit_distributed_ddl(self) -> None: + # `--timeout` covers the whole run, so ClickHouse waits only for what's left of it. + self.budget.check("the next statement") + left = int(self.budget.timeout - self.budget.elapsed) + self.client.set_client_setting( + "distributed_ddl_task_timeout", max(1, min(DDL_WAIT_LIMIT, left)) ) - schema = ( - None - if target.get("ddl_scope") - else Schema(database=target["database"], tables={target["table"]: table}) - ) - ttl = modifies_ttl(sql) - materialize = self._materializes_ttl(sql, parameters) if ttl else False - analyzed = ( - query_settings(sql, {"materialize_ttl_after_modify": "0"}) - if ttl and not materialize - else sql - ) - if classify(analyzed, schema).kind != "mutation": - return None - return "ttl" if ttl and materialize else "mutation" - - def _materializes_ttl(self, sql: str, parameters) -> bool: - value = query_setting(sql, "materialize_ttl_after_modify") - if value is None: - value = str(self.client.command("SELECT getSetting('materialize_ttl_after_modify')")) - if value.startswith("{") and isinstance(parameters, dict): - value = str(parameters[value[1:].split(":", 1)[0]]) - if value.lower() not in ("0", "1", "true", "false"): - raise WaitingError("materialize_ttl_after_modify must resolve to a boolean value") - return value.lower() in ("1", "true") - - -def _fingerprint(path: Path) -> str: - digest = hashlib.sha256(path.read_bytes()) - for statement in migration_statements(path): - digest.update(statement.sql.encode()) - return digest.hexdigest() diff --git a/docs/design/2026-10-02-ch-migrate-1.0.md b/docs/design/2026-10-02-ch-migrate-1.0.md index 67ed6e2..b9eec30 100644 --- a/docs/design/2026-10-02-ch-migrate-1.0.md +++ b/docs/design/2026-10-02-ch-migrate-1.0.md @@ -100,9 +100,8 @@ both; the README quick start contains no Python. loudly, not quietly drop its settings. - The version table cannot cause a migration to run twice. The version advances by inserting the new row and then deleting the old one, waiting for the delete. Interrupted - bookkeeping is never permission to repeat the migration: without ownership - evidence it is refused; with capability 5's journal it resumes the same - bookkeeping. The spike reproduced duplicate execution with the old asynchronous + bookkeeping is never permission to repeat the migration: overlapping heads are + refused. The spike reproduced duplicate execution with the old asynchronous `ALTER … UPDATE` approach. - The version table is created to suit the deployment: shared on ClickHouse Cloud, replicated in a replicated database or on a configured cluster, plain on a single server. @@ -113,8 +112,7 @@ both; the README quick start contains no Python. **Accept:** an integration test against a real ClickHouse server shows a `SET` in one statement taking effect in the next; an existing 0.x project upgrades with `upgrade-env` and runs `up` without editing migration files; with the version table's delete held back, -a second `up` cannot re-run the migration. It refuses unowned overlapping state, -or reattaches to proven owned bookkeeping under capability 5. +a second `up` cannot re-run the migration. It refuses the overlapping state. ### 4. Re-runnable migrations @@ -145,21 +143,23 @@ again, and reaches the same schema as a clean run. `up` does not record a migration as applied until ClickHouse has finished the work the migration started. -- After each statement that creates a mutation, `up` polls `system.mutations` until the mutation - finishes, showing progress. It does not rely on `mutations_sync`, which holds one HTTP request - open for the whole rewrite and loses the outcome if the connection drops. -- For `ON CLUSTER` DDL, `up` waits on the distributed DDL queue rather than trusting the - client-side timeout. +- After each statement that can write, `up` polls `system.mutations` until the project database + has no unfinished mutation on any replica that has to apply it, showing progress. It does not + rely on `mutations_sync`, which holds one HTTP request open for the whole rewrite. +- For `ON CLUSTER` DDL, ClickHouse waits for every host (`distributed_ddl_output_mode = 'throw'`) + within a limit below the client's read timeout, so a failing or unfinished host is an error, + not a quiet client-side timeout. - Waiting is on by default. There is no default timeout; `--timeout` sets one. - Whenever `up` stops waiting (a timeout, a killed CI job, a dropped connection), the outcome is the same: the migration is not recorded as applied, ClickHouse keeps doing the work, and the - next `up` finds the unfinished mutation and waits on it instead of issuing the statement again. - Issuing `UPDATE … SET n = n + 1` twice applies it twice. -- **Recovery exception approved on 2026-10-04:** if an interrupted statement's - outcome cannot be proven (for example, ownership evidence has expired), `up` - exits nonzero without reissuing the statement or marking the migration complete. - It identifies the unresolved work and requires operator reconciliation. Known - unfinished work still reattaches automatically. + next `up` waits for that work to finish, then runs the migration again. That relies on + capability 4: every statement must be safe to repeat, or carry a written waiver. +- **Design change, 2026-10-27 (Dan):** the design approved on 2026-10-04 recorded which work + each statement started in a journal, so a rerun would never send a statement twice. It took + about 1,300 lines, duplicated what capability 4 already guarantees, and left a killed or + rejected mutation unrecoverable without editing the journal by hand (DRY-1604). 0.7 waits by + observation instead; the [waiting design](2026-10-02-waiting.md) keeps the journal design and + its experiments for the record. - A failing mutation does not end on its own: ClickHouse retries it and holds back every later mutation on the same table. `up` reads `latest_fail_reason`, fails the migration, and prints the `KILL MUTATION` statement for the operator. ch-migrate never kills a mutation itself. diff --git a/docs/design/2026-10-02-waiting.md b/docs/design/2026-10-02-waiting.md index 9735e52..a3e2da6 100644 --- a/docs/design/2026-10-02-waiting.md +++ b/docs/design/2026-10-02-waiting.md @@ -1,5 +1,13 @@ # Waiting for mutations and distributed DDL +**Superseded for 0.7 (Dan, 2026-10-27).** 0.7 doesn't ship the journal design below. It +waits by observation: after each write, `up` polls `system.mutations` until the project +database has none unfinished, and ClickHouse waits for `ON CLUSTER` DDL itself. A rerun +waits for unfinished work, then runs the migration again, relying on the idempotency gate. +Capability 5 in the [product definition](2026-10-02-ch-migrate-1.0.md) and the README's +"Migrations that wait" describe what ships. The rest of this note is the record of the +journal design and its experiments. + **DRY-1398 · 2026-10-02 · Recovery policy approved by Dan on 2026-10-04.** A write-ahead intent plus a marker carried into ClickHouse's mutation record can diff --git a/tests/corpus/classification.yaml b/tests/corpus/classification.yaml deleted file mode 100644 index 798cd5c..0000000 --- a/tests/corpus/classification.yaml +++ /dev/null @@ -1,166 +0,0 @@ -# Server-backed classification contract. Each ALTER gets a fresh seeded table. -table_ddl: >- - CREATE TABLE {table} ( - id UInt64, x UInt64, y UInt64 DEFAULT x + 1, s String, ts DateTime, - INDEX idx id TYPE minmax GRANULARITY 1, - PROJECTION pr (SELECT id ORDER BY id) - ) ENGINE = MergeTree PARTITION BY toYYYYMM(ts) ORDER BY id - TTL ts + INTERVAL 100 YEAR -seed: >- - INSERT INTO {table} (id, x, s, ts) VALUES - (1, 10, 'one', '2020-01-01 00:00:00'), (2, 20, 'two', '2020-02-01 00:00:00') -statements: - - id: create_table - sql: CREATE TABLE IF NOT EXISTS {table} (id UInt64) ENGINE = Memory - expected: metadata - - id: create_view - sql: CREATE VIEW {db}.sample_view AS SELECT id FROM {table} - expected: metadata - - id: drop_table - sql: DROP TABLE IF EXISTS {table} - expected: metadata - - id: rename_table - sql: RENAME TABLE {table} TO {db}.renamed_table - expected: metadata - - id: create_dictionary - sql: CREATE DICTIONARY {db}.lookup (id UInt64) PRIMARY KEY id LAYOUT(FLAT()) - expected: metadata - - id: drop_view - sql: DROP VIEW IF EXISTS {db}.sample_view - expected: metadata - - id: read_only_select - sql: SELECT id FROM {table} - expected: other - - id: insert_select - sql: INSERT INTO {table} SELECT * FROM {db}.source - expected: other - - id: session_setting - sql: SET max_threads = 1 - expected: other - - id: add_column - sql: ALTER TABLE {table} ADD COLUMN added UInt64 - expected: metadata - - id: add_existing_column - sql: ALTER TABLE {table} ADD COLUMN IF NOT EXISTS x UInt64 - expected: metadata - - id: add_default_column - sql: ALTER TABLE {table} ADD COLUMN added UInt64 DEFAULT x + 1 - expected: metadata - - id: add_array_column - sql: ALTER TABLE {table} ADD COLUMN added Array(Tuple(UInt64, String)) - expected: metadata - - id: default_only - sql: ALTER TABLE {table} MODIFY COLUMN y DEFAULT x + 2 - expected: metadata - - id: same_type_default - sql: ALTER TABLE {table} MODIFY COLUMN y UInt64 DEFAULT x + 2 - expected: metadata - without_schema: mutation - - id: comment_modifier - sql: ALTER TABLE {table} MODIFY COLUMN x COMMENT 'mutation UPDATE DELETE is only text' - expected: metadata - - id: comment_column - sql: ALTER TABLE {table} COMMENT COLUMN x 'description' - expected: metadata - - id: modify_setting - sql: ALTER TABLE {table} MODIFY SETTING parts_to_throw_insert = 3000 - expected: metadata - - id: add_index - sql: ALTER TABLE {table} ADD INDEX another x TYPE minmax GRANULARITY 1 - expected: metadata - - id: add_projection - sql: ALTER TABLE {table} ADD PROJECTION another (SELECT id, x ORDER BY x) - expected: metadata - - id: drop_index - sql: ALTER TABLE {table} DROP INDEX idx - expected: mutation - - id: drop_projection - sql: ALTER TABLE {table} DROP PROJECTION pr - expected: mutation - - id: narrow_column_type - sql: ALTER TABLE {table} MODIFY COLUMN x UInt32 - expected: mutation - - id: nullable_column_type - sql: ALTER TABLE {table} MODIFY COLUMN x Nullable(UInt64) - expected: mutation - - id: low_cardinality_column_type - sql: ALTER TABLE {table} MODIFY COLUMN s LowCardinality(String) - expected: mutation - - id: drop_column - sql: ALTER TABLE {table} DROP COLUMN y - expected: mutation - - id: clear_column - sql: ALTER TABLE {table} CLEAR COLUMN y - expected: mutation - - id: materialize_column - sql: ALTER TABLE {table} MATERIALIZE COLUMN y - expected: mutation - - id: rename_column - sql: ALTER TABLE {table} RENAME COLUMN y TO renamed - expected: mutation - - id: update - sql: ALTER TABLE {table} UPDATE x = x + 1 WHERE id = 1 - expected: mutation - - id: delete - sql: ALTER TABLE {table} DELETE WHERE id = 1 - expected: mutation - - id: materialize_index - sql: ALTER TABLE {table} MATERIALIZE INDEX idx - expected: mutation - - id: clear_index - sql: ALTER TABLE {table} CLEAR INDEX idx - expected: mutation - - id: materialize_projection - sql: ALTER TABLE {table} MATERIALIZE PROJECTION pr - expected: mutation - - id: clear_projection - sql: ALTER TABLE {table} CLEAR PROJECTION pr - expected: mutation - - id: modify_ttl - sql: ALTER TABLE {table} MODIFY TTL ts + INTERVAL 99 YEAR - expected: mutation - - id: modify_ttl_without_materialization - sql: ALTER TABLE {table} MODIFY TTL ts + INTERVAL 99 YEAR SETTINGS materialize_ttl_after_modify = 0 - expected: metadata - - id: materialize_ttl - sql: ALTER TABLE {table} MATERIALIZE TTL - expected: mutation - - id: remove_ttl - sql: ALTER TABLE {table} REMOVE TTL - expected: metadata - - id: append_new_sorting_column - sql: ALTER TABLE {table} ADD COLUMN added UInt64, MODIFY ORDER BY (id, added) - expected: metadata - without_schema: rebuild - - id: append_existing_sorting_column - sql: ALTER TABLE {table} MODIFY ORDER BY (id, x) - expected: rebuild - - id: replace_sorting_key - sql: ALTER TABLE {table} MODIFY ORDER BY x - expected: rebuild - - id: change_primary_key - sql: ALTER TABLE {table} MODIFY PRIMARY KEY x - expected: rebuild - - id: change_partition_key - sql: ALTER TABLE {table} MODIFY PARTITION BY toYYYYMM(ts + INTERVAL 1 MONTH) - expected: rebuild - - id: change_engine - sql: ALTER TABLE {table} MODIFY ENGINE = ReplacingMergeTree - expected: rebuild - - id: unchanged_type - sql: ALTER TABLE {table} MODIFY COLUMN x UInt64 - expected: metadata - without_schema: mutation - - id: mixed_metadata_mutation - sql: ALTER TABLE {table} ADD COLUMN added String, MATERIALIZE COLUMN y - expected: mutation - - id: lightweight_delete - sql: DELETE FROM {table} WHERE id = 1 - expected: mutation - - id: lightweight_update - sql: UPDATE {table} SET x = x + 1 WHERE id = 1 - expected: other - detail: lightweight - - id: update_multiple_assignments - sql: ALTER TABLE {table} UPDATE x = x + 1, s = concat(s, '!') WHERE id = 1 - expected: mutation diff --git a/tests/integration/test_classification.py b/tests/integration/test_classification.py deleted file mode 100644 index 29805eb..0000000 --- a/tests/integration/test_classification.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Every corpus ALTER is classified against actual mutation or rejection evidence.""" - -from pathlib import Path - -import pytest -import yaml -from clickhouse_connect.driver.exceptions import DatabaseError - -from ch_migrate.classify import classify -from ch_migrate.introspect import get_live_schema - -CORPUS = yaml.safe_load((Path(__file__).parents[1] / "corpus/classification.yaml").read_text()) -ALTERS = [case for case in CORPUS["statements"] if case["sql"].startswith("ALTER")] -pytestmark = pytest.mark.integration - - -@pytest.mark.parametrize("case", ALTERS, ids=lambda case: case["id"]) -def test_classification_matches_real_alter_work(project, case): - values = {"db": project.database, "table": f"{project.database}.sample"} - project.client.command(CORPUS["table_ddl"].format(**values)) - project.client.command(CORPUS["seed"].format(**values)) - statement = case["sql"].format(**values) - result = classify(statement, get_live_schema(project.client, project.database)) - assert result.kind == case["expected"], (statement, result) - before = _mutation_ids(project) - if result.kind == "rebuild": - code = 36 if "sorting" in case["id"] else 62 - with pytest.raises(DatabaseError, match=f"code: {code},"): - project.client.command(statement) - assert _mutation_ids(project) == before - return - project.client.command(statement, settings={"mutations_sync": 0, "alter_sync": 0}) - created_mutations = _mutation_ids(project) - before - assert bool(created_mutations) == (result.kind == "mutation"), ( - statement, - result, - created_mutations, - ) - - -def test_classification_lightweight_update_is_synchronous_patch_work(project): - table = f"{project.database}.patches" - project.client.command( - f"CREATE TABLE {table} (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id " - "SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1" - ) - project.client.command(f"INSERT INTO {table} VALUES (1, 10), (2, 20)") - statement = f"UPDATE {table} SET x = x + 1 WHERE id = 1" - result = classify(statement) - project.client.command(statement, settings={"allow_experimental_lightweight_update": 1}) - assert project.client.query(f"SELECT id, x FROM {table} ORDER BY id").result_rows == [ - (1, 11), - (2, 20), - ] - rows = project.client.query( - "SELECT mutation_id FROM system.mutations WHERE database = {db:String} AND table = 'patches'", - parameters={"db": project.database}, - ).result_rows - assert result.kind == "other" and "lightweight" in result.detail - assert rows == [] - - -def _mutation_ids(project): - rows = project.client.query( - "SELECT DISTINCT mutation_id FROM system.mutations " - "WHERE database = {db:String} AND table = {table:String} AND " + project.REWRITES_PARTS, - parameters={"db": project.database, "table": "sample"}, - ).result_rows - return {row[0] for row in rows} diff --git a/tests/integration/test_cloud.py b/tests/integration/test_cloud.py index ab712a7..c5132ab 100644 --- a/tests/integration/test_cloud.py +++ b/tests/integration/test_cloud.py @@ -70,7 +70,7 @@ def test_cloud_version_table_is_shared_and_round_trips(project): def test_cloud_up_advances_version_only_after_mutation_finishes(project): """Cloud cannot hold a mutation with STOP MERGES; a deliberately slow one shows the wait.""" db = project.database - _slow_increment(project, seconds=2) + _slow_update(project, "x + 1", seconds=2) result = project.run("up", "it") assert result.exit_code == 0, result.output assert ( @@ -85,10 +85,10 @@ def test_cloud_up_advances_version_only_after_mutation_finishes(project): assert versioned >= mutated -def test_cloud_killed_up_reattaches_to_its_running_mutation(project): - """SIGKILL mid-wait; the rerun must resume from its journal, never resubmit the UPDATE.""" +def test_cloud_killed_up_waits_for_its_mutation_then_runs_again(project): + """SIGKILL mid-wait; the rerun waits out the running UPDATE, then sends it again.""" db = project.database - _slow_increment(project, seconds=3) + _slow_update(project, "1", seconds=3) log_path = project.root / "first-up.log" with log_path.open("w") as log: first = subprocess.Popen( @@ -104,31 +104,32 @@ def test_cloud_killed_up_reattaches_to_its_running_mutation(project): first.wait(timeout=10) resumed = project.run("up", "it") assert resumed.exit_code == 0, resumed.output - assert "not reissuing SQL" in resumed.output, resumed.output - increments = project.client.command( + value = project.client.command( f"SELECT x FROM {db}.counter", settings={"select_sequential_consistency": 1} ) - rewrites = project.client.command( - "SELECT uniqExact(mutation_id) FROM system.mutations " - "WHERE database = {db:String} AND table = {table:String} AND " + project.REWRITES_PARTS, + rewrites = project.client.query( + "SELECT create_time, is_done FROM system.mutations " + "WHERE database = {db:String} AND table = {table:String} AND " + + project.REWRITES_PARTS + + " ORDER BY create_time", parameters={"db": db, "table": "counter"}, - ) - print(f"\nCLOUD_SIGKILL increments={increments} owned_mutations={rewrites}") - assert (increments, rewrites) == (1, 1) + ).result_rows + print(f"\nCLOUD_SIGKILL value={value} mutations={len(rewrites)}") + assert value == 1 and len(rewrites) == 2 and all(done for _, done in rewrites) assert project.client.query(f"SELECT version_num FROM {db}.alembic_version").result_rows == [ ("aaaa",) ] -def _slow_increment(project, seconds): +def _slow_update(project, value, seconds): db = project.database project.client.command( f"CREATE TABLE {db}.counter (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id" ) project.client.command(f"INSERT INTO {db}.counter VALUES (1, 0)") (project.sql_dir / "slow.sql").write_text( - "-- ch-migrate: allow-non-idempotent One controlled increment\n" - f"ALTER TABLE {{db}}.counter UPDATE x = x + 1 + sleepEachRow({seconds}) WHERE id = 1;\n" + "-- ch-migrate: allow-non-idempotent One controlled, slow update\n" + f"ALTER TABLE {{db}}.counter UPDATE x = {value} + sleepEachRow({seconds}) WHERE id = 1;\n" ) project.write_revision( "aaaa", {"upgrade": "from ch_migrate import run_sql\nrun_sql('slow.sql')"} diff --git a/tests/integration/test_ddl_queue.py b/tests/integration/test_ddl_queue.py deleted file mode 100644 index 725cc18..0000000 --- a/tests/integration/test_ddl_queue.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Owned distributed DDL waits, failure propagation, and queue-identity recovery.""" - -import json -import os -import subprocess -import sys -import time -from contextlib import contextmanager - -import pytest - -pytestmark = pytest.mark.integration - - -def test_ddl_queue_create_then_alter_finishes_every_host(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _baseline(project) - _revision( - project, - [ - f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory", - f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} ADD COLUMN IF NOT EXISTS value String", - ], - ) - result = project.run("up", "it") - assert result.exit_code == 0, result.output - for client in cluster.clients.values(): - assert client.query( - "SELECT name FROM system.columns WHERE database={db:String} AND table='owned' ORDER BY position", - parameters={"db": project.database}, - ).result_rows == [("id",), ("value",)] - rows = _queue(project) - assert len({row[0] for row in rows}) == 2 - assert all(row[3] == "Finished" and row[4] == 0 for row in rows) - assert _heads(project) == ["bbbb"] - - -def test_ddl_queue_down_host_times_out_then_reattaches(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _baseline(project) - _revision( - project, - [ - f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" - ], - ) - cluster.stop_node(2) - try: - result = project.run("up", "it", "--timeout", "1") - assert result.exit_code == 1 and "Timed out" in result.output, result.output - assert cluster.hostnames[2] in result.output - before = {row[0] for row in _queue(project)} - assert len(before) == 1 - assert _heads(project) == ["aaaa"] - finally: - cluster.start_node(2) - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert {row[0] for row in _queue(project)} == before - assert all(row[3] == "Finished" and row[4] == 0 for row in _queue(project)) - assert _heads(project) == ["bbbb"] - - -def test_ddl_queue_host_exception_is_not_finished_success(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _baseline(project) - table = f"{project.database}.owned" - project.client.command( - f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" - ) - cluster.clients[2].command(f"ALTER TABLE {table} ADD COLUMN conflict String") - _revision( - project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} ADD COLUMN conflict UInt64"] - ) - result = project.run("up", "it") - assert result.exit_code == 1, result.output - assert cluster.hostnames[2] in result.output and "DUPLICATE_COLUMN" in result.output - assert _heads(project) == ["aaaa"] - assert any(row[3] == "Finished" and row[4] == 15 for row in _queue(project)) - - -def test_ddl_queue_lost_acknowledgement_uses_pre_send_marker(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _baseline(project) - _revision( - project, - [ - f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" - ], - ) - code = """import os, signal -from ch_migrate.waiting import MigrationWaiter -original = MigrationWaiter._after -def crash(self, connection, cursor, statement, parameters, context, executemany): - pending = self._pending.get(id(context)) - if pending and pending[1].get('ddl') and pending[1].get('table', [])[-1] == 'owned': - os.kill(os.getpid(), signal.SIGKILL) - return original(self, connection, cursor, statement, parameters, context, executemany) -MigrationWaiter._after = crash -from ch_migrate.cli import main -main() -""" - with _running(project, code) as (process, log): - assert process.wait(timeout=30) == -9, log.read_text() - entries = {row[0] for row in _queue(project)} - assert len(entries) == 1 - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert {row[0] for row in _queue(project)} == entries - assert _heads(project) == ["bbbb"] - - -def test_ddl_queue_and_replicated_mutation_both_must_finish(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _baseline(project) - table = f"{project.database}.owned" - project.client.command( - f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64, x UInt64) " - f"ENGINE=ReplicatedMergeTree('/clickhouse/ddl_wait/{project.database}', '{{replica}}') ORDER BY id" - ) - project.client.command(f"INSERT INTO {table} VALUES (1, 0)") - cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") - _revision( - project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} UPDATE x=x+1 WHERE id=1"] - ) - for client in cluster.clients.values(): - client.command(f"SYSTEM STOP MERGES {table}") - try: - # Long enough for the queue entry to finish on both hosts on a slow CI runner, so the - # deadline falls in the mutation wait that STOP MERGES holds open. - result = project.run("up", "it", "--timeout", "5") - assert result.exit_code == 1, result.output - rows = _queue(project) - assert rows and all(row[3] == "Finished" and row[4] == 0 for row in rows) - assert "parts_to_do=" in result.output - assert _heads(project) == ["aaaa"] - entries = {row[0] for row in rows} - finally: - for client in cluster.clients.values(): - client.command(f"SYSTEM START MERGES {table}") - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert {row[0] for row in _queue(project)} == entries - for client in cluster.clients.values(): - assert client.command(f"SELECT x FROM {table}") == 1 - assert _heads(project) == ["bbbb"] - - -def test_ddl_queue_missing_ownership_evidence_refuses_resubmission( - cluster_project, clickhouse_cluster -): - project, cluster = cluster_project, clickhouse_cluster - _baseline(project) - _revision( - project, - [ - f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" - ], - ) - cluster.stop_node(2) - try: - assert project.run("up", "it", "--timeout", "1").exit_code == 1 - entries = {row[0] for row in _queue(project)} - row = project.client.query( - f"SELECT generation, position, sequence, payload FROM {project.database}._ch_migrate_journal " - "WHERE revision='bbbb' ORDER BY sequence DESC LIMIT 1" - ).result_rows[0] - payload = json.loads(row[3]) - assert "ddl" in payload - # An operator-restored receipt whose ownership token has no server evidence. - payload["ddl"]["token"] = "missing-ownership-evidence" - project.client.insert( - f"{project.database}._ch_migrate_journal", - [["bbbb", row[0], row[1], row[2] + 1, json.dumps(payload)]], - column_names=["revision", "generation", "position", "sequence", "payload"], - ) - result = project.run("up", "it") - assert result.exit_code == 1 and "unknown" in result.output.lower(), result.output - assert "not reissued" in result.output - assert {row[0] for row in _queue(project)} == entries - assert _heads(project) == ["aaaa"] - finally: - cluster.start_node(2) - - -def test_ddl_queue_is_not_used_for_single_node_migrations(project): - project.write_revision( - "aaaa", - { - "upgrade": 'op.execute(f"CREATE TABLE IF NOT EXISTS {db}.owned (id UInt64) ENGINE=Memory")' - }, - ) - started = project.client.command("SELECT toString(now64(6))") - result = project.run("up", "it") - assert result.exit_code == 0, result.output - project.flush_logs() - # Earlier tests on a shared server (Cloud's ON CLUSTER commands) also read the queue. - reads = project.client.query( - f"SELECT count() FROM {project.query_log()} WHERE type='QueryStart' " - "AND event_time_microseconds >= toDateTime64({started:String}, 6) " - "AND position(query, 'FROM system.distributed_ddl_queue') > 0 " - "AND position(query, 'system.query_log') = 0", - parameters={"started": started}, - ).result_rows[0][0] - assert reads == 0 - - -def test_ddl_queue_ttl_metadata_and_materialization_resume(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _baseline(project) - table = f"{project.database}.expiry" - project.client.command( - f"CREATE TABLE {table} ON CLUSTER {cluster.name} (ts DateTime) " - f"ENGINE=ReplicatedMergeTree('/clickhouse/ddl_ttl/{project.database}', '{{replica}}') ORDER BY ts" - ) - project.client.command(f"INSERT INTO {table} VALUES ('2020-01-01 00:00:00')") - cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") - _revision( - project, - [f"ALTER TABLE {{db}}.expiry ON CLUSTER {cluster.name} MODIFY TTL ts + INTERVAL 100 YEAR"], - ) - for client in cluster.clients.values(): - client.command(f"SYSTEM STOP MERGES {table}") - try: - result = project.run("up", "it", "--timeout", "1") - assert result.exit_code == 1 and "parts_to_do=" in result.output, result.output - entries = {row[0] for row in _queue(project)} - assert len(entries) == 2 # The metadata and materialization have separate receipts. - assert _heads(project) == ["aaaa"] - finally: - for client in cluster.clients.values(): - client.command(f"SYSTEM START MERGES {table}") - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert {row[0] for row in _queue(project)} == entries - assert _heads(project) == ["bbbb"] - - -def test_ddl_queue_nonreplicated_mutations_wait_on_each_host(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _baseline(project) - table = f"{project.database}.owned" - project.client.command( - f"CREATE TABLE {table} ON CLUSTER {cluster.name} (x UInt64) ENGINE=MergeTree ORDER BY tuple()" - ) - for client in cluster.clients.values(): - client.command(f"INSERT INTO {table} VALUES (0)") - _revision(project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} UPDATE x=x+1 WHERE 1"]) - cluster.clients[2].command(f"SYSTEM STOP MERGES {table}") - try: - result = project.run("up", "it", "--timeout", "1") - assert result.exit_code == 1, result.output - assert cluster.hostnames[2] in result.output - assert _heads(project) == ["aaaa"] - assert cluster.clients[1].command(f"SELECT x FROM {table}") == 1 - assert cluster.clients[2].command(f"SELECT x FROM {table}") == 0 - finally: - cluster.clients[2].command(f"SYSTEM START MERGES {table}") - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - for client in cluster.clients.values(): - assert client.command(f"SELECT x FROM {table}") == 1 - - -def test_ddl_queue_cold_version_table_creation_obeys_timeout(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) - cluster.stop_node(2) - try: - result = project.run("up", "it", "--timeout", "1") - assert result.exit_code == 1 and "Timed out" in result.output, result.output - assert cluster.hostnames[2] in result.output - assert _heads(project) == [] - finally: - cluster.start_node(2) - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert _heads(project) == ["aaaa"] - - -def _baseline(project): - project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) - result = project.run("up", "it") - assert result.exit_code == 0, result.output - - -def _revision(project, statements): - path = project.sql_dir / "ddl.sql" - path.write_text( - "\n".join( - "-- ch-migrate: allow-non-idempotent Controlled distributed DDL scenario\n" + sql + ";" - for sql in statements - ) - ) - project.write_revision( - "bbbb", {"upgrade": "from ch_migrate import run_sql\nrun_sql('ddl.sql')"}, "aaaa" - ) - - -def _queue(project): - return project.client.query( - "SELECT entry, host, port, status, exception_code FROM system.distributed_ddl_queue " - "WHERE startsWith(settings['log_comment'], 'chm_mutation_') AND position(query, {db:String}) > 0 " - "AND position(query, 'alembic_version') = 0 AND position(query, '_ch_migrate_journal') = 0", - parameters={"db": project.database}, - ).result_rows - - -def _heads(project): - return [ - row[0] - for row in project.client.query( - f"SELECT version_num FROM {project.database}.alembic_version ORDER BY version_num" - ).result_rows - ] - - -@contextmanager -def _running(project, code): - path = project.root / f"ddl-{time.monotonic_ns()}.log" - with path.open("w") as log: - process = subprocess.Popen( - [sys.executable, "-c", code, "up", "it"], - cwd=project.root, - stdout=log, - stderr=subprocess.STDOUT, - env=os.environ.copy(), - ) - try: - yield process, path - finally: - if process.poll() is None: - process.kill() - process.wait(timeout=10) diff --git a/tests/integration/test_mutation_waiting.py b/tests/integration/test_mutation_waiting.py index 96257ad..2c49470 100644 --- a/tests/integration/test_mutation_waiting.py +++ b/tests/integration/test_mutation_waiting.py @@ -1,290 +1,142 @@ -"""Real-server waiting, crash recovery, failure, and timeout invariants.""" +"""Real-server waiting: `up` records a migration only after the work it started finishes.""" -import json import os import subprocess import sys import time from contextlib import contextmanager -from pathlib import Path import pytest import yaml pytestmark = pytest.mark.integration -CORPUS = yaml.safe_load((Path(__file__).parents[1] / "corpus/classification.yaml").read_text()) -MUTATIONS = [case for case in CORPUS["statements"] if case["expected"] == "mutation"] - - -@pytest.mark.parametrize("case", MUTATIONS, ids=lambda case: case["id"]) -def test_mutation_waits_for_server_corpus(project, case): - values = {"db": project.database, "table": f"{project.database}.sample"} - project.client.command(CORPUS["table_ddl"].format(**values)) - project.client.command(CORPUS["seed"].format(**values)) - if case["id"] == "lightweight_delete": - # Lightweight deletes reject projections under the server's default policy. - project.client.command( - f"ALTER TABLE {values['table']} DROP PROJECTION pr SETTINGS mutations_sync=2" - ) - sql = case["sql"].format(**values) - project.write_revision( - "aaaa", - { - "upgrade": "# ch-migrate: allow-non-idempotent Server-backed mutation corpus\n" - + f"op.execute({sql!r})" - }, - ) - result = project.run("up", "it") - assert result.exit_code == 0, result.output - rows = _mutations(project, "sample") - assert all(row[2] for row in rows), rows - assert any("chm_mutation_" in row[1] for row in rows), rows - assert _heads(project) == ["aaaa"] - -def test_mutation_failure_leaves_head_and_prints_operator_kill(project): - _seed(project) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = throwIf(x = 5) WHERE 1") - result = project.run("up", "it") - assert result.exit_code == 1, result.output - rows = _mutations(project) - failed = next(row for row in rows if row[4]) - assert failed[0] in result.output and "throwIf" in result.output - assert "KILL MUTATION WHERE database =" in result.output - assert f"mutation_id = '{failed[0]}'" in result.output - assert _heads(project) == ["aaaa"] - assert any(not row[2] and row[0] == failed[0] for row in _mutations(project)) +# Mutations ClickHouse starts and answers before finishing; STOP MERGES holds each one. +HELD = { + "update": "ALTER TABLE {db}.counter UPDATE x = 1 WHERE id = 1", + "delete": "ALTER TABLE {db}.counter DELETE WHERE id = 5", + "materialize": "ALTER TABLE {db}.counter MATERIALIZE COLUMN doubled", +} -def test_mutation_timeout_preserves_work_and_resumes(project): +@pytest.mark.parametrize("statement", HELD.values(), ids=HELD.keys()) +def test_up_records_a_migration_only_after_its_mutation(project, statement): _seed(project) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + _upgrade_sql(project, statement) project.stop_merges(f"{project.database}.counter") try: - result = project.run("up", "it", "--timeout", "0.5") - assert result.exit_code == 1, result.output - rows = _mutations(project) - ids = {row[0] for row in rows} - assert "Timed out" in result.output and "parts_to_do=" in result.output - assert all(mutation in result.output for mutation in ids) + held = project.run("up", "it", "--timeout", "1") + assert held.exit_code == 1 and "Timed out" in held.output, held.output + assert "parts_to_do=" in held.output assert _heads(project) == ["aaaa"] - assert any(not row[2] for row in rows) finally: project.client.command(f"SYSTEM START MERGES {project.database}.counter") resumed = project.run("up", "it") assert resumed.exit_code == 0, resumed.output - assert {row[0] for row in _mutations(project)} == ids - assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 1 + assert all(row[2] for row in _mutations(project)) assert _heads(project) == ["bbbb"] -@pytest.mark.parametrize("trial", range(3)) -def test_mutation_sigkill_rerun_attaches_without_second_increment(project, trial): +def test_failed_mutation_is_not_recorded_and_names_its_kill(project): _seed(project) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - project.stop_merges(f"{project.database}.counter") - try: - with _running(project, ("up", "it")) as (first, first_log): - rows = _until(lambda: _mutations(project), "mutation was not submitted") - mutation = rows[0][0] - _until( - lambda: mutation in first_log.read_text(), "first process did not report waiting" - ) - first.kill() - first.wait(timeout=10) - original = {row[0] for row in _mutations(project)} - assert _heads(project) == ["aaaa"] - with _running(project, ("up", "it")) as (resumed, log): - _until( - lambda: mutation in log.read_text(), "rerun did not report the original mutation" - ) - assert resumed.poll() is None - assert {row[0] for row in _mutations(project)} == original - project.client.command(f"SYSTEM START MERGES {project.database}.counter") - assert resumed.wait(timeout=30) == 0, log.read_text() - assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 1 - assert _heads(project) == ["bbbb"] - finally: - project.client.command(f"SYSTEM START MERGES {project.database}.counter") - - -def test_mutation_crash_before_durable_receipt_recovers_from_intent(project): - _seed(project) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - # Fault injection kills the real CLI after server acceptance, before journal acknowledgement. - code = """import os, signal -from ch_migrate.waiting import MigrationWaiter -original = MigrationWaiter._after -def crash(self, connection, cursor, statement, parameters, context, executemany): - pending = self._pending.get(id(context)) - if pending and pending[1]['kind'] == 'mutation' and pending[1]['table'][1] == 'counter': - os.kill(os.getpid(), signal.SIGKILL) - return original(self, connection, cursor, statement, parameters, context, executemany) -MigrationWaiter._after = crash -from ch_migrate.cli import main -main() -""" - project.stop_merges(f"{project.database}.counter") - try: - with _running(project, ("up", "it"), code) as (crashed, log): - assert crashed.wait(timeout=30) == -9, log.read_text() - rows = _mutations(project) - original_ids = {row[0] for row in rows} - assert rows and any("chm_mutation_" in row[1] for row in rows) - payload = project.client.query( - f"SELECT argMax(payload, sequence) FROM {project.database}._ch_migrate_journal " - "WHERE revision = 'bbbb' AND position = 1" - ).result_rows[0][0] - assert json.loads(payload)["phase"] == "intent" - project.client.command(f"SYSTEM START MERGES {project.database}.counter") - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert {row[0] for row in _mutations(project)} == original_ids - assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 1 - finally: - project.client.command(f"SYSTEM START MERGES {project.database}.counter") - - -def test_mutation_foreign_failure_is_not_adopted_or_followed(project): - _seed(project) - project.client.command( - f"ALTER TABLE {project.database}.counter UPDATE x = throwIf(x = 5) WHERE 1" - ) - _until(lambda: any(row[4] for row in _mutations(project)), "foreign mutation did not fail") - original_ids = {row[0] for row in _mutations(project)} - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = throwIf(x = 5) WHERE 1") result = project.run("up", "it") - assert result.exit_code == 1 and "foreign mutation" in result.output, result.output - assert {row[0] for row in _mutations(project)} == original_ids + assert result.exit_code == 1, result.output + failed = next(row for row in _mutations(project) if row[4]) + assert failed[0] in result.output and "throwIf" in result.output + assert f"mutation_id = '{failed[0]}'" in result.output assert _heads(project) == ["aaaa"] -def test_mutation_expired_ownership_refuses_instead_of_replaying(project): - _seed( - project, - " SETTINGS finished_mutations_to_keep=1, old_parts_lifetime=0, cleanup_delay_period=1, cleanup_delay_period_random_add=0", - ) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - project.stop_merges(f"{project.database}.counter") - try: - timed_out = project.run("up", "it", "--timeout", "0.4") - assert timed_out.exit_code == 1 - finally: - project.client.command(f"SYSTEM START MERGES {project.database}.counter") - _until(lambda: all(row[2] for row in _mutations(project)), "owned mutation did not finish") +def test_killed_mutation_then_fixed_sql_runs_again(project): + """DRY-1604: after the kill, fixing the file and running `up` is the whole recovery.""" + db = project.database project.client.command( - f"ALTER TABLE {project.database}.counter DELETE WHERE 0 SETTINGS mutations_sync=2" + f"CREATE TABLE {db}.orders (id UInt64, raw String, qty UInt64) ENGINE=MergeTree ORDER BY id" ) - _until( - lambda: not any("chm_mutation_" in row[1] for row in _mutations(project)), - "ownership did not expire", + project.client.command(f"INSERT INTO {db}.orders VALUES (1, '3', 0), (2, 'n/a', 0)") + _baseline(project) + path = _upgrade_sql(project, "ALTER TABLE {db}.orders UPDATE qty = toUInt64(raw) WHERE 1") + failed = project.run("up", "it") + assert failed.exit_code == 1 and "Cannot parse" in failed.output, failed.output + blocked = project.run("up", "it") + assert blocked.exit_code == 1, blocked.output + assert "started before this run" in blocked.output and "KILL MUTATION" in blocked.output + mutation = next(row[0] for row in _mutations(project, "orders") if row[4]) + project.client.command( + "KILL MUTATION WHERE database = {db:String} AND mutation_id = {mutation:String}", + parameters={"db": db, "mutation": mutation}, ) - resumed = project.run("up", "it") - assert resumed.exit_code == 1 and "Outcome unknown" in resumed.output, resumed.output - assert "not reissued" in resumed.output - assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 1 - assert _heads(project) == ["aaaa"] + _until(lambda: not any(row[4] for row in _mutations(project, "orders")), "kill did not land") + _write_sql(path, "ALTER TABLE {db}.orders UPDATE qty = toUInt64OrZero(raw) WHERE 1") + fixed = project.run("up", "it") + assert fixed.exit_code == 0, fixed.output + rows = project.client.query(f"SELECT id, qty FROM {db}.orders ORDER BY id").result_rows + assert rows == [(1, 3), (2, 0)] + assert _heads(project) == ["bbbb"] -def test_mutation_changed_sql_refuses_replay(project): +def test_rejected_statement_can_be_fixed_and_run_again(project): + """DRY-1604: ClickHouse refuses a sorting-key change; the corrected file then applies.""" _seed(project) - path = _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - project.stop_merges(f"{project.database}.counter") - try: - assert project.run("up", "it", "--timeout", "0.4").exit_code == 1 - before = {row[0] for row in _mutations(project)} - path.write_text(path.read_text().replace("x + 1", "x + 2")) - result = project.run("up", "it") - assert result.exit_code == 1 and "changed" in result.output, result.output - assert {row[0] for row in _mutations(project)} == before - assert _heads(project) == ["aaaa"] - finally: - project.client.command(f"SYSTEM START MERGES {project.database}.counter") + path = _upgrade_sql(project, "ALTER TABLE {db}.counter MODIFY COLUMN id String") + rejected = project.run("up", "it") + assert ( + rejected.exit_code == 1 and "ALTER_OF_COLUMN_IS_FORBIDDEN" in rejected.output + ), rejected.output + assert _heads(project) == ["aaaa"] + _write_sql(path, "ALTER TABLE {db}.counter UPDATE x = 1 WHERE id = 1") + fixed = project.run("up", "it") + assert fixed.exit_code == 0, fixed.output + assert _heads(project) == ["bbbb"] -def test_mutation_completed_prefix_is_not_repeated_and_downgrade_resets(project): +def test_unfinished_earlier_mutation_holds_the_run(project): + """Work already queued on the database finishes before any migration statement is sent.""" + db = project.database _seed(project) - project.client.command(f"CREATE TABLE {project.database}.audit (n UInt64) ENGINE=Memory") - _upgrade_sql( - project, - "INSERT INTO {db}.audit VALUES (1);\n-- ch-migrate: allow-non-idempotent increment once\nALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1", - ) - project.stop_merges(f"{project.database}.counter") + project.client.command(f"CREATE TABLE {db}.observed (value UInt8) ENGINE=Memory") + _upgrade_sql(project, "INSERT INTO {db}.observed VALUES (1)") + project.stop_merges(f"{db}.counter") try: - assert project.run("up", "it", "--timeout", "0.4").exit_code == 1 + project.client.command(f"ALTER TABLE {db}.counter UPDATE x = 7 WHERE id = 5") + held = project.run("up", "it", "--timeout", "1") + assert held.exit_code == 1 and "Timed out" in held.output, held.output + assert project.client.command(f"SELECT count() FROM {db}.observed") == 0 + assert _heads(project) == ["aaaa"] finally: - project.client.command(f"SYSTEM START MERGES {project.database}.counter") + project.client.command(f"SYSTEM START MERGES {db}.counter") resumed = project.run("up", "it") assert resumed.exit_code == 0, resumed.output - assert project.client.command(f"SELECT count() FROM {project.database}.audit") == 1 - reverted = project.run("down", "it") - assert reverted.exit_code == 0, reverted.output - reapplied = project.run("up", "it") - assert reapplied.exit_code == 0, reapplied.output - assert project.client.command(f"SELECT count() FROM {project.database}.audit") == 2 - assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 2 + assert project.client.command(f"SELECT count() FROM {db}.observed") == 1 + assert _heads(project) == ["bbbb"] -def test_mutation_waits_for_every_replica(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _seed_cluster(project, cluster) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - table = f"{project.database}.counter" - for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): - cluster.clients[2].command(f"SYSTEM STOP {action} {table}") +def test_down_waits_for_its_mutation(project): + db = project.database + _seed(project) + project.write_revision( + "bbbb", + { + "upgrade": 'op.execute("SELECT 1")', + "downgrade": "# ch-migrate: allow-non-idempotent Controlled downgrade backfill\n" + "op.execute(f'ALTER TABLE {db}.counter UPDATE x = 0 WHERE id = 5')", + }, + "aaaa", + ) + assert project.run("up", "it").exit_code == 0 + project.stop_merges(f"{db}.counter") try: - with _running(project, ("up", "it")) as (process, log): - _until(lambda: _mutations(project), "replicated mutation was not submitted") - cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table} PULL") - _until(lambda: all(row[2] for row in _mutations(project)), "initiator did not finish") - second = ( - cluster.clients[2] - .query( - "SELECT is_done FROM system.mutations WHERE database = {db:String} AND table = 'counter'", - parameters={"db": project.database}, - ) - .result_rows - ) - assert second and all(not row[0] for row in second) - _until( - lambda: "waiting on mutation" in log.read_text(), "replica wait was not reported" - ) - assert process.poll() is None - for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): - cluster.clients[2].command(f"SYSTEM START {action} {table}") + with _running(project, ("down", "it")) as (process, log): + _until(lambda: _mutations(project), "downgrade mutation was not submitted") + time.sleep(1) + assert process.poll() is None, log.read_text() + project.client.command(f"SYSTEM START MERGES {db}.counter") assert process.wait(timeout=30) == 0, log.read_text() - for client in cluster.clients.values(): - assert client.command(f"SELECT x FROM {table} WHERE id = 1") == 1 finally: - for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): - cluster.clients[2].command(f"SYSTEM START {action} {table}") - - -def test_mutation_missing_replica_is_not_success(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - _seed_cluster(project, cluster) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") - table = f"{project.database}.counter" - for client in cluster.clients.values(): - client.command(f"SYSTEM STOP MERGES {table}") - try: - first = project.run("up", "it", "--timeout", "0.5") - assert first.exit_code == 1, first.output - original = {row[0] for row in _mutations(project)} - cluster.stop_node(2) - result = project.run("up", "it", "--timeout", "1") - assert result.exit_code == 1 and "Timed out" in result.output, result.output - assert "host" in result.output - assert _heads(project) == ["aaaa"] - assert {row[0] for row in _mutations(project)} == original - finally: - if 2 in cluster.stopped: - cluster.start_node(2) - for client in cluster.clients.values(): - client.command(f"SYSTEM START MERGES {table}") - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert _heads(project) == ["bbbb"] + project.client.command(f"SYSTEM START MERGES {db}.counter") + assert all(row[2] for row in _mutations(project)) + assert _heads(project) == ["aaaa"] def test_mutation_polling_preserves_session_settings(project): @@ -347,211 +199,6 @@ def test_mutation_raw_bind_paths_are_waited(project, mode): assert _heads(project) == ["bbbb"] -def test_mutation_replaced_table_refuses_resume(project): - _seed(project) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") - project.stop_merges(f"{project.database}.counter") - result = project.run("up", "it", "--timeout", "0.4") - assert result.exit_code == 1 - project.client.command(f"DROP TABLE {project.database}.counter SYNC") - project.client.command( - f"CREATE TABLE {project.database}.counter (id UInt64, x UInt64) ENGINE=MergeTree ORDER BY id" - ) - project.client.command(f"INSERT INTO {project.database}.counter VALUES (1, 0)") - resumed = project.run("up", "it") - assert resumed.exit_code == 1 and "replaced" in resumed.output, resumed.output - assert project.client.command(f"SELECT x FROM {project.database}.counter") == 0 - assert _heads(project) == ["aaaa"] - - -def test_mutation_healthy_foreign_predecessor_is_waited_before_submission(project): - _seed(project) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") - project.stop_merges(f"{project.database}.counter") - project.client.command(f"ALTER TABLE {project.database}.counter UPDATE x=x+10 WHERE id=1") - original = {row[0] for row in _mutations(project)} - try: - blocked = project.run("up", "it", "--timeout", "0.4") - assert blocked.exit_code == 1, blocked.output - assert {row[0] for row in _mutations(project)} == original - assert all("chm_mutation_" not in row[1] for row in _mutations(project)) - assert _heads(project) == ["aaaa"] - finally: - project.client.command(f"SYSTEM START MERGES {project.database}.counter") - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id=1") == 11 - - -@pytest.mark.parametrize("scope", ["session", "query"]) -def test_mutation_ttl_materialization_setting_is_respected(project, scope): - project.client.command( - f"CREATE TABLE {project.database}.expiry (ts DateTime) ENGINE=MergeTree ORDER BY ts" - ) - project.client.command(f"INSERT INTO {project.database}.expiry VALUES ('2020-01-01 00:00:00')") - statement = "ALTER TABLE {db}.expiry MODIFY TTL ts + INTERVAL 100 YEAR" - sql = ( - "SET materialize_ttl_after_modify=0;\n" + statement - if scope == "session" - else statement + " SETTINGS materialize_ttl_after_modify=0" - ) - project.write_revision( - "aaaa", {"upgrade": "from ch_migrate import run_sql\nrun_sql('ttl.sql')"} - ) - (project.sql_dir / "ttl.sql").write_text(sql) - result = project.run("up", "it") - assert result.exit_code == 0, result.output - assert _mutations(project, "expiry") == [] - assert "TTL" in project.client.command(f"SHOW CREATE TABLE {project.database}.expiry") - - -def test_mutation_timeout_budget_is_not_reset_for_next_statement(project): - _seed(project) - project.client.command( - f"CREATE TABLE {project.database}.second (id UInt64, x UInt64) ENGINE=MergeTree ORDER BY id" - ) - project.client.command(f"INSERT INTO {project.database}.second VALUES (1, 0)") - _upgrade_sql( - project, - "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1;\n" - "-- ch-migrate: allow-non-idempotent second controlled update\n" - "ALTER TABLE {db}.second UPDATE x=x+1 WHERE id=1", - ) - project.stop_merges(f"{project.database}.counter") - project.stop_merges(f"{project.database}.second") - try: - with _running(project, ("up", "it", "--timeout", "1.5")) as (process, log): - mutation = _until(lambda: _mutations(project), "first statement did not start")[0][0] - _until(lambda: mutation in log.read_text(), "first wait was not visible") - time.sleep(0.7) - project.client.command(f"SYSTEM START MERGES {project.database}.counter") - _until(lambda: _mutations(project, "second"), "second statement did not start") - began_second = time.monotonic() - assert process.wait(timeout=10) == 1, log.read_text() - assert time.monotonic() - began_second < 1.3 - assert "second" in log.read_text() and "Timed out" in log.read_text() - assert _heads(project) == ["aaaa"] - finally: - project.client.command(f"SYSTEM START MERGES {project.database}.counter") - project.client.command(f"SYSTEM START MERGES {project.database}.second") - - -def test_mutation_version_rows_ignore_fire_and_forget_insert_settings(project): - project.write_revision( - "aaaa", - { - "upgrade": 'op.execute("SET async_insert=1, wait_for_async_insert=0, async_insert_use_adaptive_busy_timeout=0, async_insert_busy_timeout_ms=60000")' - }, - ) - project.write_revision("bbbb", {"upgrade": 'op.execute("SELECT 1")'}, "aaaa") - result = project.run("up", "it") - assert result.exit_code == 0, result.output - assert _heads(project) == ["bbbb"] - queued = project.client.query( - "SELECT count() FROM system.asynchronous_inserts WHERE database = {db:String} " - "AND table IN ('alembic_version', '_ch_migrate_journal')", - parameters={"db": project.database}, - ).result_rows[0][0] - assert queued == 0 - - -def test_mutation_version_insert_lost_receipt_finishes_bookkeeping_only(project): - _seed(project) - project.client.command(f"CREATE TABLE {project.database}.audit (n UInt64) ENGINE=Memory") - _upgrade_sql(project, "INSERT INTO {db}.audit VALUES (1)") - code = """import os, signal -from ch_migrate.waiting.versions import VersionWrites -original = VersionWrites._insert -def crash(self, step, checkpoint): - original(self, step, checkpoint) - if step['value'] == 'bbbb': - os.kill(os.getpid(), signal.SIGKILL) -VersionWrites._insert = crash -from ch_migrate.cli import main -main() -""" - with _running(project, ("up", "it"), code) as (process, log): - assert process.wait(timeout=30) == -9, log.read_text() - assert _heads(project) == ["aaaa", "bbbb"] - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert _heads(project) == ["bbbb"] - audit = project.everywhere(f"{project.database}.audit") - assert project.client.command(f"SELECT count() FROM {audit}") == 1 - - -def test_mutation_transport_loss_is_not_treated_as_server_rejection(project): - _seed(project) - _upgrade_sql(project, "CREATE TABLE IF NOT EXISTS {db}.uncertain (id UInt64) ENGINE=Memory") - code = """from ch_migrate.waiting import MigrationWaiter -from clickhouse_connect.driver.exceptions import OperationalError -original = MigrationWaiter._after -def lose_reply(self, connection, cursor, statement, parameters, context, executemany): - pending = self._pending.get(id(context)) - if pending and pending[1].get('table', [None, None])[-1] == 'uncertain': - raise OperationalError('Connection lost after server accepted the statement') - return original(self, connection, cursor, statement, parameters, context, executemany) -MigrationWaiter._after = lose_reply -from ch_migrate.cli import main -main() -""" - with _running(project, ("up", "it"), code) as (process, log): - assert process.wait(timeout=30) == 1, log.read_text() - assert project.client.command(f"EXISTS TABLE {project.database}.uncertain") == 1 - resumed = project.run("up", "it") - assert resumed.exit_code == 1 and "Outcome unknown" in resumed.output, resumed.output - assert "Read-only inspection:" in resumed.output - assert _heads(project) == ["aaaa"] - project.client.command("SYSTEM FLUSH LOGS") - # The client's text continues with the column list; a Shared database also logs its own - # re-execution of the CREATE, under another query_id, with `UUID '...'` inserted there. - prefix = f"CREATE TABLE IF NOT EXISTS {project.database}.uncertain (" - assert _requests_started(project, prefix) == 1 - - -def test_mutation_trailing_foreign_failure_does_not_hold_owned_completion(project): - _seed(project) - _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x=x+1 WHERE id=1") - code = """from ch_migrate.waiting import MigrationWaiter -original = MigrationWaiter._after -def append_foreign(self, connection, cursor, statement, parameters, context, executemany): - pending = self._pending.get(id(context)) - if pending and pending[1]['kind'] == 'mutation' and pending[1]['table'][1] == 'counter': - self.client.command(f"ALTER TABLE {self.state.database}.counter UPDATE x=throwIf(x=5) WHERE 1") - return original(self, connection, cursor, statement, parameters, context, executemany) -MigrationWaiter._after = append_foreign -from ch_migrate.cli import main -main() -""" - with _running(project, ("up", "it"), code) as (process, log): - assert process.wait(timeout=30) == 0, log.read_text() - _until(lambda: any(row[4] for row in _mutations(project)), "trailing mutation did not fail") - assert _heads(project) == ["bbbb"] - assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id=1") == 1 - - -def test_mutation_ttl_resume_does_not_materialize_twice(project): - project.client.command( - f"CREATE TABLE {project.database}.expiry (ts DateTime) ENGINE=MergeTree ORDER BY ts" - ) - project.client.command(f"INSERT INTO {project.database}.expiry VALUES ('2020-01-01 00:00:00')") - project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) - assert project.run("up", "it").exit_code == 0 - _upgrade_sql(project, "ALTER TABLE {db}.expiry MODIFY TTL ts + INTERVAL 100 YEAR") - project.stop_merges(f"{project.database}.expiry") - try: - timed_out = project.run("up", "it", "--timeout", "0.4") - assert timed_out.exit_code == 1, timed_out.output - original = {row[0] for row in _mutations(project, "expiry")} - assert any(not row[2] for row in _mutations(project, "expiry")) - finally: - project.client.command(f"SYSTEM START MERGES {project.database}.expiry") - resumed = project.run("up", "it") - assert resumed.exit_code == 0, resumed.output - assert {row[0] for row in _mutations(project, "expiry")} == original - assert _heads(project) == ["bbbb"] - - def test_mutation_piped_progress_prints_plain_lines(project): """CI logs and agents get a start line naming the mutation, never a redrawn bar.""" _seed(project) @@ -570,7 +217,7 @@ def test_mutation_piped_progress_prints_plain_lines(project): start = f" waiting on mutation_2.txt ({project.database}.counter): 0/1 parts" assert start in result.stderr.splitlines(), result.stderr assert "\x1b" not in result.stderr and "\r" not in result.stderr - assert "Run ch-migrate up it again to keep waiting" in result.stderr + assert "Run ch-migrate up it again: it waits for that work to finish" in result.stderr finally: project.client.command(f"SYSTEM START MERGES {project.database}.counter") assert _heads(project) == ["aaaa"] @@ -618,27 +265,6 @@ def test_mutation_terminal_progress_uses_a_live_line(project): project.client.command(f"SYSTEM START MERGES {project.database}.counter") -def test_mutation_parameterized_ttl_batch_preserves_each_operation(project): - project.client.command( - f"CREATE TABLE {project.database}.expiry (ts DateTime) ENGINE=MergeTree ORDER BY ts" - ) - project.client.command(f"INSERT INTO {project.database}.expiry VALUES ('2020-01-01 00:00:00')") - project.write_revision( - "aaaa", - { - "upgrade": "op.get_bind().exec_driver_sql(" - "f'ALTER TABLE {db}.expiry MODIFY TTL ts + toIntervalYear(%(years)s)', " - "[{'years': 100}, {'years': 99}])" - }, - ) - result = project.run("up", "it") - assert result.exit_code == 0, result.output - rows = _mutations(project, "expiry") - assert all(row[2] for row in rows) - assert len({row[0] for row in rows if "chm_mutation_" in row[1]}) == 2 - assert _heads(project) == ["aaaa"] - - def test_mutation_lightweight_update_finishes_without_fabricating_a_mutation(project): project.client.command( f"CREATE TABLE {project.database}.patches (id UInt64, x UInt64) " @@ -661,6 +287,66 @@ def test_mutation_lightweight_update_finishes_without_fabricating_a_mutation(pro assert _heads(project) == ["aaaa"] +def test_mutation_waits_for_every_replica(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _seed_cluster(project, cluster) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + table = f"{project.database}.counter" + for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): + cluster.clients[2].command(f"SYSTEM STOP {action} {table}") + try: + with _running(project, ("up", "it")) as (process, log): + _until(lambda: _mutations(project), "replicated mutation was not submitted") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table} PULL") + _until(lambda: all(row[2] for row in _mutations(project)), "initiator did not finish") + second = ( + cluster.clients[2] + .query( + "SELECT is_done FROM system.mutations WHERE database = {db:String} AND table = 'counter'", + parameters={"db": project.database}, + ) + .result_rows + ) + assert second and all(not row[0] for row in second) + _until( + lambda: "waiting on mutation" in log.read_text(), "replica wait was not reported" + ) + assert process.poll() is None + for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): + cluster.clients[2].command(f"SYSTEM START {action} {table}") + assert process.wait(timeout=30) == 0, log.read_text() + for client in cluster.clients.values(): + assert client.command(f"SELECT x FROM {table} WHERE id = 1") == 1 + finally: + for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): + cluster.clients[2].command(f"SYSTEM START {action} {table}") + + +def test_mutation_on_a_down_replica_is_not_success(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _seed_cluster(project, cluster) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = 1 WHERE id = 1") + table = f"{project.database}.counter" + for client in cluster.clients.values(): + client.command(f"SYSTEM STOP MERGES {table}") + try: + first = project.run("up", "it", "--timeout", "0.5") + assert first.exit_code == 1, first.output + cluster.stop_node(2) + result = project.run("up", "it", "--timeout", "1") + assert result.exit_code == 1 and "Timed out" in result.output, result.output + assert "shows 1/2 replicas" in result.output + assert _heads(project) == ["aaaa"] + finally: + if 2 in cluster.stopped: + cluster.start_node(2) + for client in cluster.clients.values(): + client.command(f"SYSTEM START MERGES {table}") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert _heads(project) == ["bbbb"] + + def _seed_cluster(project, cluster): table = f"{project.database}.counter" cluster.clients[1].command( @@ -674,11 +360,16 @@ def _seed_cluster(project, cluster): assert result.exit_code == 0, result.output -def _seed(project, settings=""): +def _seed(project): project.client.command( - f"CREATE TABLE {project.database}.counter (id UInt64, x UInt64) ENGINE=MergeTree ORDER BY id{settings}" + f"CREATE TABLE {project.database}.counter (id UInt64, x UInt64, " + "doubled UInt64 DEFAULT x * 2) ENGINE=MergeTree ORDER BY id" ) - project.client.command(f"INSERT INTO {project.database}.counter VALUES (1, 0), (5, 5)") + project.client.command(f"INSERT INTO {project.database}.counter (id, x) VALUES (1, 0), (5, 5)") + _baseline(project) + + +def _baseline(project): project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) result = project.run("up", "it") assert result.exit_code == 0, result.output @@ -686,9 +377,7 @@ def _seed(project, settings=""): def _upgrade_sql(project, sql): path = project.sql_dir / "mutation.sql" - path.write_text( - "-- ch-migrate: allow-non-idempotent Controlled waiting acceptance scenario\n" + sql + ";\n" - ) + _write_sql(path, sql) project.write_revision( "bbbb", {"upgrade": "from ch_migrate import run_sql\nrun_sql('mutation.sql')"}, @@ -697,6 +386,12 @@ def _upgrade_sql(project, sql): return path +def _write_sql(path, sql): + path.write_text( + "-- ch-migrate: allow-non-idempotent Controlled waiting acceptance scenario\n" + sql + ";\n" + ) + + def _mutations(project, table="counter"): return project.client.query( "SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason FROM system.mutations " @@ -707,21 +402,6 @@ def _mutations(project, table="counter"): ).result_rows -def _requests_started(project, prefix): - """Requests whose logged text starts with `prefix`, counted on every replica.""" - deadline = time.monotonic() + 30 - while True: - count = project.client.query( - f"SELECT uniqExact(query_id) FROM {project.query_log()} WHERE type = 'QueryStart' " - "AND startsWith(query, {prefix:String})", - parameters={"prefix": prefix}, - settings={"skip_unavailable_shards": 1}, - ).result_rows[0][0] - if count or time.monotonic() > deadline: - return count - time.sleep(1) - - def _heads(project): return [ row[0] diff --git a/tests/integration/test_on_cluster.py b/tests/integration/test_on_cluster.py new file mode 100644 index 0000000..2946a7d --- /dev/null +++ b/tests/integration/test_on_cluster.py @@ -0,0 +1,200 @@ +"""ON CLUSTER statements finish on every host before a migration is recorded.""" + +import pytest + +pytestmark = pytest.mark.integration +NOT_FINISHED = "is not finished on 1 of 2 hosts" + + +def test_on_cluster_create_then_alter_finishes_every_host(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + _revision( + project, + [ + f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory", + f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} ADD COLUMN IF NOT EXISTS value String", + ], + ) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + for client in cluster.clients.values(): + assert client.query( + "SELECT name FROM system.columns WHERE database={db:String} AND table='owned' ORDER BY position", + parameters={"db": project.database}, + ).result_rows == [("id",), ("value",)] + assert _heads(project) == ["bbbb"] + + +def test_on_cluster_down_host_times_out_then_runs_again(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + _revision( + project, + [ + f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" + ], + ) + cluster.stop_node(2) + try: + result = project.run("up", "it", "--timeout", "2") + assert result.exit_code == 1 and NOT_FINISHED in result.output, result.output + assert _heads(project) == ["aaaa"] + finally: + cluster.start_node(2) + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + for client in cluster.clients.values(): + assert client.command(f"EXISTS TABLE {project.database}.owned") == 1 + assert _heads(project) == ["bbbb"] + + +def test_on_cluster_host_exception_fails_the_run(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + table = f"{project.database}.owned" + project.client.command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" + ) + cluster.clients[2].command(f"ALTER TABLE {table} ADD COLUMN conflict String") + _revision( + project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} ADD COLUMN conflict UInt64"] + ) + result = project.run("up", "it") + assert result.exit_code == 1, result.output + assert cluster.hostnames[2] in result.output and "DUPLICATE_COLUMN" in result.output + assert _heads(project) == ["aaaa"] + + +def test_on_cluster_and_replicated_mutation_both_must_finish(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + table = f"{project.database}.owned" + project.client.command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64, x UInt64) " + f"ENGINE=ReplicatedMergeTree('/clickhouse/ddl_wait/{project.database}', '{{replica}}') ORDER BY id" + ) + project.client.command(f"INSERT INTO {table} VALUES (1, 0)") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") + _revision( + project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} UPDATE x=1 WHERE id=1"] + ) + for client in cluster.clients.values(): + client.command(f"SYSTEM STOP MERGES {table}") + try: + # Long enough for the queue entry to finish on both hosts on a slow CI runner, so the + # deadline falls in the mutation wait that STOP MERGES holds open. + result = project.run("up", "it", "--timeout", "5") + assert result.exit_code == 1 and "parts_to_do=" in result.output, result.output + assert _heads(project) == ["aaaa"] + finally: + for client in cluster.clients.values(): + client.command(f"SYSTEM START MERGES {table}") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + for client in cluster.clients.values(): + assert client.command(f"SELECT x FROM {table}") == 1 + assert _heads(project) == ["bbbb"] + + +def test_on_cluster_ttl_materialization_is_waited(cluster_project, clickhouse_cluster): + """ClickHouse holds MODIFY TTL until it's materialized, so the DDL limit bounds that wait.""" + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + table = f"{project.database}.expiry" + project.client.command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (ts DateTime) " + f"ENGINE=ReplicatedMergeTree('/clickhouse/ddl_ttl/{project.database}', '{{replica}}') ORDER BY ts" + ) + project.client.command(f"INSERT INTO {table} VALUES ('2020-01-01 00:00:00')") + cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") + _revision( + project, + [f"ALTER TABLE {{db}}.expiry ON CLUSTER {cluster.name} MODIFY TTL ts + INTERVAL 100 YEAR"], + ) + for client in cluster.clients.values(): + client.command(f"SYSTEM STOP MERGES {table}") + try: + result = project.run("up", "it", "--timeout", "3") + assert result.exit_code == 1 and "is not finished on" in result.output, result.output + assert _heads(project) == ["aaaa"] + finally: + for client in cluster.clients.values(): + client.command(f"SYSTEM START MERGES {table}") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + for client in cluster.clients.values(): + assert not client.command( + "SELECT count() FROM system.mutations WHERE database = {db:String} AND NOT is_done", + parameters={"db": project.database}, + ) + assert _heads(project) == ["bbbb"] + + +def test_on_cluster_nonreplicated_mutations_wait_on_each_host(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + _baseline(project) + table = f"{project.database}.owned" + project.client.command( + f"CREATE TABLE {table} ON CLUSTER {cluster.name} (x UInt64) ENGINE=MergeTree ORDER BY tuple()" + ) + for client in cluster.clients.values(): + client.command(f"INSERT INTO {table} VALUES (0)") + _revision(project, [f"ALTER TABLE {{db}}.owned ON CLUSTER {cluster.name} UPDATE x=1 WHERE 1"]) + cluster.clients[2].command(f"SYSTEM STOP MERGES {table}") + try: + result = project.run("up", "it", "--timeout", "2") + assert result.exit_code == 1, result.output + assert cluster.hostnames[2] in result.output + assert _heads(project) == ["aaaa"] + assert cluster.clients[1].command(f"SELECT x FROM {table}") == 1 + assert cluster.clients[2].command(f"SELECT x FROM {table}") == 0 + finally: + cluster.clients[2].command(f"SYSTEM START MERGES {table}") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + for client in cluster.clients.values(): + assert client.command(f"SELECT x FROM {table}") == 1 + + +def test_on_cluster_cold_version_table_creation_obeys_timeout(cluster_project, clickhouse_cluster): + project, cluster = cluster_project, clickhouse_cluster + project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) + cluster.stop_node(2) + try: + result = project.run("up", "it", "--timeout", "1") + assert result.exit_code == 1 and NOT_FINISHED in result.output, result.output + assert _heads(project) == [] + finally: + cluster.start_node(2) + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert _heads(project) == ["aaaa"] + + +def _baseline(project): + project.write_revision("aaaa", {"upgrade": 'op.execute("SELECT 1")'}) + result = project.run("up", "it") + assert result.exit_code == 0, result.output + + +def _revision(project, statements): + path = project.sql_dir / "ddl.sql" + path.write_text( + "\n".join( + "-- ch-migrate: allow-non-idempotent Controlled distributed DDL scenario\n" + sql + ";" + for sql in statements + ) + ) + project.write_revision( + "bbbb", {"upgrade": "from ch_migrate import run_sql\nrun_sql('ddl.sql')"}, "aaaa" + ) + + +def _heads(project): + return [ + row[0] + for row in project.client.query( + f"SELECT version_num FROM {project.database}.alembic_version ORDER BY version_num" + ).result_rows + ] diff --git a/tests/integration/test_version_table.py b/tests/integration/test_version_table.py index 7a80492..dddb7da 100644 --- a/tests/integration/test_version_table.py +++ b/tests/integration/test_version_table.py @@ -24,20 +24,11 @@ def test_version_table_interrupted_delete_does_not_replay(project): process = _start_up(project) try: _wait_for(lambda: set(_heads(project)) == {"aaaa", "bbbb"}) - original_ids = _wait_for_version_delete(project, process) + _wait_for_version_delete(project, process) assert process.poll() is None, "up must still be waiting for the version DELETE" _stop_process(process) - refused = _completed_up(project, ("--timeout", "0.4")) - assert refused[0] != 0, refused - current_ids = { - row[0] - for row in project.client.query( - "SELECT mutation_id FROM system.mutations WHERE database = {db:String} " - "AND table = 'alembic_version'", - parameters={"db": project.database}, - ).result_rows - } - assert current_ids == original_ids + refused = _completed_up(project) + assert refused[0] != 0 and "overlap" in refused[1].lower(), refused assert _probe_counts(project) == [("aaaa", 1), ("bbbb", 1)] finally: _stop_process(process) @@ -47,6 +38,7 @@ def test_version_table_interrupted_delete_does_not_replay(project): assert completed[0] == 0, completed[1] assert _probe_counts(project) == [("aaaa", 1), ("bbbb", 1), ("cccc", 1)] assert _heads(project) == ["cccc"] + _assert_version_sql(project) def test_version_table_failed_mutation_refuses_without_killing(project): @@ -211,9 +203,9 @@ def _probe_revision(project, revision, parent): ) -def _start_up(project, extra=()): +def _start_up(project): return subprocess.Popen( - [sys.executable, "-m", "ch_migrate.cli", "up", "it", *extra], + [sys.executable, "-m", "ch_migrate.cli", "up", "it"], cwd=project.root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -232,8 +224,8 @@ def _stop_process(process): return process.communicate(timeout=5)[0] -def _completed_up(project, extra=()): - process = _start_up(project, extra) +def _completed_up(project): + process = _start_up(project) try: output = process.communicate(timeout=20)[0] return process.returncode, output @@ -276,7 +268,7 @@ def _wait_for_version_delete(project, process): parameters={"db": project.database}, ).result_rows if any(not done and "DELETE" in command.upper() for _, command, done, _ in rows): - return {mutation for mutation, _, _, _ in rows} + return if process.poll() is not None: break time.sleep(0.05) @@ -294,6 +286,25 @@ def _failed_mutations(project, client=None): ).result_rows +def _assert_version_sql(project): + project.client.command("SYSTEM FLUSH LOGS") + queries = [ + row[0] + for row in project.client.query( + "SELECT DISTINCT replaceAll(query, '`', '') FROM system.query_log WHERE type = 'QueryStart' " + "AND position(query, {db:String}) > 0 AND position(query, 'alembic_version') > 0", + parameters={"db": project.database}, + ).result_rows + ] + prefix = f"{project.database}.alembic_version" + assert any(query.startswith(f"INSERT INTO {prefix}") for query in queries) + deletes = [query for query in queries if query.startswith(f"ALTER TABLE {prefix} DELETE")] + assert deletes and all( + re.search(r"mutations_sync\s*=\s*2", query) for query in deletes + ), queries + assert not any(query.startswith(f"ALTER TABLE {prefix} UPDATE") for query in queries) + + def _bootstrap_cluster_user(project, cluster, request): monkeypatch = request.getfixturevalue("monkeypatch") user = project.database + "_migration" diff --git a/tests/integration/waiting_spike_probe.py b/tests/integration/waiting_spike_probe.py deleted file mode 100644 index 7865f00..0000000 --- a/tests/integration/waiting_spike_probe.py +++ /dev/null @@ -1,558 +0,0 @@ -"""Explicit local experiments for the waiting design; not auto-collected.""" - -import secrets -import subprocess -import sys -import time -from pathlib import Path - -import clickhouse_connect -import pytest - -pytestmark = pytest.mark.integration - - -def test_waiting_metadata_and_mutation_markers(project): - for table in ("mutations", "distributed_ddl_queue", "query_log", "part_log"): - columns = project.client.query( - "SELECT name, type FROM system.columns WHERE database = 'system' AND table = {table:String} " - "AND (position(name, 'query') > 0 OR position(name, 'mutation') > 0 OR name IN ('command', 'entry', 'settings')) " - "ORDER BY position", - parameters={"table": table}, - ).result_rows - print(table, columns) - target = f"{project.database}.ownership" - project.client.command( - f"CREATE TABLE {target} (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id" - ) - project.client.command(f"INSERT INTO {target} VALUES (1, 0), (2, 0)") - project.client.command(f"SYSTEM STOP MERGES {target}") - try: - statements = [ - f"ALTER TABLE {target} UPDATE x = x + 1 WHERE id = 1 /* chm_marker_comment */", - f"ALTER TABLE {target} UPDATE x = x + 1 WHERE id = 2 AND 'chm_marker_expression' = 'chm_marker_expression'", - ] - for sql in statements: - query_id = "chm_probe_" + secrets.token_hex(8) - project.client.command( - sql, settings={"mutations_sync": 0, "query_id": query_id, "log_comment": query_id} - ) - print("submitted", query_id) - rows = project.client.query( - "SELECT mutation_id, command, is_done FROM system.mutations " - "WHERE database = {db:String} AND table = 'ownership' ORDER BY mutation_id", - parameters={"db": project.database}, - ).result_rows - print("mutation commands", rows) - project.client.command("SYSTEM FLUSH LOGS") - events = project.client.query( - "SELECT query_id, type, log_comment, query FROM system.query_log " - "WHERE startsWith(query_id, 'chm_probe_') AND position(query, {db:String}) > 0 " - "ORDER BY event_time_microseconds", - parameters={"db": project.database}, - ).result_rows - profiles = project.client.query( - "SELECT query_id, mapFilter((k, v) -> positionCaseInsensitive(k, 'mutation') > 0, ProfileEvents) " - "FROM system.query_log WHERE type = 'QueryFinish' AND startsWith(query_id, 'chm_probe_') " - "AND position(query, {db:String}) > 0", - parameters={"db": project.database}, - ).result_rows - print("mutation profile events", profiles) - print("submission events", events) - finally: - project.client.command(f"SYSTEM START MERGES {target}") - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - values = project.client.query(f"SELECT id, x FROM {target} ORDER BY id").result_rows - if values == [(1, 1), (2, 1)]: - break - time.sleep(0.05) - assert values == [(1, 1), (2, 1)] - - -def test_waiting_schema_mutation_marker_options(project, clickhouse_server, request): - clauses = [ - "MODIFY COLUMN x UInt32, DELETE WHERE 0 AND 'chm_schema_marker' = 'chm_schema_marker'", - "MATERIALIZE COLUMN y, DELETE WHERE 0 AND 'chm_schema_marker' = 'chm_schema_marker'", - "CLEAR COLUMN y, DELETE WHERE 0 AND 'chm_schema_marker' = 'chm_schema_marker'", - "RENAME COLUMN y TO renamed, DELETE WHERE 0 AND 'chm_schema_marker' = 'chm_schema_marker'", - ] - for index, clause in enumerate(clauses): - name = f"schema_probe_{index}" - table = f"{project.database}.{name}" - project.client.command( - f"CREATE TABLE {table} (id UInt64, x UInt64, y UInt64 DEFAULT x + 1) " - "ENGINE = MergeTree ORDER BY id" - ) - project.client.command(f"INSERT INTO {table} (id, x) VALUES (1, 10), (2, 20)") - project.client.command(f"SYSTEM STOP MERGES {table}") - server = clickhouse_server - worker = clickhouse_connect.get_client( - host=server.host, - port=server.port, - username=server.user, - password=server.password, - secure=server.secure, - send_receive_timeout=2, - ) - request.addfinalizer(worker.close) - query_id = "chm_schema_" + secrets.token_hex(8) - try: - try: - worker.command( - f"ALTER TABLE {table} {clause}", - settings={"mutations_sync": 0, "alter_sync": 0, "query_id": query_id}, - ) - print("combined marker request", clause, "returned") - except Exception as error: - print( - "combined marker request", - clause, - type(error).__name__, - str(error).splitlines()[0], - ) - finally: - project.client.command(f"SYSTEM START MERGES {table}") - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - active = project.client.query( - "SELECT count() FROM system.processes WHERE query_id = {id:String}", - parameters={"id": query_id}, - ).result_rows[0][0] - if not active: - break - time.sleep(0.05) - result = project.client.query( - "SELECT mutation_id, command, is_done FROM system.mutations WHERE database = {db:String} AND table = {table:String}", - parameters={"db": project.database, "table": name}, - ).result_rows - print("combined marker durable commands", clause, result) - - -def test_waiting_ddl_settings_identity(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - marker = "chm_ddl_" + secrets.token_hex(8) - cluster.stop_node(2) - try: - cluster.clients[1].command( - f"CREATE TABLE {project.database}.ddl_owned ON CLUSTER {cluster.name} (id UInt64) ENGINE = Memory", - settings={ - "distributed_ddl_task_timeout": 0, - "distributed_ddl_output_mode": "none", - "log_comment": marker, - }, - ) - rows = ( - cluster.clients[1] - .query( - "SELECT entry, host, query, settings['log_comment'], status " - "FROM system.distributed_ddl_queue WHERE position(query, {db:String}) > 0 " - "AND position(query, 'ddl_owned') > 0 ORDER BY host", - parameters={"db": project.database}, - ) - .result_rows - ) - print("DDL durable marker", marker, rows) - assert len(rows) == 2 and all(row[3] == marker for row in rows) - finally: - cluster.start_node(2) - - -def test_waiting_actual_mutation_corpus(project): - cases = [ - ("update", "ALTER TABLE {table} UPDATE x = x + 1 WHERE id = 1"), - ("delete", "ALTER TABLE {table} DELETE WHERE id = 1"), - ("lightweight_delete", "DELETE FROM {table} WHERE id = 1"), - ("lightweight_update", "UPDATE {table} SET x = x + 1 WHERE id = 1"), - ("modify_type", "ALTER TABLE {table} MODIFY COLUMN x UInt32"), - ("drop_column", "ALTER TABLE {table} DROP COLUMN y"), - ("clear_column", "ALTER TABLE {table} CLEAR COLUMN y"), - ("materialize_column", "ALTER TABLE {table} MATERIALIZE COLUMN y"), - ("rename_column", "ALTER TABLE {table} RENAME COLUMN y TO z"), - ("modify_ttl", "ALTER TABLE {table} MODIFY TTL ts + INTERVAL 1 DAY"), - ("failure", "ALTER TABLE {table} UPDATE x = throwIf(x = 5) WHERE 1"), - ] - for label, template in cases: - name = "corpus_" + label - table = f"{project.database}.{name}" - project.client.command( - f"CREATE TABLE {table} (id UInt64, x UInt64, y UInt64 DEFAULT x + 1, ts DateTime DEFAULT now()) " - "ENGINE = MergeTree ORDER BY id" - ) - project.client.command(f"INSERT INTO {table} (id, x) VALUES (1, 1), (5, 5)") - settings = {"mutations_sync": 0, "alter_sync": 0, "lightweight_deletes_sync": 0} - if label == "lightweight_update": - settings["allow_experimental_lightweight_update"] = 1 - try: - project.client.command(template.format(table=table), settings=settings) - outcome = "accepted" - except Exception as error: - outcome = type(error).__name__ + ": " + str(error).splitlines()[0] - deadline = time.monotonic() + 5 - while True: - mutations = project.client.query( - "SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason " - "FROM system.mutations WHERE database = {db:String} AND table = {table:String}", - parameters={"db": project.database, "table": name}, - ).result_rows - if ( - not mutations - or all(row[2] or row[4] for row in mutations) - or time.monotonic() > deadline - ): - break - time.sleep(0.05) - print("mutation corpus", label, outcome, mutations) - - -def test_waiting_single_node_deduplicated_intent(project): - table = f"{project.database}.intents" - project.client.command( - f"CREATE TABLE {table} (step String, owner String) ENGINE = MergeTree ORDER BY step " - "SETTINGS non_replicated_deduplication_window = 10000" - ) - for owner in ("first", "second"): - project.client.insert( - table, - [["step1", owner]], - column_names=["step", "owner"], - settings={"insert_deduplication_token": "same-step", "insert_deduplicate": 1}, - ) - rows = project.client.query(f"SELECT step, owner FROM {table}").result_rows - print("single intent dedup", rows) - assert rows == [("step1", "first")] - - -def test_waiting_cluster_deduplicated_intent(cluster_project, clickhouse_cluster): - from concurrent.futures import ThreadPoolExecutor - from threading import Barrier - - project, cluster = cluster_project, clickhouse_cluster - table = f"{project.database}.intents" - cluster.clients[1].command( - f"CREATE TABLE {table} ON CLUSTER {cluster.name} (step String, owner String) " - f"ENGINE = ReplicatedMergeTree('/clickhouse/intents/{project.database}', '{{replica}}') ORDER BY step" - ) - barrier = Barrier(2) - - def insert_owner(node): - barrier.wait() - cluster.clients[node].insert( - table, - [["step1", f"node{node}"]], - column_names=["step", "owner"], - settings={ - "insert_deduplication_token": "same-step", - "insert_deduplicate": 1, - "insert_quorum": "auto", - }, - ) - - with ThreadPoolExecutor(max_workers=2) as pool: - list(pool.map(insert_owner, (1, 2))) - for node, client in cluster.clients.items(): - client.command(f"SYSTEM SYNC REPLICA {table} LIGHTWEIGHT") - rows = client.query(f"SELECT step, owner FROM {table}").result_rows - print("cluster intent dedup", node, rows) - assert len(rows) == 1 and rows[0][1] in ("node1", "node2") - - -def test_waiting_lightweight_update_with_required_storage(project): - table = f"{project.database}.patch_probe" - project.client.command( - f"CREATE TABLE {table} (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id " - "SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1" - ) - project.client.command(f"INSERT INTO {table} VALUES (1, 10), (2, 20)") - project.client.command( - f"UPDATE {table} SET x = x + 1 WHERE id = 1", - settings={"allow_experimental_lightweight_update": 1, "mutations_sync": 0}, - ) - mutations = project.client.query( - "SELECT mutation_id, command, is_done FROM system.mutations WHERE database = {db:String} AND table = 'patch_probe'", - parameters={"db": project.database}, - ).result_rows - values = project.client.query(f"SELECT id, x FROM {table} ORDER BY id").result_rows - print("lightweight update enabled", mutations, values) - assert values == [(1, 11), (2, 20)] - - -def test_waiting_empty_fence_cost_and_retention(project): - table = f"{project.database}.fence_cost" - project.client.command( - f"CREATE TABLE {table} (id UInt64, x UInt64, payload String) ENGINE = MergeTree ORDER BY id" - ) - project.client.command( - f"INSERT INTO {table} SELECT number, number, repeat('payload', 100) FROM numbers(10000)" - ) - project.client.command( - f"ALTER TABLE {table} DELETE WHERE 0 AND 'chm_cost_marker' = 'chm_cost_marker'", - settings={"mutations_sync": 2}, - ) - project.client.command("SYSTEM FLUSH LOGS") - names = { - row[0] - for row in project.client.query( - "SELECT name FROM system.columns WHERE database = 'system' AND table = 'part_log'" - ).result_rows - } - fields = [ - name - for name in ("event_type", "query_id", "rows", "read_rows", "read_bytes", "ProfileEvents") - if name in names - ] - if fields: - events = project.client.query( - "SELECT " + ", ".join(fields) + " FROM system.part_log " - "WHERE database = {db:String} AND table = 'fence_cost' AND toString(event_type) LIKE '%Mutat%'", - parameters={"db": project.database}, - ).result_rows - print("empty fence physical cost", fields, events) - mutations = project.client.query( - "SELECT command, is_done, parts_to_do FROM system.mutations WHERE database = {db:String} AND table = 'fence_cost'", - parameters={"db": project.database}, - ).result_rows - print("empty fence retained", mutations) - assert project.client.command(f"SELECT count() FROM {table}") == 10000 - - -@pytest.mark.parametrize("trial", range(3)) -def test_waiting_kill_before_receipt_and_resume_from_intent(project, trial): - table = f"{project.database}.counter" - project.client.command(f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY tuple()") - project.client.command(f"INSERT INTO {table} VALUES (0)") - table_uuid = str( - project.client.query( - "SELECT uuid FROM system.tables WHERE database = {db:String} AND name = 'counter'", - parameters={"db": project.database}, - ).result_rows[0][0] - ) - project.client.command( - f"CREATE TABLE {project.database}.waiting_intent (token String, table_uuid String) " - "ENGINE = MergeTree ORDER BY token SETTINGS non_replicated_deduplication_window = 10000" - ) - token = "chm_recovery_" + secrets.token_hex(8) - project.client.insert( - f"{project.database}.waiting_intent", - [[token, table_uuid]], - column_names=["token", "table_uuid"], - settings={"insert_deduplication_token": "revision-step", "insert_deduplicate": 1}, - ) - project.client.command(f"SYSTEM STOP MERGES {table}") - scripts = Path(__file__).parents[2] / "docs/design/spikes/2026-10-02-waiting" - submitted = subprocess.Popen( - [sys.executable, str(scripts / "submit_without_ack.py"), token], - cwd=project.root, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - resumed = None - try: - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - rows = project.client.query( - "SELECT mutation_id, command, is_done FROM system.mutations " - "WHERE database = {db:String} AND table = 'counter' ORDER BY mutation_id", - parameters={"db": project.database}, - ).result_rows - if any(token in row[1] for row in rows): - break - time.sleep(0.05) - assert len(rows) == 2 and any(token in row[1] for row in rows), rows - original_ids = [row[0] for row in rows] - print("pre-receipt crash", trial, rows) - submitted.kill() - submitted.communicate(timeout=5) - resumed = subprocess.Popen( - [sys.executable, str(scripts / "resume_fence.py")], - cwd=project.root, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - deadline = time.monotonic() + 10 - while not (project.root / "resumed_marker").exists() and time.monotonic() < deadline: - time.sleep(0.05) - assert (project.root / "resumed_marker").exists() - assert resumed.poll() is None - project.client.command(f"SYSTEM START MERGES {table}") - output = resumed.communicate(timeout=20)[0] - assert resumed.returncode == 0, output - print("recovered", trial, output.strip()) - ids = [ - row[0] - for row in project.client.query( - "SELECT mutation_id FROM system.mutations WHERE database = {db:String} " - "AND table = 'counter' ORDER BY mutation_id", - parameters={"db": project.database}, - ).result_rows - ] - assert ids == original_ids - assert project.client.command(f"SELECT x FROM {table}") == 1 - finally: - for process in (submitted, resumed): - if process is not None and process.poll() is None: - process.kill() - process.communicate(timeout=5) - project.client.command(f"SYSTEM START MERGES {table}") - - -def test_waiting_foreign_failure_blocks_later_mutation(project): - table = f"{project.database}.foreign_probe" - project.client.command( - f"CREATE TABLE {table} (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id" - ) - project.client.command(f"INSERT INTO {table} VALUES (1, 1), (5, 5)") - project.client.command( - f"ALTER TABLE {table} UPDATE x = throwIf(x = 5) WHERE 1", settings={"mutations_sync": 0} - ) - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - rows = project.client.query( - "SELECT mutation_id, command, is_done, latest_fail_reason FROM system.mutations " - "WHERE database = {db:String} AND table = 'foreign_probe' ORDER BY mutation_id", - parameters={"db": project.database}, - ).result_rows - if rows and rows[0][3]: - break - time.sleep(0.05) - assert rows[0][3] - project.client.command( - f"ALTER TABLE {table} UPDATE x = x + 1 WHERE id = 1", settings={"mutations_sync": 0} - ) - rows = project.client.query( - "SELECT mutation_id, command, is_done, parts_to_do, latest_fail_reason FROM system.mutations " - "WHERE database = {db:String} AND table = 'foreign_probe' ORDER BY mutation_id", - parameters={"db": project.database}, - ).result_rows - print("foreign failure and queued follower", rows) - assert len(rows) == 2 and all(not row[2] for row in rows) - assert project.client.query(f"SELECT id, x FROM {table} ORDER BY id").result_rows == [ - (1, 1), - (5, 5), - ] - - -def test_waiting_requires_every_replica(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - table = f"{project.database}.replica_wait" - cluster.clients[1].command( - f"CREATE TABLE {table} ON CLUSTER {cluster.name} (id UInt64, x UInt64) " - f"ENGINE = ReplicatedMergeTree('/clickhouse/wait/{project.database}', '{{replica}}') ORDER BY id" - ) - cluster.clients[1].command(f"INSERT INTO {table} VALUES (1, 0)") - cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") - for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): - cluster.clients[2].command(f"SYSTEM STOP {action} {table}") - try: - cluster.clients[1].command( - f"ALTER TABLE {table} UPDATE x = x + 1 WHERE 1", settings={"mutations_sync": 0} - ) - cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table} PULL") - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - states = {} - for node, client in cluster.clients.items(): - states[node] = client.query( - "SELECT mutation_id, is_done, parts_to_do FROM system.mutations " - "WHERE database = {db:String} AND table = 'replica_wait'", - parameters={"db": project.database}, - ).result_rows - if states[1] and states[2] and states[1][0][1] and not states[2][0][1]: - break - time.sleep(0.05) - print("replica completion divergence", states) - assert states[1][0][1] == 1 and states[2][0][1] == 0 - assert states[1][0][0] == states[2][0][0] - finally: - for action in ("MERGES", "FETCHES", "REPLICATION QUEUES"): - cluster.clients[2].command(f"SYSTEM START {action} {table}") - cluster.clients[2].command(f"SYSTEM SYNC REPLICA {table}") - assert cluster.clients[2].query(f"SELECT x FROM {table}").result_rows == [(1,)] - - -def test_waiting_managed_sync_settings_override(project): - table = f"{project.database}.sync_settings" - project.client.command(f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY tuple()") - project.client.command(f"INSERT INTO {table} VALUES (0)") - project.client.command(f"SYSTEM STOP MERGES {table}") - try: - project.client.command( - f"ALTER TABLE {table} UPDATE x = x + 1 WHERE 1, " - "DELETE WHERE 0 AND 'chm_settings_marker' = 'chm_settings_marker' " - "SETTINGS mutations_sync = 2, mutations_sync = 0, alter_sync = 0" - ) - rows = project.client.query( - "SELECT mutation_id, command, is_done FROM system.mutations " - "WHERE database = {db:String} AND table = 'sync_settings'", - parameters={"db": project.database}, - ).result_rows - print("managed sync override", rows) - assert rows and all(row[2] == 0 for row in rows) - finally: - project.client.command(f"SYSTEM START MERGES {table}") - - -def test_waiting_completed_ownership_marker_can_expire(project): - table = f"{project.database}.retention_probe" - project.client.command( - f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY tuple() " - "SETTINGS finished_mutations_to_keep = 1, old_parts_lifetime = 0, " - "cleanup_delay_period = 1, cleanup_delay_period_random_add = 0" - ) - project.client.command(f"INSERT INTO {table} VALUES (0)") - project.client.command( - f"ALTER TABLE {table} UPDATE x = x + 1 WHERE " "'chm_expired_owner' = 'chm_expired_owner'", - settings={"mutations_sync": 2}, - ) - project.client.command( - f"ALTER TABLE {table} DELETE WHERE 0", - settings={"mutations_sync": 2}, - ) - deadline = time.monotonic() + 30 - while time.monotonic() < deadline: - rows = project.client.query( - "SELECT mutation_id, command FROM system.mutations " - "WHERE database = {db:String} AND table = 'retention_probe'", - parameters={"db": project.database}, - ).result_rows - if not any("chm_expired_owner" in row[1] for row in rows): - break - time.sleep(0.1) - print("expired ownership marker", rows, "x =", project.client.command(f"SELECT x FROM {table}")) - assert rows and not any("chm_expired_owner" in row[1] for row in rows) - assert project.client.command(f"SELECT x FROM {table}") == 1 - - -def test_waiting_ddl_failure_names_host(cluster_project, clickhouse_cluster): - project, cluster = cluster_project, clickhouse_cluster - table = f"{project.database}.ddl_failure" - for client in cluster.clients.values(): - client.command(f"CREATE TABLE {table} (id UInt64, x String) ENGINE = MergeTree ORDER BY id") - cluster.clients[2].command(f"ALTER TABLE {table} ADD COLUMN conflict String") - token = "chm_ddl_failure_" + secrets.token_hex(8) - cluster.clients[1].command( - f"ALTER TABLE {table} ON CLUSTER {cluster.name} ADD COLUMN conflict UInt64", - settings={ - "distributed_ddl_task_timeout": 0, - "distributed_ddl_output_mode": "none", - "log_comment": token, - }, - ) - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - rows = ( - cluster.clients[1] - .query( - "SELECT host, status, exception_code, exception_text FROM system.distributed_ddl_queue " - "WHERE settings['log_comment'] = {token:String} ORDER BY host", - parameters={"token": token}, - ) - .result_rows - ) - if len(rows) == 2 and any(row[2] for row in rows): - break - time.sleep(0.05) - print("DDL host exception", rows) - assert len(rows) == 2 and any(row[2] and row[3] for row in rows) diff --git a/tests/test_classify.py b/tests/test_classify.py deleted file mode 100644 index 8fcaf6b..0000000 --- a/tests/test_classify.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Corpus classifications and uncertain or quoted SQL boundaries.""" - -from pathlib import Path - -import pytest -import yaml - -from ch_migrate.classify import classify -from ch_migrate.introspect import Schema, parse_create_table -from ch_migrate.statements import MigrationStatement - -CORPUS = yaml.safe_load((Path(__file__).parent / "corpus/classification.yaml").read_text()) -VALUES = {"db": "example", "table": "example.sample"} - - -@pytest.fixture -def schema(): - table = parse_create_table(CORPUS["table_ddl"].format(**VALUES)) - return Schema(database="example", tables={"sample": table}) - - -@pytest.mark.parametrize("case", CORPUS["statements"], ids=lambda case: case["id"]) -def test_classification_corpus(case, schema): - sql = case["sql"].format(**VALUES) - result = classify(sql, schema) - assert result.kind == case["expected"], (sql, result) - if "detail" in case: - assert case["detail"] in result.detail - if sql.startswith("ALTER"): - assert result.table == "example.sample" - - -@pytest.mark.parametrize( - "case", - [case for case in CORPUS["statements"] if "without_schema" in case], - ids=lambda case: case["id"], -) -def test_classification_without_live_schema_is_conservative(case): - result = classify(case["sql"].format(**VALUES)) - assert result.kind == case["without_schema"] - if "MODIFY COLUMN" in case["sql"]: - assert "if the type changes" in result.detail - - -def test_classification_ignores_comments_and_literal_keywords(schema): - statement = MigrationStatement( - "/* UPDATE x */ ALTER TABLE \"example\".`sample` ON CLUSTER 'local' " - "MODIFY COLUMN x COMMENT 'DROP COLUMN y, MODIFY ENGINE = Memory'", - "migration.sql", - 9, - (), - "upgrade", - ) - result = classify(statement, schema) - assert result.kind == "metadata" - assert result.table == "example.sample" - - -def test_classification_different_database_cannot_supply_a_type(schema): - result = classify("ALTER TABLE elsewhere.sample MODIFY COLUMN x UInt64", schema) - assert result.kind == "mutation" - assert "if the type changes" in result.detail - - -def test_classification_unknown_action_is_not_metadata(schema): - result = classify("ALTER TABLE example.sample SOMETHING UNRECOGNIZED", schema) - assert result.kind == "other" - - -def test_classification_preserves_identifier_case(schema): - result = classify("ALTER TABLE example.sample MODIFY ORDER BY ID", schema) - assert result.kind == "rebuild" - - -def test_classification_empty_and_read_only_have_no_table(): - assert classify("/* nothing */").kind == "other" - assert classify("SELECT 'ALTER TABLE x DELETE WHERE 1'").table is None diff --git a/tests/test_waiting.py b/tests/test_waiting.py index a4c25e6..6cab874 100644 --- a/tests/test_waiting.py +++ b/tests/test_waiting.py @@ -1,96 +1,77 @@ -"""Ownership SQL and deadline/completion boundaries that could silently replay work.""" +"""Waiting decisions without a server: the deadline, what counts as a write, and completion.""" + +from types import SimpleNamespace import pytest -from click.testing import CliRunner -from ch_migrate.cli import main from ch_migrate.waiting.budget import WaitBudget -from ch_migrate.waiting.errors import WaitTimeout +from ch_migrate.waiting.display import WaitDisplay +from ch_migrate.waiting.errors import MutationFailed, WaitTimeout from ch_migrate.waiting.mutations import MutationWaiter -from ch_migrate.waiting.sql import ( - is_session_or_read, - mutation_sql, - query_setting, - statement_table, -) +from ch_migrate.waiting.sql import is_write -@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf", "-inf"]) -def test_wait_timeout_rejects_invalid_values_before_project_access(tmp_path, monkeypatch, timeout): - # Click's public command signature, not a database mock, decides usage errors. - monkeypatch.chdir(tmp_path) - result = CliRunner().invoke(main, ["up", "not-configured", "--timeout", timeout]) - assert result.exit_code == 2 - assert "--timeout" in result.output +def test_deadline_is_shared_across_waits(monkeypatch): + now = [10.0] + monkeypatch.setattr("ch_migrate.waiting.budget.time.monotonic", lambda: now[0]) + budget = WaitBudget(timeout=2, started=now[0]) + now[0] = 11.5 + assert budget.elapsed == 1.5 + now[0] = 12.0 + with pytest.raises(WaitTimeout, match="second mutation"): + budget.check("second mutation") -def test_mutation_settings_override_ignores_comment_and_literal_keywords(): - sql = ( - "ALTER TABLE db.t UPDATE value = 'SETTINGS -- not a comment', settings=settings+1 " - "WHERE id=1 -- trailing predicate comment\n" - "SETTINGS /* policy */ mutations_sync=2, max_threads=3 -- trailing setting comment\n;" - ) - rewritten = mutation_sql(sql, "owned-token") - assert query_setting(rewritten, "mutations_sync") == "0" - assert query_setting(rewritten, "max_threads") == "3" - assert "'SETTINGS -- not a comment'" in rewritten - assert "settings=settings+1 WHERE id=1" in rewritten - assert "DELETE WHERE 0" in rewritten +@pytest.mark.parametrize( + ("sql", "write"), + [ + ("SELECT 1", False), + (" -- note\n select count() FROM t", False), + ("/* why */ SET max_threads = 1", False), + ("SHOW TABLES", False), + ("EXISTS TABLE t", False), + ("ALTER TABLE t UPDATE x = 1 WHERE 1", True), + ("INSERT INTO t SELECT 1", True), + ("WITH source AS (SELECT 1) INSERT INTO t SELECT * FROM source", True), + ("CREATE TABLE t (x UInt8) ENGINE = Memory", True), + ("", True), + ], +) +def test_only_reads_and_session_statements_skip_the_wait(sql, write): + assert is_write(sql) is write -def test_settings_column_is_not_a_settings_clause(): - rewritten = mutation_sql("ALTER TABLE db.t UPDATE settings=2 WHERE settings=1", "owned-token") - assert "UPDATE settings=2 WHERE settings=1" in rewritten - assert query_setting(rewritten, "mutations_sync") == "0" +def test_zero_parts_left_is_not_completion(): + waiter = _waiter([_row("h1", "t", "mutation_1.txt", done=0, parts=0)]) + with pytest.raises(WaitTimeout, match="mutation_1.txt host h1; parts_to_do=0"): + waiter.wait() -def test_qualified_identifiers_preserve_dots_inside_quoted_names(): - assert statement_table( - 'ALTER TABLE "data.base".`some.table` UPDATE x=1 WHERE 1', "default" - ) == ("data.base", "some.table") +def test_a_replica_without_the_mutation_is_still_pending(): + waiter = _waiter([_row("r1", "t", "0000000001", done=1)], hosts={"t": ("r1", "r2")}) + with pytest.raises(WaitTimeout, match="0000000001 host r2; not received yet"): + waiter.wait() -@pytest.mark.parametrize( - "sql,readonly", - [ - ("WITH 1 AS x SELECT x", True), - ("WITH source AS (SELECT 1) INSERT INTO t SELECT * FROM source", False), - ("INSERT INTO t SELECT 'SELECT'", False), - ("SET max_threads=3", True), - ], -) -def test_statement_tracking_distinguishes_reads_from_writes(sql, readonly): - assert is_session_or_read(sql) is readonly +def test_a_failed_mutation_stops_with_the_scoped_kill(): + waiter = _waiter([_row("h1", "t", "mutation_2.txt", reason="Cannot parse 'n/a'")]) + with pytest.raises(MutationFailed) as raised: + waiter.wait(before_run=True) + failure = raised.value.failure + assert failure.foreign and failure.reason == "Cannot parse 'n/a'" + assert failure.kill.endswith("AND table = 't' AND mutation_id = 'mutation_2.txt'") -def test_deadline_is_shared_across_waits(monkeypatch): - now = [10.0] - monkeypatch.setattr("ch_migrate.waiting.budget.time.monotonic", lambda: now[0]) - budget = WaitBudget(timeout=2, started=now[0]) - now[0] = 11.5 - assert budget.elapsed == 1.5 - now[0] = 12.0 - with pytest.raises(WaitTimeout, match="second mutation"): - budget.check("second mutation") +def _waiter(rows, hosts=None): + state = SimpleNamespace(database="db", health_cluster=None) + budget = WaitBudget(timeout=0, display=WaitDisplay()) + waiter = MutationWaiter(None, state, budget) + # The real wait loop runs against these stubs: the rows the server returns, and which + # hosts each table's mutations must finish on. + waiter._rows = lambda expected: [waiter._row(row) for row in rows] + waiter._resolve_hosts = lambda table: (hosts or {}).get(table) + return waiter -def test_zero_parts_is_not_mutation_completion(monkeypatch): - waiter = MutationWaiter(None, None, WaitBudget(timeout=0)) - receipt = { - "database": "db", - "table": "t", - "engine": "MergeTree", - "initiator": "host", - "token": "owned", - "uuids": {"host": "uuid"}, - "ids": {"host": ["mutation_1.txt"]}, - "done_hosts": [], - "baseline": {"host": []}, - } - monkeypatch.setattr(waiter, "_identities", lambda _: {"host": "uuid"}) - monkeypatch.setattr( - waiter, "_rows", lambda _: [("host", "mutation_1.txt", "owned", 0, 0, "", [])] - ) - with pytest.raises(WaitTimeout, match="mutation_1.txt"): - waiter.wait(receipt, lambda: None) - assert receipt["done_hosts"] == [] +def _row(host, table, mutation, done=0, parts=1, reason=""): + return (host, table, mutation, done, parts, reason, []) diff --git a/tests/test_waiting_display.py b/tests/test_waiting_display.py index e438457..b8317d8 100644 --- a/tests/test_waiting_display.py +++ b/tests/test_waiting_display.py @@ -8,8 +8,8 @@ from ch_migrate.migration_runner import Upgrade, _report_failure, _report_interrupt from ch_migrate.runner import _Progress from ch_migrate.sql import SqlStatementError -from ch_migrate.waiting.display import DDLProgress, HostParts, MutationProgress, WaitDisplay -from ch_migrate.waiting.errors import FailedMutation, MutationFailed, UnknownOutcome, WaitTimeout +from ch_migrate.waiting.display import HostParts, MutationProgress, WaitDisplay +from ch_migrate.waiting.errors import FailedMutation, MutationFailed, WaitTimeout GIB = 1 << 30 @@ -83,15 +83,6 @@ def test_each_replica_gets_its_own_row(clock): assert "· 2 replicas" in buffer.getvalue().splitlines()[-1] -def test_on_cluster_ddl_counts_hosts(clock): - display, buffer = _display(terminal=True) - display.update(DDLProgress("query-0000000012", 2, 3)) - assert "2/3 hosts" in buffer.getvalue() - piped, lines = _display(terminal=False) - piped.update(DDLProgress("query-0000000012", 2, 3)) - assert lines.getvalue() == " waiting on ON CLUSTER DDL query-0000000012: 2/3 hosts\n" - - def test_failed_mutation_names_reason_retry_and_kill(capsys): failure = FailedMutation( "demo_dev.events", @@ -109,7 +100,10 @@ def test_failed_mutation_names_reason_retry_and_kill(capsys): ) assert "Cannot parse string 'n/a' as UInt32" in err assert "keeps retrying a failed mutation" in err and failure.kill in err - assert "If the cause is temporary" in err and "reconciles the journal" in err + assert ( + "If the cause is temporary" in err + and "then fixes the SQL and runs ch-migrate up dev" in err + ) def test_timeout_says_the_work_continues_and_how_to_resume(capsys): @@ -120,22 +114,14 @@ def test_timeout_says_the_work_continues_and_how_to_resume(capsys): err = capsys.readouterr().err assert err.startswith("✗ Timed out after 3.0s waiting on 3f9c2a1b backfill") assert "Pending: demo_dev.events mutation mutation_7.txt; parts_to_do=3" in err - assert "ch-migrate up dev again to keep waiting; it won't reissue the statement" in err - - -def test_unknown_outcome_shows_the_operator_guidance(capsys): - unknown = UnknownOutcome("Outcome unknown for demo_dev.events\nRead-only inspection: SELECT 1") - _report_failure(Upgrade("dev"), _current("3f9c2a1b backfill"), _statement_error(unknown)) - err = capsys.readouterr().err - assert "outcome unknown, so the statement was not reissued" in err - assert " Read-only inspection: SELECT 1" in err + assert "ch-migrate up dev again: it waits for that work to finish" in err def test_interrupt_says_the_work_continues(capsys): _report_interrupt("dev", _current("3f9c2a1b backfill")) err = capsys.readouterr().err assert err.startswith("✗ Interrupted during 3f9c2a1b backfill") - assert "Run ch-migrate up dev to pick up where it stopped" in err + assert "Run ch-migrate up dev again: it waits for that work to finish" in err def _display(terminal): From 41cb3a5df0800c915831244a712f4a69fb4973ee Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 13:59:52 -0400 Subject: [PATCH 25/33] [test] Describe the status contract without a client The status test's docstring named the downstream project whose CI relies on status exiting 0. The contract stands on its own, so it now says what any required CI status job can rely on. Docstring only. Solves: client references in the public repo Tests: unit suite on 3.10-3.14 --- tests/test_status_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_status_command.py b/tests/test_status_command.py index c53e80e..afe3bda 100644 --- a/tests/test_status_command.py +++ b/tests/test_status_command.py @@ -1,7 +1,7 @@ """`ch-migrate status` is a report: CI runs it as a non-blocking check. -Sazabi's required "ClickHouse Migration Status" job relies on status exiting 0 when -the database is unreachable or migrations are pending; only a broken project fails. +A required CI status job can rely on status exiting 0 when the database is +unreachable or migrations are pending; only a broken project fails. """ import pytest From c481468c7320cbc6f2bab56e0c870bc4f9178a1f Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 14:23:12 -0400 Subject: [PATCH 26/33] [gate] Move up's gate enforcement into gate.py env.py is about to enforce the same gate for a direct `alembic upgrade`, so the function that prints the pending upgrades' findings and refuses the run moves from the CLI into gate.py, beside lint_pending_up. Moved as is; `ch-migrate up` calls it from there. Solves: DRY-1609 Tests: unit suite in a clean checkout of this commit --- ch_migrate/cli.py | 39 +++------------------------------------ ch_migrate/gate.py | 35 ++++++++++++++++++++++++++++++++++- tests/test_cli_output.py | 10 ++++------ 3 files changed, 41 insertions(+), 43 deletions(-) diff --git a/ch_migrate/cli.py b/ch_migrate/cli.py index 88b7e87..041019c 100644 --- a/ch_migrate/cli.py +++ b/ch_migrate/cli.py @@ -43,41 +43,6 @@ def render_template(template_path: Path, **kwargs: str) -> str: return content -def _enforce_up_gate(environment: str, skip_mv_check: bool) -> None: - from ch_migrate.gate import lint_pending_up - - try: - report = lint_pending_up(Path.cwd(), environment, skip_mv_check) - except Exception as error: - raise ui.Failure(f"Migration preflight failed: {error}") from error - if _print_gate_findings(report): - raise ui.Failure( - "Migration gate failed; nothing was applied.", - "Fix SQL or add a reasoned in-file waiver.", - ) - - -def _print_gate_findings(report) -> bool: - from ch_migrate.display import lint_finding_line - from ch_migrate.lint import GATE_RULES, Severity - - blocked = False - for finding in report.results: - gate_error = finding.rule in GATE_RULES and finding.severity == Severity.ERROR - blocked |= gate_error - severity = finding.severity.value - if severity == "error" and not gate_error: - severity = "warn" - position = finding.file or "config.yaml" - if finding.line is not None: - position += f":{finding.line}" - console = ui.err if gate_error else ui.out - console.print(lint_finding_line(finding, location=position, severity=severity)) - if finding.statement: - ui.detail(finding.statement, stderr=gate_error) - return blocked - - def _require_current_env() -> None: from ch_migrate.alembic_env import has_current_env @@ -289,7 +254,9 @@ def up( if timeout is not None and not math.isfinite(timeout): raise click.BadParameter("must be finite", param_hint="--timeout") _require_current_env() - _enforce_up_gate(environment, skip_mv_check) + from ch_migrate.gate import enforce_up_gate + + enforce_up_gate(Path.cwd(), environment, skip_mv_check) from ch_migrate.migration_runner import Upgrade, run_upgrade sys.exit(run_upgrade(Upgrade(environment, revision, timeout, verbose))) diff --git a/ch_migrate/gate.py b/ch_migrate/gate.py index 1492140..7f7cede 100644 --- a/ch_migrate/gate.py +++ b/ch_migrate/gate.py @@ -4,13 +4,46 @@ from dotenv import load_dotenv +from ch_migrate import ui from ch_migrate.config import get_env_config, load_config from ch_migrate.connection import get_migration_state -from ch_migrate.lint import LintConfig, LintReport, Severity, lint_migrations +from ch_migrate.display import lint_finding_line +from ch_migrate.lint import GATE_RULES, LintConfig, LintReport, Severity, lint_migrations from ch_migrate.rebase import build_revision_graph from ch_migrate.statements import pending_revisions +def enforce_up_gate(project_root: Path, environment: str, skip_mv_check: bool = False) -> None: + """Print the pending upgrades' findings, and refuse the run if a gate rule failed.""" + try: + report = lint_pending_up(project_root, environment, skip_mv_check) + except Exception as error: + raise ui.Failure(f"Migration preflight failed: {error}") from error + if print_gate_findings(report): + raise ui.Failure( + "Migration gate failed; nothing was applied.", + "Fix SQL or add a reasoned in-file waiver.", + ) + + +def print_gate_findings(report: LintReport) -> bool: + blocked = False + for finding in report.results: + gate_error = finding.rule in GATE_RULES and finding.severity == Severity.ERROR + blocked |= gate_error + severity = finding.severity.value + if severity == "error" and not gate_error: + severity = "warn" + position = finding.file or "config.yaml" + if finding.line is not None: + position += f":{finding.line}" + console = ui.err if gate_error else ui.out + console.print(lint_finding_line(finding, location=position, severity=severity)) + if finding.statement: + ui.detail(finding.statement, stderr=gate_error) + return blocked + + def lint_pending_up( project_root: Path, environment: str, skip_mv_check: bool = False ) -> LintReport: diff --git a/tests/test_cli_output.py b/tests/test_cli_output.py index fb6bd5e..e6e66b9 100644 --- a/tests/test_cli_output.py +++ b/tests/test_cli_output.py @@ -7,7 +7,8 @@ from click.testing import CliRunner from ch_migrate import ui -from ch_migrate.cli import _print_gate_findings, main +from ch_migrate.cli import main +from ch_migrate.gate import print_gate_findings from ch_migrate.lint import LintResult, Severity @@ -70,12 +71,9 @@ def test_gate_findings_use_lint_markers(capsys): ), ] ) - assert _print_gate_findings(report) is True + assert print_gate_findings(report) is True captured = capsys.readouterr() - assert ( - captured.err - == "✗ a.up.sql:2 Use IF NOT EXISTS [idempotency]\n CREATE TABLE t\n" - ) + assert captured.err == "✗ a.up.sql:2 Use IF NOT EXISTS [idempotency]\n CREATE TABLE t\n" assert captured.out == ( "! config.yaml Drops a column [destructive_changes]\n" "✓ b.up.sql:5 Idempotency waiver: copies once [idempotency]\n" From c07fcbc18b90d266a505843c546bca984c9d72d0 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 14:25:37 -0400 Subject: [PATCH 27/33] [gate] Check every way a migration sends SQL The gate read SQL only from string literals passed to op.execute and from run_sql files. op.execute(text(...)), op.get_bind().execute(...) and exec_driver_sql(...) went unchecked, as did any SQL built at run time, so a non-idempotent UPDATE written that way passed with no finding (2 of 5 forms in a probe). Now text() is unwrapped, literal SQL sent through execute or exec_driver_sql on any connection is checked like op.execute, and SQL that only exists at run time becomes an unreadable statement that needs a reasoned waiver. Only the idempotency rule sees unreadable statements; the SQL-parsing rules skip them. Solves: DRY-1609 Tests: unit 675; Docker single node 53 passed, 4 skipped --- ch_migrate/lint.py | 342 ++++++++++++++++++++++++++++----------- ch_migrate/statements.py | 53 +++++- tests/test_statements.py | 53 ++++++ 3 files changed, 348 insertions(+), 100 deletions(-) diff --git a/ch_migrate/lint.py b/ch_migrate/lint.py index 0769ec5..a0efddb 100644 --- a/ch_migrate/lint.py +++ b/ch_migrate/lint.py @@ -132,8 +132,7 @@ def check( client: Any | None = None, database: str | None = None, graph: RevisionGraph | None = None, - ) -> list[LintResult]: - ... + ) -> list[LintResult]: ... # --------------------------------------------------------------------------- @@ -142,31 +141,176 @@ def check( # Subset of CH reserved words that commonly collide with column names. # Full list is version-dependent; these are the most common traps. -_CH_RESERVED_WORDS = frozenset({ - "add", "after", "alias", "all", "alter", "and", "anti", "any", "array", - "as", "asc", "attach", "between", "both", "by", "case", "cast", "check", - "cluster", "collate", "column", "comment", "constraint", "create", - "cross", "cube", "current", "database", "databases", "date", "day", - "default", "delete", "desc", "describe", "detach", "dictionaries", - "dictionary", "distinct", "distributed", "drop", "else", "end", "engine", - "events", "except", "exists", "explain", "expression", "extract", "fetch", - "final", "first", "flush", "following", "for", "format", "from", "full", - "function", "global", "granularity", "group", "having", "hour", "if", - "ilike", "in", "index", "inject", "inner", "insert", "interval", "into", - "is", "join", "key", "kill", "last", "layout", "leading", "left", "like", - "limit", "live", "local", "logs", "materialize", "materialized", "max", - "merges", "min", "minute", "modify", "month", "move", "mutation", "no", - "not", "null", "nulls", "offset", "on", "optimize", "or", "order", - "outer", "outfile", "over", "partition", "populate", "preceding", - "primary", "prewhere", "projection", "quarter", "range", "reload", - "remove", "rename", "replace", "right", "rollup", "row", "rows", - "sample", "second", "select", "semi", "set", "settings", "show", - "source", "start", "stop", "system", "table", "tables", "temporary", - "test", "then", "ties", "timestamp", "to", "top", "totals", "trailing", - "trim", "truncate", "type", "unbounded", "union", "update", "use", - "using", "uuid", "values", "view", "volume", "watch", "week", "when", - "where", "window", "with", "year", -}) +_CH_RESERVED_WORDS = frozenset( + { + "add", + "after", + "alias", + "all", + "alter", + "and", + "anti", + "any", + "array", + "as", + "asc", + "attach", + "between", + "both", + "by", + "case", + "cast", + "check", + "cluster", + "collate", + "column", + "comment", + "constraint", + "create", + "cross", + "cube", + "current", + "database", + "databases", + "date", + "day", + "default", + "delete", + "desc", + "describe", + "detach", + "dictionaries", + "dictionary", + "distinct", + "distributed", + "drop", + "else", + "end", + "engine", + "events", + "except", + "exists", + "explain", + "expression", + "extract", + "fetch", + "final", + "first", + "flush", + "following", + "for", + "format", + "from", + "full", + "function", + "global", + "granularity", + "group", + "having", + "hour", + "if", + "ilike", + "in", + "index", + "inject", + "inner", + "insert", + "interval", + "into", + "is", + "join", + "key", + "kill", + "last", + "layout", + "leading", + "left", + "like", + "limit", + "live", + "local", + "logs", + "materialize", + "materialized", + "max", + "merges", + "min", + "minute", + "modify", + "month", + "move", + "mutation", + "no", + "not", + "null", + "nulls", + "offset", + "on", + "optimize", + "or", + "order", + "outer", + "outfile", + "over", + "partition", + "populate", + "preceding", + "primary", + "prewhere", + "projection", + "quarter", + "range", + "reload", + "remove", + "rename", + "replace", + "right", + "rollup", + "row", + "rows", + "sample", + "second", + "select", + "semi", + "set", + "settings", + "show", + "source", + "start", + "stop", + "system", + "table", + "tables", + "temporary", + "test", + "then", + "ties", + "timestamp", + "to", + "top", + "totals", + "trailing", + "trim", + "truncate", + "type", + "unbounded", + "union", + "update", + "use", + "using", + "uuid", + "values", + "view", + "volume", + "watch", + "week", + "when", + "where", + "window", + "with", + "year", + } +) # --------------------------------------------------------------------------- @@ -180,12 +324,8 @@ class DestructiveChangeRule(LintRule): name = "destructive_changes" default_severity = Severity.WARN - _RE_DROP_TABLE = re.compile( - r"\bDROP\s+TABLE\b", re.IGNORECASE - ) - _RE_DROP_COLUMN = re.compile( - r"\bDROP\s+COLUMN\b", re.IGNORECASE - ) + _RE_DROP_TABLE = re.compile(r"\bDROP\s+TABLE\b", re.IGNORECASE) + _RE_DROP_COLUMN = re.compile(r"\bDROP\s+COLUMN\b", re.IGNORECASE) def check(self, sql: str, **kwargs: Any) -> list[LintResult]: config = kwargs.get("config") or LintConfig() @@ -197,24 +337,28 @@ def check(self, sql: str, **kwargs: Any) -> list[LintResult]: file_path = kwargs.get("file_path") for match in self._RE_DROP_TABLE.finditer(sql): - line = sql[:match.start()].count("\n") + 1 - results.append(LintResult( - rule=self.name, - message="DROP TABLE is destructive and irreversible", - severity=severity, - file=file_path, - line=line, - )) + line = sql[: match.start()].count("\n") + 1 + results.append( + LintResult( + rule=self.name, + message="DROP TABLE is destructive and irreversible", + severity=severity, + file=file_path, + line=line, + ) + ) for match in self._RE_DROP_COLUMN.finditer(sql): - line = sql[:match.start()].count("\n") + 1 - results.append(LintResult( - rule=self.name, - message="DROP COLUMN is destructive and irreversible", - severity=severity, - file=file_path, - line=line, - )) + line = sql[: match.start()].count("\n") + 1 + results.append( + LintResult( + rule=self.name, + message="DROP COLUMN is destructive and irreversible", + severity=severity, + file=file_path, + line=line, + ) + ) return results @@ -233,6 +377,12 @@ def check(self, sql: str, **kwargs: Any) -> list[LintResult]: elif reason is not None: severity = Severity.INFO message = f"Idempotency waiver: {reason}" + elif not kwargs.get("readable", True): + message = ( + "This SQL only exists at run time, so ch-migrate can't check that running it " + "twice is safe. Pass it as a string literal, or put " + "# ch-migrate: allow-non-idempotent on the line above" + ) else: check = classify_idempotency(sql) if check.status == "ok": @@ -290,14 +440,16 @@ def check(self, sql: str, **kwargs: Any) -> list[LintResult]: for match in self._RE_COLUMN_DEF.finditer(sql): col_name = match.group(1) if col_name.lower() in _CH_RESERVED_WORDS: - line = sql[:match.start()].count("\n") + 1 - results.append(LintResult( - rule=self.name, - message=f"Column '{col_name}' is a ClickHouse reserved word", - severity=severity, - file=file_path, - line=line, - )) + line = sql[: match.start()].count("\n") + 1 + results.append( + LintResult( + rule=self.name, + message=f"Column '{col_name}' is a ClickHouse reserved word", + severity=severity, + file=file_path, + line=line, + ) + ) return results @@ -325,17 +477,19 @@ def check(self, sql: str, **kwargs: Any) -> list[LintResult]: for match in self._RE_DDL.finditer(sql): # Check if ON CLUSTER or {on_cluster} appears nearby - rest = sql[match.end():match.end() + 200] + rest = sql[match.end() : match.end() + 200] if not re.search(r"(?:ON\s+CLUSTER|{on_cluster})", rest, re.IGNORECASE): - line = sql[:match.start()].count("\n") + 1 + line = sql[: match.start()].count("\n") + 1 stmt_type = match.group(0).strip() - results.append(LintResult( - rule=self.name, - message=f"{stmt_type} without ON CLUSTER or {{on_cluster}} placeholder", - severity=severity, - file=file_path, - line=line, - )) + results.append( + LintResult( + rule=self.name, + message=f"{stmt_type} without ON CLUSTER or {{on_cluster}} placeholder", + severity=severity, + file=file_path, + line=line, + ) + ) return results @@ -386,17 +540,19 @@ def check(self, sql: str, **kwargs: Any) -> list[LintResult]: if result.result_rows: row_count = result.result_rows[0][0] if row_count > threshold: - line = sql[:match.start()].count("\n") + 1 - results.append(LintResult( - rule=self.name, - message=( - f"ALTER on '{table_name}' which has {row_count:,} parts " - f"(threshold: {threshold:,})" - ), - severity=severity, - file=file_path, - line=line, - )) + line = sql[: match.start()].count("\n") + 1 + results.append( + LintResult( + rule=self.name, + message=( + f"ALTER on '{table_name}' which has {row_count:,} parts " + f"(threshold: {threshold:,})" + ), + severity=severity, + file=file_path, + line=line, + ) + ) except Exception: pass @@ -449,28 +605,24 @@ def check(self, sql: str, **kwargs: Any) -> list[LintResult]: for table_name, match in tables_to_check: affected = dep_graph.affected_by_drop(table_name) if affected: - mv_names = [ - n.name for n in affected if n.obj_type == "materialized_view" - ] - dict_names = [ - n.name for n in affected if n.obj_type == "dictionary" - ] + mv_names = [n.name for n in affected if n.obj_type == "materialized_view"] + dict_names = [n.name for n in affected if n.obj_type == "dictionary"] if mv_names or dict_names: - line = sql[:match.start()].count("\n") + 1 + line = sql[: match.start()].count("\n") + 1 deps = [] if mv_names: deps.append(f"MVs: {', '.join(mv_names)}") if dict_names: deps.append(f"Dicts: {', '.join(dict_names)}") - results.append(LintResult( - rule=self.name, - message=( - f"'{table_name}' has dependent objects: {'; '.join(deps)}" - ), - severity=severity, - file=file_path, - line=line, - )) + results.append( + LintResult( + rule=self.name, + message=(f"'{table_name}' has dependent objects: {'; '.join(deps)}"), + severity=severity, + file=file_path, + line=line, + ) + ) return results @@ -605,6 +757,9 @@ def _lint_statement( for rule in rules: if isinstance(rule, MVDeclarationRule) or rule.get_severity(scope.config) == Severity.OFF: continue + # SQL that only exists at run time can't be parsed; only its waiver is checked. + if not statement.readable and not isinstance(rule, IdempotencyRule): + continue findings = rule.check( statement.sql, file_path=statement.source, @@ -614,6 +769,7 @@ def _lint_statement( graph=scope.graph, comments=statement.comments, session_safe=scope.session_safe, + readable=statement.readable, ) for finding in findings: finding.file = statement.source diff --git a/ch_migrate/statements.py b/ch_migrate/statements.py index a5b416b..16e8455 100644 --- a/ch_migrate/statements.py +++ b/ch_migrate/statements.py @@ -22,10 +22,17 @@ class MigrationStatement: line: int comments: tuple[str, ...] direction: MigrationDirection + # False when the SQL only exists at run time; `sql` then holds the call's source. + readable: bool = True def migration_statements(path: Path) -> list[MigrationStatement]: - """Collect literal SQL in upgrade/downgrade bodies, preserving its source.""" + """Collect the SQL in upgrade/downgrade bodies, preserving its source. + + Literal SQL is returned as statements. A call that sends SQL known only at run time + (a variable, an expression) comes back as one unreadable statement, so callers can + insist on a waiver instead of passing it silently. + """ content = path.read_text() root = path.parent.parent if root.name == "migrations": @@ -45,7 +52,7 @@ def migration_statements(path: Path) -> list[MigrationStatement]: for call in calls: if _call_name(call) in ("read_sql", "run_sql"): result.extend(_file_statements(call, source, direction)) - elif _call_name(call) == "op.execute": + elif _sends_sql(call): result.extend(_inline_statements(call, source, direction)) return result @@ -79,14 +86,19 @@ def _call_name(call: ast.Call) -> str: return "" +def _sends_sql(call: ast.Call) -> bool: + """`op.execute`, and `execute`/`exec_driver_sql` on a connection such as `op.get_bind()`.""" + return isinstance(call.func, ast.Attribute) and call.func.attr in ("execute", "exec_driver_sql") + + def _file_statements( call: ast.Call, source: _Source, direction: MigrationDirection ) -> list[MigrationStatement]: - if not call.args or not isinstance(call.args[0], ast.Constant): - return [] - path_value = call.args[0].value + path_value = ( + call.args[0].value if call.args and isinstance(call.args[0], ast.Constant) else None + ) if not isinstance(path_value, str): - return [] + return [_unreadable(call, source, direction)] sql_path = source.path.parent.parent / "sql" / path_value statements = split_statements(sql_path.read_text()) filename = sql_path.relative_to(source.root).as_posix() @@ -103,7 +115,13 @@ def _inline_statements( return [] text = _literal_sql(call.args[0], source.content) if text is None: - return [] + # `op.execute(read_sql("file.sql"))`: the file's statements are collected from read_sql. + if isinstance(call.args[0], ast.Call) and _call_name(call.args[0]) in ( + "read_sql", + "run_sql", + ): + return [] + return [_unreadable(call, source, direction)] comments = _python_comments(source.lines, call.lineno) filename = source.path.relative_to(source.root).as_posix() result = [] @@ -114,7 +132,18 @@ def _inline_statements( return result +def _unreadable( + call: ast.Call, source: _Source, direction: MigrationDirection +) -> MigrationStatement: + code = ast.get_source_segment(source.content, call) or ast.unparse(call) + filename = source.path.relative_to(source.root).as_posix() + comments = _python_comments(source.lines, call.lineno) + return MigrationStatement(code, filename, call.lineno, comments, direction, readable=False) + + def _literal_sql(node: ast.AST, content: str) -> str | None: + if _is_text_call(node): + return _literal_sql(node.args[0], content) if isinstance(node, ast.Constant) and isinstance(node.value, str): return node.value if not isinstance(node, ast.JoinedStr): @@ -135,6 +164,16 @@ def _literal_sql(node: ast.AST, content: str) -> str | None: return "".join(pieces) +def _is_text_call(node: ast.AST) -> bool: + """SQLAlchemy's `text("...")` or `sa.text("...")` around the SQL.""" + if not isinstance(node, ast.Call) or len(node.args) != 1: + return False + func = node.func + return (isinstance(func, ast.Name) and func.id == "text") or ( + isinstance(func, ast.Attribute) and func.attr == "text" + ) + + def _python_comments(lines: list[str], lineno: int) -> tuple[str, ...]: comments = [] index = lineno - 2 diff --git a/tests/test_statements.py b/tests/test_statements.py index fe403bc..fffabdf 100644 --- a/tests/test_statements.py +++ b/tests/test_statements.py @@ -127,6 +127,59 @@ def test_missing_referenced_file_is_not_silently_ignored(project): migration_statements(path) +def test_every_way_of_sending_sql_is_collected(project): + path = _revision( + project, + "def upgrade():\n" + " op.execute(text('UPDATE {db}.t SET x = 1 WHERE 1'))\n" + " op.get_bind().execute(sa.text(f'DELETE FROM {db}.t WHERE id = 1'))\n" + " op.get_bind().exec_driver_sql('ALTER TABLE {db}.t UPDATE x = 2 WHERE 1')\n", + ) + statements = migration_statements(path) + assert [(s.sql, s.line, s.readable) for s in statements] == [ + ("UPDATE {db}.t SET x = 1 WHERE 1", 4, True), + ("DELETE FROM {db}.t WHERE id = 1", 5, True), + ("ALTER TABLE {db}.t UPDATE x = 2 WHERE 1", 6, True), + ] + + +def test_sql_known_only_at_run_time_is_unreadable_not_skipped(project): + path = _revision( + project, + "def upgrade():\n" + " # ch-migrate: allow-non-idempotent built from a reviewed list\n" + " op.execute(statement)\n" + " run_sql(name)\n" + " connection.exec_driver_sql(build())\n", + ) + statements = migration_statements(path) + assert [(s.sql, s.line, s.readable) for s in statements] == [ + ("op.execute(statement)", 5, False), + ("run_sql(name)", 6, False), + ("connection.exec_driver_sql(build())", 7, False), + ] + assert statements[0].comments == ( + "ch-migrate: allow-non-idempotent built from a reviewed list", + ) + + +def test_unreadable_sql_needs_a_waiver_and_skips_sql_rules(project): + _revision( + project, + "def upgrade():\n" + " op.execute(statement)\n" + " # ch-migrate: allow-non-idempotent each table is filled once\n" + " op.execute(other)\n", + ) + report = lint_migrations(project / "migrations/versions") + assert [(r.rule, r.severity.value, r.line) for r in report.results] == [ + ("idempotency", "error", 4), + ("idempotency", "info", 6), + ] + assert "only exists at run time" in report.results[0].message + assert report.results[1].message == "Idempotency waiver: each table is filled once" + + def _revision(project: Path, body: str) -> Path: path = project / "migrations/versions/a.py" path.write_text("revision = 'a'\ndown_revision = None\n" + body) From 7c070fa57ce5fe2d998040734e3b9c858dfa0127 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 14:38:50 -0400 Subject: [PATCH 28/33] [waiting] Stop unsafe reruns of interrupted migrations A rerun runs an interrupted migration again from its first statement. That is safe only for statements the idempotency gate vouches for; a waived UPDATE n = n + 1 whose wait was cut short applied twice and up reported success. Each step is now noted in _ch_migrate_in_progress before it runs and cleared once its version is recorded. A noted step reruns on its own only if every statement passes the gate without a waiver; otherwise up or down stops before sending anything, names the statements, and needs --rerun-interrupted REV (-x rerun_interrupted with alembic). Downgrades are judged by the same rules. The marker table follows the version table's engine (Shared, Replicated database, ON CLUSTER with a Keeper path, or MergeTree), is read with a replica sync or sequential consistency, and is left out of mutation waiting. The same step hook runs the gate before a direct alembic upgrade's first migration; ch-migrate up has already run it. Solves: DRY-1609 Tests: unit 686; Docker single 58 passed, 4 skipped; cluster 16 --- CHANGELOG.md | 11 +- README.md | 45 +++-- ch_migrate/alembic_env.py | 38 ++++- ch_migrate/cli.py | 35 +++- ch_migrate/migration_runner.py | 25 ++- ch_migrate/waiting/mutations.py | 11 +- ch_migrate/waiting/reruns.py | 182 +++++++++++++++++++++ docs/design/2026-10-02-ch-migrate-1.0.md | 5 +- tests/integration/test_cloud.py | 7 +- tests/integration/test_mutation_waiting.py | 125 ++++++++++++-- tests/integration/test_on_cluster.py | 17 +- tests/test_reruns.py | 123 ++++++++++++++ tests/test_waiting_display.py | 20 +++ 13 files changed, 590 insertions(+), 54 deletions(-) create mode 100644 ch_migrate/waiting/reruns.py create mode 100644 tests/test_reruns.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 61a6b96..5d55207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,15 @@ Changes are recorded in [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) 0.7 waits for the work a migration starts. `up` doesn't record a migration as applied until the mutations and `ON CLUSTER` DDL its statements started have finished, and it shows their progress while it waits. A normal `pip install -U` reaches it from 0.6, so read this before upgrading: 1. **`up` now blocks until that work finishes, with no default timeout.** If a CI job's time limit stops it mid-wait, ClickHouse keeps working, and the next `up` waits for that work before it runs the migration again. `--timeout SECONDS` sets a limit of your own. -2. **A rerun runs the stopped migration again from its first statement.** 0.6's idempotency gate already requires statements that are safe to repeat, or a written waiver; check what a waived statement changed before you rerun it. -3. **Two grants, only where `bootstrap` couldn't grant `SELECT ON system.*`**, such as on ClickHouse Cloud. As the admin user, replacing `` with your project name: +2. **A rerun runs the stopped migration again from its first statement, but only on its own when every statement passes the idempotency gate without a waiver.** An interrupted migration with a waived statement, which includes every `UPDATE`, `DELETE` and `INSERT ... SELECT`, stops the next run until you check what the earlier one changed and name it with `--rerun-interrupted REV`. CI jobs that retry `up` after a timeout need that step. +3. **The gate now reads more of your migrations.** SQL in `text("...")` and SQL sent through `op.get_bind().execute(...)` or `exec_driver_sql(...)` is checked, and SQL built at run time needs a waiver. `lint` and `up` can report migrations that passed before; waive them or keep them behind `lint.gate_baseline`. +4. **Two grants, only where `bootstrap` couldn't grant `SELECT ON system.*`**, such as on ClickHouse Cloud. As the admin user, replacing `` with your project name: ```sql GRANT CURRENT GRANTS(SELECT ON system.parts) TO _migration_role; GRANT CURRENT GRANTS(SELECT ON system.replicas) TO _migration_role; ``` `system.parts` gives the progress bar its bytes and time estimate; without it the bar counts parts. `system.replicas` is read only for self-hosted replicated tables, so Cloud needs just the first. -4. **No `upgrade-env` this time.** The environment 0.6 wrote is current. +5. **No `upgrade-env` this time.** The environment 0.6 wrote is current. `up` creates one small table, `_ch_migrate_in_progress`, in the project database. To stay on 0.6, pin `ch-migrate-cli==0.6.0`. @@ -24,13 +25,15 @@ To stay on 0.6, pin `ch-migrate-cli==0.6.0`. - `up` waits before it records a migration as applied. After each statement that can write, it polls `system.mutations` on the migration session until the project database has no unfinished mutation on any replica that has to apply it, instead of holding a request open with `mutations_sync`. Mutations another writer started count too, and ones already queued are waited for before the first migration runs. The version table's own mutations stay with its 0.6 bookkeeping. - `ON CLUSTER` statements wait inside ClickHouse: `up` sends them with `distributed_ddl_output_mode = 'throw'` and a limit of 270 seconds, or what's left of `--timeout` if that's less, so a host that fails or doesn't finish in time fails the migration with ClickHouse's message. -- A failed mutation fails the migration with ClickHouse's `latest_fail_reason` and prints the scoped `KILL MUTATION` statement for an operator; `ch-migrate` never kills a mutation itself. A failed mutation from before the run blocks it the same way. After a kill, fix the SQL and run `up` again. +- A failed mutation fails the migration with ClickHouse's `latest_fail_reason` and prints the scoped `KILL MUTATION` statement for an operator; `ch-migrate` never kills a mutation itself. A failed mutation from before the run blocks it the same way. After a kill, fix the SQL and run `up` again, naming the revision with `--rerun-interrupted` if it's waived. - `up` runs Alembic in its own process, so killing `up` can't leave a child process advancing the version table. `down` and a direct `alembic upgrade` go through the same environment and wait too; `--timeout` exists only on `ch-migrate up`. +- The idempotency gate checks literal SQL in `text("...")` and in `execute`/`exec_driver_sql` on any connection, requires a waiver for SQL that only exists at run time, and runs from `env.py` for a direct `alembic upgrade`. ### Added - `up --timeout SECONDS`: one waiting budget for the whole run. When it runs out, `up` exits 1, names what is still pending and leaves the work running; the next `up` waits on it again, then runs the migration. - Progress while `up` waits: in a terminal, a live bar per mutation weighted by bytes from `system.parts`, with parts and time left, and one row per replica. Piped output gets plain lines when a wait starts, at each 10% or every 30 seconds, and when it finishes. Without access to `system.parts` the bar counts parts only. +- `up --rerun-interrupted REV` and `down --rerun-interrupted REV` (`-x rerun_interrupted=REV` with `alembic`): run an interrupted migration again after checking what its earlier run changed. Each migration is noted in `_ch_migrate_in_progress` while it runs, created with the version table's engine. - `bootstrap` grants `SELECT` on `system.replicas` and `system.parts`. ## [0.6.0] diff --git a/README.md b/README.md index 7fb5efd..3c9f2f7 100644 --- a/README.md +++ b/README.md @@ -296,24 +296,34 @@ when it finishes: Part sizes come from `system.parts`. If the migration role can't read it, the bar counts parts only and says so once; progress never changes whether `up` succeeds. -**When `up` stops short.** After a timeout, Ctrl-C, dropped connection or killed -process, ClickHouse keeps working and the migration isn't recorded. Run `up` -again: it waits for that work to finish, then runs the migration again from its -first statement. That's safe because the idempotency gate below requires every -statement to be safe to repeat, or to carry a written waiver; before rerunning a -waived statement, check what the first attempt changed. +**When `up` stops short.** After a timeout, Ctrl-C, dropped connection, killed +process or failed statement, ClickHouse keeps working and the migration isn't +recorded. `up` and `down` note each migration in `_ch_migrate_in_progress` when it +starts and clear the note once its version is recorded, so the next run knows the +migration was interrupted. That run waits for unfinished work, then: + +- **Runs the migration again on its own** when every statement in it passes the + idempotency gate without a waiver, so a repeat leaves the same result. +- **Stops before sending anything** when the migration has a waived statement or + SQL the gate can't read, because that statement may already have applied. It + names each one. Check what the earlier run changed, fix the SQL if needed, then + name the revision: `ch-migrate up ENV --rerun-interrupted REV` (or `down`), or + `-x rerun_interrupted=REV` with `alembic`. Naming it accepts the repeat. + +Every `UPDATE`, `DELETE` and `INSERT ... SELECT` needs a waiver, so an interrupted +backfill always takes that second path. **Failed mutation.** ClickHouse's `latest_fail_reason` stops the run, and `up` prints the scoped `KILL MUTATION` statement for an operator; ch-migrate never runs it. ClickHouse keeps retrying a failed mutation, and later mutations on that table queue behind it. If the cause is temporary, such as memory or disk, fix it and run `up` again. If the statement itself is wrong, kill the mutation, fix the -SQL and run `up` again. Killing a mutation doesn't undo the parts it already -rewrote. +SQL and run `up` again, naming the revision with `--rerun-interrupted` if it's +waived. Killing a mutation doesn't undo the parts it already rewrote. Run one migration runner per database, using a CI concurrency key or an external -lock. Re-run `bootstrap` when upgrading a restricted migration user to obtain the -`system.replicas` and `system.parts` grants. +lock; the in-progress notes aren't a lock. Re-run `bootstrap` when upgrading a +restricted migration user to obtain the `system.replicas` and `system.parts` grants. ### Re-runnable migrations @@ -321,7 +331,14 @@ lock. Re-run `bootstrap` when upgrading a restricted migration user to obtain th them. Idempotency and standalone-SET errors refuse the whole run and print the source file, line, statement, and suggested fix. There is no command-line bypass; `--skip-mv-check` does not skip this gate. Other findings are warnings in `up`; standalone `lint` -retains their configured severities. +retains their configured severities. A direct `alembic upgrade` runs the same gate +from `env.py` before its first migration. + +The gate reads SQL in `run_sql` files and in literal strings, f-strings and +`text("...")` passed to `op.execute`, or to `execute` and `exec_driver_sql` on a +connection such as `op.get_bind()`. SQL that only exists at run time, such as a +variable, a built string or `run_sql(name)`, can't be checked, so it needs a +reasoned waiver like any other statement the gate can't vouch for. Static lint rejects a standalone `SET` when `migrations/env.py` lacks the `CH_MIGRATE_ENV_VERSION = 2` marker: that statement would be ignored on the old @@ -355,7 +372,7 @@ INSERT INTO {db}.target SELECT id FROM {db}.source; ``` For inline Python SQL, use the same directive in a `#` comment directly above -`op.execute`. Empty reasons remain errors. Waivers print as INFO with their +the call that sends it. Empty reasons remain errors. Waivers print as INFO with their reasons in `lint` and `up`; they do not make the statement idempotent. `MODIFY`, `MATERIALIZE`, `CLEAR COLUMN`, `TRUNCATE`, `OPTIMIZE`, `SYSTEM`, grants, revokes, `REPLACE PARTITION`, attaching or detaching a whole partition, and @@ -368,8 +385,8 @@ when the revision completes. A dropped or renamed column stays locked until that change's mutation finishes, and `up` waits for it before running the next statement, so a retry after a later failure doesn't hit "affected by mutation ... which is not finished yet". If `up` was stopped while it waited, run it again: it -waits for that mutation, then runs the revision again. Before retrying waived -operations, review what the earlier attempt actually changed. +waits for that mutation, then runs the revision again, unless the revision has a +statement the gate can't vouch for (see below). **Existing history:** `upgrade-env` records the current local script head(s) in `config.yaml` as `lint.gate_baseline`, preserving YAML comments. It records diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index 8a9e330..a0d6aac 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -4,6 +4,7 @@ import logging import os +from functools import partial from logging.config import fileConfig from pathlib import Path from types import SimpleNamespace @@ -18,6 +19,7 @@ from sqlalchemy import URL, Column, MetaData, String, Table, create_engine, pool, text from sqlalchemy.sql.dml import Insert +from ch_migrate import ui from ch_migrate.config import get_env_config from ch_migrate.hooks import HookRegistry, run_hooks from ch_migrate.rebase import _literal_assignment @@ -29,6 +31,7 @@ sync_version_replica, ) from ch_migrate.waiting import MigrationWaiter +from ch_migrate.waiting.reruns import RerunGuard ENV_VERSION = 2 DEFAULT_SESSION_TIMEOUT = 1800 @@ -113,7 +116,7 @@ def run() -> None: if context.is_offline_mode(): _run_offline(env_config) else: - _run_online(env_config) + _run_online(env_config, environment) def has_current_env(path: Path) -> bool: @@ -153,7 +156,7 @@ def _run_offline(env_config: dict[str, Any]) -> None: context.run_migrations() -def _run_online(env_config: dict[str, Any]) -> None: +def _run_online(env_config: dict[str, Any], environment: str) -> None: # The session lives on whichever replica its connection reached: ClickHouse Cloud routes # each TCP connection, not each session id. A pool of its own keeps the session on one # connection instead of sharing the driver's default pool with other in-process clients. @@ -164,13 +167,13 @@ def _run_online(env_config: dict[str, Any]) -> None: ) try: with engine.connect() as connection: - _run_connection(connection, env_config) + _run_connection(connection, env_config, environment) finally: engine.dispose() session_pool.clear() -def _run_connection(connection, env_config: dict[str, Any]) -> None: +def _run_connection(connection, env_config: dict[str, Any], environment: str) -> None: hooks = HookRegistry.from_config(env_config.get("hooks")) state = _checked_session(connection, env_config) context.configure( @@ -182,6 +185,7 @@ def _run_connection(connection, env_config: dict[str, Any]) -> None: ch_migrate_version_state=state, ) waiter = MigrationWaiter(connection, state, context.config.attributes.get("ch_migrate_timeout")) + guard = _rerun_guard(waiter.client, state, environment) try: waiter.install(context.config) with context.begin_transaction(): @@ -194,11 +198,37 @@ def _run_connection(connection, env_config: dict[str, Any]) -> None: phase="pre_migrate", revision="all", ) + runtime = context.get_context() + runtime._migrations_fn = guard.steps(runtime._migrations_fn) context.run_migrations() finally: waiter.close() +def _rerun_guard(client, state: VersionTableState, environment: str) -> RerunGuard: + attributes = context.config.attributes + # `ch-migrate up` passes --rerun-interrupted directly; `down` and a direct alembic + # command pass it as -x rerun_interrupted=REV[,REV]. + named = attributes.get("ch_migrate_rerun_interrupted") or context.get_x_argument( + as_dictionary=True + ).get("rerun_interrupted", "").split(",") + guard = RerunGuard(client, state, frozenset(name.strip() for name in named if name.strip())) + if not attributes.get("ch_migrate_gate_checked"): + # A direct `alembic upgrade` hasn't been through `ch-migrate up`'s gate. + guard.before_first_upgrade = partial(_enforce_gate, environment) + return guard + + +def _enforce_gate(environment: str) -> None: + from ch_migrate.gate import enforce_up_gate # lint imports this module + + try: + enforce_up_gate(Path.cwd(), environment) + except ui.Failure as failure: + failure.show() + raise SystemExit(1) from None + + def _checked_session(connection, env_config: dict[str, Any]) -> VersionTableState: # Establish the session before requiring its existence on every later request. connection.exec_driver_sql("SELECT 1") diff --git a/ch_migrate/cli.py b/ch_migrate/cli.py index 041019c..48ef4ef 100644 --- a/ch_migrate/cli.py +++ b/ch_migrate/cli.py @@ -239,9 +239,21 @@ def bootstrap(environment: str, dry_run: bool, verbose: bool) -> None: is_flag=True, help="Skip materialized view declaration validation", ) +@click.option( + "--rerun-interrupted", + "rerun_interrupted", + multiple=True, + metavar="REVISION", + help="Run this interrupted revision again after checking what its earlier run changed", +) @click.option("--verbose", is_flag=True, help="Show the full traceback if a migration fails") def up( - environment: str, revision: str, skip_mv_check: bool, timeout: float | None, verbose: bool + environment: str, + revision: str, + skip_mv_check: bool, + timeout: float | None, + rerun_interrupted: tuple[str, ...], + verbose: bool, ) -> None: """Apply pending migrations. @@ -250,6 +262,8 @@ def up( Idempotency and standalone-SET gate errors refuse the run before Alembic. Other findings are warnings; --skip-mv-check only skips MV declaration checks. + A revision interrupted on an earlier run reruns on its own only if every statement + in it passes the gate without a waiver; otherwise name it with --rerun-interrupted. """ if timeout is not None and not math.isfinite(timeout): raise click.BadParameter("must be finite", param_hint="--timeout") @@ -259,21 +273,34 @@ def up( enforce_up_gate(Path.cwd(), environment, skip_mv_check) from ch_migrate.migration_runner import Upgrade, run_upgrade - sys.exit(run_upgrade(Upgrade(environment, revision, timeout, verbose))) + sys.exit(run_upgrade(Upgrade(environment, revision, timeout, verbose, rerun_interrupted))) @main.command() @click.argument("environment") @click.option("--revision", "-r", default="-1", help="Revision to downgrade to (default: -1)") +@click.option( + "--rerun-interrupted", + "rerun_interrupted", + multiple=True, + metavar="REVISION", + help="Run this interrupted revision again after checking what its earlier run changed", +) @click.option("--verbose", is_flag=True, help="Show the full traceback if a migration fails") -def down(environment: str, revision: str, verbose: bool) -> None: +def down( + environment: str, revision: str, rerun_interrupted: tuple[str, ...], verbose: bool +) -> None: """Rollback migrations. By default, rolls back the last migration. Use --revision to specify a target. + A downgrade interrupted on an earlier run reruns on its own only if every statement + in it passes the gate without a waiver; otherwise name it with --rerun-interrupted. """ _require_current_env() _refuse_irreversible_downgrade(environment, revision) - sys.exit(run_migrations(environment, ["downgrade", revision], verbose=verbose)) + # The downgrade runs in an Alembic subprocess; -x passes the names on to env.py. + named = ["-x", f"rerun_interrupted={','.join(rerun_interrupted)}"] if rerun_interrupted else [] + sys.exit(run_migrations(environment, [*named, "downgrade", revision], verbose=verbose)) @main.command() diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py index c535716..8735c01 100644 --- a/ch_migrate/migration_runner.py +++ b/ch_migrate/migration_runner.py @@ -36,6 +36,7 @@ WaitingError, WaitTimeout, ) +from ch_migrate.waiting.reruns import InterruptedStep _LOCATION = re.compile( r"^(?Pmigrations/sql/\S+) \(statement \d+ of \d+, line (?P\d+)\): " @@ -44,12 +45,13 @@ @dataclass(frozen=True) class Upgrade: - """One `ch-migrate up`: where, how far, how long to wait, and how much to show.""" + """One `ch-migrate up`: where, how far, how long to wait, what to rerun, what to show.""" environment: str revision: str = "head" timeout: float | None = None verbose: bool = False + rerun_interrupted: tuple[str, ...] = () def run_upgrade(upgrade: Upgrade) -> int: @@ -102,6 +104,9 @@ def _alembic_config(upgrade: Upgrade, progress: _Progress, display: WaitDisplay) config.attributes.update( ch_migrate_environment=upgrade.environment, ch_migrate_command="upgrade", + # `ch-migrate up` has already run the gate; env.py runs it only for a direct upgrade. + ch_migrate_gate_checked=True, + ch_migrate_rerun_interrupted=upgrade.rerun_interrupted, ch_migrate_timeout=upgrade.timeout, ch_migrate_display=display, ch_migrate_log_handler=_MigrationLog(progress, display), @@ -128,6 +133,8 @@ def _report_failure(upgrade: Upgrade, progress: _Progress, error: Exception) -> _report_mutation_failure(upgrade.environment, where, waiting.failure) elif isinstance(waiting, WaitTimeout): _report_timeout(upgrade.environment, where, waiting) + elif isinstance(waiting, InterruptedStep): + _report_interrupted_step(upgrade.environment, waiting) elif waiting is not None: ui.error(f"{where}: {waiting}") else: @@ -172,6 +179,22 @@ def _report_timeout(environment: str, where: str, timeout: WaitTimeout) -> None: ) +def _report_interrupted_step(environment: str, stop: InterruptedStep) -> None: + revision = stop.step.revision[:8] + ui.error( + f"{revision} was interrupted on an earlier run, and ch-migrate can't vouch for " + "running it again" + ) + for item in stop.unvouched: + where = f"{item.statement.source}:{item.statement.line}" + ui.detail(f"{where} {item.why}: {item.statement.sql.splitlines()[0]}", stderr=True) + ui.hint( + "Some of its statements may already have applied. Check what the earlier run changed, " + f"then run `ch-migrate up {environment} --rerun-interrupted {revision}`.", + stderr=True, + ) + + def _report_statement_failure(upgrade: Upgrade, where: str, error: Exception) -> None: ui.error(f"{where} failed") ui.detail(_reason(error), stderr=True) diff --git a/ch_migrate/waiting/mutations.py b/ch_migrate/waiting/mutations.py index 83b78f8..00d5763 100644 --- a/ch_migrate/waiting/mutations.py +++ b/ch_migrate/waiting/mutations.py @@ -15,14 +15,15 @@ from clickhouse_connect.driver.exceptions import DatabaseError +from ch_migrate.waiting import reruns from ch_migrate.waiting.display import HostParts, MutationProgress from ch_migrate.waiting.errors import FailedMutation, MutationFailed, WaitingError from ch_migrate.waiting.sql import sql_string SHARED = "shared" -# version_table.py owns the version table's mutations: its DELETE waits on its own request, -# and a stuck one is refused before the run starts. -VERSION_TABLE = "alembic_version" +# ch-migrate's own tables: version_table.py waits on the version table's DELETE itself and +# refuses a stuck one before the run starts, and the rerun markers' DELETE waits the same way. +INTERNAL_TABLES = ("alembic_version", reruns.TABLE) class MutationWaiter: @@ -58,11 +59,11 @@ def _rows(self, expected: set[tuple[str, str]]) -> list[_Row]: rows = self.client.query( "SELECT materialize(hostName()), table, mutation_id, is_done, parts_to_do, " f"latest_fail_reason, parts_to_do_names FROM {source} WHERE database = {{db:String}} " - "AND table != {version:String} " + "AND NOT has({internal:Array(String)}, table) " "AND (NOT is_done OR has({expected:Array(String)}, concat(table, '/', mutation_id)))", parameters={ "db": self.state.database, - "version": VERSION_TABLE, + "internal": list(INTERNAL_TABLES), "cluster": cluster, "expected": sorted(f"{table}/{mutation}" for table, mutation in expected), }, diff --git a/ch_migrate/waiting/reruns.py b/ch_migrate/waiting/reruns.py new file mode 100644 index 0000000..a814880 --- /dev/null +++ b/ch_migrate/waiting/reruns.py @@ -0,0 +1,182 @@ +"""Stop a rerun of an interrupted migration step that ch-migrate can't vouch for. + +Before a step runs, `up` and `down` record that it started; once its version is recorded, +the record goes. A record still there on the next run means the step was interrupted, so +some of its statements may already have applied. The step then runs again on its own only +if every statement in it passes the idempotency gate without a waiver. Otherwise the run +stops until someone checks what the earlier attempt changed and names the revision with +`--rerun-interrupted`. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import quote +from uuid import uuid4 + +from clickhouse_connect.driver.binding import quote_identifier + +from ch_migrate.idempotency import classify_idempotency, waiver_reason +from ch_migrate.statements import MigrationStatement, migration_statements +from ch_migrate.waiting.errors import WaitingError +from ch_migrate.waiting.sql import sql_string + +TABLE = "_ch_migrate_in_progress" +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Step: + revision: str + direction: str # "upgrade" or "downgrade" + path: Path + + +@dataclass(frozen=True) +class Unvouched: + """A statement in an interrupted step that might not be safe to run again.""" + + statement: MigrationStatement + why: str + + +class InterruptedStep(WaitingError): + """An interrupted step has statements that might not be safe to run again.""" + + def __init__(self, step: Step, unvouched: list[Unvouched]): + self.step, self.unvouched = step, unvouched + listed = "; ".join(f"{_where(item.statement)} ({item.why})" for item in unvouched) + super().__init__( + f"{step.revision[:8]} was interrupted on an earlier run, and ch-migrate can't vouch " + f"for running it again: {listed}. Check what the earlier run changed, then run it " + f"again with --rerun-interrupted {step.revision[:8]} " + f"(with alembic, -x rerun_interrupted={step.revision[:8]})." + ) + + +class RerunGuard: + """Records each step as it starts and refuses an unsafe rerun of an interrupted one.""" + + def __init__(self, client, state, acknowledged: frozenset[str]): + self.client, self.state, self.acknowledged = client, state, acknowledged + self.before_first_upgrade = None # set by env.py when the gate hasn't run yet + self._ready = False + + def steps(self, original): + """Wrap Alembic's step iterator: check and record each step, clear it once recorded.""" + + def iterate(heads, context): + for step in original(heads, context): + if not hasattr(step, "revision"): + yield step # Stamp steps run no migration body. + continue + direction = "upgrade" if step.is_upgrade else "downgrade" + current = Step(step.revision.revision, direction, Path(step.revision.path)) + self.begin(current) + # Alembic asks for the next step only once this one's version is recorded. + yield step + self.finish(current) + + return iterate + + def begin(self, step: Step) -> None: + if step.direction == "upgrade" and self.before_first_upgrade is not None: + check, self.before_first_upgrade = self.before_first_upgrade, None + check() + self._prepare() + if self._started(step): + unvouched = unvouched_statements(step.path, step.direction) + if unvouched and not self._acknowledged(step.revision): + raise InterruptedStep(step, unvouched) + logger.warning( + f"{step.revision[:8]} was interrupted on an earlier run; running it again" + ) + self._record(step) + + def finish(self, step: Step) -> None: + self.client.command( + f"ALTER TABLE {self._table()} DELETE WHERE revision = {sql_string(step.revision)} " + "SETTINGS mutations_sync = 2" + ) + + def _acknowledged(self, revision: str) -> bool: + return any(len(given) >= 4 and revision.startswith(given) for given in self.acknowledged) + + def _prepare(self) -> None: + if self._ready: + return + if not self.client.command(f"EXISTS TABLE {self._table()}"): + self.client.command(self._create_sql()) + if self._replicated(): + # Fetch markers another replica wrote; there are no mutations here to wait for. + self.client.command(f"SYSTEM SYNC REPLICA {self._table()} LIGHTWEIGHT") + self._ready = True + + def _started(self, step: Step) -> bool: + return bool( + self.client.command( + f"SELECT count() FROM {self._table()} " + "WHERE revision = {revision:String} AND direction = {direction:String}", + parameters={"revision": step.revision, "direction": step.direction}, + settings={"select_sequential_consistency": 1} if self._shared() else None, + ) + ) + + def _record(self, step: Step) -> None: + self.client.command( + f"INSERT INTO {self._table()} (revision, direction) " + "SELECT {revision:String}, {direction:String}", + parameters={"revision": step.revision, "direction": step.direction}, + # Replicated and Shared MergeTree drop a block identical to an earlier insert. + settings={ + "insert_deduplication_token": f"chm_started_{uuid4().hex}", + "async_insert": 0, + }, + ) + + def _create_sql(self) -> str: + engine, on_cluster = "MergeTree", "" + if self.state.database_engine == "Replicated": + engine = "ReplicatedMergeTree" + elif not self._shared() and self.state.cluster: + path = f"/clickhouse/ch_migrate/{quote(self.state.database, safe='')}/{{shard}}/{TABLE}" + engine = f"ReplicatedMergeTree({sql_string(path)}, '{{replica}}')" + on_cluster = f" ON CLUSTER {quote_identifier(self.state.on_cluster)}" + return ( + f"CREATE TABLE IF NOT EXISTS {self._table()}{on_cluster} (revision String, " + "direction LowCardinality(String), started DateTime64(3) DEFAULT now64(3)) " + f"ENGINE = {engine} ORDER BY (revision, direction)" + ) + + def _table(self) -> str: + return f"{quote_identifier(self.state.database)}.{quote_identifier(TABLE)}" + + def _shared(self) -> bool: + return self.state.database_engine.startswith("Shared") + + def _replicated(self) -> bool: + return self.state.database_engine == "Replicated" or ( + not self._shared() and bool(self.state.cluster) + ) + + +def unvouched_statements(path: Path, direction: str) -> list[Unvouched]: + """Statements in this step the gate can't vouch for: waived, unreadable, or unsafe.""" + found = [] + for statement in migration_statements(path): + if statement.direction != direction: + continue + reason = waiver_reason(statement.comments) + if reason is not None: + found.append(Unvouched(statement, f"waiver: {reason}" if reason else "empty waiver")) + elif not statement.readable: + found.append(Unvouched(statement, "SQL that only exists at run time")) + elif classify_idempotency(statement.sql).status != "ok": + found.append(Unvouched(statement, "not safe to run twice")) + return found + + +def _where(statement: MigrationStatement) -> str: + return f"{statement.source}:{statement.line}" diff --git a/docs/design/2026-10-02-ch-migrate-1.0.md b/docs/design/2026-10-02-ch-migrate-1.0.md index b9eec30..9e5cb67 100644 --- a/docs/design/2026-10-02-ch-migrate-1.0.md +++ b/docs/design/2026-10-02-ch-migrate-1.0.md @@ -153,7 +153,10 @@ started. - Whenever `up` stops waiting (a timeout, a killed CI job, a dropped connection), the outcome is the same: the migration is not recorded as applied, ClickHouse keeps doing the work, and the next `up` waits for that work to finish, then runs the migration again. That relies on - capability 4: every statement must be safe to repeat, or carry a written waiver. + capability 4: every statement must be safe to repeat, or carry a written waiver. A + migration noted as started but not recorded reruns on its own only if every statement + passes the gate without a waiver; otherwise `up` stops and names the statements until + someone checks and passes `--rerun-interrupted REV` (Dan, 2026-10-27, DRY-1609). - **Design change, 2026-10-27 (Dan):** the design approved on 2026-10-04 recorded which work each statement started in a journal, so a rerun would never send a statement twice. It took about 1,300 lines, duplicated what capability 4 already guarantees, and left a killed or diff --git a/tests/integration/test_cloud.py b/tests/integration/test_cloud.py index c5132ab..4b78791 100644 --- a/tests/integration/test_cloud.py +++ b/tests/integration/test_cloud.py @@ -86,7 +86,7 @@ def test_cloud_up_advances_version_only_after_mutation_finishes(project): def test_cloud_killed_up_waits_for_its_mutation_then_runs_again(project): - """SIGKILL mid-wait; the rerun waits out the running UPDATE, then sends it again.""" + """SIGKILL mid-wait; the waived UPDATE's rerun stops until named, then waits and resends.""" db = project.database _slow_update(project, "1", seconds=3) log_path = project.root / "first-up.log" @@ -102,7 +102,10 @@ def test_cloud_killed_up_waits_for_its_mutation_then_runs_again(project): finally: first.kill() first.wait(timeout=10) - resumed = project.run("up", "it") + stopped = project.run("up", "it") + assert stopped.exit_code == 1, stopped.output + assert "was interrupted on an earlier run" in stopped.output + resumed = project.run("up", "it", "--rerun-interrupted", "aaaa") assert resumed.exit_code == 0, resumed.output value = project.client.command( f"SELECT x FROM {db}.counter", settings={"select_sequential_consistency": 1} diff --git a/tests/integration/test_mutation_waiting.py b/tests/integration/test_mutation_waiting.py index 2c49470..c630f18 100644 --- a/tests/integration/test_mutation_waiting.py +++ b/tests/integration/test_mutation_waiting.py @@ -31,12 +31,111 @@ def test_up_records_a_migration_only_after_its_mutation(project, statement): assert _heads(project) == ["aaaa"] finally: project.client.command(f"SYSTEM START MERGES {project.database}.counter") - resumed = project.run("up", "it") + resumed = project.run("up", "it", "--rerun-interrupted", "bbbb") assert resumed.exit_code == 0, resumed.output assert all(row[2] for row in _mutations(project)) assert _heads(project) == ["bbbb"] +def test_interrupted_step_with_a_waiver_stops_until_it_is_named(project): + """A waived statement may already have applied, so a rerun needs someone to check first.""" + db = project.database + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1") + project.stop_merges(f"{db}.counter") + try: + held = project.run("up", "it", "--timeout", "1") + assert held.exit_code == 1 and "Timed out" in held.output, held.output + finally: + project.client.command(f"SYSTEM START MERGES {db}.counter") + _until(lambda: all(row[2] for row in _mutations(project)), "first increment did not finish") + stopped = project.run("up", "it") + assert stopped.exit_code == 1, stopped.output + assert "was interrupted on an earlier run" in stopped.output + assert "waiver: Controlled waiting acceptance scenario" in stopped.output + assert "--rerun-interrupted bbbb" in stopped.output + assert project.client.command(f"SELECT x FROM {db}.counter WHERE id = 1") == 1 + assert _heads(project) == ["aaaa"] + named = project.run("up", "it", "--rerun-interrupted", "bbbb") + assert named.exit_code == 0, named.output + # Naming the revision accepts the repeat: this increment runs a second time. + assert project.client.command(f"SELECT x FROM {db}.counter WHERE id = 1") == 2 + assert _markers(project) == 0 + assert _heads(project) == ["bbbb"] + + +def test_interrupted_step_the_gate_passes_runs_again_on_its_own(project): + db = project.database + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter MATERIALIZE COLUMN doubled", waived=False) + project.stop_merges(f"{db}.counter") + try: + held = project.run("up", "it", "--timeout", "1") + assert held.exit_code == 1 and "Timed out" in held.output, held.output + finally: + project.client.command(f"SYSTEM START MERGES {db}.counter") + resumed = project.run("up", "it") + assert resumed.exit_code == 0, resumed.output + assert "was interrupted on an earlier run; running it again" in resumed.output + assert _markers(project) == 0 + assert _heads(project) == ["bbbb"] + + +def test_completed_steps_leave_no_marker(project): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter MATERIALIZE COLUMN doubled", waived=False) + assert project.run("up", "it").exit_code == 0 + assert _markers(project) == 0 + + +def test_direct_alembic_upgrade_runs_the_gate(project): + _seed(project) + _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 1", waived=False) + result = subprocess.run( + [sys.executable, "-m", "alembic", "upgrade", "head"], + cwd=project.root, + env={**os.environ, "CH_ENVIRONMENT": "it"}, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 1, result.stdout + result.stderr + assert "Migration gate failed; nothing was applied." in result.stderr + assert "[idempotency]" in result.stdout + result.stderr + assert project.client.command(f"SELECT x FROM {project.database}.counter WHERE id = 1") == 0 + assert _heads(project) == ["aaaa"] + + +def test_interrupted_downgrade_with_a_waiver_stops_until_it_is_named(project): + db = project.database + _seed(project) + project.write_revision( + "bbbb", + { + "upgrade": 'op.execute("SELECT 1")', + "downgrade": "# ch-migrate: allow-non-idempotent Controlled downgrade backfill\n" + "op.execute(f'ALTER TABLE {db}.counter UPDATE x = x + 1 WHERE id = 5')", + }, + "aaaa", + ) + assert project.run("up", "it").exit_code == 0 + project.stop_merges(f"{db}.counter") + try: + with _running(project, ("down", "it")) as (process, log): + _until(lambda: _mutations(project), "downgrade mutation was not submitted") + process.kill() + finally: + project.client.command(f"SYSTEM START MERGES {db}.counter") + _until(lambda: all(row[2] for row in _mutations(project)), "downgrade mutation did not finish") + stopped = project.run("down", "it") + assert stopped.exit_code == 1, stopped.output + assert "was interrupted on an earlier run" in stopped.output + assert _heads(project) == ["bbbb"] + named = project.run("down", "it", "--rerun-interrupted", "bbbb") + assert named.exit_code == 0, named.output + assert _heads(project) == ["aaaa"] + + def test_failed_mutation_is_not_recorded_and_names_its_kill(project): _seed(project) _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = throwIf(x = 5) WHERE 1") @@ -69,7 +168,10 @@ def test_killed_mutation_then_fixed_sql_runs_again(project): ) _until(lambda: not any(row[4] for row in _mutations(project, "orders")), "kill did not land") _write_sql(path, "ALTER TABLE {db}.orders UPDATE qty = toUInt64OrZero(raw) WHERE 1") - fixed = project.run("up", "it") + # The killed backfill may have rewritten some parts, so the waived rerun is named. + stopped = project.run("up", "it") + assert stopped.exit_code == 1 and "was interrupted on an earlier run" in stopped.output + fixed = project.run("up", "it", "--rerun-interrupted", "bbbb") assert fixed.exit_code == 0, fixed.output rows = project.client.query(f"SELECT id, qty FROM {db}.orders ORDER BY id").result_rows assert rows == [(1, 3), (2, 0)] @@ -85,7 +187,7 @@ def test_rejected_statement_can_be_fixed_and_run_again(project): rejected.exit_code == 1 and "ALTER_OF_COLUMN_IS_FORBIDDEN" in rejected.output ), rejected.output assert _heads(project) == ["aaaa"] - _write_sql(path, "ALTER TABLE {db}.counter UPDATE x = 1 WHERE id = 1") + _write_sql(path, "ALTER TABLE {db}.counter ADD COLUMN IF NOT EXISTS note String", waived=False) fixed = project.run("up", "it") assert fixed.exit_code == 0, fixed.output assert _heads(project) == ["bbbb"] @@ -342,7 +444,7 @@ def test_mutation_on_a_down_replica_is_not_success(cluster_project, clickhouse_c cluster.start_node(2) for client in cluster.clients.values(): client.command(f"SYSTEM START MERGES {table}") - resumed = project.run("up", "it") + resumed = project.run("up", "it", "--rerun-interrupted", "bbbb") assert resumed.exit_code == 0, resumed.output assert _heads(project) == ["bbbb"] @@ -375,9 +477,9 @@ def _baseline(project): assert result.exit_code == 0, result.output -def _upgrade_sql(project, sql): +def _upgrade_sql(project, sql, waived=True): path = project.sql_dir / "mutation.sql" - _write_sql(path, sql) + _write_sql(path, sql, waived) project.write_revision( "bbbb", {"upgrade": "from ch_migrate import run_sql\nrun_sql('mutation.sql')"}, @@ -386,10 +488,13 @@ def _upgrade_sql(project, sql): return path -def _write_sql(path, sql): - path.write_text( - "-- ch-migrate: allow-non-idempotent Controlled waiting acceptance scenario\n" + sql + ";\n" - ) +def _write_sql(path, sql, waived=True): + waiver = "-- ch-migrate: allow-non-idempotent Controlled waiting acceptance scenario\n" + path.write_text((waiver if waived else "") + sql + ";\n") + + +def _markers(project): + return project.client.command(f"SELECT count() FROM {project.database}._ch_migrate_in_progress") def _mutations(project, table="counter"): diff --git a/tests/integration/test_on_cluster.py b/tests/integration/test_on_cluster.py index 2946a7d..ed345bb 100644 --- a/tests/integration/test_on_cluster.py +++ b/tests/integration/test_on_cluster.py @@ -34,6 +34,7 @@ def test_on_cluster_down_host_times_out_then_runs_again(cluster_project, clickho [ f"CREATE TABLE IF NOT EXISTS {{db}}.owned ON CLUSTER {cluster.name} (id UInt64) ENGINE=Memory" ], + waived=False, ) cluster.stop_node(2) try: @@ -90,7 +91,7 @@ def test_on_cluster_and_replicated_mutation_both_must_finish(cluster_project, cl finally: for client in cluster.clients.values(): client.command(f"SYSTEM START MERGES {table}") - resumed = project.run("up", "it") + resumed = project.run("up", "it", "--rerun-interrupted", "bbbb") assert resumed.exit_code == 0, resumed.output for client in cluster.clients.values(): assert client.command(f"SELECT x FROM {table}") == 1 @@ -111,6 +112,7 @@ def test_on_cluster_ttl_materialization_is_waited(cluster_project, clickhouse_cl _revision( project, [f"ALTER TABLE {{db}}.expiry ON CLUSTER {cluster.name} MODIFY TTL ts + INTERVAL 100 YEAR"], + waived=False, ) for client in cluster.clients.values(): client.command(f"SYSTEM STOP MERGES {table}") @@ -151,7 +153,7 @@ def test_on_cluster_nonreplicated_mutations_wait_on_each_host(cluster_project, c assert cluster.clients[2].command(f"SELECT x FROM {table}") == 0 finally: cluster.clients[2].command(f"SYSTEM START MERGES {table}") - resumed = project.run("up", "it") + resumed = project.run("up", "it", "--rerun-interrupted", "bbbb") assert resumed.exit_code == 0, resumed.output for client in cluster.clients.values(): assert client.command(f"SELECT x FROM {table}") == 1 @@ -178,14 +180,11 @@ def _baseline(project): assert result.exit_code == 0, result.output -def _revision(project, statements): +def _revision(project, statements, waived=True): + """Waived statements need --rerun-interrupted after an interruption; the rest rerun alone.""" + waiver = "-- ch-migrate: allow-non-idempotent Controlled distributed DDL scenario\n" path = project.sql_dir / "ddl.sql" - path.write_text( - "\n".join( - "-- ch-migrate: allow-non-idempotent Controlled distributed DDL scenario\n" + sql + ";" - for sql in statements - ) - ) + path.write_text("\n".join((waiver if waived else "") + sql + ";" for sql in statements)) project.write_revision( "bbbb", {"upgrade": "from ch_migrate import run_sql\nrun_sql('ddl.sql')"}, "aaaa" ) diff --git a/tests/test_reruns.py b/tests/test_reruns.py new file mode 100644 index 0000000..e599aed --- /dev/null +++ b/tests/test_reruns.py @@ -0,0 +1,123 @@ +"""Rerunning an interrupted step: which steps ch-migrate vouches for, and the marker table.""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from ch_migrate.waiting.reruns import InterruptedStep, RerunGuard, Step, unvouched_statements + + +def test_a_first_run_records_the_step_before_it_runs(tmp_path): + client = _Client(started=0) + _guard(client).begin(_step(tmp_path, "op.execute('SELECT 1')")) + assert any(sql.startswith("INSERT INTO `db`.`_ch_migrate_in_progress`") for sql in client.sent) + + +def test_an_interrupted_step_the_gate_passes_runs_again(tmp_path, caplog): + client = _Client(started=1) + step = _step(tmp_path, "op.execute('ALTER TABLE t ADD COLUMN IF NOT EXISTS c UInt8')") + _guard(client).begin(step) + assert "was interrupted on an earlier run; running it again" in caplog.text + assert any(sql.startswith("INSERT INTO") for sql in client.sent) + + +def test_an_interrupted_step_with_a_waiver_stops(tmp_path): + client = _Client(started=1) + step = _step( + tmp_path, + "# ch-migrate: allow-non-idempotent counted once per deploy\n" + " op.execute('ALTER TABLE t UPDATE n = n + 1 WHERE 1')", + ) + with pytest.raises(InterruptedStep) as stopped: + _guard(client).begin(step) + assert [item.why for item in stopped.value.unvouched] == ["waiver: counted once per deploy"] + assert "--rerun-interrupted bbbbbbbb" in str(stopped.value) + assert not any(sql.startswith("INSERT INTO") for sql in client.sent) + + +def test_naming_the_revision_lets_an_interrupted_step_run_again(tmp_path): + client = _Client(started=1) + step = _step(tmp_path, "op.execute(statement)") + _guard(client, acknowledged={"bbbbbbbb"}).begin(step) + assert any(sql.startswith("INSERT INTO") for sql in client.sent) + + +def test_a_recorded_step_clears_every_marker_for_its_revision(tmp_path): + client = _Client(started=0) + _guard(client).finish(_step(tmp_path, "op.execute('SELECT 1')")) + assert client.sent == [ + "ALTER TABLE `db`.`_ch_migrate_in_progress` DELETE WHERE revision = " + "'bbbbbbbbbbbb' SETTINGS mutations_sync = 2" + ] + + +def test_downgrades_are_judged_by_the_same_rules(tmp_path): + path = _revision( + tmp_path, + "op.execute('CREATE TABLE IF NOT EXISTS t (x UInt8) ENGINE = Memory')", + downgrade="op.execute('INSERT INTO t SELECT 1')", + ) + assert unvouched_statements(path, "upgrade") == [] + assert [item.why for item in unvouched_statements(path, "downgrade")] == [ + "not safe to run twice" + ] + + +@pytest.mark.parametrize( + ("engine", "cluster", "expected"), + [ + ("Atomic", None, "ENGINE = MergeTree ORDER BY"), + ("Shared", None, "ENGINE = MergeTree ORDER BY"), + ("Replicated", None, "ENGINE = ReplicatedMergeTree ORDER BY"), + ( + "Atomic", + "main", + "ON CLUSTER `main` (revision String, direction LowCardinality(String), started " + "DateTime64(3) DEFAULT now64(3)) ENGINE = ReplicatedMergeTree(" + "'/clickhouse/ch_migrate/db/{shard}/_ch_migrate_in_progress', '{replica}')", + ), + ], +) +def test_the_marker_table_follows_the_deployment(engine, cluster, expected): + state = _state(engine, cluster) + assert expected in RerunGuard(_Client(started=0), state, frozenset())._create_sql() + + +class _Client: + def __init__(self, started: int): + self.started, self.sent = started, [] + + def command(self, sql, parameters=None, settings=None): + if sql.startswith("EXISTS TABLE"): + return 1 + if sql.startswith("SELECT count()"): + return self.started + self.sent.append(sql) + return None + + +def _guard(client, acknowledged=()): + return RerunGuard(client, _state("Atomic", None), frozenset(acknowledged)) + + +def _state(engine, cluster): + on_cluster = cluster if engine == "Atomic" else None + return SimpleNamespace( + database="db", database_engine=engine, cluster=cluster, on_cluster=on_cluster + ) + + +def _step(tmp_path: Path, upgrade: str) -> Step: + return Step("bbbbbbbbbbbb", "upgrade", _revision(tmp_path, upgrade)) + + +def _revision(tmp_path: Path, upgrade: str, downgrade: str = "pass") -> Path: + versions = tmp_path / "migrations" / "versions" + versions.mkdir(parents=True, exist_ok=True) + path = versions / "bbbb.py" + path.write_text( + "revision = 'bbbbbbbbbbbb'\ndown_revision = None\n\n" + f"def upgrade():\n {upgrade}\n\ndef downgrade():\n {downgrade}\n" + ) + return path diff --git a/tests/test_waiting_display.py b/tests/test_waiting_display.py index b8317d8..0275674 100644 --- a/tests/test_waiting_display.py +++ b/tests/test_waiting_display.py @@ -8,8 +8,10 @@ from ch_migrate.migration_runner import Upgrade, _report_failure, _report_interrupt from ch_migrate.runner import _Progress from ch_migrate.sql import SqlStatementError +from ch_migrate.statements import MigrationStatement from ch_migrate.waiting.display import HostParts, MutationProgress, WaitDisplay from ch_migrate.waiting.errors import FailedMutation, MutationFailed, WaitTimeout +from ch_migrate.waiting.reruns import InterruptedStep, Step, Unvouched GIB = 1 << 30 @@ -117,6 +119,24 @@ def test_timeout_says_the_work_continues_and_how_to_resume(capsys): assert "ch-migrate up dev again: it waits for that work to finish" in err +def test_interrupted_step_names_what_it_cannot_vouch_for(capsys): + statement = MigrationStatement( + "ALTER TABLE {db}.events UPDATE n = n + 1 WHERE 1", + "migrations/sql/bump.up.sql", + 2, + (), + "upgrade", + ) + stop = InterruptedStep( + Step("3f9c2a1b5d6e", "upgrade", None), [Unvouched(statement, "waiver: counted once")] + ) + _report_failure(Upgrade("dev"), _current("3f9c2a1b bump"), stop) + err = capsys.readouterr().err + assert err.startswith("✗ 3f9c2a1b was interrupted on an earlier run") + assert "migrations/sql/bump.up.sql:2 waiver: counted once: ALTER TABLE {db}.events" in err + assert "ch-migrate up dev --rerun-interrupted 3f9c2a1b" in err + + def test_interrupt_says_the_work_continues(capsys): _report_interrupt("dev", _current("3f9c2a1b backfill")) err = capsys.readouterr().err From e53c723ff5dcb3e38d17a79b9f22911a7a8255d1 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 14:47:06 -0400 Subject: [PATCH 29/33] [ui] Say a timed-out rerun may stop for a waived statement The timeout and Ctrl-C hints promised that the next up runs the migration again. Since 7c070fa it does that only when the gate vouches for every statement; a waived backfill stops first and asks to be named. The hints now say so. Solves: DRY-1609 Tests: unit 686; piped and terminal progress integration tests --- ch_migrate/migration_runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ch_migrate/migration_runner.py b/ch_migrate/migration_runner.py index 8735c01..f444fc6 100644 --- a/ch_migrate/migration_runner.py +++ b/ch_migrate/migration_runner.py @@ -119,7 +119,8 @@ def _report_interrupt(environment: str, progress: _Progress) -> None: ui.hint( "Work ClickHouse already started keeps running, and the migration is not recorded as " f"applied. Run `ch-migrate up {environment}` again: it waits for that work to finish, " - "then runs the migration again.", + "then runs the migration again, or stops first if a waived statement may already have " + "applied.", stderr=True, ) @@ -174,7 +175,7 @@ def _report_timeout(environment: str, where: str, timeout: WaitTimeout) -> None: ui.hint( "ClickHouse keeps doing the work, and the migration is not recorded as applied. " f"Run `ch-migrate up {environment}` again: it waits for that work to finish, then runs " - "the migration again.", + "the migration again, or stops first if a waived statement may already have applied.", stderr=True, ) From 5fead891b3633007d5d1bd8b6cc1fc45ad9db1f7 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 14:53:42 -0400 Subject: [PATCH 30/33] [test] Wait for a failure reason to reach this replica On Cloud the test's own client can reach a replica that hasn't received the failed mutation's reason yet, so reading it once right after up exits found nothing (1 failure in the 7c070fa Cloud run). A probe showed the reason on every replica within 8 seconds; the test now polls for it. up itself is unaffected: it keeps waiting until its replica reports the failure. Solves: DRY-1609 Tests: the test on Docker, and three passing runs on Cloud --- tests/integration/test_mutation_waiting.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_mutation_waiting.py b/tests/integration/test_mutation_waiting.py index c630f18..3029298 100644 --- a/tests/integration/test_mutation_waiting.py +++ b/tests/integration/test_mutation_waiting.py @@ -141,7 +141,11 @@ def test_failed_mutation_is_not_recorded_and_names_its_kill(project): _upgrade_sql(project, "ALTER TABLE {db}.counter UPDATE x = throwIf(x = 5) WHERE 1") result = project.run("up", "it") assert result.exit_code == 1, result.output - failed = next(row for row in _mutations(project) if row[4]) + # On Cloud this client may reach a replica that hasn't received the failure reason yet. + failed = _until( + lambda: next((row for row in _mutations(project) if row[4]), None), + "the failed mutation's reason is not visible", + ) assert failed[0] in result.output and "throwIf" in result.output assert f"mutation_id = '{failed[0]}'" in result.output assert _heads(project) == ["aaaa"] From fa19ffed7eed9f2e144d971ec95739c91a080668 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 15:16:07 -0400 Subject: [PATCH 31/33] [style] Order the rerun guard code by call order Reordering only; every line is unchanged. RerunGuard is the reason reruns.py exists, so it now leads the module, followed by the value types in the order it uses them, and _acknowledged follows _started, as begin calls them. In env.py, _rerun_guard and _enforce_gate move below _checked_session and _post_migrate, which _run_connection calls first. In gate.py, lint_pending_up moves above print_gate_findings, matching enforce_up_gate's calls. Solves: python-style pass over DRY-1609 Tests: unit 686 on 3.10 and 3.14; Docker single node 58 passed, 4 skipped; cluster 16 passed --- ch_migrate/alembic_env.py | 48 +++++++++++++-------------- ch_migrate/gate.py | 30 ++++++++--------- ch_migrate/waiting/reruns.py | 64 ++++++++++++++++++------------------ 3 files changed, 71 insertions(+), 71 deletions(-) diff --git a/ch_migrate/alembic_env.py b/ch_migrate/alembic_env.py index a0d6aac..8d55bff 100644 --- a/ch_migrate/alembic_env.py +++ b/ch_migrate/alembic_env.py @@ -205,30 +205,6 @@ def _run_connection(connection, env_config: dict[str, Any], environment: str) -> waiter.close() -def _rerun_guard(client, state: VersionTableState, environment: str) -> RerunGuard: - attributes = context.config.attributes - # `ch-migrate up` passes --rerun-interrupted directly; `down` and a direct alembic - # command pass it as -x rerun_interrupted=REV[,REV]. - named = attributes.get("ch_migrate_rerun_interrupted") or context.get_x_argument( - as_dictionary=True - ).get("rerun_interrupted", "").split(",") - guard = RerunGuard(client, state, frozenset(name.strip() for name in named if name.strip())) - if not attributes.get("ch_migrate_gate_checked"): - # A direct `alembic upgrade` hasn't been through `ch-migrate up`'s gate. - guard.before_first_upgrade = partial(_enforce_gate, environment) - return guard - - -def _enforce_gate(environment: str) -> None: - from ch_migrate.gate import enforce_up_gate # lint imports this module - - try: - enforce_up_gate(Path.cwd(), environment) - except ui.Failure as failure: - failure.show() - raise SystemExit(1) from None - - def _checked_session(connection, env_config: dict[str, Any]) -> VersionTableState: # Establish the session before requiring its existence on every later request. connection.exec_driver_sql("SELECT 1") @@ -275,3 +251,27 @@ def after_revision(ctx, step, heads, run_args): ) return after_revision + + +def _rerun_guard(client, state: VersionTableState, environment: str) -> RerunGuard: + attributes = context.config.attributes + # `ch-migrate up` passes --rerun-interrupted directly; `down` and a direct alembic + # command pass it as -x rerun_interrupted=REV[,REV]. + named = attributes.get("ch_migrate_rerun_interrupted") or context.get_x_argument( + as_dictionary=True + ).get("rerun_interrupted", "").split(",") + guard = RerunGuard(client, state, frozenset(name.strip() for name in named if name.strip())) + if not attributes.get("ch_migrate_gate_checked"): + # A direct `alembic upgrade` hasn't been through `ch-migrate up`'s gate. + guard.before_first_upgrade = partial(_enforce_gate, environment) + return guard + + +def _enforce_gate(environment: str) -> None: + from ch_migrate.gate import enforce_up_gate # lint imports this module + + try: + enforce_up_gate(Path.cwd(), environment) + except ui.Failure as failure: + failure.show() + raise SystemExit(1) from None diff --git a/ch_migrate/gate.py b/ch_migrate/gate.py index 7f7cede..d28d9fe 100644 --- a/ch_migrate/gate.py +++ b/ch_migrate/gate.py @@ -26,6 +26,21 @@ def enforce_up_gate(project_root: Path, environment: str, skip_mv_check: bool = ) +def lint_pending_up( + project_root: Path, environment: str, skip_mv_check: bool = False +) -> LintReport: + load_dotenv(project_root / ".env.local") + config_path = project_root / "config.yaml" + config = LintConfig.from_config(load_config(config_path)) + if skip_mv_check: + config.rules["mv_declarations"] = Severity.OFF + versions = project_root / "migrations" / "versions" + graph = build_revision_graph(versions) + env_config = get_env_config(environment, config_path) + pending = pending_revisions(graph, get_migration_state(env_config).heads) + return lint_migrations(versions, config=config, revisions=pending) + + def print_gate_findings(report: LintReport) -> bool: blocked = False for finding in report.results: @@ -42,18 +57,3 @@ def print_gate_findings(report: LintReport) -> bool: if finding.statement: ui.detail(finding.statement, stderr=gate_error) return blocked - - -def lint_pending_up( - project_root: Path, environment: str, skip_mv_check: bool = False -) -> LintReport: - load_dotenv(project_root / ".env.local") - config_path = project_root / "config.yaml" - config = LintConfig.from_config(load_config(config_path)) - if skip_mv_check: - config.rules["mv_declarations"] = Severity.OFF - versions = project_root / "migrations" / "versions" - graph = build_revision_graph(versions) - env_config = get_env_config(environment, config_path) - pending = pending_revisions(graph, get_migration_state(env_config).heads) - return lint_migrations(versions, config=config, revisions=pending) diff --git a/ch_migrate/waiting/reruns.py b/ch_migrate/waiting/reruns.py index a814880..c6c2a26 100644 --- a/ch_migrate/waiting/reruns.py +++ b/ch_migrate/waiting/reruns.py @@ -27,35 +27,6 @@ logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class Step: - revision: str - direction: str # "upgrade" or "downgrade" - path: Path - - -@dataclass(frozen=True) -class Unvouched: - """A statement in an interrupted step that might not be safe to run again.""" - - statement: MigrationStatement - why: str - - -class InterruptedStep(WaitingError): - """An interrupted step has statements that might not be safe to run again.""" - - def __init__(self, step: Step, unvouched: list[Unvouched]): - self.step, self.unvouched = step, unvouched - listed = "; ".join(f"{_where(item.statement)} ({item.why})" for item in unvouched) - super().__init__( - f"{step.revision[:8]} was interrupted on an earlier run, and ch-migrate can't vouch " - f"for running it again: {listed}. Check what the earlier run changed, then run it " - f"again with --rerun-interrupted {step.revision[:8]} " - f"(with alembic, -x rerun_interrupted={step.revision[:8]})." - ) - - class RerunGuard: """Records each step as it starts and refuses an unsafe rerun of an interrupted one.""" @@ -101,9 +72,6 @@ def finish(self, step: Step) -> None: "SETTINGS mutations_sync = 2" ) - def _acknowledged(self, revision: str) -> bool: - return any(len(given) >= 4 and revision.startswith(given) for given in self.acknowledged) - def _prepare(self) -> None: if self._ready: return @@ -124,6 +92,9 @@ def _started(self, step: Step) -> bool: ) ) + def _acknowledged(self, revision: str) -> bool: + return any(len(given) >= 4 and revision.startswith(given) for given in self.acknowledged) + def _record(self, step: Step) -> None: self.client.command( f"INSERT INTO {self._table()} (revision, direction) " @@ -162,6 +133,35 @@ def _replicated(self) -> bool: ) +@dataclass(frozen=True) +class Step: + revision: str + direction: str # "upgrade" or "downgrade" + path: Path + + +class InterruptedStep(WaitingError): + """An interrupted step has statements that might not be safe to run again.""" + + def __init__(self, step: Step, unvouched: list[Unvouched]): + self.step, self.unvouched = step, unvouched + listed = "; ".join(f"{_where(item.statement)} ({item.why})" for item in unvouched) + super().__init__( + f"{step.revision[:8]} was interrupted on an earlier run, and ch-migrate can't vouch " + f"for running it again: {listed}. Check what the earlier run changed, then run it " + f"again with --rerun-interrupted {step.revision[:8]} " + f"(with alembic, -x rerun_interrupted={step.revision[:8]})." + ) + + +@dataclass(frozen=True) +class Unvouched: + """A statement in an interrupted step that might not be safe to run again.""" + + statement: MigrationStatement + why: str + + def unvouched_statements(path: Path, direction: str) -> list[Unvouched]: """Statements in this step the gate can't vouch for: waived, unreadable, or unsafe.""" found = [] From b3d93ec2326f3ae1cc7a120438e3bd6c1dce1dd7 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 15:18:54 -0400 Subject: [PATCH 32/33] [test] Name the kill step in the Cloud SIGKILL test The test reached 36 lines once it checked the stop and the named rerun. Starting up and killing it once its mutation runs is one step, so it moves to _kill_up_once_its_mutation_runs and the test reads as the scenario. Solves: python-style pass over DRY-1609 Tests: Cloud "integration and cloud" 4 passed (CLOUD_SIGKILL value=1, mutations=2); no leftover databases --- tests/integration/test_cloud.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_cloud.py b/tests/integration/test_cloud.py index 4b78791..15dcae5 100644 --- a/tests/integration/test_cloud.py +++ b/tests/integration/test_cloud.py @@ -89,19 +89,7 @@ def test_cloud_killed_up_waits_for_its_mutation_then_runs_again(project): """SIGKILL mid-wait; the waived UPDATE's rerun stops until named, then waits and resends.""" db = project.database _slow_update(project, "1", seconds=3) - log_path = project.root / "first-up.log" - with log_path.open("w") as log: - first = subprocess.Popen( - [sys.executable, "-m", "ch_migrate.cli", "up", "it"], - cwd=project.root, - stdout=log, - stderr=subprocess.STDOUT, - ) - try: - _until_running_mutation(project) - finally: - first.kill() - first.wait(timeout=10) + _kill_up_once_its_mutation_runs(project) stopped = project.run("up", "it") assert stopped.exit_code == 1, stopped.output assert "was interrupted on an earlier run" in stopped.output @@ -139,6 +127,21 @@ def _slow_update(project, value, seconds): ) +def _kill_up_once_its_mutation_runs(project): + with (project.root / "first-up.log").open("w") as log: + first = subprocess.Popen( + [sys.executable, "-m", "ch_migrate.cli", "up", "it"], + cwd=project.root, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + _until_running_mutation(project) + finally: + first.kill() + first.wait(timeout=10) + + def _until_running_mutation(project): deadline = time.monotonic() + 60 while time.monotonic() < deadline: From 3367b9d32dc3eb7b03a3eb31828e385244921e19 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Tue, 27 Oct 2026 15:39:30 -0400 Subject: [PATCH 33/33] [style] Format the package and tests with black Formatting only. pyproject.toml configures black at 100 columns, but 14 files in ch_migrate/ and tests/ weren't formatted, so each one touched in a later change dragged a reformat into that diff (as lint.py did in c07fcbc). Now `black --check ch_migrate tests` passes. The archived spike scripts under docs/design/spikes/ and the 0.4.1 env.py fixture stay as they were written. Solves: python-style pass over DRY-1609 Tests: unit 686 on 3.10-3.14; Docker single node 58 passed, 4 skipped; cluster 16 passed --- ch_migrate/deps.py | 48 +++--- ch_migrate/diff.py | 217 +++++++++++++++---------- ch_migrate/display.py | 4 +- ch_migrate/introspect.py | 27 +-- ch_migrate/mv_validate.py | 57 ++----- ch_migrate/scaffold.py | 8 +- ch_migrate/secrets.py | 3 +- tests/integration/test_lint_pending.py | 4 +- tests/test_deps.py | 14 +- tests/test_diff.py | 82 +++++++--- tests/test_hooks.py | 82 ++++++---- tests/test_lint.py | 96 +++++------ tests/test_mv_validate.py | 86 +++++----- tests/test_snapshot.py | 7 +- 14 files changed, 401 insertions(+), 334 deletions(-) diff --git a/ch_migrate/deps.py b/ch_migrate/deps.py index c355feb..a71a071 100644 --- a/ch_migrate/deps.py +++ b/ch_migrate/deps.py @@ -67,17 +67,21 @@ def validate_migration(sql: str, graph: DependencyGraph) -> list[MigrationWarnin data_flow_deps.append(edge.target) if schema_deps: - warnings.append(MigrationWarning( - severity="error", - message=f"DROP TABLE {table_name} breaks schema dependencies: {', '.join(schema_deps)}", - affected_objects=schema_deps, - )) + warnings.append( + MigrationWarning( + severity="error", + message=f"DROP TABLE {table_name} breaks schema dependencies: {', '.join(schema_deps)}", + affected_objects=schema_deps, + ) + ) if data_flow_deps: - warnings.append(MigrationWarning( - severity="warning", - message=f"DROP TABLE {table_name} breaks data flow to: {', '.join(data_flow_deps)}", - affected_objects=data_flow_deps, - )) + warnings.append( + MigrationWarning( + severity="warning", + message=f"DROP TABLE {table_name} breaks data flow to: {', '.join(data_flow_deps)}", + affected_objects=data_flow_deps, + ) + ) # Check DROP VIEW / DROP MATERIALIZED VIEW for m in _RE_DROP_VIEW.finditer(sql): @@ -86,11 +90,13 @@ def validate_migration(sql: str, graph: DependencyGraph) -> list[MigrationWarnin affected = graph.affected_by_drop(view_name) if affected: affected_names = [n.name for n in affected] - warnings.append(MigrationWarning( - severity="warning", - message=f"DROP VIEW {view_name} affects: {', '.join(affected_names)}", - affected_objects=affected_names, - )) + warnings.append( + MigrationWarning( + severity="warning", + message=f"DROP VIEW {view_name} affects: {', '.join(affected_names)}", + affected_objects=affected_names, + ) + ) # Check DROP DICTIONARY for m in _RE_DROP_DICT.finditer(sql): @@ -99,11 +105,13 @@ def validate_migration(sql: str, graph: DependencyGraph) -> list[MigrationWarnin affected = graph.affected_by_drop(dict_name) if affected: affected_names = [n.name for n in affected] - warnings.append(MigrationWarning( - severity="warning", - message=f"DROP DICTIONARY {dict_name} affects: {', '.join(affected_names)}", - affected_objects=affected_names, - )) + warnings.append( + MigrationWarning( + severity="warning", + message=f"DROP DICTIONARY {dict_name} affects: {', '.join(affected_names)}", + affected_objects=affected_names, + ) + ) return warnings diff --git a/ch_migrate/diff.py b/ch_migrate/diff.py index 1cd7ee8..f591b4c 100644 --- a/ch_migrate/diff.py +++ b/ch_migrate/diff.py @@ -52,48 +52,60 @@ def _compare_columns( remote_col = remote_map.get(name) if local_col and not remote_col: - diffs.append(FieldDiff( - field_name=f"column '{name}'", - local_value=local_col.type, - remote_value=None, - message=f"column '{name}' {local_col.type} is in the snapshot but not in the database", - )) + diffs.append( + FieldDiff( + field_name=f"column '{name}'", + local_value=local_col.type, + remote_value=None, + message=f"column '{name}' {local_col.type} is in the snapshot but not in the database", + ) + ) elif remote_col and not local_col: - diffs.append(FieldDiff( - field_name=f"column '{name}'", - local_value=None, - remote_value=remote_col.type, - message=f"column '{name}' {remote_col.type} is in the database but not in the snapshot", - )) + diffs.append( + FieldDiff( + field_name=f"column '{name}'", + local_value=None, + remote_value=remote_col.type, + message=f"column '{name}' {remote_col.type} is in the database but not in the snapshot", + ) + ) elif local_col and remote_col: if local_col.type != remote_col.type: - diffs.append(FieldDiff( - field_name=f"column '{name}' type", - local_value=local_col.type, - remote_value=remote_col.type, - message=f"column '{name}' type differs: {local_col.type} vs {remote_col.type}", - )) + diffs.append( + FieldDiff( + field_name=f"column '{name}' type", + local_value=local_col.type, + remote_value=remote_col.type, + message=f"column '{name}' type differs: {local_col.type} vs {remote_col.type}", + ) + ) if local_col.default_kind != remote_col.default_kind: - diffs.append(FieldDiff( - field_name=f"column '{name}' default_kind", - local_value=local_col.default_kind, - remote_value=remote_col.default_kind, - message=f"column '{name}' default kind differs", - )) + diffs.append( + FieldDiff( + field_name=f"column '{name}' default_kind", + local_value=local_col.default_kind, + remote_value=remote_col.default_kind, + message=f"column '{name}' default kind differs", + ) + ) if local_col.default_expr != remote_col.default_expr: - diffs.append(FieldDiff( - field_name=f"column '{name}' default_expr", - local_value=local_col.default_expr, - remote_value=remote_col.default_expr, - message=f"column '{name}' default expression differs", - )) + diffs.append( + FieldDiff( + field_name=f"column '{name}' default_expr", + local_value=local_col.default_expr, + remote_value=remote_col.default_expr, + message=f"column '{name}' default expression differs", + ) + ) if local_col.codec != remote_col.codec: - diffs.append(FieldDiff( - field_name=f"column '{name}' codec", - local_value=local_col.codec, - remote_value=remote_col.codec, - message=f"column '{name}' codec differs: {local_col.codec} vs {remote_col.codec}", - )) + diffs.append( + FieldDiff( + field_name=f"column '{name}' codec", + local_value=local_col.codec, + remote_value=remote_col.codec, + message=f"column '{name}' codec differs: {local_col.codec} vs {remote_col.codec}", + ) + ) return diffs @@ -104,51 +116,61 @@ def _compare_tables(local: TableDefinition, remote: TableDefinition) -> list[Fie # Engine if local.engine != remote.engine: - diffs.append(FieldDiff( - field_name="engine", - local_value=local.engine, - remote_value=remote.engine, - message=f"engine differs: {local.engine} vs {remote.engine}", - )) + diffs.append( + FieldDiff( + field_name="engine", + local_value=local.engine, + remote_value=remote.engine, + message=f"engine differs: {local.engine} vs {remote.engine}", + ) + ) # Columns diffs.extend(_compare_columns(local.columns, remote.columns)) # ORDER BY if local.order_by != remote.order_by: - diffs.append(FieldDiff( - field_name="order_by", - local_value=", ".join(local.order_by), - remote_value=", ".join(remote.order_by), - message=f"ORDER BY differs", - )) + diffs.append( + FieldDiff( + field_name="order_by", + local_value=", ".join(local.order_by), + remote_value=", ".join(remote.order_by), + message=f"ORDER BY differs", + ) + ) # PARTITION BY if local.partition_by != remote.partition_by: - diffs.append(FieldDiff( - field_name="partition_by", - local_value=local.partition_by, - remote_value=remote.partition_by, - message=f"PARTITION BY differs", - )) + diffs.append( + FieldDiff( + field_name="partition_by", + local_value=local.partition_by, + remote_value=remote.partition_by, + message=f"PARTITION BY differs", + ) + ) # TTL if local.ttl != remote.ttl: - diffs.append(FieldDiff( - field_name="ttl", - local_value=local.ttl, - remote_value=remote.ttl, - message=f"TTL differs", - )) + diffs.append( + FieldDiff( + field_name="ttl", + local_value=local.ttl, + remote_value=remote.ttl, + message=f"TTL differs", + ) + ) # Settings if local.settings != remote.settings: - diffs.append(FieldDiff( - field_name="settings", - local_value=str(local.settings), - remote_value=str(remote.settings), - message=f"SETTINGS differ", - )) + diffs.append( + FieldDiff( + field_name="settings", + local_value=str(local.settings), + remote_value=str(remote.settings), + message=f"SETTINGS differ", + ) + ) return diffs @@ -156,6 +178,7 @@ def _compare_tables(local: TableDefinition, remote: TableDefinition) -> list[Fie def _normalize_ddl(raw: str) -> str: """Normalize raw DDL for fallback string comparison.""" import re + s = raw.strip() s = re.sub(r"\s+", " ", s) return s @@ -164,12 +187,14 @@ def _normalize_ddl(raw: str) -> str: def _compare_raw_ddl(local_ddl: str, remote_ddl: str) -> list[FieldDiff]: """Fallback: normalized string comparison for unparseable objects.""" if _normalize_ddl(local_ddl) != _normalize_ddl(remote_ddl): - return [FieldDiff( - field_name="raw_ddl", - local_value=local_ddl[:200] if local_ddl else None, - remote_value=remote_ddl[:200] if remote_ddl else None, - message="DDL definition differs (raw comparison)", - )] + return [ + FieldDiff( + field_name="raw_ddl", + local_value=local_ddl[:200] if local_ddl else None, + remote_value=remote_ddl[:200] if remote_ddl else None, + message="DDL definition differs (raw comparison)", + ) + ] return [] @@ -200,18 +225,30 @@ def compare_schemas(local: Schema, live: Schema) -> list[SchemaDiff]: live_obj = live_objs.get(name) if local_obj and not live_obj: - results.append(SchemaDiff( - name=name, obj_type=obj_type, status=DiffStatus.LOCAL_ONLY, - )) + results.append( + SchemaDiff( + name=name, + obj_type=obj_type, + status=DiffStatus.LOCAL_ONLY, + ) + ) elif live_obj and not local_obj: - results.append(SchemaDiff( - name=name, obj_type=obj_type, status=DiffStatus.REMOTE_ONLY, - )) + results.append( + SchemaDiff( + name=name, + obj_type=obj_type, + status=DiffStatus.REMOTE_ONLY, + ) + ) else: # Both exist — compare field_diffs: list[FieldDiff] = [] - if obj_type == "table" and isinstance(local_obj, TableDefinition) and isinstance(live_obj, TableDefinition): + if ( + obj_type == "table" + and isinstance(local_obj, TableDefinition) + and isinstance(live_obj, TableDefinition) + ): field_diffs = _compare_tables(local_obj, live_obj) else: # Fallback to raw DDL comparison for views, MVs, dicts @@ -220,13 +257,21 @@ def compare_schemas(local: Schema, live: Schema) -> list[SchemaDiff]: field_diffs = _compare_raw_ddl(local_ddl, live_ddl) if field_diffs: - results.append(SchemaDiff( - name=name, obj_type=obj_type, - status=DiffStatus.MODIFIED, field_diffs=field_diffs, - )) + results.append( + SchemaDiff( + name=name, + obj_type=obj_type, + status=DiffStatus.MODIFIED, + field_diffs=field_diffs, + ) + ) else: - results.append(SchemaDiff( - name=name, obj_type=obj_type, status=DiffStatus.IN_SYNC, - )) + results.append( + SchemaDiff( + name=name, + obj_type=obj_type, + status=DiffStatus.IN_SYNC, + ) + ) return results diff --git a/ch_migrate/display.py b/ch_migrate/display.py index c90f915..3573282 100644 --- a/ch_migrate/display.py +++ b/ch_migrate/display.py @@ -326,7 +326,9 @@ def render_diff_report( remote_only = [d for d in diffs if d.status == DiffStatus.REMOTE_ONLY] if not (modified or local_only or remote_only): - console.print(Text.assemble(("✓ ", "green"), f"All {_count(len(in_sync), 'object')} in sync.")) + console.print( + Text.assemble(("✓ ", "green"), f"All {_count(len(in_sync), 'object')} in sync.") + ) return for d in modified: diff --git a/ch_migrate/introspect.py b/ch_migrate/introspect.py index 48a9c2b..d185d79 100644 --- a/ch_migrate/introspect.py +++ b/ch_migrate/introspect.py @@ -172,9 +172,7 @@ def affected_by_drop(self, name: str) -> list[ObjectNode]: _RE_SETTINGS = re.compile(r"SETTINGS\s+(.+)$", re.IGNORECASE | re.MULTILINE) _RE_CREATE_VIEW = re.compile( - r"CREATE\s+VIEW\s+(?:IF\s+NOT\s+EXISTS\s+)?" - r"(?:`?(\w+)`?\.)?`?(\w+)`?" - r"\s+AS\s+", + r"CREATE\s+VIEW\s+(?:IF\s+NOT\s+EXISTS\s+)?" r"(?:`?(\w+)`?\.)?`?(\w+)`?" r"\s+AS\s+", re.IGNORECASE, ) @@ -266,9 +264,7 @@ def _parse_single_column(line: str) -> ColumnDefinition | None: # Extract DEFAULT/MATERIALIZED/ALIAS default_kind = None default_expr = None - default_match = re.search( - r"\b(DEFAULT|MATERIALIZED|ALIAS)\s+(.+)$", rest, re.IGNORECASE - ) + default_match = re.search(r"\b(DEFAULT|MATERIALIZED|ALIAS)\s+(.+)$", rest, re.IGNORECASE) if default_match: default_kind = default_match.group(1).upper() default_expr = default_match.group(2).strip() @@ -481,15 +477,11 @@ def parse_create_dictionary(ddl: str) -> DictDefinition | None: source_type = source_match.group(1).lower() # Extract table from SOURCE(CLICKHOUSE(TABLE '...' DB '...')) - tbl_match = re.search( - r"TABLE\s+'([^']+)'", ddl[source_match.start() :], re.IGNORECASE - ) + tbl_match = re.search(r"TABLE\s+'([^']+)'", ddl[source_match.start() :], re.IGNORECASE) if tbl_match: source_table = tbl_match.group(1) - db_match = re.search( - r"DB\s+'([^']+)'", ddl[source_match.start() :], re.IGNORECASE - ) + db_match = re.search(r"DB\s+'([^']+)'", ddl[source_match.start() :], re.IGNORECASE) if db_match: source_db = db_match.group(1) @@ -548,9 +540,7 @@ def parse_create_statement( # --------------------------------------------------------------------------- -def list_objects( - client: Any, database: str, obj_type: str = "table" -) -> list[str]: +def list_objects(client: Any, database: str, obj_type: str = "table") -> list[str]: """List objects of a given type in a database. Args: @@ -575,16 +565,13 @@ def list_objects( }.get(obj_type, "1=1") result = client.query( - f"SELECT name FROM system.tables " - f"WHERE database = {{db:String}} AND {engine_filter}", + f"SELECT name FROM system.tables " f"WHERE database = {{db:String}} AND {engine_filter}", parameters={"db": database}, ) return [row[0] for row in result.result_rows] -def get_create_statement( - client: Any, database: str, name: str, obj_type: str = "table" -) -> str: +def get_create_statement(client: Any, database: str, name: str, obj_type: str = "table") -> str: """Get the CREATE statement for an object. Args: diff --git a/ch_migrate/mv_validate.py b/ch_migrate/mv_validate.py index 0ba25bf..50aaa6d 100644 --- a/ch_migrate/mv_validate.py +++ b/ch_migrate/mv_validate.py @@ -121,9 +121,7 @@ def _extract_create_date(path: Path) -> str | None: # Handles: {db}.table, db.table, table (no db prefix) # Handles backtick-quoted identifiers and comma-separated user lists _RE_GRANT_INSERT = re.compile( - r"GRANT\s+INSERT\s+ON\s+" - r"(?:(?:\{[^}]*\}|`?\w+`?)\.)?`?(\w+)`?" - r"\s+TO\s+(.+?)(?:;|\n|\Z)", + r"GRANT\s+INSERT\s+ON\s+" r"(?:(?:\{[^}]*\}|`?\w+`?)\.)?`?(\w+)`?" r"\s+TO\s+(.+?)(?:;|\n|\Z)", re.IGNORECASE, ) @@ -162,11 +160,7 @@ def _parse_user_list(users_str: str) -> list[str]: Handles backtick-quoted identifiers and strips whitespace. """ - return [ - u.strip().strip("`").lower() - for u in users_str.split(",") - if u.strip().strip("`") - ] + return [u.strip().strip("`").lower() for u in users_str.split(",") if u.strip().strip("`")] def _find_grant_inserts(content: str) -> list[tuple[str, str]]: @@ -203,8 +197,7 @@ def _find_row_policies(content: str) -> list[tuple[str, str]]: Returns list of (policy_name, table_name) tuples, lowercased. """ return [ - (m.group(1).lower(), m.group(2).lower()) - for m in _RE_CREATE_ROW_POLICY.finditer(content) + (m.group(1).lower(), m.group(2).lower()) for m in _RE_CREATE_ROW_POLICY.finditer(content) ] @@ -236,9 +229,7 @@ def _read_migration_sql_from_content(content: str, path: Path) -> str: """ sql_parts: list[str] = [] - for match in re.finditer( - r'op\.execute\(\s*(?:f?"""(.*?)"""|f?"([^"]*)")', content, re.DOTALL - ): + for match in re.finditer(r'op\.execute\(\s*(?:f?"""(.*?)"""|f?"([^"]*)")', content, re.DOTALL): sql_parts.append(match.group(1) or match.group(2) or "") for match in re.finditer(r'(?:read_sql|run_sql)\(\s*["\']([^"\']+)["\']', content): @@ -308,18 +299,13 @@ def _check_grant_inserts( target = decl.target_table.lower() for user in decl.inserting_users: - has_grant = any( - table == target and u == user.lower() for table, u in grant_inserts - ) + has_grant = any(table == target and u == user.lower() for table, u in grant_inserts) if not has_grant: errors.append( MVValidationError( file=file_name, mv_name=decl.mv_name, - message=( - f"Missing: GRANT INSERT ON {{db}}.{decl.target_table} " - f"TO {user}" - ), + message=(f"Missing: GRANT INSERT ON {{db}}.{decl.target_table} " f"TO {user}"), ) ) @@ -336,18 +322,13 @@ def _check_grant_selects( target = decl.target_table.lower() for user in decl.select_users: - has_grant = any( - table == target and u == user.lower() for table, u in grant_selects - ) + has_grant = any(table == target and u == user.lower() for table, u in grant_selects) if not has_grant: errors.append( MVValidationError( file=file_name, mv_name=decl.mv_name, - message=( - f"Missing: GRANT SELECT ON {{db}}.{decl.target_table} " - f"TO {user}" - ), + message=(f"Missing: GRANT SELECT ON {{db}}.{decl.target_table} " f"TO {user}"), ) ) @@ -365,8 +346,7 @@ def _check_source_rls( for user in decl.source_rls_users: has_policy = any( - table == source - and user.lower() in [u.strip() for u in users_str.split(",")] + table == source and user.lower() in [u.strip() for u in users_str.split(",")] for table, users_str in permissive_policies ) if not has_policy: @@ -395,8 +375,7 @@ def _check_target_rls( for policy_name in decl.target_rls_policies: has_policy = any( - table == target and policy_name.lower() in policy - for policy, table in row_policies + table == target and policy_name.lower() in policy for policy, table in row_policies ) if not has_policy: errors.append( @@ -531,24 +510,16 @@ def validate_mv_migrations( continue # Core check: GRANT INSERT on target for each inserting user - errors.extend( - _check_grant_inserts(decl, grant_inserts, file_name) - ) + errors.extend(_check_grant_inserts(decl, grant_inserts, file_name)) # Optional checks (only if the declaration includes them) if decl.select_users: - errors.extend( - _check_grant_selects(decl, grant_selects, file_name) - ) + errors.extend(_check_grant_selects(decl, grant_selects, file_name)) if decl.source_rls_users: - errors.extend( - _check_source_rls(decl, permissive_policies, file_name) - ) + errors.extend(_check_source_rls(decl, permissive_policies, file_name)) if decl.target_rls_policies: - errors.extend( - _check_target_rls(decl, row_policies, file_name) - ) + errors.extend(_check_target_rls(decl, row_policies, file_name)) return errors diff --git a/ch_migrate/scaffold.py b/ch_migrate/scaffold.py index bb18028..958e10f 100644 --- a/ch_migrate/scaffold.py +++ b/ch_migrate/scaffold.py @@ -31,9 +31,7 @@ def fetch_current_ddl(env_config: dict[str, Any], table_name: str) -> str | None return None -def find_dependent_dictionaries( - env_config: dict[str, Any], table_name: str -) -> list[str]: +def find_dependent_dictionaries(env_config: dict[str, Any], table_name: str) -> list[str]: """Find dictionaries that use this table as a source. Queries system.dictionaries to find any dictionary whose source @@ -91,9 +89,7 @@ def _make_shadow_ddl(ddl: str, table_name: str) -> str: return shadow -def generate_exchange_sql( - table_name: str, current_ddl: str | None = None -) -> str: +def generate_exchange_sql(table_name: str, current_ddl: str | None = None) -> str: """Generate the SQL file content for an EXCHANGE TABLES migration. This creates the shadow table DDL that the user should modify with diff --git a/ch_migrate/secrets.py b/ch_migrate/secrets.py index e90577e..b54a725 100644 --- a/ch_migrate/secrets.py +++ b/ch_migrate/secrets.py @@ -28,8 +28,7 @@ def _get_ssm_client(region: Optional[str] = None): # type: ignore[no-untyped-de import boto3 # type: ignore[import-not-found,import-untyped] except ImportError: raise ImportError( - "boto3 is required for SSM support. " - "Install with: pip install ch-migrate-cli[ssm]" + "boto3 is required for SSM support. " "Install with: pip install ch-migrate-cli[ssm]" ) if region: return boto3.client("ssm", region_name=region) diff --git a/tests/integration/test_lint_pending.py b/tests/integration/test_lint_pending.py index d549542..5abdaad 100644 --- a/tests/integration/test_lint_pending.py +++ b/tests/integration/test_lint_pending.py @@ -12,9 +12,7 @@ def test_lint_pending_reports_only_second_revision(project, monkeypatch): (project.sql_dir / "b.sql").write_text( "-- pending change\n\nDROP TABLE IF EXISTS {db}.pending_only;\n" ) - project.write_revision( - "aaaa", {"upgrade": "from ch_migrate import run_sql\nrun_sql('a.sql')"} - ) + project.write_revision("aaaa", {"upgrade": "from ch_migrate import run_sql\nrun_sql('a.sql')"}) project.write_revision( "bbbb", {"upgrade": "from ch_migrate import run_sql\nrun_sql('b.sql')"}, diff --git a/tests/test_deps.py b/tests/test_deps.py index 36f7feb..00eb217 100644 --- a/tests/test_deps.py +++ b/tests/test_deps.py @@ -34,7 +34,9 @@ def _make_graph() -> DependencyGraph: graph.edges = [ DependencyEdge(source="events", target="hourly_events", dep_type=DepType.DATA_FLOW), DependencyEdge(source="events", target="hourly_events", dep_type=DepType.SCHEMA), - DependencyEdge(source="hourly_events", target="hourly_events_dest", dep_type=DepType.DATA_FLOW), + DependencyEdge( + source="hourly_events", target="hourly_events_dest", dep_type=DepType.DATA_FLOW + ), DependencyEdge(source="users", target="dict_users", dep_type=DepType.SCHEMA), ] return graph @@ -136,6 +138,7 @@ def test_renders_empty_graph(self): def test_shows_edge_count(self): import re as _re + output = StringIO() console = Console(file=output, force_terminal=True, width=100) graph = _make_graph() @@ -207,13 +210,15 @@ def test_handles_circular_dependency(self): @pytest.fixture def deps_runner(tmp_path: Path, monkeypatch): monkeypatch.chdir(tmp_path) - (tmp_path / "config.yaml").write_text(""" + (tmp_path / "config.yaml").write_text( + """ environments: dev: host: localhost database: testdb user: test -""") +""" + ) monkeypatch.setenv("CH_DEV_PASSWORD", "pass") return CliRunner() @@ -226,6 +231,7 @@ def test_shows_graph(self, mock_deps, mock_client, deps_runner): mock_deps.return_value = _make_graph() from ch_migrate.cli import main + result = deps_runner.invoke(main, ["deps", "dev"]) assert result.exit_code == 0 assert "Dependency Graph" in result.output @@ -240,6 +246,7 @@ def test_validate_safe_migration(self, mock_deps, mock_client, deps_runner, tmp_ sql_file.write_text("ALTER TABLE users ADD COLUMN email String") from ch_migrate.cli import main + result = deps_runner.invoke(main, ["deps", "dev", "--validate", str(sql_file)]) assert result.exit_code == 0 assert "passed" in result.output @@ -254,5 +261,6 @@ def test_validate_breaking_migration(self, mock_deps, mock_client, deps_runner, sql_file.write_text("DROP TABLE events") from ch_migrate.cli import main + result = deps_runner.invoke(main, ["deps", "dev", "--validate", str(sql_file)]) assert result.exit_code == 1 diff --git a/tests/test_diff.py b/tests/test_diff.py index 6ad6996..85926d8 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -84,40 +84,52 @@ def test_detects_added_column(self): local = Schema(database="db") live = Schema(database="db") local.tables["users"] = self._base_table() - live.tables["users"] = self._base_table(columns=[ - ColumnDefinition(name="id", type="UInt64"), - ColumnDefinition(name="name", type="String"), - ColumnDefinition(name="email", type="String"), - ]) + live.tables["users"] = self._base_table( + columns=[ + ColumnDefinition(name="id", type="UInt64"), + ColumnDefinition(name="name", type="String"), + ColumnDefinition(name="email", type="String"), + ] + ) diffs = compare_schemas(local, live) modified = [d for d in diffs if d.status == DiffStatus.MODIFIED] assert len(modified) == 1 - assert any("email" in fd.message and "in the database but not in the snapshot" in fd.message for fd in modified[0].field_diffs) + assert any( + "email" in fd.message and "in the database but not in the snapshot" in fd.message + for fd in modified[0].field_diffs + ) def test_detects_removed_column(self): local = Schema(database="db") live = Schema(database="db") - local.tables["users"] = self._base_table(columns=[ - ColumnDefinition(name="id", type="UInt64"), - ColumnDefinition(name="name", type="String"), - ColumnDefinition(name="phone", type="String"), - ]) + local.tables["users"] = self._base_table( + columns=[ + ColumnDefinition(name="id", type="UInt64"), + ColumnDefinition(name="name", type="String"), + ColumnDefinition(name="phone", type="String"), + ] + ) live.tables["users"] = self._base_table() diffs = compare_schemas(local, live) modified = [d for d in diffs if d.status == DiffStatus.MODIFIED] assert len(modified) == 1 - assert any("phone" in fd.message and "in the snapshot but not in the database" in fd.message for fd in modified[0].field_diffs) + assert any( + "phone" in fd.message and "in the snapshot but not in the database" in fd.message + for fd in modified[0].field_diffs + ) def test_detects_column_type_change(self): local = Schema(database="db") live = Schema(database="db") local.tables["users"] = self._base_table() - live.tables["users"] = self._base_table(columns=[ - ColumnDefinition(name="id", type="UInt64"), - ColumnDefinition(name="name", type="Nullable(String)"), - ]) + live.tables["users"] = self._base_table( + columns=[ + ColumnDefinition(name="id", type="UInt64"), + ColumnDefinition(name="name", type="Nullable(String)"), + ] + ) diffs = compare_schemas(local, live) modified = [d for d in diffs if d.status == DiffStatus.MODIFIED] @@ -193,8 +205,12 @@ def test_detects_partition_by_change(self): def test_views_fallback_to_raw_ddl(self): local = Schema(database="db") live = Schema(database="db") - local.views["v"] = ViewDefinition(name="v", select_query="SELECT 1", raw_ddl="CREATE VIEW v AS SELECT 1") - live.views["v"] = ViewDefinition(name="v", select_query="SELECT 2", raw_ddl="CREATE VIEW v AS SELECT 2") + local.views["v"] = ViewDefinition( + name="v", select_query="SELECT 1", raw_ddl="CREATE VIEW v AS SELECT 1" + ) + live.views["v"] = ViewDefinition( + name="v", select_query="SELECT 2", raw_ddl="CREATE VIEW v AS SELECT 2" + ) diffs = compare_schemas(local, live) modified = [d for d in diffs if d.status == DiffStatus.MODIFIED] @@ -218,7 +234,9 @@ def test_mixed_object_types(self): local.tables["t1"] = self._base_table("t1") live.tables["t1"] = self._base_table("t1") - local.views["v1"] = ViewDefinition(name="v1", select_query="SELECT 1", raw_ddl="CREATE VIEW v1 AS SELECT 1") + local.views["v1"] = ViewDefinition( + name="v1", select_query="SELECT 1", raw_ddl="CREATE VIEW v1 AS SELECT 1" + ) # v1 is local_only (not in live) live.dictionaries["d1"] = DictDefinition(name="d1", raw_ddl="CREATE DICTIONARY d1 (...)") @@ -239,13 +257,15 @@ def test_mixed_object_types(self): @pytest.fixture def diff_runner(tmp_path: Path, monkeypatch): monkeypatch.chdir(tmp_path) - (tmp_path / "config.yaml").write_text(""" + (tmp_path / "config.yaml").write_text( + """ environments: dev: host: localhost database: testdb user: test -""") +""" + ) monkeypatch.setenv("CH_DEV_PASSWORD", "pass") # Create a snapshot @@ -277,6 +297,7 @@ def test_in_sync_exits_0(self, mock_schema, mock_client, diff_runner, tmp_path): mock_schema.return_value = live from ch_migrate.cli import main + result = diff_runner.invoke(main, ["diff", "dev"]) assert result.exit_code == 0 @@ -298,6 +319,7 @@ def test_drift_exits_1(self, mock_schema, mock_client, diff_runner, tmp_path): mock_schema.return_value = live from ch_migrate.cli import main + result = diff_runner.invoke(main, ["diff", "dev"]) assert result.exit_code == 1 @@ -305,16 +327,19 @@ def test_drift_exits_1(self, mock_schema, mock_client, diff_runner, tmp_path): @patch("ch_migrate.introspect.get_live_schema") def test_no_snapshot_exits_1(self, mock_schema, mock_client, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) - (tmp_path / "config.yaml").write_text(""" + (tmp_path / "config.yaml").write_text( + """ environments: dev: host: localhost database: testdb user: test -""") +""" + ) monkeypatch.setenv("CH_DEV_PASSWORD", "pass") from ch_migrate.cli import main + runner = CliRunner() result = runner.invoke(main, ["diff", "dev"]) assert result.exit_code == 1 @@ -342,9 +367,14 @@ def test_render_with_drift(self): console = Console(file=output, force_terminal=True, width=80) diffs = [ - SchemaDiff(name="users", obj_type="table", status=DiffStatus.MODIFIED, field_diffs=[ - FieldDiff("engine", "MergeTree", "ReplacingMergeTree", "engine differs"), - ]), + SchemaDiff( + name="users", + obj_type="table", + status=DiffStatus.MODIFIED, + field_diffs=[ + FieldDiff("engine", "MergeTree", "ReplacingMergeTree", "engine differs"), + ], + ), SchemaDiff(name="new_table", obj_type="table", status=DiffStatus.REMOTE_ONLY), SchemaDiff(name="old_view", obj_type="view", status=DiffStatus.LOCAL_ONLY), ] diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 2261a31..9abc30a 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -22,38 +22,46 @@ def test_from_empty_dict(self): assert not registry.has_hooks def test_from_post_migrate_only(self): - registry = HookRegistry.from_config({ - "post_migrate": [ - "SYSTEM RELOAD DICTIONARY {db}.dict_regions ON CLUSTER default", - ] - }) + registry = HookRegistry.from_config( + { + "post_migrate": [ + "SYSTEM RELOAD DICTIONARY {db}.dict_regions ON CLUSTER default", + ] + } + ) assert registry.pre_migrate == [] assert len(registry.post_migrate) == 1 assert registry.has_hooks def test_from_pre_and_post(self): - registry = HookRegistry.from_config({ - "pre_migrate": ["SELECT 1"], - "post_migrate": [ - "SYSTEM RELOAD DICTIONARY {db}.dict_a ON CLUSTER default", - "SELECT count() FROM {db}.users", - ], - }) + registry = HookRegistry.from_config( + { + "pre_migrate": ["SELECT 1"], + "post_migrate": [ + "SYSTEM RELOAD DICTIONARY {db}.dict_a ON CLUSTER default", + "SELECT count() FROM {db}.users", + ], + } + ) assert len(registry.pre_migrate) == 1 assert len(registry.post_migrate) == 2 assert registry.has_hooks def test_coerces_single_string_to_list(self): - registry = HookRegistry.from_config({ - "post_migrate": "SYSTEM RELOAD DICTIONARY {db}.dict_a ON CLUSTER default", - }) + registry = HookRegistry.from_config( + { + "post_migrate": "SYSTEM RELOAD DICTIONARY {db}.dict_a ON CLUSTER default", + } + ) assert len(registry.post_migrate) == 1 def test_handles_none_values(self): - registry = HookRegistry.from_config({ - "pre_migrate": None, - "post_migrate": None, - }) + registry = HookRegistry.from_config( + { + "pre_migrate": None, + "post_migrate": None, + } + ) assert registry.pre_migrate == [] assert registry.post_migrate == [] @@ -72,9 +80,7 @@ def test_executes_hooks_in_order(self): assert connection.commit.call_count == 2 # Verify the SQL was resolved - executed_sql = [ - str(c.args[0]) for c in connection.execute.call_args_list - ] + executed_sql = [str(c.args[0]) for c in connection.execute.call_args_list] assert "SYSTEM RELOAD DICTIONARY mydb.dict_a ON CLUSTER default" in executed_sql[0] assert "SELECT count() FROM mydb.users" in executed_sql[1] @@ -125,7 +131,8 @@ def test_hooks_parsed_from_config_yaml(self, tmp_path: Path, monkeypatch): from ch_migrate.config import get_env_config config_file = tmp_path / "config.yaml" - config_file.write_text(""" + config_file.write_text( + """ environments: dev: host: localhost @@ -135,7 +142,8 @@ def test_hooks_parsed_from_config_yaml(self, tmp_path: Path, monkeypatch): hooks: post_migrate: - "SYSTEM RELOAD DICTIONARY {db}.dict_regions ON CLUSTER default" -""") +""" + ) monkeypatch.setenv("CH_DEV_PASSWORD", "pass") env_config = get_env_config("dev", config_file) @@ -145,20 +153,25 @@ def test_hooks_parsed_from_config_yaml(self, tmp_path: Path, monkeypatch): registry = HookRegistry.from_config(env_config.get("hooks")) assert registry.has_hooks - assert registry.post_migrate[0] == "SYSTEM RELOAD DICTIONARY {db}.dict_regions ON CLUSTER default" + assert ( + registry.post_migrate[0] + == "SYSTEM RELOAD DICTIONARY {db}.dict_regions ON CLUSTER default" + ) def test_no_hooks_section_works(self, tmp_path: Path, monkeypatch): """Existing configs without hooks section continue to work.""" from ch_migrate.config import get_env_config config_file = tmp_path / "config.yaml" - config_file.write_text(""" + config_file.write_text( + """ environments: dev: host: localhost database: testdb user: test -""") +""" + ) monkeypatch.setenv("CH_DEV_PASSWORD", "pass") env_config = get_env_config("dev", config_file) @@ -174,14 +187,18 @@ class TestHookExecutionOrder: def test_pre_hooks_fire_before_post_hooks(self): """Pre-migrate hooks should fire before post-migrate hooks.""" connection = MagicMock() - registry = HookRegistry.from_config({ - "pre_migrate": ["SELECT 'pre'"], - "post_migrate": ["SELECT 'post'"], - }) + registry = HookRegistry.from_config( + { + "pre_migrate": ["SELECT 'pre'"], + "post_migrate": ["SELECT 'post'"], + } + ) # Simulate the env.py execution order: pre-hooks, then post-hooks run_hooks(connection, registry.pre_migrate, db="mydb", phase="pre_migrate", revision="all") - run_hooks(connection, registry.post_migrate, db="mydb", phase="post_migrate", revision="abc123") + run_hooks( + connection, registry.post_migrate, db="mydb", phase="post_migrate", revision="abc123" + ) assert connection.execute.call_count == 2 calls = [str(c.args[0]) for c in connection.execute.call_args_list] @@ -224,6 +241,7 @@ def test_upgrade_env_creates_new_env_py(self, tmp_path: Path): # command is registered and validates missing migrations dir # Test with no migrations dir in cwd assert result.exit_code != 0 or "Updated" in result.output or "not found" in result.output + def test_upgrade_env_backs_up_existing(self, tmp_path: Path, monkeypatch): """upgrade-env creates a .bak backup of existing env.py.""" from click.testing import CliRunner diff --git a/tests/test_lint.py b/tests/test_lint.py index bafdc68..5797e3a 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -42,25 +42,29 @@ def test_from_config_empty(self): assert config.large_table_threshold == 100_000_000 def test_from_config_with_values(self): - config = LintConfig.from_config({ - "lint": { - "large_table_threshold": 50_000, - "rules": { - "destructive_changes": "error", - "missing_on_cluster": "off", - }, + config = LintConfig.from_config( + { + "lint": { + "large_table_threshold": 50_000, + "rules": { + "destructive_changes": "error", + "missing_on_cluster": "off", + }, + } } - }) + ) assert config.large_table_threshold == 50_000 assert config.rules["destructive_changes"] == Severity.ERROR assert config.rules["missing_on_cluster"] == Severity.OFF def test_from_config_invalid_severity_ignored(self): - config = LintConfig.from_config({ - "lint": { - "rules": {"destructive_changes": "invalid_value"}, + config = LintConfig.from_config( + { + "lint": { + "rules": {"destructive_changes": "invalid_value"}, + } } - }) + ) assert "destructive_changes" not in config.rules @@ -77,10 +81,12 @@ def test_empty_report(self): assert report.warning_count == 0 def test_report_with_errors(self): - report = LintReport(results=[ - LintResult(rule="test", message="bad", severity=Severity.ERROR), - LintResult(rule="test", message="meh", severity=Severity.WARN), - ]) + report = LintReport( + results=[ + LintResult(rule="test", message="bad", severity=Severity.ERROR), + LintResult(rule="test", message="meh", severity=Severity.WARN), + ] + ) assert report.has_errors assert report.error_count == 1 assert report.warning_count == 1 @@ -111,10 +117,12 @@ def test_no_flags_on_safe_sql(self): assert results == [] def test_multiple_drops(self): - sql = textwrap.dedent("""\ + sql = textwrap.dedent( + """\ DROP TABLE mydb.old_events; ALTER TABLE mydb.users DROP COLUMN phone; - """) + """ + ) results = DestructiveChangeRule().check(sql) assert len(results) == 2 @@ -191,25 +199,29 @@ def test_passes_create_mv_if_not_exists(self): class TestReservedWordRule: def test_flags_reserved_column_name(self): - sql = textwrap.dedent("""\ + sql = textwrap.dedent( + """\ CREATE TABLE mydb.t ( `id` UInt64, `key` String, `select` String ) - """) + """ + ) results = ReservedWordRule().check(sql) reserved_names = {r.message.split("'")[1] for r in results} assert "key" in reserved_names assert "select" in reserved_names def test_passes_non_reserved_names(self): - sql = textwrap.dedent("""\ + sql = textwrap.dedent( + """\ CREATE TABLE mydb.t ( `user_id` UInt64, `event_name` String ) - """) + """ + ) results = ReservedWordRule().check(sql) assert results == [] @@ -251,10 +263,12 @@ def test_passes_with_placeholder(self): def test_flags_alter_and_drop(self): config = LintConfig(rules={"missing_on_cluster": Severity.WARN}) - sql = textwrap.dedent("""\ + sql = textwrap.dedent( + """\ ALTER TABLE mydb.t ADD COLUMN foo String; DROP TABLE mydb.t; - """) + """ + ) results = MissingOnClusterRule().check(sql, config=config) assert len(results) == 2 @@ -275,27 +289,21 @@ def _make_client(self, row_count: int) -> MagicMock: def test_flags_large_table(self): client = self._make_client(200_000_000) sql = "ALTER TABLE mydb.users ADD COLUMN phone String" - results = LargeTableMutationRule().check( - sql, client=client, database="mydb" - ) + results = LargeTableMutationRule().check(sql, client=client, database="mydb") assert len(results) == 1 assert "200,000,000" in results[0].message def test_passes_small_table(self): client = self._make_client(1000) sql = "ALTER TABLE mydb.users ADD COLUMN phone String" - results = LargeTableMutationRule().check( - sql, client=client, database="mydb" - ) + results = LargeTableMutationRule().check(sql, client=client, database="mydb") assert results == [] def test_respects_custom_threshold(self): client = self._make_client(5000) config = LintConfig(large_table_threshold=1000) sql = "ALTER TABLE mydb.users ADD COLUMN phone String" - results = LargeTableMutationRule().check( - sql, client=client, database="mydb", config=config - ) + results = LargeTableMutationRule().check(sql, client=client, database="mydb", config=config) assert len(results) == 1 def test_skips_without_client(self): @@ -339,9 +347,7 @@ def test_flags_drop_on_mv_source(self): sql = "DROP TABLE IF EXISTS events" with patch("ch_migrate.introspect.get_dependencies", return_value=graph): - results = MVDependencyRule().check( - sql, client=client, database="mydb" - ) + results = MVDependencyRule().check(sql, client=client, database="mydb") assert len(results) == 1 assert "hourly_mv" in results[0].message @@ -353,9 +359,7 @@ def test_passes_on_unrelated_table(self): sql = "DROP TABLE IF EXISTS unrelated_table" with patch("ch_migrate.introspect.get_dependencies", return_value=graph): - results = MVDependencyRule().check( - sql, client=client, database="mydb" - ) + results = MVDependencyRule().check(sql, client=client, database="mydb") assert results == [] @@ -400,7 +404,8 @@ def _create_migration(self, tmp_path: Path, name: str, sql_content: str) -> Path versions_dir.mkdir(exist_ok=True) rev_id = name[:12].ljust(12, "0") - content = textwrap.dedent(f"""\ + content = textwrap.dedent( + f"""\ \"\"\"Migration {name} Revision ID: {rev_id} @@ -418,16 +423,15 @@ def upgrade(): def downgrade(): pass - """) + """ + ) file_path = versions_dir / f"{rev_id}_{name}.py" file_path.write_text(content) return versions_dir def test_static_lint_finds_issues(self, tmp_path: Path): - versions_dir = self._create_migration( - tmp_path, "drop_users", "DROP TABLE mydb.users" - ) + versions_dir = self._create_migration(tmp_path, "drop_users", "DROP TABLE mydb.users") report = lint_migrations(versions_dir) assert report.warning_count > 0 @@ -442,9 +446,7 @@ def test_static_lint_clean(self, tmp_path: Path): assert report.warning_count == 0 def test_lint_with_error_severity(self, tmp_path: Path): - versions_dir = self._create_migration( - tmp_path, "drop_bad", "DROP TABLE mydb.users" - ) + versions_dir = self._create_migration(tmp_path, "drop_bad", "DROP TABLE mydb.users") config = LintConfig(rules={"destructive_changes": Severity.ERROR}) report = lint_migrations(versions_dir, config=config) assert report.has_errors diff --git a/tests/test_mv_validate.py b/tests/test_mv_validate.py index 9916925..ba97eba 100644 --- a/tests/test_mv_validate.py +++ b/tests/test_mv_validate.py @@ -85,23 +85,27 @@ def _write_migration( parts.append("") parts.append(extra_python) - parts.extend([ - "", - "def upgrade() -> None:", - " db = get_db()", - ]) + parts.extend( + [ + "", + "def upgrade() -> None:", + " db = get_db()", + ] + ) if sql: parts.append(f' op.execute(f"""{sql}""")') else: parts.append(" pass") - parts.extend([ - "", - "def downgrade() -> None:", - " pass", - "", - ]) + parts.extend( + [ + "", + "def downgrade() -> None:", + " pass", + "", + ] + ) content = "\n".join(parts) file_path = versions_dir / f"{rev_id}_{name}.py" @@ -116,9 +120,7 @@ def _write_migration( class TestCreateMVDetection: def test_detects_create_mv(self): - assert _sql_has_create_mv( - "CREATE MATERIALIZED VIEW mydb.mv TO mydb.dest AS SELECT 1" - ) + assert _sql_has_create_mv("CREATE MATERIALIZED VIEW mydb.mv TO mydb.dest AS SELECT 1") def test_detects_create_mv_if_not_exists(self): assert _sql_has_create_mv( @@ -126,9 +128,7 @@ def test_detects_create_mv_if_not_exists(self): ) def test_case_insensitive(self): - assert _sql_has_create_mv( - "create materialized view mydb.mv TO mydb.dest AS SELECT 1" - ) + assert _sql_has_create_mv("create materialized view mydb.mv TO mydb.dest AS SELECT 1") def test_ignores_regular_view(self): assert not _sql_has_create_mv("CREATE VIEW mydb.v AS SELECT 1") @@ -142,9 +142,7 @@ def test_ignores_empty(self): assert not _sql_has_create_mv("") def test_ignores_drop_mv(self): - assert not _sql_has_create_mv( - "DROP MATERIALIZED VIEW IF EXISTS mydb.mv" - ) + assert not _sql_has_create_mv("DROP MATERIALIZED VIEW IF EXISTS mydb.mv") class TestGrantInsertDetection: @@ -153,16 +151,16 @@ def test_finds_basic_grant(self): assert ("my_table", "my_user") in grants def test_finds_db_placeholder(self): - grants = _find_grant_inserts( - "GRANT INSERT ON {db}.my_table TO my_user" - ) + grants = _find_grant_inserts("GRANT INSERT ON {db}.my_table TO my_user") assert ("my_table", "my_user") in grants def test_finds_multiple(self): - sql = textwrap.dedent("""\ + sql = textwrap.dedent( + """\ GRANT INSERT ON {db}.table_a TO user_a; GRANT INSERT ON {db}.table_b TO user_b; - """) + """ + ) grants = _find_grant_inserts(sql) assert ("table_a", "user_a") in grants assert ("table_b", "user_b") in grants @@ -172,9 +170,7 @@ def test_case_insensitive(self): assert ("mytable", "myuser") in grants def test_comma_separated_users(self): - grants = _find_grant_inserts( - "GRANT INSERT ON {db}.my_table TO user_a, user_b;" - ) + grants = _find_grant_inserts("GRANT INSERT ON {db}.my_table TO user_a, user_b;") assert ("my_table", "user_a") in grants assert ("my_table", "user_b") in grants @@ -189,17 +185,13 @@ def test_finds_basic_grant(self): assert ("my_table", "reader") in grants def test_finds_column_grant(self): - grants = _find_grant_selects( - "GRANT SELECT(col1, col2) ON {db}.my_table TO reader" - ) + grants = _find_grant_selects("GRANT SELECT(col1, col2) ON {db}.my_table TO reader") assert ("my_table", "reader") in grants class TestRowPolicyDetection: def test_finds_basic_policy(self): - policies = _find_row_policies( - "CREATE ROW POLICY my_deny ON {db}.my_table USING 0 TO ALL" - ) + policies = _find_row_policies("CREATE ROW POLICY my_deny ON {db}.my_table USING 0 TO ALL") assert ("my_deny", "my_table") in policies def test_finds_if_not_exists(self): @@ -837,7 +829,8 @@ def test_mv_in_sql_file_detected(self, tmp_path: Path): versions.mkdir(parents=True, exist_ok=True) rev_id = "abc123456789" - content = textwrap.dedent(f"""\ + content = textwrap.dedent( + f"""\ \"\"\"create my_mv Revision ID: {rev_id} @@ -859,7 +852,8 @@ def upgrade() -> None: def downgrade() -> None: pass - """) + """ + ) (versions / f"{rev_id}_create_my_mv.py").write_text(content) errors = validate_mv_migrations(versions) @@ -875,13 +869,17 @@ def test_up_does_not_block_on_mv_warnings(self, tmp_path: Path, monkeypatch): from ch_migrate.cli import main # Create minimal project structure - (tmp_path / "config.yaml").write_text(textwrap.dedent("""\ + (tmp_path / "config.yaml").write_text( + textwrap.dedent( + """\ environments: dev: host: localhost database: testdb migration_user: test - """)) + """ + ) + ) (tmp_path / ".env.local").write_text("CH_DEV_MIGRATION_PASSWORD=pass\n") versions = tmp_path / "migrations" / "versions" @@ -909,13 +907,17 @@ def test_up_skip_mv_check_bypasses(self, tmp_path: Path, monkeypatch): from click.testing import CliRunner from ch_migrate.cli import main - (tmp_path / "config.yaml").write_text(textwrap.dedent("""\ + (tmp_path / "config.yaml").write_text( + textwrap.dedent( + """\ environments: dev: host: localhost database: testdb migration_user: test - """)) + """ + ) + ) (tmp_path / ".env.local").write_text("CH_DEV_MIGRATION_PASSWORD=pass\n") versions = tmp_path / "migrations" / "versions" @@ -928,9 +930,7 @@ def test_up_skip_mv_check_bypasses(self, tmp_path: Path, monkeypatch): monkeypatch.chdir(tmp_path) runner = CliRunner() # With --skip-mv-check, validation is bypassed before the env-version guard. - result = runner.invoke( - main, ["up", "dev", "--skip-mv-check"], catch_exceptions=False - ) + result = runner.invoke(main, ["up", "dev", "--skip-mv-check"], catch_exceptions=False) # The validation finding must not be reported output = result.output assert "create_mv000_create_mv.py:" not in output diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index eee1d44..8665254 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -58,13 +58,15 @@ def runner(tmp_path: Path, monkeypatch): """Set up a CLI runner in a tmp project directory.""" monkeypatch.chdir(tmp_path) (tmp_path / "migrations" / "sql" / "snapshots").mkdir(parents=True) - (tmp_path / "config.yaml").write_text(""" + (tmp_path / "config.yaml").write_text( + """ environments: dev: host: localhost database: testdb user: test -""") +""" + ) monkeypatch.setenv("CH_DEV_PASSWORD", "pass") return CliRunner() @@ -179,6 +181,7 @@ def test_timestamped_dirs_no_overwrite(self, mock_schema, mock_client, runner, t # Ensure a second snapshot creates a new directory (different timestamp) import time + time.sleep(1.1) result2 = runner.invoke(main, ["snapshot", "dev"])