From 81857f16420c6db2781596cb663eaafe211d896f Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Tue, 21 Jul 2026 14:52:49 +0000 Subject: [PATCH 1/7] Expose local membership generation monitor --- CHANGELOG.md | 2 ++ lib/group.ex | 20 ++++++++++++++++++++ test/group_test.exs | 15 +++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e01b07f..dbdc0dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ ## Unreleased +- Add `Group.monitor_generation/1` so long-lived registration owners can + terminate and re-register when the local membership ETS generation is lost. - **Breaking**: `Group.disconnect/3` now discards the complete local view of each departed cluster — remote entries included, and monitors receive `:unregistered`/`:left` events for them — instead of removing only locally owned rows. Reconnecting resyncs through the normal diff --git a/lib/group.ex b/lib/group.ex index 27ce1eb..6a4c6b2 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -233,6 +233,26 @@ defmodule Group do def start_link(opts), do: Group.Supervisor.start_link(opts) + @doc """ + Monitors the process that owns the local Group membership generation. + + All local registry and process-group entries are stored in ETS tables owned + by this process. If it exits, those entries no longer exist even when their + owner processes remain alive. Long-lived owners can use this monitor to + terminate and re-register against the next Group generation. + + A generation that exits between lookup and monitor creation still produces + the normal immediate `:DOWN` message for the returned monitor reference. + + Returns `{:ok, pid, monitor_ref}` or `{:error, :not_running}`. + """ + def monitor_generation(name) when is_atom(name) do + case GenServer.whereis(Data.data_name(name)) do + pid when is_pid(pid) -> {:ok, pid, Process.monitor(pid)} + nil -> {:error, :not_running} + end + end + # =========================================================================== # Cluster Management (Node <-> Cluster) # =========================================================================== diff --git a/test/group_test.exs b/test/group_test.exs index d0c1c22..6bc6b80 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -49,6 +49,21 @@ defmodule GroupTest do end end + describe "monitor_generation/1" do + test "notifies long-lived owners when local membership storage exits", %{name: name} do + assert {:ok, generation_pid, monitor_ref} = Group.monitor_generation(name) + + Process.exit(generation_pid, :kill) + + assert_receive {:DOWN, ^monitor_ref, :process, ^generation_pid, :killed} + end + + test "returns not_running for an unknown Group" do + name = :"missing_group_#{System.unique_integer([:positive])}" + assert {:error, :not_running} = Group.monitor_generation(name) + end + end + describe "join/3 and leave/2" do test "joined process appears in members/2", %{name: name} do key = "chat/room/#{System.unique_integer([:positive])}" From fe2cfc940a74b452608a735ec3c041d533c15124 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Fri, 31 Jul 2026 05:31:15 +0000 Subject: [PATCH 2/7] Add nonblocking anti-entropy replication --- CHANGELOG.md | 32 +- README.md | 138 +- lib/group.ex | 62 +- lib/group/replica.ex | 3234 ++++++++++++++++----- lib/group/replica/data.ex | 1009 ++++++- lib/group/replica/protocol.ex | 30 + lib/group/replica/transport.ex | 97 + lib/group/supervisor.ex | 66 +- priv/bench/README.md | 66 +- priv/bench/lib/group_bench/distributed.ex | 253 +- priv/bench/lib/group_bench/local.ex | 121 +- priv/bench/lib/group_bench/replica.ex | 173 +- test/README.md | 32 +- test/distributed_test.exs | 1539 +++++++++- test/group_test.exs | 180 +- test/replica_adversarial_test.exs | 358 +++ test/support/test_cluster.ex | 273 ++ test/support/test_replica_transport.ex | 140 + 18 files changed, 6793 insertions(+), 1010 deletions(-) create mode 100644 lib/group/replica/protocol.ex create mode 100644 lib/group/replica/transport.ex create mode 100644 test/replica_adversarial_test.exs create mode 100644 test/support/test_replica_transport.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index dbdc0dc..22721e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,23 @@ ## Unreleased +- Replace replica state sends/snapshots with per-origin, generation- and + cluster-epoch-fenced streams: sequenced deltas repair gaps from a bounded + oplog and fall back to exact origin snapshots after pruning. Replica data now + uses a pluggable nonblocking transport (dist Erlang by default via + `send_nosuspend`), while dist Erlang remains the control plane. Nonblocking + control heartbeats lease peer state, requesting a fresh authoritative hello + on generation or epoch-revision changes, so a stopped Group on a connected + VM cannot leave permanent registry or membership rows. Reconnects also sweep + superseded per-shard receive cursors and reconstruct epochless PG rows, so + reordered cluster controls cannot strand live rows from an older epoch. Full + epoch authority is installed once by shard 0; matching data shards exchange + constant-size lane hellos and retain shard-to-shard transport ordering. + Authority capture is serialized with epoch activation, and exact versus + incrementally observed revisions are tracked separately so a concurrent + partial snapshot cannot be mistaken for complete authority. +- Registry authority is retained per origin separately from the visible + winner. Conflict callbacks select the winner; Group now records and + propagates an authoritative loser delete, and each owner node terminates only + its own losing process. This also applies to custom conflict callbacks. - Add `Group.monitor_generation/1` so long-lived registration owners can terminate and re-register when the local membership ETS generation is lost. - **Breaking**: `Group.disconnect/3` now discards the complete local view of each departed @@ -6,14 +25,13 @@ them — instead of removing only locally owned rows. Reconnecting resyncs through the normal snapshot exchange. `connect`/`disconnect` also raise `ArgumentError` for non-binary cluster names instead of silently tolerating them. -- The built-in registry conflict resolver now consistently includes the winner's metadata in - the losing process's `{:group_registry_conflict, key, winner_meta}` exit reason. Custom - `resolve_registry_conflict` callbacks remain responsible for any process exits they require. +- The registry conflict resolver now consistently includes the winner's metadata in + the losing process's `{:group_registry_conflict, key, winner_meta}` exit reason. - **Breaking**: `Group.dispatch/4` remote sends and process-DOWN replication are now - non-suspending and never auto-connect — on a busy or disconnected distribution link the - message is dropped, the link is force-disconnected, and bounded reconnect retries begin (the - same policy replication lanes have used since 0.1.8). Previously dispatch could block the - caller and initiate new connections. + non-suspending and never auto-connect. Busy dispatch drops still force a disconnect and + bounded reconnect retry; replica frames are dropped and repaired by anti-entropy without + disturbing the dist-Erlang control connection. Previously dispatch could block the caller + and initiate new connections. - Configured function-form `extract_meta` callbacks are now applied on reads and lifecycle events (previously they were silently ignored and full metadata was exposed), and invalid `:extract_meta` values raise `ArgumentError` at startup. diff --git a/README.md b/README.md index c3d8fce..0943994 100644 --- a/README.md +++ b/README.md @@ -212,14 +212,16 @@ delivers an event with `previous_meta` set to the old value. All operations are **eventually consistent**: - Writes (`register`, `join`, etc.) return immediately after updating local ETS. -- Changes replicate to other nodes asynchronously over Erlang distribution. +- Changes replicate asynchronously over a configurable, nonblocking replica + transport. Erlang distribution remains the membership/control plane. - During network partitions, nodes may have divergent views. -- When partitions heal, state is re-synced via `cluster_state` messages. +- When connectivity returns, per-origin stream heads repair missing sequence + ranges from a bounded oplog; a lag beyond the retained prefix falls back to + an exact snapshot of that origin's shard/cluster slice. - Registry conflicts (same key registered on two nodes during a partition) can be resolved with a configurable `resolve_registry_conflict` callback. The - built-in resolver kills the losing process with - `{:group_registry_conflict, key, winner_meta}`; custom resolvers control any - process exits themselves. + callback selects a winner; each origin retires and terminates only its own + losing process with `{:group_registry_conflict, key, winner_meta}`. ## Configuration @@ -238,7 +240,11 @@ All operations are **eventually consistent**: replicated_sender_flush_interval: 5, busy_dist_retry_attempts: 300, busy_dist_retry_interval: 1_000, - replicated_pg_receiver_local_request_quota: 8 + replicated_pg_receiver_local_request_quota: 8, + replica_transport: Group.Replica.Transport.Distribution, + replicated_oplog_max_entries: 65_536, + replicated_anti_entropy_interval: 1_000, + replicated_peer_lease_timeout: 15_000 } ``` @@ -257,9 +263,10 @@ All operations are **eventually consistent**: - **`resolve_registry_conflict`** — `{module, function, extra_args}` callback invoked as `apply(mod, fun, [name, key, {pid1, meta1, time1}, {pid2, meta2, time2} | extra_args])`. Called when partition healing or concurrent registration finds the same key - registered on two nodes. Must return the winning pid and is responsible for - any process exits it requires. Runs synchronously inside the shard GenServer — - must return quickly and never block. + registered on two nodes. Must return the winning pid (or neither pid to + reject both). Group records an authoritative delete and terminates a losing + owner only on that owner's local node. The callback runs synchronously inside + the shard GenServer, so it must return quickly and never block. - **`extract_meta`** — `{module, function, args}` or `fun(meta)` applied to metadata on reads and lifecycle events. Useful for stripping internal fields. - **`replicated_pg_receiver_buffer_size`** — max buffered replicated PG @@ -275,18 +282,35 @@ All operations are **eventually consistent**: - **`replicated_sender_flush_interval`** — max outbound buffer age in milliseconds. Defaults to 5. - **`busy_dist_retry_attempts`** — reconnect attempts after a non-suspending - remote send reports a busy link. Defaults to 300. -- **`busy_dist_retry_interval`** — milliseconds between busy-link reconnect - attempts. Defaults to 1,000. -- **`replicated_pg_receiver_local_request_quota`** — local PG requests drained - after each replicated receiver turn. Defaults to 8. + remote dispatch reports a busy dist link. Defaults to 300. Replica transport + frames are simply dropped and repaired instead of forcing a disconnect. +- **`busy_dist_retry_interval`** — milliseconds between dispatch busy-link + reconnect attempts. Defaults to 1,000. +- **`replicated_pg_receiver_local_request_quota`** — legacy-named quota for + queued local shard requests drained per fairness turn while replica data or + cluster controls are busy. Defaults to 8. +- **`replica_transport`** — a module implementing + `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses + `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, + or `:disconnected`. Dropped and busy frames are repaired by anti-entropy. +- **`replicated_oplog_max_entries`** — maximum retained replica records per + shard across all local streams. Defaults to 65,536. Pruning never waits for + peer acknowledgements; a peer behind the retained floor receives an exact + snapshot. +- **`replicated_anti_entropy_interval`** — interval in milliseconds for stream + head advertisements and nonblocking control heartbeats. Defaults to 1,000. +- **`replicated_peer_lease_timeout`** — time without a dist-Erlang control + heartbeat before state owned by that Group peer is purged. Defaults to 15,000 + and must exceed the anti-entropy interval. Probes continue after expiry so a + Group restart on a still-connected VM recovers automatically. ## Architecture ``` Group.Supervisor (:"my_app_group_sup") -├── Group.Replica.Data — owns ETS tables and serializes membership writes -├── Group.PeerReconnect — bounded recovery after busy distribution links +├── optional transport child — sideband adapter listener/pool +├── Group.Replica.Data — owns ETS, journal, generations, and epochs +├── Group.PeerReconnect — bounded recovery after busy remote dispatch ├── Group.Replica.Supervisor — supervises N shard GenServers │ ├── Group.Replica (shard 0) │ ├── Group.Replica (shard 1) @@ -310,7 +334,7 @@ contention for unrelated keys. ### ETS Tables -Each shard owns 4 ETS tables: +Each shard has materialized read indexes plus authority/recovery indexes: | Table | Type | Key | Purpose | |---|---|---|---| @@ -319,6 +343,12 @@ Each shard owns 4 ETS tables: | `pg_by_key` | `:ordered_set` | `{cluster, key, pid}` | Group membership lookup | | `pg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | Reverse index for death cleanup | +Registry claim tables retain one authoritative claim per origin independently +of the visible winner. Stream metadata, oplog, append-order, and receive-cursor +tables support crash replay and gap repair. Keeping claims separate from the +single visible `reg_by_key` projection prevents a losing-but-still-live remote +claim from being forgotten before its owner emits an authoritative delete. + Plus 3 shared tables: - `cluster_nodes` (`:bag`, cluster→nodes) @@ -340,29 +370,66 @@ nodes. This handshake: 1. Validates that shard counts match (raises on mismatch). 2. Exchanges cluster membership lists. -3. Each shard sends its locally owned registry and group slice in a - `cluster_state` message for each shared cluster. - -This is how a new node catches up to the existing cluster state. +3. Shard 0 exchanges protocol version, origin generation, and one complete + active named-cluster epoch snapshot per node. Matching data shards exchange + only constant-size lane/transport descriptors tied to that authority + revision. + +Constant-size heartbeats renew the peer lease. If an origin generation or +cluster-epoch revision changes, the receiver requests a fresh authoritative +hello; if heartbeats stop, lease expiry purges that origin's complete local +view and discovery probes allow it to rejoin later. + +Incremental cluster open/close controls are generation fenced, receiver +batched, and installed by shard 0 into one node-wide authority table. The +highest observed revision keeps heartbeats constant-size during a burst; after +the burst becomes quiet, one authoritative hello closes any gaps left by +dropped or reordered controls. Per-shard view rows record only constant-size +lane readiness; they do not copy the epoch map. Snapshot capture is serialized +with local epoch activation, so its revision and epoch rows are one coherent +point-in-time value. The highest observed incremental revision is tracked +separately and can never promote a partial view to exact authority. Discovery +hints never mutate membership on their own. Authority installation fans a +local fence to every lane, which sweeps only that lane's retained receive +streams. Because PG rows intentionally do not carry protocol epochs, a +superseded origin/cluster slice is cleared and its current cursor reset so the +next head reconstructs it from retained deltas or an exact snapshot. + +Replica state itself does not travel on the control plane. Once the hello is +fenced, stream-head exchange on the replica transport catches the peer up. ### Replication -After the initial sync, steady-state changes propagate through separate sender -and receiver batching lanes: - -- local writes enqueue outbound registry or PG replication in shard-local sender - buffers -- sender flushes group those ops by target node and send one - `replicate_registry_batch` or `replicate_pg_batch` message per remote node -- remote shards buffer those replicated registry / PG ops receiver-side, apply - them in FIFO order with bulk ETS operations, then take a bounded fairness turn - before yielding back to the mailbox - -The sender flush timer is mainly a fallback for idle periods. Outbound buffers -also flush immediately when they hit the configured size, when a new enqueue +Every local mutation is first appended to a stream identified by +`{group, origin_node, origin_generation, shard, cluster, cluster_epoch}` and a +strictly increasing sequence number. It is then applied to the materialized +ETS view and batched into one delta frame per target. Process-death registry +and PG removals can share one record and retain their one-event-batch behavior. + +Receivers advance a cursor only across a contiguous sequence prefix. A gap +requests the missing suffix. Repeated head advertisements recover a dropped +tail even when no later write occurs. If the requested sequence is older than +the bounded oplog floor, the origin sends an exact snapshot of only its own +registry claims and PG memberships; absence from that snapshot is a delete. + +There are no leaders, quorum acknowledgements, tombstones, or known-membership +retention barriers. Oplog memory is bounded locally and independently of slow +peers. Deletes are normal ordered records while retained, and exact snapshots +close gaps after pruning. + +The sender flush timer is mainly a fallback for idle periods. The unified +outbound buffer also flushes immediately when it hits the configured size, when a new enqueue finds the buffer already past its flush interval, and before control or routing work such as cluster connect/disconnect or peer-protocol handling. +Transport ordering is not required for correctness: each shard serializes +writes, each stream numbers them, and receivers reject gaps and duplicates. +Per-shard ordered delivery is still a useful fast path. Cross-stream order is +not a correctness dependency; cluster epochs reject data racing a disconnect +or reconnect, and generation fencing rejects data from a restarted origin. +An alternative sideband adapter authenticates the peer as a dist-Erlang node +and calls `Group.Replica.Transport.deliver/4` locally. + ### Named Cluster TTL Leases Named-cluster TTLs are a local way to reduce replication fanout to nodes that @@ -383,7 +450,8 @@ no longer care about a cluster. Shards monitor all registered/joined processes. On `DOWN`, the shard: 1. Removes entries from both the primary and reverse-index ETS tables. -2. Groups removed entries by peer and sends one non-suspending process-down batch per peer. +2. Appends authoritative unregister/leave mutations before deleting the rows, + then sends one non-suspending sequenced delta batch per peer. 3. Fires `:unregistered` / `:left` events to local monitors. ### Node Disconnect diff --git a/lib/group.ex b/lib/group.ex index 6a4c6b2..f125efd 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -10,15 +10,16 @@ defmodule Group do ## Consistency Model - All operations are **eventually consistent**. The built-in replication layer uses - Erlang distribution to propagate state across nodes, which means: + All operations are **eventually consistent**. Erlang distribution remains + the membership/control plane; replica state uses a configurable nonblocking + transport with sequenced anti-entropy streams. This means: - Writes (register, join, etc.) return immediately after local update - - Other nodes receive updates asynchronously via Erlang distribution + - Other nodes receive updates asynchronously over the replica transport - During network partitions, nodes may have divergent views - When partitions heal, conflicts are resolved. The built-in resolver kills - the losing process with `{:group_registry_conflict, key, winner_meta}`; - custom resolvers control any process exits themselves + each losing origin records an authoritative delete and terminates only its + own local process with `{:group_registry_conflict, key, winner_meta}` ## Clusters @@ -195,10 +196,9 @@ defmodule Group do - `:resolve_registry_conflict` — `{module, function, extra_args}` callback invoked when two nodes hold the same registry key (partition heal or concurrent registration). Called as `apply(module, function, [name, key, {pid1, meta1, time1}, {pid2, meta2, time2} | extra_args])`. - Must return the winner pid and is responsible for any process exits it requires. When - no callback is configured, the built-in resolver kills the loser with - `{:group_registry_conflict, key, winner_meta}`. **Important:** This callback runs - synchronously inside the shard GenServer — it must + Must return the winner pid (or neither contender to reject both). Group records + an authoritative delete and terminates a losing process only on its owner node. + **Important:** This callback runs synchronously inside the shard GenServer — it must return quickly and never block. Any information needed for the decision should be carried in the registration metadata, not fetched at resolution time. - `:extract_meta` — `{module, function, args}` or a one-argument function to @@ -218,13 +218,26 @@ defmodule Group do buffer replicated outbound ops before flushing during idle periods. Sender buffers also flush on size, overdue enqueue, and control/routing barriers (default: `5`) - - `:busy_dist_retry_attempts` — max reconnect attempts after a shard hits - `send_nosuspend == false` to a remote node and forces a disconnect + - `:busy_dist_retry_attempts` — max reconnect attempts after a remote + `Group.dispatch/4` send reports a busy dist link and forces a disconnect (default: `300`) - - `:busy_dist_retry_interval` — interval in milliseconds between reconnect - attempts after a busy-dist disconnect (default: `1_000`) - - `:replicated_pg_receiver_local_request_quota` — max queued local PG shard requests - drained after each replicated PG flush before yielding (default: `8`) + - `:busy_dist_retry_interval` — interval in milliseconds between dispatch + busy-link reconnect attempts (default: `1_000`) + - `:replicated_pg_receiver_local_request_quota` — legacy-named quota for queued + local shard requests drained in each fairness turn, including while replica + data or cluster controls are busy (default: `8`) + - `:replica_transport` — replica data transport module or `{module, opts}` tuple. + Defaults to `Group.Replica.Transport.Distribution`. The transport must be + nonblocking and may return `:busy`; anti-entropy repairs dropped frames. + - `:replicated_oplog_max_entries` — maximum retained replica records per shard + before old prefixes are pruned and lagging peers require a snapshot + (default: `65_536`) + - `:replicated_anti_entropy_interval` — milliseconds between repeated stream + head advertisements (default: `1_000`) + - `:replicated_peer_lease_timeout` — milliseconds without a dist-Erlang + replica heartbeat before remote Group state is purged (default: `15_000`). + Must be greater than `:replicated_anti_entropy_interval`. Discovery probes + an expired peer so a restarted Group can recover automatically. """ def child_spec(opts) do name = Keyword.fetch!(opts, :name) @@ -1094,6 +1107,7 @@ defmodule Group do @doc false def connect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do + _epochs = Data.activate_local_clusters(name, clusters) Data.add_cluster_node(name, clusters, node()) notify_shard = :rand.uniform(get_config(name).num_shards) - 1 @@ -1108,17 +1122,25 @@ defmodule Group do @doc false def disconnect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do + _epochs = Data.deactivate_local_clusters(name, clusters) Data.remove_cluster_node(name, clusters, node()) num_shards = get_config(name).num_shards shard_names = for i <- 0..(num_shards - 1), do: Replica.shard_name(name, i) - Replica.local_request_all( - shard_names, - {:cluster_disconnect, clusters}, - timeout - ) + result = + Replica.local_request_all( + shard_names, + {:cluster_disconnect, clusters}, + timeout + ) + + # Keep the remote routing rows through the shard barrier so any buffered + # records are dispatched before the cluster-close control message. Once + # every shard has crossed the barrier, no cluster rows may remain locally. + Data.remove_clusters(name, clusters) + result end # =========================================================================== diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 933c795..95bb6b6 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -6,428 +6,124 @@ defmodule Group.Replica do @process_down_batch_size 32 @replicated_pg_receiver_flush_timer :flush_replicated_pg_receiver_buffer @replicated_registry_receiver_flush_timer :flush_replicated_registry_receiver_buffer - @replicated_pg_broadcast_flush_timer :flush_replicated_pg_broadcast_buffer - @replicated_registry_broadcast_flush_timer :flush_replicated_registry_broadcast_buffer + @replica_broadcast_flush_timer :flush_replica_broadcast_buffer + @anti_entropy_timer :group_replica_anti_entropy @local_request_tag :group_local_request @local_reply_tag :group_local_reply + @protocol_version Group.Replica.Protocol.version() _archdoc = ~S""" - Sharded GenServer: peer discovery, replication, monitoring, conflict resolution. - - One per shard. Registered as :"#{name}_replica_#{shard_index}". - - ## Message Protocol - - | Message | Direction | Purpose | - |------------------------------------------------------------|----------------|----------------------------------| - | `{:peer_connect, pid, shard, num_shards, clusters}` | A→B (per-shard)| Establish peer relationship | - | `{:peer_connect_ack, pid, shard, num_shards, clusters}` | B→A (per-shard)| Acknowledge peer | - | `{:cluster_state, cluster, reg_data, pg_data}` | both | Per-cluster data snapshot | - | `{:replicate_registry_batch, ops}` | broadcast | Propagate batched registry ops | - | `{:replicate_pg_batch, ops}` | broadcast | Propagate batched PG ops | - | `{:cluster_connect, clusters, pid}` | S→remote S | Node joining named clusters | - | `{:cluster_connect_ack, clusters, pid, cluster_data}` | S→remote S | Ack + bundled shard data | - | `{:cluster_disconnect, clusters, pid}` | shard 0→remote | Node leaving named clusters | - | `{:send_cluster_data, clusters, target_node}` | local fan-out | Notify siblings: send shard data | - | `{:group_dispatch, pids, message}` | caller→remote | Per-node fan-out for dispatch | - - ## Protocol Flows - - ### 1. Peer Discovery (nodeup or init) - - Triggered by `nodeup` or `init`. Each shard independently discovers its - counterpart on the remote node. Both sides exchange cluster lists, then send - per-cluster `cluster_state` snapshots for shared clusters (always includes - nil). Merge applies data; conflicts with local entries go through - `resolve_conflict`. - - Node A shard i Node B shard i - ──────────── ──────────── - │ │ - │ {:peer_connect, pid, i, N, clusters} │ - │──────────────────────────────────────>│ - │ │── add A to nil cluster ETS - │ │── compute shared clusters - │ │── add A to shared named clusters - │ │── monitor A's shard pid - │ │ - │ {:peer_connect_ack, pid, i, N, clusters} - │<──────────────────────────────────────│ - │── add B to nil cluster ETS │ - │── compute shared clusters │ {:cluster_state, C, reg, pg} - │── add B to shared named clusters │──────────────────────────────>│ - │── monitor B's shard pid │ (one per shared cluster) │ - │ │ │ - │ {:cluster_state, C, reg, pg} │ │ - │──────────────────────────────>│ │ │ - │ (one per shared cluster) │ │ │ - │ │ │ │ - ▼ ▼ ▼ ▼ - merge_remote_cluster_data merge_remote_cluster_data - ├─ no conflict: insert ├─ no conflict: insert - ├─ local vs remote: resolve_conflict (default kills loser; re-broadcast winner) - └─ both remote: timestamp wins └─ both remote: timestamp wins - - ### 2. Steady-State Replication - - After peer discovery, local writes enqueue outbound registry or PG replication - in separate sender buffers. Flushes group those ops by target node and send - one `{:replicate_registry_batch, ops}` or `{:replicate_pg_batch, ops}` per - remote node. The nil cluster uses `remote_shards` (per-shard map); named - clusters use `cluster_nodes` ETS. Reads (`lookup`, `members`) go directly to - ETS — no GenServer involved. - - Node A shard i Node B shard i - ──────────── ──────────── - │ │ - Group.register(name, key, meta) │ - │── ETS insert (by_key + by_pid) │ - │── monitor pid │ - │── enqueue sender-side registry op │ - │ │ - │ {:replicate_registry_batch, [ ... ]} - │──────────────────────────────────────>│ - │ │── enqueue receiver-side - │ │ registry ops - │ │── flush in FIFO order - │ │ ├─ nil: insert - │ │ ├─ same pid: update - │ │ ├─ local conflict: - │ │ │ resolve_conflict() - │ │ └─ both remote: - │ │ timestamp wins - │ │ - Group.join(name, group, meta) │ - │── ETS insert (by_key + by_pid) │ - │── enqueue sender-side PG op │ - │ │ - │ {:replicate_pg_batch, [ ... ]} - │──────────────────────────────────────>│ - │ │── enqueue receiver-side PG - │ │ ops - │ │── flush in FIFO order - │ │ (no overwrite conflict - │ │ for PG) - │ │ - Group.dispatch(name, group, msg) │ - │── send directly to local pids │ - │── group remote pids by node │ - │── hash self() to pick shard j │ - │ │ - │ {:group_dispatch, pids, msg} Node B shard j - │────────────────────────────────────────>│ - │ │── send msg to each local pid - │ │ - - Dispatch groups remote PG members by node and sends one - `:group_dispatch` message per remote node, reducing cross-node - messages from O(members) to O(nodes). The target shard is chosen - by hashing the caller's pid (`phash2(self(), num_shards)`), so - back-to-back dispatches from the same caller always route through - the same shard, preserving per-sender message ordering. - - ### 3. Named Cluster Connect (random shard S + fan-out) - - `Group.connect/2` adds local node to ETS, picks random shard S, and sends - one GenServer.call. Shard S notifies remote shard S, which acks with bundled - data and fans out to siblings. Randomizing S load-balances across shards - when many concurrent connects happen. - - Node A Node B - ────── ────── - Group.connect(name, "game") - │── ETS: add self to "game" - │── pick random shard S - │ - Shard S Shard S - ─────── ─────── - │ │ - │ {:cluster_connect, ["game"], pid} │ - │──────────────────────────────────────>│ - │ │── ETS: add A to "game" - │ │── bundle shard S data - │ │ - │ {:cluster_connect_ack, ["game"], pid, [{cluster, reg, pg}]} - │<──────────────────────────────────────│ - │ │ - │ Shard S sends to siblings: - │ {:send_cluster_data, ["game"], A} - │ │ - │ Shards 0..N (except S): - │ │── {:cluster_state, "game", reg, pg} - │ │──────────────────────────────>│ - │ │ (to matching A shard) │ - │ │ │ - │── merge bundled ack data │ - │── ETS: add B to "game" │ - │── send shard S cluster_state ────────>│ - │── fan out to siblings: │ - │ {:send_cluster_data, ["game"], B} │ - │ │ - Shards 0..N (except S): │ - │── {:cluster_state, "game", reg, pg} │ - │──────────────────────────────────────>│ - │ (to matching B shard) │ - - ### 4. Named Cluster Disconnect (all shards local + shard 0 broadcast) - - `Group.disconnect/2` removes local node from ETS, then calls ALL local shards - to purge their entries. Only shard 0 broadcasts to remote shard 0, which fans - out to siblings for per-shard purge. - - Node A Node B - ────── ────── - Group.disconnect(name, "game") - │── ETS: remove self from "game" - │ - Shards 0..N (all called): - │── purge own entries for "game"+A - │── dispatch :unregistered/:left events - │ - Shard 0 only: - │ {:cluster_disconnect, ["game"], pid} - │──────────────────────────────────────>│ Shard 0 - │ │── ETS: remove A from "game" - │ │── fan out to siblings: - │ │ {:cluster_disconnect, ["game"], pid} - │ │ - │ Shards 0..N: - │ │── purge entries for "game"+A - │ │── dispatch events - - ### 5. Partition Heal (peer discovery re-runs + conflict resolution) - - When a partition heals, `nodeup` triggers peer discovery on both sides. - Both exchange `cluster_state` snapshots. Registry key conflicts where the - existing entry is local go through `resolve_conflict` — the same path used - for live contention. The default resolver kills the loser process. - - The tiebreaker must be deterministic regardless of which node is resolving. - The default uses timestamp comparison, with pid ordering as a tiebreaker - when timestamps are equal (`pid2 > pid1`). Erlang pids have a total order - (by node name then id), so this produces the same winner on all nodes. - Using a perspective-dependent tiebreaker (e.g. "remote wins on ties") would - cause mutual kill — both nodes pick the other's pid, both processes die. - - Node A Node B - ────── ────── - (partition: A and B both register key K) - A has: {K, pid_a, time_a, local} B has: {K, pid_b, time_b, local} - │ │ - ─────── partition heals (nodeup) ──────────── - │ │ - │ peer_connect / peer_connect_ack │ - │<─────────────────────────────────────>│ - │ │ - │ {:cluster_state, nil, [{K, pid_b, ...}], []} - │<──────────────────────────────────────│ - │ │ - │ {:cluster_state, nil, [{K, pid_a, ...}], []} - │──────────────────────────────────────>│ - │ │ - merge: K exists locally merge: K exists locally - resolve_conflict( resolve_conflict( - local={pid_a, time_a}, local={pid_b, time_b}, - remote={pid_b, time_b}) remote={pid_a, time_a}) - │ │ - (assuming time_b > time_a): (assuming time_b > time_a): - pid_b wins (remote) pid_b wins (local) - ├─ kill pid_a ├─ kill pid_a (cross-node, idempotent) - ├─ delete pid_a entry ├─ re-insert pid_b with new timestamp - ├─ insert pid_b └─ re-broadcast pid_b - ├─ demonitor pid_a in a registry batch - ├─ dispatch :unregistered(pid_a) │──────────────────────────>│ - │ │ (arrives as same-pid │ - │ │ update — harmless) │ - - ### 6. Nodedown / Process Death Cleanup - - `nodedown` purges all remote node data. Local process `DOWN` purges the pid's - entries and broadcasts unregister/leave to cluster members. - - Node A Node B dies - ────── ────────── - │ X - {:nodedown, B} │ - │ │ - All shards (each independently): │ - │── purge_cluster_node(B) │ - │ (remove B from all cluster_nodes │ - │ and node_clusters — idempotent, │ - │ guards against late peer_connect) │ - │ │ - │── purge_node(shard, B) │ - │ (scan by_key for node==B, │ - │ delete from both by_key + by_pid) │ - │── dispatch :unregistered/:left events │ - │── remove B from remote_shards │ - - ────────────────────────────────────────────── - - Local process dies Node B - ────────────────── ────── - {:DOWN, mref, :process, pid, reason} │ - │ │ - Owning shard: │ - │── delete_all_for_pid(shard, pid) │ - │ (scan by_pid, delete from by_key, │ - │ match_delete from by_pid) │ - │── enqueue unregister/leave ops │ - │ into replicated sender batches ──────>│── delete if pid matches - │── demonitor pid │ - │── dispatch events │ - - ## Cluster Membership Tracking - - The nil cluster is tracked in ETS (cluster_nodes table), maintained by the - peer_connect protocol. Nodes are added on peer discovery and removed on - nodedown/shard death. This allows Group.nodes/1 to return actual Group peers - rather than all Erlang nodes. - - ## Sharding - - Each key is routed to a shard via `:erlang.phash2({cluster, key}, num_shards)`. - Including `cluster` in the hash input means the same key string in different - clusters may land on different shards — this is intentional so named-cluster - operations don't create false contention with nil-cluster operations. - - `phash2` produces near-uniform distribution across shards for diverse keyspaces. - With 10K distinct keys across 2–8 shards, observed deviation from perfect - uniformity is <2%. In practice, real workloads with varied key prefixes will - see balanced shard load. - - **Hot keys:** A single extremely popular key (e.g. a chat room - with thousands of joins/leaves) always hashes to one shard, so all *writes* - for that key serialize through that shard's GenServer. However, *reads* — - `Group.lookup/3` and `Group.members/3` — go directly to ETS and bypass the - GenServer entirely. Since reads typically dominate, a hot key's impact on - overall throughput is limited to write-heavy scenarios. Adding more shards - does not help a single hot key (it still lands on one shard), but it does - reduce contention between unrelated keys. - - Shard counts must match across all nodes in a cluster. The peer_connect - handshake validates `num_shards` and raises on mismatch, since a disagreement - would route the same key to different shards on different nodes, breaking - replication consistency. - - ## Conflict Resolution is Synchronous - - The `:resolve_registry_conflict` callback runs synchronously inside the shard - GenServer's `handle_info` (during replicated registry apply or - `merge_remote_cluster_data`). - This is intentional: the resolver's return value determines ETS mutations (delete - loser entry, insert winner, demonitor evicted local pid, re-broadcast winner) that - must happen atomically within a single `handle_info` turn. Making the resolver async - would open a window where another replicated registry update, `DOWN`, or `cluster_state` - for the same key could race with the pending resolution, corrupting the dual-index - ETS tables. - - Consequence: a blocking resolver stalls the **entire shard** — no registrations, - joins, replication, or cleanup can proceed on that shard until the callback returns. - Callers must ensure their resolver returns quickly. Any information needed for the - decision (e.g. priority, version, creation time) should be carried in the - registration metadata, not fetched at resolution time. - - ## Monitor Event Delivery - - Lifecycle events (`:registered`, `:unregistered`, `:joined`, `:left`) are delivered - to `Group.monitor/3` subscribers in a batched diff of `{:group, events, info}` tuples. - Each GenServer handler invocation is a natural batch boundary: - - - **Single local operations** (register, join, leave, unregister): build one - event, deliver one tuple with one event per matching subscriber. - - **Buffered replicated operations** (batched replicated registry and PG - ops): receiver shards may accumulate several ops before flushing, then - deliver one tuple per subscriber containing the ordered events from that - flush. - - **Bulk operations** (nodedown, process DOWN, cluster_disconnect, cluster_state - merge): accumulate events into a local variable, then deliver one tuple per - subscriber containing all matching events from that handler turn. - - Events are built by `build_event/6`, accumulated in reverse via prepend, and - flushed by `notify_monitors/2` which reverses once, resolves only the monitor - keys that can match each event (`:all`, `{:exact, key}`, and the key's - slash-terminated prefixes), caches those lookups per batch, and sends one - `{:group, events, %{name: name}}` per subscriber. Both functions are private to - this module. - - `resolve_conflict/5` returns `{state, event_or_nil}` so callers can accumulate - the event. `merge_remote_cluster_data/5` threads `{state, events}` through its - reduce, generating `:registered`/`:joined` events for new entries and conflict - events for existing ones. This means `cluster_state` merges (peer discovery, - partition heal, `Group.connect`) produce batched diffs with all new entries. - `build_purged_events/5` takes an events accumulator and prepends purged-entry - events to it. - - ## Replicated Sender Buffering - - Local writes stage outbound replicated registry and PG operations in separate - sender buffers. On flush, those ops are grouped by target node so one shard - send can carry many logical replication updates. - - Sender-side buffering is not timer-only. A sender buffer flushes when: - - - the lane reaches `replicated_sender_buffer_size` - - a new enqueue notices the lane is already older than - `replicated_sender_flush_interval` - - a control or topology path crosses the sender barrier (`cluster_connect`, - `cluster_disconnect`, peer protocol, `cluster_state`, `DOWN`, `nodedown`, - process-down cleanup, explicit mailbox barriers) - - terminate runs - - The timer is therefore a fallback for idle periods, not the only flush - trigger. A very long single GenServer callback can still delay all of these - flush paths until that callback returns; batching and fairness only apply - between mailbox turns. - - Remote shard sends use `send_nosuspend(..., [:noconnect])`. If a send returns - `false`, Group treats that as a degraded link: it drops the unsent message, - force-disconnects the Erlang node, and enters a bounded reconnect loop for - that node. Recovery only starts from this explicit busy-send path; ordinary - `nodedown` events do not start reconnect retries on their own. - - ## Replicated Receiver Fairness - - Receiver-side batching solves the apply-cost problem for both replicated PG - and replicated registry traffic, but a hot stream in either lane can still - monopolize the shard if every completed replication turn is immediately - followed by another one. - - To keep local latency-sensitive writes from sitting behind an unbounded remote - backlog, the shard gives a bounded local request turn after each completed - replicated PG or replicated registry apply turn: - - - one bounded replicated lane turn (PG or registry) - - then drain any already-waiting cluster/protocol messages - - then drain up to `replicated_pg_receiver_local_request_quota` local PG - `join` / `leave` requests, or one local non-PG request - - then yield back to the GenServer loop - - Contiguous local PG `join` / `leave` requests from that bounded local turn are - staged against an in-memory view and applied with bulk ETS operations, while - replicated registry flushes are staged against an in-memory view per - `{cluster, key}` and applied with bulk ETS operations. Other local request - types still execute sequentially in FIFO order. - - Local callers use an explicit request/reply lane (`send` + monitor + tagged - reply) rather than `GenServer.call/3`, so the replica can selectively receive - one local request turn without reaching into `'$gen_call'` internals. - - The fairness model ensures ordering is preserved where correctness matters: - - - all public local shard calls get protection from replicated PG and registry - backlog, but earlier cluster/protocol messages still run first, avoiding - stale ordering around disconnect, peer discovery, and cluster sync - - FIFO is preserved within the local lane because the selective receive matches - a single broad `@local_request_tag` shape and therefore takes the oldest - queued local request in the mailbox - - local PG batching does not reorder within that local lane; it only batches - contiguous `join` / `leave` messages already collected in FIFO order + Sharded control process for local writes, replica transport, anti-entropy, + process monitoring, and registry conflict projection. + + There is one process per shard, registered as + :"#{name}_replica_#{shard_index}". Reads bypass it and use the materialized + ETS indexes owned by Group.Replica.Data. + + ## Authority and identity + + Each locally owned mutation belongs to one stream: + + {group, origin_node, origin_generation, shard, cluster, cluster_epoch} + + The shard assigns a strictly increasing sequence number and appends the record + to its write-ahead oplog before changing materialized ETS. Data owns the + journal, so a shard crash replays any appended-but-unapplied record before + rebuilding local process monitors. + + A registry's authoritative claims are stored per origin separately from its + single visible winner. Conflict selection folds claims in a stable order. + When a local claim loses, only its owner node appends the authoritative + unregister and terminates the local process. Retaining hidden remote claims + until their origin deletes them prevents a later winner change from orphaning + or permanently forgetting a live claim. + + PG memberships need no winner projection: the origin stream owns exactly the + rows whose member processes live on that origin node. + + ## Wire protocol + + Dist Erlang remains the control plane: + + - peer_connect / peer_connect_ack discover matching shards and clusters. + - shard 0 exchanges replica_hello authority containing the origin generation + and complete active named-cluster epoch set exactly once per node. + - matching nonzero shards exchange constant-size replica_lane_hello messages + containing their transport descriptor and the authority revision they use. + - replica_cluster_open / replica_cluster_close fence named-cluster lifetimes. + - constant-size periodic heartbeats provide a bounded peer lease without + creating remote process monitors. A generation/epoch-revision mismatch + requests a fresh authoritative hello. + + Replica state uses the configured Group.Replica.Transport: + + - heads advertises {stream, retained_floor, head}. + - delta_batch carries one or more contiguous stream runs. + - need requests the receiver's next missing sequence. + - snapshot exactly replaces one origin's registry claims and PG slice when + the requested prefix has already been pruned. + + Every stream field is validated against the authenticated source node and + current generation/epoch. An old generation, a closed epoch, a wrong shard, + or a transitive claim for another node's pid is rejected. Control/data + reordering is safe: early frames are ignored and repeated heads repair them; + late frames fail their generation or epoch fence. + + ## Bounded recovery + + The oplog is bounded per shard, not by peer acknowledgements. A dropped tail + is found by periodic heads. A gap inside the retained range is repaired with + bounded delta batches. A gap below the retained floor receives the existing + full-sync primitive, narrowed to an exact origin/shard/cluster snapshot. + Absence from that snapshot is deletion, so no tombstones are required. + + There is no leader, quorum, retention ACK, or requirement to know all members. + A slow or disconnected peer cannot pin memory. When it returns it repairs from + deltas when possible and a snapshot otherwise. + + ## Nonblocking transport and ordering + + All cross-node control messages use :erlang.send_nosuspend/3 with :noconnect. + The default replica adapter does the same. Transport callbacks return :ok, + :busy, or :disconnected; failure drops the frame and anti-entropy repairs it. + Replica shards never remotely monitor or exit member processes. + + The transport need not order frames for correctness. The local shard + serializes writes, sequence numbers establish per-stream order, and receivers + reject duplicates and gaps. TCP shard-to-shard ordering remains the efficient + fast path. No semantic operation spans clusters, so cross-stream ordering is + unnecessary; generation and cluster-epoch fences cover lifecycle races. + + ## Batching and fairness + + Local writes share one outbound sender buffer so registry and PG mutations + retain mailbox order. Flushes group records by target and stream. Size, age, + control/routing barriers, and idle timers bound the delay. + + Incremental cluster controls are generation fenced, receiver batched, and + installed by shard 0 into the node-wide authority table. Their observed + revision suppresses full-hello storms during bursts; a quiet authoritative + hello repairs any missing or reordered controls. Snapshot capture is + serialized with epoch activation, and the last exact revision is distinct + from the highest incrementally observed revision. Shard-local lane readiness + is separate from shared authority, so no epoch map is copied per shard. + + Incoming PG mutations retain the bulk receiver lane. Contiguous registry + records in one stream run are projected together and emit one monitor event + batch. A mixed process-down record applies maximal same-domain segments in + wire order and emits one combined batch. + + After replicated work, the shard takes a bounded local-request turn before + yielding. FIFO is preserved within the local request lane, while protocol and + cluster barriers flush earlier buffered state first. + + Receive-only handlers for the previous direct batch/snapshot messages remain + for rolling compatibility and tests; protocol v1 never emits them. """ require Logger - alias Group.Replica.Data + alias Group.Replica.{Data, Protocol} defstruct [ :name, @@ -440,24 +136,30 @@ defmodule Group.Replica do :replicated_sender_buffer_size, :replicated_sender_flush_interval, :replicated_pg_receiver_local_request_quota, + :replicated_oplog_max_entries, + :replicated_anti_entropy_interval, + :replicated_peer_lease_timeout, + :replica_transport, + :replica_transport_opts, + :anti_entropy_ref, :pending_replicated_pg_started_at, :pending_replicated_pg_flush_ref, :pending_replicated_registry_started_at, :pending_replicated_registry_flush_ref, - :pending_replicated_pg_broadcast_started_at, - :pending_replicated_pg_broadcast_flush_ref, - :pending_replicated_registry_broadcast_started_at, - :pending_replicated_registry_broadcast_flush_ref, + :pending_replica_broadcast_started_at, + :pending_replica_broadcast_flush_ref, pending_replicated_pg_len: 0, pending_replicated_pg_ops: [], pending_replicated_registry_len: 0, pending_replicated_registry_ops: [], - pending_replicated_pg_broadcast_len: 0, - pending_replicated_pg_broadcast_ops: [], - pending_replicated_registry_broadcast_len: 0, - pending_replicated_registry_broadcast_ops: [], + pending_replica_broadcast_len: 0, + pending_replica_broadcast_ops: [], remote_shards: %{}, - monitors: %{} + peer_last_seen: %{}, + cluster_control_dirty: %{}, + authority_dirty_notified: MapSet.new(), + monitors: %{}, + peer_transports: %{} ] def start_link(opts) do @@ -536,9 +238,20 @@ defmodule Group.Replica do replicated_sender_buffer_size: config.replicated_sender_buffer_size, replicated_sender_flush_interval: config.replicated_sender_flush_interval, replicated_pg_receiver_local_request_quota: - config.replicated_pg_receiver_local_request_quota + config.replicated_pg_receiver_local_request_quota, + replicated_oplog_max_entries: config.replicated_oplog_max_entries, + replicated_anti_entropy_interval: config.replicated_anti_entropy_interval, + replicated_peer_lease_timeout: config.replicated_peer_lease_timeout, + replica_transport: elem(config.replica_transport, 0), + replica_transport_opts: elem(config.replica_transport, 1) } + state = schedule_anti_entropy(state) + + # Complete any write-ahead record left unapplied by a shard crash, then + # rebuild local process monitors from the surviving materialized tables. + state = replay_local_journal(state) + # Rebuild monitors from any surviving ETS data (after shard crash/restart) state = rebuild_monitors(state) @@ -595,11 +308,21 @@ defmodule Group.Replica do # Cluster connect/disconnect (broadcast to all shards, rare operation) # ===================================================================== + def handle_call({:cluster_connect, _, _} = request, _from, state) do + {reply, state} = process_local_request(state, request) + {:reply, reply, state} + end + def handle_call({:cluster_connect, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end + def handle_call({:cluster_disconnect, _, _} = request, _from, state) do + {reply, state} = process_local_request(state, request) + {:reply, reply, state} + end + def handle_call({:cluster_disconnect, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} @@ -628,6 +351,428 @@ defmodule Group.Replica do {:noreply, state} end + def handle_info( + {:replica_hello, remote_pid, version, generation, epoch_revision, cluster_epochs, + transport_id, transport_descriptor}, + %{shard_index: 0} = state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + known_generation = Data.remote_generation(state.name, remote_node) + observed_revision = Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + authoritative_revision = Data.remote_cluster_epoch_revision(state.name, remote_node) + + exact_revision = Data.remote_cluster_epoch_exact_revision(state.name, remote_node) + + stale_revision? = + known_generation == generation and + Enum.any?([observed_revision, authoritative_revision], fn + revision when is_integer(revision) -> epoch_revision < revision + _ -> false + end) + + cond do + version != Protocol.version() or transport_id != state.replica_transport.id() -> + Logger.error( + "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" + ) + + {:noreply, state} + + stale_revision? -> + {:noreply, state} + + known_generation == generation and exact_revision == epoch_revision -> + # Requests and heartbeats may race while one large authority snapshot + # is being installed. Once this exact revision is present, another + # identical hello is only a lease/descriptor refresh; reinstalling its + # full epoch set would serialize every shard behind redundant ETS work. + state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) + + state = %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + peer_transports: + Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) + } + + {:noreply, state} + + true -> + {:noreply, + install_replica_authority( + state, + remote_pid, + generation, + epoch_revision, + cluster_epochs, + transport_id, + transport_descriptor + )} + end + end + + def handle_info( + {:replica_hello, remote_pid, _version, _generation, _epoch_revision, _cluster_epochs, + _transport_id, _transport_descriptor}, + state + ) do + # Full authority is installed only by shard 0. A full hello delivered to a + # data lane cannot be used as its lane identity because remote_pid belongs + # to the remote control shard, not this matching shard. + {:noreply, request_replica_authority(state, node(remote_pid))} + end + + def handle_info( + {:replica_lane_hello, remote_pid, version, generation, epoch_revision, transport_id, + transport_descriptor}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + if version == Protocol.version() and transport_id == state.replica_transport.id() do + if function_exported?(state.replica_transport, :peer_up, 4) do + :ok = + state.replica_transport.peer_up( + state.name, + remote_node, + transport_descriptor, + state.replica_transport_opts + ) + end + + state = %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_transports: + Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) + } + + if replica_authority_current?(state, remote_node, generation, epoch_revision) do + :ok = + Data.put_remote_view_info( + state.name, + state.shard_index, + remote_node, + generation, + Data.remote_cluster_epoch_exact_revision(state.name, remote_node), + epoch_revision + ) + + state = + state + |> purge_remote_streams_outside_authority(remote_node) + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + + {:noreply, state} + else + {:noreply, request_replica_authority(state, remote_node)} + end + else + Logger.error( + "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" + ) + + {:noreply, state} + end + end + + def handle_info( + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + if replica_authority_current?(state, remote_node, generation, epoch_revision) do + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + + state = + if old_generation == generation do + state + |> purge_closed_remote_epochs(remote_node, stale_epochs) + |> purge_remote_streams_outside_authority(remote_node) + else + state + end + + state = %{ + state + | cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, remote_node) + } + + state = + if Map.has_key?(state.remote_shards, remote_node) do + state + |> touch_replica_peer(remote_node) + |> send_replica_heads(remote_node) + else + state + end + + {:noreply, state} + else + {:noreply, state} + end + end + + def handle_info({:replica_authority_removed_local, remote_node}, state) do + state = flush_pending_replicated_message_barrier(state) + {:noreply, expire_replica_peer(state, remote_node)} + end + + def handle_info( + {:replica_cluster_open, remote_pid, generation, revision, epochs}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + controls = + collect_replica_cluster_controls( + :replica_cluster_open, + remote_pid, + generation, + [{revision, epochs}], + state.replicated_sender_buffer_size - 1 + ) + + case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + {:accept, observed_revision, epochs} -> + stale = + Data.put_remote_cluster_epochs( + state.name, + state.shard_index, + remote_node, + observed_revision, + epochs + ) + + shared = + Enum.filter(epochs, fn {cluster, _epoch} -> + node() in Data.cluster_nodes(state.name, cluster) + end) + + Data.add_cluster_node(state.name, Enum.map(shared, &elem(&1, 0)), remote_node) + + fan_out_to_siblings( + state, + {:replica_cluster_open_control_local, remote_node, generation, observed_revision, + epochs, stale, Enum.map(shared, &elem(&1, 0))} + ) + + state = + state + |> mark_authority_dirty(remote_node) + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + + state = send_replica_heads(state, remote_node, Enum.map(shared, &elem(&1, 0))) + {:noreply, take_one_local_request_turn(state)} + + :stale -> + {:noreply, take_one_local_request_turn(state)} + + :refresh -> + {:noreply, + state + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + end + + def handle_info({:replica_authority_dirty_local, remote_node}, %{shard_index: 0} = state) do + {:noreply, mark_cluster_control_dirty(state, remote_node)} + end + + def handle_info({:replica_authority_dirty_local, remote_node}, state) do + send(shard_name(state.name, 0), {:replica_authority_dirty_local, remote_node}) + {:noreply, state} + end + + def handle_info( + {:replica_cluster_open_control_local, remote_node, generation, revision, epochs, stale, + shared}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + state = + if replica_authority_current?(state, remote_node, generation, revision) do + state + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + |> send_replica_heads(remote_node, shared) + else + state + end + + {:noreply, take_one_local_request_turn(state)} + end + + def handle_info({:replica_cluster_stale_epochs_local, remote_node, stale}, state) do + state = flush_pending_replicated_message_barrier(state) + state = purge_closed_remote_epochs(state, remote_node, stale) + {:noreply, send_replica_heads(state, remote_node)} + end + + def handle_info( + {:replica_cluster_close, remote_pid, generation, revision, epochs}, + %{shard_index: 0} = state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + controls = + collect_replica_cluster_controls( + :replica_cluster_close, + remote_pid, + generation, + [{revision, epochs}], + state.replicated_sender_buffer_size - 1 + ) + + case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + {:accept, observed_revision, epochs} -> + closed = + Data.close_remote_cluster_epochs( + state.name, + 0, + remote_node, + observed_revision, + epochs + ) + + if state.shard_index == 0 do + Data.remove_cluster_node(state.name, Enum.map(closed, &elem(&1, 0)), remote_node) + end + + fan_out_to_siblings( + state, + {:replica_cluster_close_control_local, remote_node, generation, observed_revision, + closed} + ) + + state = + state + |> mark_cluster_control_dirty(remote_node) + |> purge_closed_remote_epochs(remote_node, closed) + + {:noreply, take_one_local_request_turn(state)} + + :stale -> + {:noreply, take_one_local_request_turn(state)} + + :refresh -> + {:noreply, + state + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + end + + def handle_info({:replica_cluster_close, remote_pid, _generation, _revision, _epochs}, state) do + {:noreply, request_replica_authority(state, node(remote_pid))} + end + + def handle_info( + {:replica_cluster_close_control_local, remote_node, generation, revision, closed}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + state = + if replica_authority_current?(state, remote_node, generation, revision) do + purge_closed_remote_epochs(state, remote_node, closed) + else + state + end + + {:noreply, take_one_local_request_turn(state)} + end + + def handle_info({:replica_cluster_close_local, remote_node, closed}, state) do + state = flush_pending_replicated_message_barrier(state) + + :ok = + Data.forget_remote_cluster_epochs(state.name, state.shard_index, remote_node, closed) + + {:noreply, purge_closed_remote_epochs(state, remote_node, closed)} + end + + def handle_info( + {:replica_heartbeat, remote_pid, version, generation, epoch_revision}, + state + ) do + remote_node = node(remote_pid) + + state = + if version == Protocol.version() and + replica_authority_current?(state, remote_node, generation, epoch_revision) do + :ok = + Data.put_remote_view_info( + state.name, + state.shard_index, + remote_node, + generation, + Data.remote_cluster_epoch_exact_revision(state.name, remote_node), + epoch_revision + ) + + state + |> put_remote_shard(remote_node, remote_pid) + |> touch_replica_peer(remote_node) + else + request_replica_authority(state, remote_node) + end + + {:noreply, state} + end + + def handle_info({:replica_hello_request, remote_pid}, state) do + if state.shard_index == 0 do + {:noreply, send_replica_hello(state, node(remote_pid))} + else + send(shard_name(state.name, 0), {:replica_hello_request, remote_pid}) + {:noreply, state} + end + end + + def handle_info({:group_replica_frame, remote_pid, frame}, state) when is_pid(remote_pid) do + remote_node = node(remote_pid) + state = handle_replica_frame(state, remote_node, frame) + {:noreply, take_priority_turn(state)} + end + + def handle_info({:group_replica_frame, remote_node, frame}, state) when is_atom(remote_node) do + state = handle_replica_frame(state, remote_node, frame) + {:noreply, take_priority_turn(state)} + end + + def handle_info({@anti_entropy_timer, ref}, state) do + state = + if state.anti_entropy_ref == ref do + state + |> expire_stale_replica_peers() + |> probe_replica_peers() + |> request_quiet_cluster_hellos() + |> broadcast_replica_heartbeats() + |> broadcast_replica_heads() + |> schedule_anti_entropy() + else + state + end + + {:noreply, state} + end + def handle_info({@local_request_tag, caller_pid, ref, request}, state) when is_pid(caller_pid) and is_reference(ref) do {:noreply, process_local_request_turn(state, [{{:send, caller_pid, ref}, request}])} @@ -655,21 +800,17 @@ defmodule Group.Replica do %{name: name, shard_index: shard} = state remote_node = node(remote_pid) - # Compute shared clusters and add the remote node to nil plus every shared - # named cluster in one serialized membership mutation. + # Compute shared clusters for diagnostics only. The generation-fenced hello + # is the sole authority that mutates peer and cluster membership. Keeping + # discovery hints side-effect free prevents a delayed pre-restart + # peer_connect from permanently re-adding stale cluster rows. my_clusters = Data.my_clusters(name) shared = compute_shared_clusters(my_clusters, remote_clusters) - Data.add_cluster_node(name, [nil | Enum.reject(shared, &is_nil/1)], remote_node) - - already_known = Map.has_key?(state.remote_shards, remote_node) - state = - if already_known do - state - else - Process.monitor(remote_pid) - %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} - end + # Replica peers are addressed by registered `{name, node}` and established + # only by replica_hello. Do not remotely monitor the shard PID: creating a + # remote monitor itself emits a distribution signal and may suspend on a + # busy dist connection. # Send ack with our cluster list send_to_peer( @@ -678,14 +819,12 @@ defmodule Group.Replica do {:peer_connect_ack, self(), shard, state.num_shards, my_clusters} ) + send_replica_hello(state, remote_node) + log_once(state, fn -> "#{log_prefix(state)} peer_connect from #{remote_node} (#{length(shared)} shared clusters)" end) - # Send cluster_state for all shared clusters in one pass (single table scan - # instead of one scan per cluster — O(N) vs O(C×N)) - send_cluster_states(state, shared, remote_node) - {:noreply, state} end @@ -709,28 +848,16 @@ defmodule Group.Replica do %{name: name} = state remote_node = node(remote_pid) - # Compute shared clusters and add the remote node to nil plus every shared - # named cluster in one serialized membership mutation. + # Discovery acknowledgements are hints only; replica_hello is the sole + # generation-fenced authority for peer and cluster membership. my_clusters = Data.my_clusters(name) shared = compute_shared_clusters(my_clusters, remote_clusters) - Data.add_cluster_node(name, [nil | Enum.reject(shared, &is_nil/1)], remote_node) - - already_known = Map.has_key?(state.remote_shards, remote_node) - - state = - if already_known do - state - else - Process.monitor(remote_pid) - %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} - end log_once(state, fn -> "#{log_prefix(state)} peer_connect_ack from #{remote_node} (#{length(shared)} shared clusters)" end) - # Send cluster_state for all shared clusters in one pass - send_cluster_states(state, shared, remote_node) + send_replica_hello(state, remote_node) {:noreply, state} end @@ -787,20 +914,9 @@ defmodule Group.Replica do if shared != [] do Data.add_cluster_node(name, shared, remote_node) - # Bundle this shard's cluster data directly into the ack (one cross-node - # message instead of ack + N separate cluster_state messages) - {reg_by_cluster, pg_by_cluster} = - Data.local_data_by_cluster(name, state.shard_index, shared) - - cluster_data = - for cluster <- shared do - reg_data = Map.get(reg_by_cluster, cluster, []) - pg_data = Map.get(pg_by_cluster, cluster, []) - {cluster, reg_data, pg_data} - end - - send_to_peer(state, remote_node, {:cluster_connect_ack, shared, self(), cluster_data}) - fan_out_to_siblings(state, {:send_cluster_data, shared, remote_node}) + # Membership is a control-plane handshake. Replica state follows on the + # data transport via heads/deltas (or an exact snapshot fallback). + send_to_peer(state, remote_node, {:cluster_connect_ack, shared, self(), []}) end {:noreply, state} @@ -821,7 +937,8 @@ defmodule Group.Replica do if active != [] do Data.add_cluster_node(name, active, remote_node) - # Merge the data bundled in the ack + # The empty data list is the v1 contract. Retain merge support for a + # rolling peer that still bundles legacy cluster data. {new_state, events} = Enum.reduce(cluster_data, {state, []}, fn {cluster, reg_data, pg_data}, {acc_state, acc_events} -> @@ -832,9 +949,7 @@ defmodule Group.Replica do end end) - send_cluster_states(new_state, active, remote_node) - fan_out_to_siblings(new_state, {:send_cluster_data, active, remote_node}) - {new_state, events} + {send_replica_heads(new_state, remote_node), events} else {state, []} end @@ -860,10 +975,24 @@ defmodule Group.Replica do fan_out_to_siblings(state, {:cluster_disconnect, clusters, remote_pid}) end - events = - Enum.reduce(clusters, [], fn cluster, acc -> + {state, events} = + Enum.reduce(clusters, {state, []}, fn cluster, {outer_state, acc} -> + affected_keys = + Data.purge_registry_claims_for_cluster(name, shard, cluster, remote_node) + {purged_reg, purged_pg} = purge_cluster_entries(name, shard, cluster, remote_node) - build_purged_events(name, purged_reg, purged_pg, :cluster_disconnect, acc) + + acc = build_purged_events(name, purged_reg, purged_pg, :cluster_disconnect, acc) + + Enum.reduce(affected_keys, {outer_state, acc}, fn key, {inner_state, inner_events} -> + reconcile_registry_projection( + inner_state, + cluster, + key, + :cluster_disconnect, + inner_events + ) + end) end) notify_monitors(name, events) @@ -900,14 +1029,36 @@ defmodule Group.Replica do # Purge all data from the dead node {purged_reg, purged_pg} = Data.purge_node(name, shard, dead_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, dead_node) log_once(state, fn -> "#{log_prefix(state)} nodedown #{dead_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg entries)" end) events = build_purged_events(name, purged_reg, purged_pg, :nodedown) + + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + notify_monitors(name, events) - state = %{state | remote_shards: Map.delete(state.remote_shards, dead_node)} + + state = %{ + state + | remote_shards: Map.delete(state.remote_shards, dead_node), + peer_last_seen: Map.delete(state.peer_last_seen, dead_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node) + } + + Data.delete_replica_cursors_for_origin(name, shard, dead_node) + Data.delete_remote_replica_info(name, shard, dead_node) + + if function_exported?(state.replica_transport, :peer_down, 3) do + :ok = state.replica_transport.peer_down(name, dead_node, state.replica_transport_opts) + end + + state = %{state | peer_transports: Map.delete(state.peer_transports, dead_node)} {:noreply, state} end @@ -926,12 +1077,25 @@ defmodule Group.Replica do # Unconditional (not gated on shard 0) — same reasoning as nodedown handler. Data.purge_cluster_node(name, remote_node) {purged_reg, purged_pg} = Data.purge_node(name, shard, remote_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, remote_node) log_verbose(state, fn -> "#{log_prefix_shard(state)} remote_shard_down #{remote_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg)" end) events = build_purged_events(name, purged_reg, purged_pg, {:nodedown, remote_node}) + + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection( + acc, + cluster, + key, + {:nodedown, remote_node}, + inner_events + ) + end) + notify_monitors(name, events) state = %{state | remote_shards: Map.delete(state.remote_shards, remote_node)} state = %{state | monitors: Map.delete(state.monitors, pid)} @@ -947,13 +1111,26 @@ defmodule Group.Replica do pids = Enum.map(downs, &elem(&1, 0)) reason_by_pid = Map.new(downs) + {visible_reg, pending_pg} = Data.entries_for_pids(name, shard, pids) + + claimed_reg = + Data.local_registry_claims_by_pids(name, shard, pids) + |> Enum.map(fn {pid, cluster, key, meta, _generation, _epoch} -> + {pid, cluster, key, meta} + end) + + pending_reg = Enum.uniq(visible_reg ++ claimed_reg) + + sequenced_downs = + append_process_down_records(state, reason_by_pid, pending_reg, pending_pg) + {purged_reg, purged_pg} = Data.delete_all_for_pids(name, shard, pids) log_verbose(state, fn -> "#{log_prefix_shard(state)} process_down_batch pids=#{length(downs)} (#{length(purged_reg) + length(purged_pg)} entries cleaned)" end) - broadcast_process_down_batch(state, reason_by_pid, purged_reg, purged_pg) + state = finish_process_down_records(state, sequenced_downs) events = build_process_down_events(name, purged_reg, purged_pg, reason_by_pid) notify_monitors(name, events) state = %{state | monitors: Map.drop(monitors, pids)} @@ -1024,21 +1201,10 @@ defmodule Group.Replica do {:noreply, state} end - def handle_info({@replicated_pg_broadcast_flush_timer, flush_ref}, state) do - state = - if state.pending_replicated_pg_broadcast_flush_ref == flush_ref do - flush_pending_replicated_pg_broadcast(state) - else - state - end - - {:noreply, state} - end - - def handle_info({@replicated_registry_broadcast_flush_timer, flush_ref}, state) do + def handle_info({@replica_broadcast_flush_timer, flush_ref}, state) do state = - if state.pending_replicated_registry_broadcast_flush_ref == flush_ref do - flush_pending_replicated_registry_broadcast(state) + if state.pending_replica_broadcast_flush_ref == flush_ref do + flush_pending_replica_broadcast(state) else state end @@ -1180,16 +1346,10 @@ defmodule Group.Replica do defp process_local_request_turn( state, - [{_reply_to, request} | _] = initial_messages + initial_messages ) do remaining = - case local_request_domain(request) do - :pg -> - max(state.replicated_pg_receiver_local_request_quota - length(initial_messages), 0) - - :other -> - 0 - end + max(state.replicated_pg_receiver_local_request_quota - length(initial_messages), 0) messages = collect_local_request_messages(initial_messages, remaining) process_local_request_messages(state, messages) @@ -1285,8 +1445,14 @@ defmodule Group.Replica do {:cluster_connect, clusters} -> do_cluster_connect(state, clusters) + {:cluster_connect, clusters, epochs} -> + do_cluster_connect(state, clusters, epochs) + {:cluster_disconnect, clusters} -> do_cluster_disconnect(state, clusters) + + {:cluster_disconnect, clusters, epochs} -> + do_cluster_disconnect(state, clusters, epochs) end end @@ -1334,60 +1500,14 @@ defmodule Group.Replica do |> flush_pending_replicated_barrier() end - defp flush_pending_replicated_sender_barrier( - %{ - pending_replicated_pg_broadcast_len: 0, - pending_replicated_registry_broadcast_len: 0 - } = state - ), - do: state + defp flush_pending_replicated_sender_barrier(%{pending_replica_broadcast_len: 0} = state), + do: state - defp flush_pending_replicated_sender_barrier( - %{pending_replicated_pg_broadcast_len: 0, pending_replicated_registry_broadcast_len: len} = - state - ) - when len > 0, - do: flush_pending_replicated_registry_broadcast(state) + defp flush_pending_replicated_sender_barrier(state), do: flush_pending_replica_broadcast(state) - defp flush_pending_replicated_sender_barrier( - %{pending_replicated_pg_broadcast_len: len, pending_replicated_registry_broadcast_len: 0} = - state - ) - when len > 0, - do: flush_pending_replicated_pg_broadcast(state) - - defp flush_pending_replicated_sender_barrier(state) do - if state.pending_replicated_pg_broadcast_started_at <= - state.pending_replicated_registry_broadcast_started_at do - state - |> flush_pending_replicated_pg_broadcast() - |> flush_pending_replicated_registry_broadcast() - else - state - |> flush_pending_replicated_registry_broadcast() - |> flush_pending_replicated_pg_broadcast() - end - end - - defp flush_pending_replicated_pg_broadcast_barrier( - %{pending_replicated_pg_broadcast_len: 0} = state - ), - do: state - - defp flush_pending_replicated_pg_broadcast_barrier(state), - do: flush_pending_replicated_pg_broadcast(state) - - defp flush_pending_replicated_registry_broadcast_barrier( - %{pending_replicated_registry_broadcast_len: 0} = state - ), - do: state - - defp flush_pending_replicated_registry_broadcast_barrier(state), - do: flush_pending_replicated_registry_broadcast(state) - - defp process_pg_local_request_batch(state, messages) do - %{name: name, shard_index: shard} = state - local_node = node() + defp process_pg_local_request_batch(state, messages) do + %{name: name, shard_index: shard} = state + local_node = node() {entries, replies, events, broadcasts, new_monitors, maybe_demonitor_pids} = Enum.reduce( @@ -1475,11 +1595,21 @@ defmodule Group.Replica do end ) + sequenced_broadcasts = + broadcasts + |> Enum.reverse() + |> Enum.map(&append_local_replica_record(state, &1)) + {insert_entries, delete_entries} = pg_batch_diff(entries) Data.pg_delete_many(name, shard, delete_entries) Data.pg_insert_many(name, shard, insert_entries) state = finalize_local_batch_monitors(state, new_monitors, maybe_demonitor_pids) - state = send_local_batch_broadcasts(state, broadcasts) + + state = + Enum.reduce(sequenced_broadcasts, state, fn record, acc -> + finish_local_replica_record(acc, record, :pg) + end) + notify_monitors(name, events) reply_local_requests(replies) state @@ -1604,19 +1734,63 @@ defmodule Group.Replica do end defp enqueue_broadcast_op(state, {:register, _cluster, _key, _pid, _meta, _time, _node} = op), - do: enqueue_replicated_registry_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :registry) defp enqueue_broadcast_op(state, {:unregister, _cluster, _key, _pid, _meta, _reason} = op), - do: enqueue_replicated_registry_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :registry) defp enqueue_broadcast_op( state, {:join, _cluster, _key, _pid, _meta, _time, _reason, _node} = op ), - do: enqueue_replicated_pg_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :pg) defp enqueue_broadcast_op(state, {:leave, _cluster, _key, _pid, _meta, _reason} = op), - do: enqueue_replicated_pg_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :pg) + + defp sequence_and_enqueue_broadcast(state, op, domain) do + record = append_local_replica_record(state, op) + finish_local_replica_record(state, record, domain) + end + + defp append_local_replica_record(state, op) do + cluster = Protocol.op_cluster(op) + + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + nil + + stream_id -> + {seq, mutations} = + Data.append_replica_record(state.name, state.shard_index, stream_id, [op]) + + {:sequenced, stream_id, seq, mutations} + end + end + + defp finish_local_replica_record(state, nil, _domain), do: state + + defp finish_local_replica_record( + state, + {:sequenced, stream_id, seq, mutations} = sequenced, + domain + ) do + apply_registry_claim_mutations(state, stream_id, seq, mutations) + + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + case domain do + :registry -> enqueue_replicated_registry_broadcast(state, sequenced) + :pg -> enqueue_replicated_pg_broadcast(state, sequenced) + end + end defp reply_local_requests(replies) do Enum.each(Enum.reverse(replies), fn {reply_to, reply} -> @@ -1634,6 +1808,8 @@ defmodule Group.Replica do case Data.registry_lookup(name, shard, cluster, key) do nil -> time = System.system_time() + op = {:register, cluster, key, pid, meta, time, node(pid)} + record = append_local_replica_record(state, op) mref = monitor_pid(state, pid) Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) @@ -1641,11 +1817,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} register key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:register, cluster, key, pid, meta, time, node(pid)} - ) + state = finish_local_replica_record(state, record, :registry) state = put_monitor(state, pid, mref) @@ -1660,17 +1832,15 @@ defmodule Group.Replica do {^pid, old_meta, _time, _node} -> time = System.system_time() + op = {:register, cluster, key, pid, meta, time, node(pid)} + record = append_local_replica_record(state, op) Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) log_verbose(state, fn -> "#{log_prefix_shard(state)} re-register key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:register, cluster, key, pid, meta, time, node(pid)} - ) + state = finish_local_replica_record(state, record, :registry) event = build_event(name, :registered, key, pid, meta, %{ @@ -1691,6 +1861,8 @@ defmodule Group.Replica do case Data.registry_lookup(name, shard, cluster, key) do {pid, meta, _time, entry_node} when entry_node == node() -> + op = {:unregister, cluster, key, pid, meta, :unregister} + record = append_local_replica_record(state, op) Data.registry_delete(name, shard, cluster, key, pid) state = maybe_demonitor_pid(state, name, shard, pid) @@ -1698,11 +1870,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} unregister key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:unregister, cluster, key, pid, meta, :unregister} - ) + state = finish_local_replica_record(state, record, :registry) event = build_event(name, :unregistered, key, pid, meta, %{ @@ -1727,6 +1895,8 @@ defmodule Group.Replica do case Data.pg_lookup(name, shard, cluster, key, pid) do nil -> time = System.system_time() + op = {:join, cluster, key, pid, meta, time, :join, node(pid)} + record = append_local_replica_record(state, op) mref = monitor_pid(state, pid) Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) @@ -1734,11 +1904,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} join key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:join, cluster, key, pid, meta, time, :join, node(pid)} - ) + state = finish_local_replica_record(state, record, :pg) state = put_monitor(state, pid, mref) @@ -1753,17 +1919,15 @@ defmodule Group.Replica do {old_meta, _time, _node} -> time = System.system_time() + op = {:join, cluster, key, pid, meta, time, :update, node(pid)} + record = append_local_replica_record(state, op) Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) log_verbose(state, fn -> "#{log_prefix_shard(state)} re-join key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:join, cluster, key, pid, meta, time, :update, node(pid)} - ) + state = finish_local_replica_record(state, record, :pg) event = build_event(name, :joined, key, pid, meta, %{previous_meta: old_meta, cluster: cluster}) @@ -1781,6 +1945,8 @@ defmodule Group.Replica do {{:error, :not_in_group}, state} {meta, _time, _node} -> + op = {:leave, cluster, key, pid, meta, :leave} + record = append_local_replica_record(state, op) Data.pg_delete(name, shard, cluster, key, pid) state = maybe_demonitor_pid(state, name, shard, pid) @@ -1788,11 +1954,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} leave key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:leave, cluster, key, pid, meta, :leave} - ) + state = finish_local_replica_record(state, record, :pg) event = build_event(name, :left, key, pid, meta, %{reason: :leave, cluster: cluster}) notify_monitors(name, [event]) @@ -1800,7 +1962,15 @@ defmodule Group.Replica do end end - defp do_cluster_connect(state, clusters) do + defp do_cluster_connect(state, clusters), + do: + do_cluster_connect( + state, + clusters, + Enum.map(clusters, &{&1, Data.local_cluster_epoch(state.name, &1)}) + ) + + defp do_cluster_connect(state, clusters, epochs) do state = flush_pending_replicated_sender_barrier(state) %{name: name} = state @@ -1811,13 +1981,33 @@ defmodule Group.Replica do peers = Data.cluster_nodes(name, nil) -- [node()] for target_node <- peers do - send_remote_shard_message(state, target_node, {:cluster_connect, clusters, self()}) + shared = + Enum.filter(clusters, fn cluster -> + not is_nil(Data.remote_cluster_epoch(name, target_node, cluster)) + end) + + Data.add_cluster_node(name, shared, target_node) + + send_remote_shard_message( + state, + target_node, + {:replica_cluster_open, self(), Data.generation(name), + Data.local_cluster_epoch_revision(name), epochs} + ) end {:ok, state} end - defp do_cluster_disconnect(state, clusters) do + defp do_cluster_disconnect(state, clusters), + do: + do_cluster_disconnect( + state, + clusters, + Enum.map(clusters, &{&1, Data.closed_local_cluster_epoch(state.name, &1)}) + ) + + defp do_cluster_disconnect(state, clusters, epochs) do state = flush_pending_replicated_sender_barrier(state) %{name: name, shard_index: shard} = state @@ -1827,7 +2017,9 @@ defmodule Group.Replica do {events, local_pids} = Enum.reduce(clusters, {[], MapSet.new()}, fn cluster, {events, local_pids} -> - {purged_reg, purged_pg} = purge_cluster_entries(name, shard, cluster, :all) + affected_keys = Data.purge_registry_claims_for_cluster(name, shard, cluster) + purged_reg = Data.delete_registry_keys(name, shard, cluster, affected_keys) + purged_pg = Data.delete_pg_cluster(name, shard, cluster) local_pids = Enum.reduce(purged_reg ++ purged_pg, local_pids, fn @@ -1847,10 +2039,28 @@ defmodule Group.Replica do maybe_demonitor_pid(acc, name, shard, pid) end) + # Disconnect purges every origin's materialized rows for these clusters. + # Forget their receive cursors as well: if this node later reconnects while + # a remote origin kept the same epoch, its advertised head must rebuild the + # rows instead of being mistaken for data we still retain. + :ok = Data.delete_replica_cursors_for_clusters(name, shard, clusters) + if shard == 0 do - broadcast_to_peers(state, {:cluster_disconnect, clusters, self()}) + broadcast_to_peers( + state, + {:replica_cluster_close, self(), Data.generation(name), + Data.local_cluster_epoch_revision(name), epochs} + ) end + Enum.each(epochs, fn + {cluster, epoch} when not is_nil(epoch) -> + Data.drop_local_stream(name, shard, cluster, epoch) + + _ -> + :ok + end) + notify_monitors(name, events) {:ok, state} end @@ -1919,66 +2129,35 @@ defmodule Group.Replica do end defp enqueue_replicated_pg_broadcast(state, op), - do: enqueue_replicated_pg_broadcasts(state, [op]) - - defp enqueue_replicated_pg_broadcasts(state, ops) do - state = flush_pending_replicated_registry_broadcast_barrier(state) - now = System.monotonic_time(:millisecond) - ops_len = length(ops) - - state = - case state.pending_replicated_pg_broadcast_len do - 0 -> - %{state | pending_replicated_pg_broadcast_started_at: now} - |> schedule_replicated_pg_broadcast_flush() - |> Map.put(:pending_replicated_pg_broadcast_ops, Enum.reverse(ops)) - |> Map.put(:pending_replicated_pg_broadcast_len, ops_len) - - len -> - %{ - state - | pending_replicated_pg_broadcast_ops: - Enum.reverse(ops, state.pending_replicated_pg_broadcast_ops), - pending_replicated_pg_broadcast_len: len + ops_len - } - end - - if state.pending_replicated_pg_broadcast_len >= state.replicated_sender_buffer_size or - pending_replicated_pg_broadcast_due?(state, now) do - flush_pending_replicated_pg_broadcast(state) - else - state - end - end + do: enqueue_replica_broadcasts(state, [op]) defp enqueue_replicated_registry_broadcast(state, op), - do: enqueue_replicated_registry_broadcasts(state, [op]) + do: enqueue_replica_broadcasts(state, [op]) - defp enqueue_replicated_registry_broadcasts(state, ops) do - state = flush_pending_replicated_pg_broadcast_barrier(state) + defp enqueue_replica_broadcasts(state, ops) do now = System.monotonic_time(:millisecond) ops_len = length(ops) state = - case state.pending_replicated_registry_broadcast_len do + case state.pending_replica_broadcast_len do 0 -> - %{state | pending_replicated_registry_broadcast_started_at: now} - |> schedule_replicated_registry_broadcast_flush() - |> Map.put(:pending_replicated_registry_broadcast_ops, Enum.reverse(ops)) - |> Map.put(:pending_replicated_registry_broadcast_len, ops_len) + %{state | pending_replica_broadcast_started_at: now} + |> schedule_replica_broadcast_flush() + |> Map.put(:pending_replica_broadcast_ops, Enum.reverse(ops)) + |> Map.put(:pending_replica_broadcast_len, ops_len) len -> %{ state - | pending_replicated_registry_broadcast_ops: - Enum.reverse(ops, state.pending_replicated_registry_broadcast_ops), - pending_replicated_registry_broadcast_len: len + ops_len + | pending_replica_broadcast_ops: + Enum.reverse(ops, state.pending_replica_broadcast_ops), + pending_replica_broadcast_len: len + ops_len } end - if state.pending_replicated_registry_broadcast_len >= state.replicated_sender_buffer_size or - pending_replicated_registry_broadcast_due?(state, now) do - flush_pending_replicated_registry_broadcast(state) + if state.pending_replica_broadcast_len >= state.replicated_sender_buffer_size or + pending_replica_broadcast_due?(state, now) do + flush_pending_replica_broadcast(state) else state end @@ -2025,6 +2204,55 @@ defmodule Group.Replica do state = process_inline_priority_message(state, msg) take_priority_control_turn(state) + {:replica_hello, _remote_pid, _version, _generation, _epoch_revision, _cluster_epochs, + _transport_id, _descriptor} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_lane_hello, _remote_pid, _version, _generation, _epoch_revision, _transport_id, + _descriptor} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_authority_installed_local, _remote_node, _generation, _epoch_revision, + _old_generation, _stale_epochs} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_authority_removed_local, _remote_node} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_authority_dirty_local, _remote_node} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_cluster_open_control_local, _remote_node, _generation, _revision, _epochs, _stale, + _shared} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_cluster_close_control_local, _remote_node, _generation, _revision, _closed} = + msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_heartbeat, _remote_pid, _version, _generation, _epoch_revision} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_hello_request, _remote_pid} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_cluster_open, _remote_pid, _generation, _revision, _epochs} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_cluster_close, _remote_pid, _generation, _revision, _epochs} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + {:send_cluster_data, _clusters, _target_node} = msg -> state = process_inline_priority_message(state, msg) take_priority_control_turn(state) @@ -2098,38 +2326,20 @@ defmodule Group.Replica do %{state | pending_replicated_registry_flush_ref: flush_ref} end - defp schedule_replicated_pg_broadcast_flush(%{replicated_sender_flush_interval: 0} = state) do - %{state | pending_replicated_pg_broadcast_flush_ref: nil} - end - - defp schedule_replicated_pg_broadcast_flush(state) do - flush_ref = make_ref() - - Process.send_after( - self(), - {@replicated_pg_broadcast_flush_timer, flush_ref}, - state.replicated_sender_flush_interval - ) - - %{state | pending_replicated_pg_broadcast_flush_ref: flush_ref} - end - - defp schedule_replicated_registry_broadcast_flush( - %{replicated_sender_flush_interval: 0} = state - ) do - %{state | pending_replicated_registry_broadcast_flush_ref: nil} + defp schedule_replica_broadcast_flush(%{replicated_sender_flush_interval: 0} = state) do + %{state | pending_replica_broadcast_flush_ref: nil} end - defp schedule_replicated_registry_broadcast_flush(state) do + defp schedule_replica_broadcast_flush(state) do flush_ref = make_ref() Process.send_after( self(), - {@replicated_registry_broadcast_flush_timer, flush_ref}, + {@replica_broadcast_flush_timer, flush_ref}, state.replicated_sender_flush_interval ) - %{state | pending_replicated_registry_broadcast_flush_ref: flush_ref} + %{state | pending_replica_broadcast_flush_ref: flush_ref} end defp pending_replicated_pg_due?(%{pending_replicated_pg_len: 0}, _now), do: false @@ -2147,24 +2357,12 @@ defmodule Group.Replica do state.replicated_registry_receiver_flush_interval end - defp pending_replicated_pg_broadcast_due?(%{pending_replicated_pg_broadcast_len: 0}, _now), + defp pending_replica_broadcast_due?(%{pending_replica_broadcast_len: 0}, _now), do: false - defp pending_replicated_pg_broadcast_due?(state, now) do - state.replicated_sender_flush_interval == 0 or - now - state.pending_replicated_pg_broadcast_started_at >= - state.replicated_sender_flush_interval - end - - defp pending_replicated_registry_broadcast_due?( - %{pending_replicated_registry_broadcast_len: 0}, - _now - ), - do: false - - defp pending_replicated_registry_broadcast_due?(state, now) do + defp pending_replica_broadcast_due?(state, now) do state.replicated_sender_flush_interval == 0 or - now - state.pending_replicated_registry_broadcast_started_at >= + now - state.pending_replica_broadcast_started_at >= state.replicated_sender_flush_interval end @@ -2222,47 +2420,24 @@ defmodule Group.Replica do } end - defp flush_pending_replicated_pg_broadcast(%{pending_replicated_pg_broadcast_len: 0} = state), + defp flush_pending_replica_broadcast(%{pending_replica_broadcast_len: 0} = state), do: state - defp flush_pending_replicated_pg_broadcast(state) do - ops = Enum.reverse(state.pending_replicated_pg_broadcast_ops) - - log_verbose(state, fn -> - "#{log_prefix_shard(state)} flush_replicated_pg_broadcast_buffer ops=#{length(ops)}" - end) - - send_replicated_pg_batches(state, ops) - - %{ - state - | pending_replicated_pg_broadcast_ops: [], - pending_replicated_pg_broadcast_len: 0, - pending_replicated_pg_broadcast_started_at: nil, - pending_replicated_pg_broadcast_flush_ref: nil - } - end - - defp flush_pending_replicated_registry_broadcast( - %{pending_replicated_registry_broadcast_len: 0} = state - ), - do: state - - defp flush_pending_replicated_registry_broadcast(state) do - ops = Enum.reverse(state.pending_replicated_registry_broadcast_ops) + defp flush_pending_replica_broadcast(state) do + ops = Enum.reverse(state.pending_replica_broadcast_ops) log_verbose(state, fn -> - "#{log_prefix_shard(state)} flush_replicated_registry_broadcast_buffer ops=#{length(ops)}" + "#{log_prefix_shard(state)} flush_replica_broadcast_buffer ops=#{length(ops)}" end) - send_replicated_registry_batches(state, ops) + send_replicated_batches(state, ops) %{ state - | pending_replicated_registry_broadcast_ops: [], - pending_replicated_registry_broadcast_len: 0, - pending_replicated_registry_broadcast_started_at: nil, - pending_replicated_registry_broadcast_flush_ref: nil + | pending_replica_broadcast_ops: [], + pending_replica_broadcast_len: 0, + pending_replica_broadcast_started_at: nil, + pending_replica_broadcast_flush_ref: nil } end @@ -2381,22 +2556,28 @@ defmodule Group.Replica do member = {cluster, key, pid} {initial, current} = replicated_pg_entry(entries, name, shard, member) - previous_meta = - case {current, reason} do - {{old_meta, _old_time, _old_node}, :update} -> old_meta - _ -> nil - end + case current do + {^meta, ^time, ^entry_node} -> + {entries, events} - event = - build_event(name, :joined, key, pid, meta, %{ - previous_meta: previous_meta, - cluster: cluster - }) + _ -> + previous_meta = + case {current, reason} do + {{old_meta, _old_time, _old_node}, :update} -> old_meta + _ -> nil + end + + event = + build_event(name, :joined, key, pid, meta, %{ + previous_meta: previous_meta, + cluster: cluster + }) - updated_entries = - Map.put(entries, member, {initial, {meta, time, entry_node}}) + updated_entries = + Map.put(entries, member, {initial, {meta, time, entry_node}}) - {updated_entries, [event | events]} + {updated_entries, [event | events]} + end {:leave, cluster, key, pid, meta, reason}, {entries, events} -> member = {cluster, key, pid} @@ -2477,20 +2658,33 @@ defmodule Group.Replica do end) end - defp send_replicated_pg_batches(state, ops) do + defp send_replicated_batches(state, ops) do ops - |> group_broadcast_ops_by_target(state, &pg_op_cluster/1) + |> group_broadcast_ops_by_target(state, &sequenced_op_cluster/1) |> Enum.each(fn {target_node, target_ops} -> - send_remote_shard_message(state, target_node, {:replicate_pg_batch, target_ops}) + send_replica_delta_batch(state, target_node, target_ops) end) end - defp send_replicated_registry_batches(state, ops) do - ops - |> group_broadcast_ops_by_target(state, ®istry_op_cluster/1) - |> Enum.each(fn {target_node, target_ops} -> - send_remote_shard_message(state, target_node, {:replicate_registry_batch, target_ops}) - end) + defp send_replica_delta_batch(state, target_node, sequenced_ops) do + runs = + sequenced_ops + |> Enum.group_by(fn {:sequenced, stream_id, _seq, _mutations} -> stream_id end) + |> Enum.map(fn {stream_id, records} -> + records = + records + |> Enum.map(fn {:sequenced, ^stream_id, seq, mutations} -> {seq, mutations} end) + |> Enum.sort_by(&elem(&1, 0)) + + {first_seq, _mutations} = hd(records) + + {_floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + {stream_id, first_seq, records, head} + end) + + try_send_replica_frame(state, target_node, {:delta_batch, Protocol.version(), runs}) end defp group_broadcast_ops_by_target(ops, state, cluster_fun) do @@ -2521,7 +2715,7 @@ defmodule Group.Replica do target_nodes = case cluster do nil -> - for {target_node, _pid} <- state.remote_shards, do: target_node + for {target_node, _last_seen} <- state.peer_last_seen, do: target_node _cluster -> for target_node <- Data.cluster_nodes(state.name, cluster), @@ -2538,11 +2732,19 @@ defmodule Group.Replica do defp pg_op_cluster({:leave, cluster, _key, _pid, _meta, _reason}), do: cluster + defp pg_op_cluster({:sequenced, _stream_id, _seq, [op | _]}), do: pg_op_cluster(op) + defp registry_op_cluster({:register, cluster, _key, _pid, _meta, _time, _entry_node}), do: cluster defp registry_op_cluster({:unregister, cluster, _key, _pid, _meta, _reason}), do: cluster + defp registry_op_cluster({:sequenced, _stream_id, _seq, [op | _]}), + do: registry_op_cluster(op) + + defp sequenced_op_cluster({:sequenced, stream_id, _seq, _mutations}), + do: Protocol.stream_cluster(stream_id) + defp replicated_op_for_active_cluster?(name, op, cluster_fun) when is_function(cluster_fun, 1) do case cluster_fun.(op) do @@ -2557,7 +2759,7 @@ defmodule Group.Replica do defp broadcast_to_peers(state, message) do for {target_node, _pid} <- state.remote_shards do - send_remote_shard_message(state, target_node, message) + send_remote_control_message(state, target_node, message) end end @@ -2569,82 +2771,1183 @@ defmodule Group.Replica do :ok false -> - Group.PeerReconnect.busy_link(state.name, target_node) - :ok + :busy end end - defp broadcast_process_down_batch(state, reason_by_pid, reg_entries, pg_entries) do - messages = - Enum.reduce(reg_entries, %{}, fn {pid, cluster, key, meta}, acc -> - accumulate_process_down_entry( - acc, - process_down_targets(state, cluster), - {:reg, pid, cluster, key, meta, Map.fetch!(reason_by_pid, pid)} - ) - end) - |> then(fn acc -> - Enum.reduce(pg_entries, acc, fn {pid, cluster, key, meta}, inner -> - accumulate_process_down_entry( - inner, - process_down_targets(state, cluster), - {:pg, pid, cluster, key, meta, Map.fetch!(reason_by_pid, pid)} - ) - end) - end) + defp send_remote_control_message(state, target_node, message) do + control_name = shard_name(state.name, 0) - Enum.each(messages, fn {target_node, {reg_entries, pg_entries}} -> - send_remote_shard_message( - state, - target_node, - {:replicate_process_down_batch, Enum.reverse(reg_entries), Enum.reverse(pg_entries)} - ) - end) + case :erlang.send_nosuspend({control_name, target_node}, message, [:noconnect]) do + true -> :ok + false -> :busy + end end - defp process_down_targets(state, nil) do - for {target_node, _pid} <- state.remote_shards, do: target_node - end + defp try_send_replica_frame(state, target_node, frame) do + case state.replica_transport.try_send( + state.name, + target_node, + state.shard_index, + frame, + state.replica_transport_opts + ) do + :ok -> :ok + :busy -> :ok + :disconnected -> :ok + end - defp process_down_targets(%{name: name}, cluster) do - for target_node <- Data.cluster_nodes(name, cluster), target_node != node(), do: target_node + state end - defp accumulate_process_down_entry(acc, target_nodes, {:reg, pid, cluster, key, meta, reason}) do - Enum.reduce(target_nodes, acc, fn target_node, inner -> - Map.update( - inner, - target_node, - {[{pid, cluster, key, meta, reason}], []}, - fn {reg_entries, pg_entries} -> - {[{pid, cluster, key, meta, reason} | reg_entries], pg_entries} - end + defp install_replica_authority( + state, + remote_pid, + generation, + epoch_revision, + cluster_epochs, + transport_id, + transport_descriptor + ) do + remote_node = node(remote_pid) + + shared = + cluster_epochs + |> Enum.map(&elem(&1, 0)) + |> compute_shared_clusters(Data.my_clusters(state.name)) + + previous_shared = Data.clusters_for_node(state.name, remote_node) -- [nil] + shared_set = MapSet.new(shared) + departed = Enum.reject(previous_shared, &MapSet.member?(shared_set, &1)) + Data.remove_cluster_node(state.name, departed, remote_node) + + Data.add_cluster_node( + state.name, + [nil | Enum.reject(shared, &is_nil/1)], + remote_node + ) + + {old_generation, stale_epochs} = + Data.put_remote_replica_info( + state.name, + 0, + remote_node, + generation, + epoch_revision, + cluster_epochs ) - end) + + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) + + state = + if old_generation == generation do + state + |> purge_closed_remote_epochs(remote_node, stale_epochs) + |> purge_remote_streams_outside_authority(remote_node) + else + state + end + + state = %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + peer_transports: + Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) + } + + fan_out_to_siblings( + state, + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs} + ) + + send_replica_heads(state, remote_node) end - defp accumulate_process_down_entry(acc, target_nodes, {:pg, pid, cluster, key, meta, reason}) do - Enum.reduce(target_nodes, acc, fn target_node, inner -> - Map.update( - inner, - target_node, - {[], [{pid, cluster, key, meta, reason}]}, - fn {reg_entries, pg_entries} -> - {reg_entries, [{pid, cluster, key, meta, reason} | pg_entries]} - end - ) - end) + defp notify_replica_transport_peer_up(state, remote_node, transport_descriptor) do + if function_exported?(state.replica_transport, :peer_up, 4) do + :ok = + state.replica_transport.peer_up( + state.name, + remote_node, + transport_descriptor, + state.replica_transport_opts + ) + end + + state end - defp collect_local_process_downs(acc, monitors, 0), do: {Enum.reverse(acc), monitors} + defp send_replica_hello(state, target_node) do + descriptor = state.replica_transport.descriptor(state.name, state.replica_transport_opts) - defp collect_local_process_downs(acc, monitors, remaining) do - receive do - {:DOWN, _mref, :process, pid, reason} when is_map_key(monitors, pid) -> - collect_local_process_downs([{pid, reason} | acc], monitors, remaining - 1) - after - 0 -> - {Enum.reverse(acc), monitors} + if state.shard_index == 0 do + {generation, epoch_revision, cluster_epochs} = + Data.local_replica_authority(state.name) + + send_remote_control_message( + state, + target_node, + {:replica_hello, self(), Protocol.version(), generation, epoch_revision, cluster_epochs, + state.replica_transport.id(), descriptor} + ) + else + send_remote_shard_message( + state, + target_node, + {:replica_lane_hello, self(), Protocol.version(), Data.generation(state.name), + Data.local_cluster_epoch_revision(state.name), state.replica_transport.id(), descriptor} + ) + end + + state + end + + defp request_replica_authority(state, remote_node) do + send_remote_control_message(state, remote_node, {:replica_hello_request, self()}) + state + end + + defp replica_authority_current?(state, remote_node, generation, epoch_revision) do + Data.remote_generation(state.name, remote_node) == generation and + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) == epoch_revision + end + + defp schedule_anti_entropy(state) do + ref = make_ref() + + Process.send_after( + self(), + {@anti_entropy_timer, ref}, + state.replicated_anti_entropy_interval + ) + + %{state | anti_entropy_ref: ref} + end + + defp monotonic_millis, do: System.monotonic_time(:millisecond) + + defp put_remote_shard(state, remote_node, remote_pid) do + %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} + end + + defp touch_replica_peer(state, remote_node) do + %{state | peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis())} + end + + defp collect_replica_cluster_controls(_tag, _remote_pid, _generation, acc, 0), + do: Enum.reverse(acc) + + defp collect_replica_cluster_controls(tag, remote_pid, generation, acc, remaining) do + receive do + {^tag, ^remote_pid, ^generation, revision, epochs} -> + collect_replica_cluster_controls( + tag, + remote_pid, + generation, + [{revision, epochs} | acc], + remaining - 1 + ) + after + 0 -> Enum.reverse(acc) + end + end + + defp accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + case Data.remote_view_generation(state.name, state.shard_index, remote_node) do + ^generation -> + authoritative_revision = + Data.remote_view_cluster_epoch_revision( + state.name, + state.shard_index, + remote_node + ) + + accepted = + Enum.filter(controls, fn {revision, _epochs} -> + is_nil(authoritative_revision) or revision > authoritative_revision + end) + + case accepted do + [] -> + :stale + + accepted -> + accepted = Enum.sort_by(accepted, &elem(&1, 0)) + observed_revision = accepted |> List.last() |> elem(0) + + epochs = + accepted + |> Enum.flat_map(&elem(&1, 1)) + |> Map.new() + |> Map.to_list() + + {:accept, observed_revision, epochs} + end + + _other_generation -> + :refresh + end + end + + defp mark_cluster_control_dirty(state, remote_node) do + %{ + state + | cluster_control_dirty: + Map.put(state.cluster_control_dirty, remote_node, monotonic_millis()) + } + end + + defp mark_authority_dirty(%{shard_index: 0} = state, remote_node) do + mark_cluster_control_dirty(state, remote_node) + end + + defp mark_authority_dirty(state, remote_node) do + if MapSet.member?(state.authority_dirty_notified, remote_node) do + state + else + send(shard_name(state.name, 0), {:replica_authority_dirty_local, remote_node}) + + %{ + state + | authority_dirty_notified: MapSet.put(state.authority_dirty_notified, remote_node) + } + end + end + + defp request_quiet_cluster_hellos(state) do + now = monotonic_millis() + + dirty = + Enum.reduce(state.cluster_control_dirty, %{}, fn {remote_node, last_activity}, acc -> + if now - last_activity >= state.replicated_anti_entropy_interval do + request_replica_authority(state, remote_node) + Map.put(acc, remote_node, now) + else + Map.put(acc, remote_node, last_activity) + end + end) + + %{state | cluster_control_dirty: dirty} + end + + defp broadcast_replica_heartbeats(state) do + Enum.reduce(state.remote_shards, state, fn {target_node, _pid}, acc -> + send_remote_shard_message( + acc, + target_node, + {:replica_heartbeat, self(), Protocol.version(), Data.generation(acc.name), + Data.local_cluster_epoch_revision(acc.name)} + ) + + acc + end) + end + + defp probe_replica_peers(state) do + Enum.each(Node.list(), fn remote_node -> + unless Map.has_key?(state.remote_shards, remote_node) do + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), state.shard_index, state.num_shards, + Data.my_clusters(state.name)} + ) + end + end) + + state + end + + defp broadcast_replica_heads(state) do + Enum.reduce(state.peer_last_seen, state, fn {target_node, _last_seen}, acc -> + send_replica_heads(acc, target_node) + end) + end + + defp expire_stale_replica_peers(state) do + now = monotonic_millis() + + Enum.reduce(state.peer_last_seen, state, fn {remote_node, last_seen}, acc -> + if now - last_seen > acc.replicated_peer_lease_timeout do + expire_replica_peer(acc, remote_node) + else + acc + end + end) + end + + defp expire_replica_peer(state, remote_node) do + %{name: name, shard_index: shard} = state + + if shard == 0 do + Data.purge_cluster_node(name, remote_node) + end + + {purged_reg, purged_pg} = Data.purge_node(name, shard, remote_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, remote_node) + events = build_purged_events(name, purged_reg, purged_pg, :peer_lease_expired) + + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :peer_lease_expired, inner_events) + end) + + notify_monitors(name, events) + Data.delete_replica_cursors_for_origin(name, shard, remote_node) + Data.delete_remote_replica_info(name, shard, remote_node) + + if shard == 0 do + fan_out_to_siblings(state, {:replica_authority_removed_local, remote_node}) + end + + if function_exported?(state.replica_transport, :peer_down, 3) do + :ok = state.replica_transport.peer_down(name, remote_node, state.replica_transport_opts) + end + + %{ + state + | remote_shards: Map.delete(state.remote_shards, remote_node), + peer_last_seen: Map.delete(state.peer_last_seen, remote_node), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, remote_node), + peer_transports: Map.delete(state.peer_transports, remote_node) + } + end + + defp send_replica_heads(state, target_node) do + send_replica_heads(state, target_node, :all) + end + + defp send_replica_heads(state, target_node, clusters) do + heads = replica_heads_for_clusters(state, target_node, clusters) + + if heads == [] do + state + else + try_send_replica_frame(state, target_node, {:heads, Protocol.version(), heads}) + end + end + + defp replica_heads_for_clusters(state, target_node, :all) do + state.name + |> Data.replica_stream_heads(state.shard_index) + |> Enum.filter(fn {stream_id, _floor, _head} -> + replica_stream_target?(state, stream_id, target_node) + end) + end + + defp replica_heads_for_clusters(state, target_node, clusters) when is_list(clusters) do + Enum.flat_map(clusters, fn cluster -> + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + [] + + stream_id -> + {floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + if head > 0 and replica_stream_target?(state, stream_id, target_node) do + [{stream_id, floor, head}] + else + [] + end + end + end) + end + + defp replica_stream_target?(state, stream_id, target_node) do + Protocol.stream_name(stream_id) == state.name and + Protocol.stream_origin(stream_id) == node() and + Protocol.stream_shard(stream_id) == state.shard_index and + Protocol.stream_generation(stream_id) == Data.generation(state.name) and + Protocol.stream_epoch(stream_id) == + Data.local_cluster_epoch(state.name, Protocol.stream_cluster(stream_id)) and + case Protocol.stream_cluster(stream_id) do + nil -> Map.has_key?(state.peer_last_seen, target_node) + cluster -> target_node in Data.cluster_nodes(state.name, cluster) + end + end + + defp valid_remote_stream?(state, source_node, stream_id) do + cluster = Protocol.stream_cluster(stream_id) + + Protocol.stream_name(stream_id) == state.name and + Protocol.stream_origin(stream_id) == source_node and + Protocol.stream_shard(stream_id) == state.shard_index and + Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and + Protocol.stream_epoch(stream_id) == + Data.remote_cluster_epoch(state.name, source_node, cluster) and + (is_nil(cluster) or cluster_member?(state.name, cluster)) + end + + defp handle_replica_frame(state, source_node, {:heads, version, heads}) + when version == @protocol_version do + needs = + Enum.flat_map(heads, fn {stream_id, _floor, head} -> + if valid_remote_stream?(state, source_node, stream_id) do + cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) + if head > cursor, do: [{stream_id, cursor + 1}], else: [] + else + [] + end + end) + + needs + |> Enum.chunk_every(state.replicated_sender_buffer_size) + |> Enum.reduce(state, fn chunk, acc -> + try_send_replica_frame(acc, source_node, {:needs, Protocol.version(), chunk}) + end) + end + + defp handle_replica_frame(state, source_node, {:delta_batch, version, runs}) + when version == @protocol_version do + state = flush_pending_replicated_sender_barrier(state) + + Enum.reduce(runs, state, fn {stream_id, _first_seq, records, advertised_head}, acc -> + apply_replica_delta_run(acc, source_node, stream_id, records, advertised_head) + end) + end + + defp handle_replica_frame(state, source_node, {:need, version, stream_id, next_seq}) + when version == @protocol_version do + if Protocol.stream_origin(stream_id) == node() and + Protocol.stream_shard(stream_id) == state.shard_index and + replica_stream_target?(state, stream_id, source_node) do + send_replica_repair(state, source_node, stream_id, next_seq) + else + state + end + end + + defp handle_replica_frame(state, source_node, {:needs, version, needs}) + when version == @protocol_version do + send_replica_repairs(state, source_node, needs) + end + + defp handle_replica_frame( + state, + source_node, + {:snapshot, version, stream_id, snapshot_seq, reg_data, pg_data} + ) + when version == @protocol_version do + if valid_remote_stream?(state, source_node, stream_id) and + snapshot_seq >= Data.replica_cursor(state.name, state.shard_index, stream_id) do + state = flush_pending_replicated_barrier(state) + cluster = Protocol.stream_cluster(stream_id) + + reg_data = + Enum.filter(reg_data, fn {key, pid, _meta, _time} -> + node(pid) == source_node and + shard_index_for(cluster, key, state.num_shards) == state.shard_index + end) + + affected_registry_keys = + Data.replace_registry_claims_for_stream( + state.name, + state.shard_index, + stream_id, + snapshot_seq, + reg_data + ) + + {state, events} = + Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) + end) + + events = replace_remote_pg_snapshot(state, source_node, cluster, pg_data, events) + :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, snapshot_seq) + notify_monitors(state.name, events) + state + else + state + end + end + + defp handle_replica_frame(state, _source_node, _frame), do: state + + defp apply_replica_delta_run(state, source_node, stream_id, records, advertised_head) do + if valid_remote_stream?(state, source_node, stream_id) do + cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) + records = Enum.drop_while(records, fn {seq, _mutations} -> seq <= cursor end) + + case records do + [] -> + state + + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + request_replica_need(state, source_node, stream_id, cursor + 1) + + _ -> + {contiguous, _next_seq} = take_contiguous_replica_records(records, cursor + 1, []) + + {accepted, rejected} = + Enum.split_while(contiguous, fn {_seq, mutations} -> + valid_replica_mutations?(stream_id, mutations) + end) + + if rejected != [] do + Logger.error( + "#{log_prefix_shard(state)} rejected replica record with invalid origin/cluster authority from #{inspect(source_node)}" + ) + end + + state = + apply_received_replica_records(state, stream_id, accepted) + |> flush_pending_replicated_barrier() + + case List.last(accepted) do + nil -> + if rejected == [] do + state + else + request_replica_need(state, source_node, stream_id, cursor + 1) + end + + {last_seq, _mutations} -> + :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, last_seq) + + if last_seq < advertised_head or length(accepted) < length(records) do + request_replica_need(state, source_node, stream_id, last_seq + 1) + else + state + end + end + end + else + state + end + end + + defp take_contiguous_replica_records([], next_seq, acc), do: {Enum.reverse(acc), next_seq} + + defp take_contiguous_replica_records([{seq, mutations} | rest], seq, acc) do + take_contiguous_replica_records(rest, seq + 1, [{seq, mutations} | acc]) + end + + defp take_contiguous_replica_records(_records, next_seq, acc), + do: {Enum.reverse(acc), next_seq} + + defp valid_replica_mutations?(stream_id, mutations) do + origin = Protocol.stream_origin(stream_id) + cluster = Protocol.stream_cluster(stream_id) + + mutations != [] and Enum.all?(mutations, &valid_replica_mutation?(&1, cluster, origin)) + end + + defp valid_replica_mutation?( + {:register, cluster, _key, pid, _meta, _time, entry_node}, + cluster, + origin + ), + do: node(pid) == origin and entry_node == origin + + defp valid_replica_mutation?( + {:unregister, cluster, _key, pid, _meta, _reason}, + cluster, + origin + ), + do: node(pid) == origin + + defp valid_replica_mutation?( + {:join, cluster, _key, pid, _meta, _time, _reason, entry_node}, + cluster, + origin + ), + do: node(pid) == origin and entry_node == origin + + defp valid_replica_mutation?({:leave, cluster, _key, pid, _meta, _reason}, cluster, origin), + do: node(pid) == origin + + defp valid_replica_mutation?(_mutation, _cluster, _origin), do: false + + defp apply_received_replica_records(state, stream_id, records) do + records + |> Enum.chunk_by(fn {_seq, mutations} -> replica_record_domain(mutations) end) + |> Enum.reduce(state, fn records, acc -> + case records |> hd() |> elem(1) |> replica_record_domain() do + :registry -> + acc = flush_pending_replicated_pg_barrier(acc) + + {acc, events} = + Enum.reduce(records, {acc, []}, fn {seq, mutations}, {inner, events} -> + {inner, record_events} = + apply_received_registry_claims(inner, stream_id, seq, mutations) + + {inner, record_events ++ events} + end) + + notify_monitors(acc.name, events) + acc + + _pg_or_mixed -> + Enum.reduce(records, acc, fn {seq, mutations}, inner -> + apply_received_replica_record(inner, stream_id, seq, mutations) + end) + end + end) + end + + defp replica_record_domain(mutations) do + case mutations |> Enum.map(&replica_mutation_domain/1) |> Enum.uniq() do + [domain] -> domain + [] -> :empty + _ -> :mixed + end + end + + defp enqueue_received_replica_mutation(state, {:register, _, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_registry_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:unregister, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_registry_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:join, _, _, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_pg_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:leave, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_pg_ops(state, [op]) + state + end + + defp apply_received_replica_record(state, stream_id, seq, mutations) do + domains = mutations |> Enum.map(&replica_mutation_domain/1) |> Enum.uniq() + + if length(domains) > 1 do + apply_received_mixed_replica_record(state, stream_id, seq, mutations) + else + apply_received_homogeneous_replica_record(state, stream_id, seq, mutations, domains) + end + end + + defp apply_received_homogeneous_replica_record( + state, + stream_id, + seq, + mutations, + [:registry] + ) do + state = flush_pending_replicated_pg_barrier(state) + {state, _events} = apply_received_registry_claims(state, stream_id, seq, mutations) + state + end + + defp apply_received_homogeneous_replica_record(state, _stream_id, _seq, mutations, [:pg]) do + Enum.reduce(mutations, state, fn op, acc -> + enqueue_received_replica_mutation(acc, op) + end) + end + + defp apply_received_homogeneous_replica_record(state, _stream_id, _seq, [], []), do: state + + # Process death may remove registry and PG rows in one authoritative record. + # Apply its maximal same-domain segments in wire order and emit one monitor + # batch, preserving the existing process-down batching contract. + defp apply_received_mixed_replica_record(state, stream_id, seq, mutations) do + state = flush_pending_replicated_barrier(state) + + {state, events} = + mutations + |> Enum.chunk_by(&replica_mutation_domain/1) + |> Enum.reduce({state, []}, fn segment, {acc, events} -> + case replica_mutation_domain(hd(segment)) do + :registry -> + {acc, segment_events} = apply_received_registry_claims(acc, stream_id, seq, segment) + {acc, segment_events ++ events} + + :pg -> + {insert_entries, delete_entries, segment_events} = + apply_replicated_pg_ops(acc.name, acc.shard_index, segment) + + Data.pg_delete_many(acc.name, acc.shard_index, delete_entries) + Data.pg_insert_many(acc.name, acc.shard_index, insert_entries) + {acc, segment_events ++ events} + end + end) + + notify_monitors(state.name, events) + state + end + + defp replica_mutation_domain({:register, _, _, _, _, _, _}), do: :registry + defp replica_mutation_domain({:unregister, _, _, _, _, _}), do: :registry + defp replica_mutation_domain({:join, _, _, _, _, _, _, _}), do: :pg + defp replica_mutation_domain({:leave, _, _, _, _, _}), do: :pg + + defp apply_received_registry_claims(state, stream_id, seq, ops) do + keys = + Enum.map(ops, fn + {:register, _cluster, key, pid, meta, time, _entry_node} -> + Data.put_registry_claim( + state.name, + state.shard_index, + stream_id, + seq, + key, + pid, + meta, + time + ) + + key + + {:unregister, _cluster, key, pid, _meta, _reason} -> + Data.delete_registry_claim(state.name, state.shard_index, stream_id, seq, key, pid) + key + end) + |> Enum.uniq() + + cluster = Protocol.stream_cluster(stream_id) + + Enum.reduce(keys, {state, []}, fn key, {acc, events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, events) + end) + end + + defp send_replica_repair(state, target_node, stream_id, next_seq) do + send_replica_repairs(state, target_node, [{stream_id, next_seq}]) + end + + defp request_replica_need(state, target_node, stream_id, next_seq) do + try_send_replica_frame( + state, + target_node, + {:needs, Protocol.version(), [{stream_id, next_seq}]} + ) + end + + defp send_replica_repairs(state, target_node, needs) do + {state, runs} = + Enum.reduce(needs, {state, []}, fn {stream_id, next_seq}, {acc, runs} -> + if Protocol.stream_origin(stream_id) == node() and + Protocol.stream_shard(stream_id) == acc.shard_index and + replica_stream_target?(acc, stream_id, target_node) do + case replica_repair(acc, target_node, stream_id, next_seq) do + {:run, run} -> {acc, [run | runs]} + {:state, acc} -> {acc, runs} + end + else + {acc, runs} + end + end) + + case runs do + [] -> + state + + runs -> + try_send_replica_frame( + state, + target_node, + {:delta_batch, Protocol.version(), Enum.reverse(runs)} + ) + end + end + + defp replica_repair(state, target_node, stream_id, next_seq) do + {floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + cond do + next_seq > head -> + {:state, state} + + next_seq >= floor -> + records = + Data.replica_records( + state.name, + state.shard_index, + stream_id, + next_seq, + state.replicated_sender_buffer_size + ) + + case records do + [] -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + + [{first_seq, _} | _] -> + {:run, {stream_id, first_seq, records, head}} + end + + true -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + end + end + + defp send_replica_snapshot(state, target_node, stream_id, head) do + cluster = Protocol.stream_cluster(stream_id) + + {_reg_by_cluster, pg_by_cluster} = + Data.local_data_by_cluster(state.name, state.shard_index, [cluster]) + + reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) + + try_send_replica_frame( + state, + target_node, + {:snapshot, Protocol.version(), stream_id, head, reg_data, + Map.get(pg_by_cluster, cluster, [])} + ) + end + + defp replace_remote_pg_snapshot(state, source_node, cluster, pg_data, events) do + current = + state.name + |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) + |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + + desired = + pg_data + |> Enum.filter(fn {key, pid, _meta, _time} -> + node(pid) == source_node and + shard_index_for(cluster, key, state.num_shards) == state.shard_index + end) + |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + + {inserts, deletes, events} = + current + |> Map.keys() + |> Kernel.++(Map.keys(desired)) + |> Enum.uniq() + |> Enum.reduce({[], [], events}, fn {key, pid}, {inserts, deletes, acc} -> + case {Map.get(current, {key, pid}), Map.get(desired, {key, pid})} do + {same, same} -> + {inserts, deletes, acc} + + {{old_meta, _old_time}, nil} -> + event = + build_event(state.name, :left, key, pid, old_meta, %{ + reason: :reconcile, + cluster: cluster + }) + + {inserts, [{cluster, key, pid} | deletes], [event | acc]} + + {nil, {meta, time}} -> + event = build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}) + + {[{cluster, key, pid, meta, time, source_node} | inserts], deletes, [event | acc]} + + {{old_meta, _old_time}, {meta, time}} -> + event = + if old_meta == meta do + nil + else + build_event(state.name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + end + + acc = if event, do: [event | acc], else: acc + {[{cluster, key, pid, meta, time, source_node} | inserts], deletes, acc} + end + end) + + Data.pg_delete_many(state.name, state.shard_index, deletes) + Data.pg_insert_many(state.name, state.shard_index, inserts) + events + end + + defp maybe_purge_remote_generation(state, _remote_node, nil, _generation), do: state + + defp maybe_purge_remote_generation(state, _remote_node, generation, generation), do: state + + defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do + {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) + + affected = + Data.purge_registry_claims_for_origin( + state.name, + state.shard_index, + remote_node + ) + + {state, events} = + Enum.reduce(affected, {state, []}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + + notify_monitors(state.name, events) + Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) + state + end + + defp purge_closed_remote_epochs(state, _remote_node, []), do: state + + defp purge_closed_remote_epochs(state, remote_node, cluster_epochs) do + generation = Data.remote_generation(state.name, remote_node) + + stream_ids = + Enum.map(cluster_epochs, fn {cluster, epoch} -> + Protocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + epoch + ) + end) + + affected_keys = + Data.purge_registry_claims_for_streams( + state.name, + state.shard_index, + stream_ids + ) + + Enum.each(stream_ids, fn stream_id -> + :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) + end) + + clusters = cluster_epochs |> Enum.map(&elem(&1, 0)) |> Enum.uniq() + + purged_pg = + Data.delete_pg_for_origin_clusters( + state.name, + state.shard_index, + clusters, + remote_node + ) + + events = build_purged_events(state.name, [], purged_pg, :cluster_disconnect, []) + + {state, events} = + Enum.reduce(affected_keys, {state, events}, fn {cluster, key}, + {inner_state, inner_events} -> + reconcile_registry_projection( + inner_state, + cluster, + key, + :cluster_disconnect, + inner_events + ) + end) + + notify_monitors(state.name, events) + state + end + + defp purge_superseded_remote_streams(state, remote_node, current_epochs) do + generation = Data.remote_generation(state.name, remote_node) + + current_epochs = Map.new(current_epochs) + + superseded = + Enum.flat_map(current_epochs, fn {cluster, current_epoch} -> + current_stream = + Protocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + current_epoch + ) + + state.name + |> Data.replica_cursor_streams_for_origin_cluster( + state.shard_index, + remote_node, + cluster + ) + |> Enum.reject(&(&1 == current_stream)) + end) + + purge_superseded_remote_streams(state, remote_node, current_epochs, superseded) + end + + defp purge_remote_streams_outside_authority(state, remote_node) do + generation = Data.remote_generation(state.name, remote_node) + + streams = + Data.replica_cursor_streams_for_origin( + state.name, + state.shard_index, + remote_node + ) + + # A shard only needs authority for clusters for which it has retained + # receive state. This keeps local fanout proportional to actual shard data, + # rather than rebuilding the node-wide epoch map in every lane. + current_epochs = + streams + |> Enum.map(&Protocol.stream_cluster/1) + |> Enum.uniq() + |> Map.new(fn cluster -> + {cluster, Data.remote_cluster_epoch(state.name, remote_node, cluster)} + end) + + superseded = + Enum.reject(streams, fn stream_id -> + Protocol.stream_generation(stream_id) == generation and + Map.get(current_epochs, Protocol.stream_cluster(stream_id)) == + Protocol.stream_epoch(stream_id) + end) + + purge_superseded_remote_streams(state, remote_node, current_epochs, superseded) + end + + defp purge_superseded_remote_streams(state, _remote_node, _current_epochs, []), do: state + + defp purge_superseded_remote_streams( + state, + remote_node, + current_epochs, + superseded + ) do + generation = Data.remote_generation(state.name, remote_node) + + superseded + |> Enum.group_by(&Protocol.stream_cluster/1) + |> Enum.reduce(state, fn {cluster, cluster_streams}, acc -> + affected_keys = + Data.purge_registry_claims_for_streams( + state.name, + state.shard_index, + cluster_streams + ) + + Enum.each(cluster_streams, fn stream_id -> + :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) + end) + + # PG rows do not carry their stream epoch. Remove the origin/cluster + # slice and reset the current cursor so its exact state is rebuilt by + # the next head advertisement (delta when retained, snapshot after + # pruning). Registry claims do carry epochs and are removed narrowly. + purged_pg = + Data.delete_pg_for_origin_clusters( + state.name, + state.shard_index, + [cluster], + remote_node + ) + + case Map.get(current_epochs, cluster) do + nil -> + :ok + + current_epoch -> + current_stream = + Protocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + current_epoch + ) + + :ok = Data.delete_replica_cursor(state.name, state.shard_index, current_stream) + end + + events = build_purged_events(state.name, [], purged_pg, :cluster_disconnect, []) + + {acc, events} = + Enum.reduce(affected_keys, {acc, events}, fn {affected_cluster, key}, + {inner, inner_events} -> + reconcile_registry_projection( + inner, + affected_cluster, + key, + :cluster_disconnect, + inner_events + ) + end) + + notify_monitors(state.name, events) + acc + end) + end + + defp append_process_down_records(state, reason_by_pid, reg_entries, pg_entries) do + mutations_by_cluster = + Enum.reduce(reg_entries, %{}, fn {pid, cluster, key, meta}, acc -> + op = {:unregister, cluster, key, pid, meta, Map.fetch!(reason_by_pid, pid)} + Map.update(acc, cluster, [op], &[op | &1]) + end) + |> then(fn acc -> + Enum.reduce(pg_entries, acc, fn {pid, cluster, key, meta}, inner -> + op = {:leave, cluster, key, pid, meta, Map.fetch!(reason_by_pid, pid)} + Map.update(inner, cluster, [op], &[op | &1]) + end) + end) + + Enum.flat_map(mutations_by_cluster, fn {cluster, mutations} -> + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + [] + + stream_id -> + {seq, mutations} = + Data.append_replica_record( + state.name, + state.shard_index, + stream_id, + Enum.reverse(mutations) + ) + + [{:sequenced, stream_id, seq, mutations}] + end + end) + end + + defp finish_process_down_records(state, records) do + Enum.each(records, fn {:sequenced, stream_id, seq, mutations} -> + apply_registry_claim_mutations(state, stream_id, seq, mutations) + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + end) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + records + |> Enum.reduce(%{}, fn {:sequenced, _stream_id, _seq, [op | _]} = record, acc -> + cluster = Protocol.op_cluster(op) + + Enum.reduce(process_down_targets(state, cluster), acc, fn target_node, inner -> + Map.update(inner, target_node, [record], &[record | &1]) + end) + end) + |> Enum.reduce(state, fn {target_node, target_records}, acc -> + send_replica_delta_batch(acc, target_node, Enum.reverse(target_records)) + end) + end + + defp process_down_targets(state, nil) do + for {target_node, _last_seen} <- state.peer_last_seen, do: target_node + end + + defp process_down_targets(%{name: name}, cluster) do + for target_node <- Data.cluster_nodes(name, cluster), target_node != node(), do: target_node + end + + defp collect_local_process_downs(acc, monitors, 0), do: {Enum.reverse(acc), monitors} + + defp collect_local_process_downs(acc, monitors, remaining) do + receive do + {:DOWN, _mref, :process, pid, reason} when is_map_key(monitors, pid) -> + collect_local_process_downs([{pid, reason} | acc], monitors, remaining - 1) + after + 0 -> + {Enum.reverse(acc), monitors} end end @@ -2768,6 +4071,111 @@ defmodule Group.Replica do %{state | monitors: monitors} end + defp replay_local_journal(state) do + state.name + |> Data.local_replica_unapplied(state.shard_index) + |> Enum.each(fn {stream_id, seq, mutations} -> + if current_local_stream?(state, stream_id) do + apply_registry_claim_mutations(state, stream_id, seq, mutations) + Enum.each(mutations, &replay_local_mutation(state, &1)) + end + + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + end) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + state + end + + defp current_local_stream?(state, stream_id) do + cluster = Protocol.stream_cluster(stream_id) + + Protocol.stream_name(stream_id) == state.name and + Protocol.stream_origin(stream_id) == node() and + Protocol.stream_generation(stream_id) == Data.generation(state.name) and + Protocol.stream_shard(stream_id) == state.shard_index and + Protocol.stream_epoch(stream_id) == Data.local_cluster_epoch(state.name, cluster) + end + + defp apply_registry_claim_mutations(state, stream_id, seq, mutations) do + Enum.each(mutations, fn + {:register, _cluster, key, pid, meta, time, _entry_node} -> + Data.put_registry_claim( + state.name, + state.shard_index, + stream_id, + seq, + key, + pid, + meta, + time + ) + + {:unregister, _cluster, key, pid, _meta, _reason} -> + Data.delete_registry_claim(state.name, state.shard_index, stream_id, seq, key, pid) + + _pg_mutation -> + :ok + end) + + :ok + end + + defp replay_local_mutation(state, {:register, cluster, key, pid, meta, time, entry_node}) do + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + entry_node + ) + end + + defp replay_local_mutation(state, {:unregister, cluster, key, pid, meta, reason}) do + Data.registry_delete_matching_many( + state.name, + state.shard_index, + [{pid, cluster, key, meta, reason}] + ) + + :ok + end + + defp replay_local_mutation( + state, + {:join, cluster, key, pid, meta, time, _reason, entry_node} + ) do + Data.pg_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + entry_node + ) + end + + defp replay_local_mutation(state, {:leave, cluster, key, pid, meta, reason}) do + Data.pg_delete_matching_many( + state.name, + state.shard_index, + [{pid, cluster, key, meta, reason}] + ) + + :ok + end + defp cluster_member?(name, cluster) do node() in Data.cluster_nodes(name, cluster) end @@ -2882,6 +4290,7 @@ defmodule Group.Replica do cond do winner_pid == remote_pid -> + exit_local_conflict_loser(local_pid, key, remote_meta) time = System.system_time() event = @@ -2893,7 +4302,10 @@ defmodule Group.Replica do { Map.put(entries, entry, {initial, {remote_pid, remote_meta, time, node(remote_pid)}}), [event | events], - broadcasts, + [ + {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} + | broadcasts + ], MapSet.put(maybe_demonitor_pids, local_pid) } @@ -2908,6 +4320,8 @@ defmodule Group.Replica do } true -> + exit_local_conflict_loser(local_pid, key, nil) + event = build_event(state.name, :unregistered, key, local_pid, local_meta, %{ reason: :resolve_conflict, @@ -2923,6 +4337,214 @@ defmodule Group.Replica do end end + defp reconcile_registry_projection(state, cluster, key, reason, events) do + claims = Data.registry_claims(state.name, state.shard_index, cluster, key) + winner = select_registry_claim_winner(state, cluster, key, claims) + + {state, retired?} = retire_local_registry_losers(state, cluster, key, claims, winner) + + winner = + if retired? do + state.name + |> Data.registry_claims(state.shard_index, cluster, key) + |> then(&select_registry_claim_winner(state, cluster, key, &1)) + else + winner + end + + current = Data.registry_lookup(state.name, state.shard_index, cluster, key) + + projection_reason = if retired?, do: :resolve_conflict, else: reason + + project_registry_winner( + state, + cluster, + key, + current, + winner, + projection_reason, + events + ) + end + + defp select_registry_claim_winner(_state, _cluster, _key, []), do: nil + defp select_registry_claim_winner(_state, _cluster, _key, [claim]), do: claim + + defp select_registry_claim_winner(state, cluster, key, claims) do + claims = + Enum.sort_by(claims, fn {pid, _meta, time, origin_node, generation, epoch, _seq} -> + {time, pid, origin_node, generation, epoch} + end) + + Enum.reduce_while(tl(claims), hd(claims), fn claim, winner -> + {winner_pid, winner_meta, winner_time, _origin, _generation, _epoch, _seq} = winner + {pid, meta, time, _origin, _generation, _epoch, _seq} = claim + + selected = + resolve_conflict_winner( + state, + cluster, + key, + {winner_pid, winner_meta, winner_time}, + {pid, meta, time} + ) + + cond do + selected == winner_pid -> {:cont, winner} + selected == pid -> {:cont, claim} + true -> {:halt, nil} + end + end) + end + + defp retire_local_registry_losers(state, cluster, key, claims, winner) do + winner_pid = if winner, do: elem(winner, 0), else: nil + + local_losers = + Enum.filter(claims, fn {pid, _meta, _time, origin_node, _generation, _epoch, _seq} -> + origin_node == node() and pid != winner_pid + end) + + state = + Enum.reduce(local_losers, state, fn + {pid, meta, _time, _origin_node, _generation, _epoch, _seq}, acc -> + op = {:unregister, cluster, key, pid, meta, :resolve_conflict} + record = append_local_replica_record(acc, op) + acc = finish_local_replica_record(acc, record, :registry) + winner_meta = if winner, do: elem(winner, 1), else: nil + exit_local_conflict_loser(pid, key, winner_meta) + acc + end) + + {state, local_losers != []} + end + + defp project_registry_winner(state, _cluster, _key, nil, nil, _reason, events), + do: {state, events} + + defp project_registry_winner( + state, + cluster, + key, + nil, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + _reason, + events + ) do + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) + + event = build_event(state.name, :registered, key, pid, meta, %{cluster: cluster}) + {state, [event | events]} + end + + defp project_registry_winner( + state, + cluster, + key, + {pid, old_meta, old_time, old_node}, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + _reason, + events + ) do + if old_meta == meta and old_time == time and old_node == origin_node do + {state, events} + else + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) + + event = + build_event(state.name, :registered, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + + {state, [event | events]} + end + end + + defp project_registry_winner( + state, + cluster, + key, + {old_pid, old_meta, _old_time, old_node}, + nil, + reason, + events + ) do + Data.registry_delete(state.name, state.shard_index, cluster, key, old_pid) + + state = + if old_node == node() do + maybe_demonitor_pid(state, state.name, state.shard_index, old_pid) + else + state + end + + event = + build_event(state.name, :unregistered, key, old_pid, old_meta, %{ + reason: reason, + cluster: cluster + }) + + {state, [event | events]} + end + + defp project_registry_winner( + state, + cluster, + key, + {old_pid, old_meta, _old_time, old_node}, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + reason, + events + ) do + Data.registry_delete(state.name, state.shard_index, cluster, key, old_pid) + + state = + if old_node == node() do + maybe_demonitor_pid(state, state.name, state.shard_index, old_pid) + else + state + end + + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) + + unregistered = + build_event(state.name, :unregistered, key, old_pid, old_meta, %{ + reason: reason, + cluster: cluster + }) + + registered = build_event(state.name, :registered, key, pid, meta, %{cluster: cluster}) + {state, [registered, unregistered | events]} + end + defp resolve_conflict( state, cluster, @@ -2943,6 +4565,7 @@ defmodule Group.Replica do cond do winner_pid == remote_pid -> + exit_local_conflict_loser(local_pid, key, remote_meta) # Remote wins — replace local entry Data.registry_delete(name, shard, cluster, key, local_pid) state = maybe_demonitor_pid(state, name, shard, local_pid) @@ -2968,6 +4591,12 @@ defmodule Group.Replica do cluster: cluster }) + state = + enqueue_broadcast_op( + state, + {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} + ) + {state, event} winner_pid == local_pid -> @@ -2986,7 +4615,7 @@ defmodule Group.Replica do ) state = - enqueue_replicated_registry_broadcast( + enqueue_broadcast_op( state, {:register, cluster, key, local_pid, local_meta, time, node(local_pid)} ) @@ -2994,12 +4623,13 @@ defmodule Group.Replica do {state, nil} true -> + exit_local_conflict_loser(local_pid, key, nil) # Neither wins — remove both Data.registry_delete(name, shard, cluster, key, local_pid) state = maybe_demonitor_pid(state, name, shard, local_pid) state = - enqueue_replicated_registry_broadcast( + enqueue_broadcast_op( state, {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} ) @@ -3055,7 +4685,7 @@ defmodule Group.Replica do # causing mutual kill (both processes die, key becomes unregistered). # Erlang pids have a total order (by node name then id), so pid comparison # gives a consistent tiebreaker across all nodes. - {winner_pid, winner_meta, loser_pid} = + {winner_pid, _winner_meta, _loser_pid} = if time2 > time1 or (time2 == time1 and pid2 > pid1) do {pid2, meta2, pid1} else @@ -3067,12 +4697,18 @@ defmodule Group.Replica do "pid1=#{inspect(pid1)}, pid2=#{inspect(pid2)}, picking #{inspect(winner_pid)} as winner" end) - Process.exit(loser_pid, {:group_registry_conflict, key, winner_meta}) winner_pid end - # Gather local data for all shared clusters in ONE table scan (instead of C scans) - # and send per-cluster cluster_state messages. One O(N) scan vs C × O(N) scans. + defp exit_local_conflict_loser(pid, key, winner_meta) when node(pid) == node() do + Process.exit(pid, {:group_registry_conflict, key, winner_meta}) + :ok + end + + defp exit_local_conflict_loser(_pid, _key, _winner_meta), do: :ok + + # Legacy receive-only compatibility: gather local data for all requested + # clusters in one scan before emitting the old cluster_state messages. defp send_cluster_states(state, clusters, target_node) do %{name: name, shard_index: shard} = state {reg_by_cluster, pg_by_cluster} = Data.local_data_by_cluster(name, shard, clusters) diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index edfd9a0..e6e4c81 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -10,8 +10,9 @@ defmodule Group.Replica.Data do ## ETS Table Layout - Each shard owns 4 tables. There are also 3 shared tables per Group instance: - 2 for cluster membership and 1 for local named-cluster TTL leases. + Each shard has materialized registry/PG indexes, authoritative registry-claim + indexes, and replica stream/oplog/cursor tables. Shared tables hold cluster + membership, local named-cluster TTL leases, generations, and cluster epochs. ### reg_by_key — `:set`, keyed by `{cluster, key}` @@ -139,8 +140,8 @@ defmodule Group.Replica.Data do All tables are `:public` with `read_concurrency: true`. Reads happen directly from any process (the Replica GenServer, Group API callers, etc.). Writes are serialized through the Replica GenServer for each shard, ensuring consistent paired updates to both the - by_key and by_pid tables. The Data GenServer itself only owns the tables (for crash - survival via rest_for_one) — it handles no messages after init. + by_key and by_pid tables. The Data GenServer owns the tables (for shard-crash survival + via rest_for_one) and serializes cross-shard generation, epoch, and cluster-node changes. """ def start_link(opts) do @@ -151,6 +152,371 @@ defmodule Group.Replica.Data do def data_name(name), do: :"#{name}_data" + # ===================================================================== + # Replica generations, epochs, journal, and cursors + # ===================================================================== + + def generation(name) do + :ets.lookup_element(replication_meta_table(name), :generation, 2) + end + + def local_cluster_epoch_revision(name) do + :ets.lookup_element(replication_meta_table(name), :cluster_epoch_revision, 2) + end + + def local_cluster_epoch(name, nil), do: generation(name) + + def local_cluster_epoch(name, cluster) do + case :ets.lookup(local_cluster_epochs_table(name), cluster) do + [{^cluster, epoch}] -> epoch + [] -> nil + end + end + + def local_cluster_epochs(name) do + [{nil, generation(name)} | :ets.tab2list(local_cluster_epochs_table(name))] + end + + def local_replica_authority(name) do + GenServer.call(data_name(name), :local_replica_authority, :infinity) + end + + def closed_local_cluster_epoch(name, cluster) do + case :ets.lookup(closed_local_cluster_epochs_table(name), cluster) do + [{^cluster, epoch}] -> epoch + [] -> nil + end + end + + def remote_generation(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_generation, remote_node}) do + [{{:remote_generation, ^remote_node}, generation}] -> generation + [] -> nil + end + end + + def remote_cluster_epoch(name, remote_node, nil), do: remote_generation(name, remote_node) + + def remote_cluster_epoch(name, remote_node, cluster) do + case :ets.lookup(remote_cluster_epochs_table(name), {remote_node, cluster}) do + [{{^remote_node, ^cluster}, epoch}] -> epoch + [] -> nil + end + end + + def remote_cluster_epoch_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_revision, remote_node}) do + [{{:remote_epoch_revision, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + def remote_cluster_epoch_exact_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_exact, remote_node}) do + [{{:remote_epoch_exact, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + def remote_cluster_epoch_observed_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_observed, remote_node}) do + [{{:remote_epoch_observed, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + @doc false + def remote_authority_install_count(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_authority_installs, remote_node}) do + [{{:remote_authority_installs, ^remote_node}, count}] -> count + [] -> 0 + end + end + + def remote_view_generation(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, generation, _revision, _observed}] -> + generation + + [] -> + nil + end + end + + def remote_view_cluster_epoch_revision(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, _generation, revision, _observed}] -> + revision + + [] -> + nil + end + end + + def remote_view_observed_revision(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, _generation, _revision, observed}] -> + observed + + [] -> + nil + end + end + + def put_remote_replica_info(name, shard, remote_node, generation, epoch_revision, epochs) do + GenServer.call( + data_name(name), + {:put_remote_replica_info, shard, remote_node, generation, epoch_revision, epochs}, + :infinity + ) + end + + def put_remote_view_info(name, shard, remote_node, generation, authoritative, observed) do + GenServer.call( + data_name(name), + {:put_remote_view_info, shard, remote_node, generation, authoritative, observed}, + :infinity + ) + end + + def put_remote_cluster_epochs(name, shard, remote_node, revision, epochs) do + GenServer.call( + data_name(name), + {:put_remote_cluster_epochs, shard, remote_node, revision, epochs}, + :infinity + ) + end + + def close_remote_cluster_epochs(name, shard, remote_node, revision, epochs) do + GenServer.call( + data_name(name), + {:close_remote_cluster_epochs, shard, remote_node, revision, epochs}, + :infinity + ) + end + + def forget_remote_cluster_epochs(name, shard, remote_node, epochs) do + GenServer.call( + data_name(name), + {:forget_remote_cluster_epochs, shard, remote_node, epochs}, + :infinity + ) + end + + def delete_remote_replica_info(name, shard, remote_node) do + GenServer.call( + data_name(name), + {:delete_remote_replica_info, shard, remote_node}, + :infinity + ) + end + + def activate_local_clusters(name, clusters) do + GenServer.call(data_name(name), {:activate_local_clusters, clusters}, :infinity) + end + + def deactivate_local_clusters(name, clusters) do + GenServer.call(data_name(name), {:deactivate_local_clusters, clusters}, :infinity) + end + + def local_stream_id(name, shard, cluster) do + case local_cluster_epoch(name, cluster) do + nil -> + nil + + epoch -> + Group.Replica.Protocol.stream_id( + name, + node(), + generation(name), + shard, + cluster, + epoch + ) + end + end + + def append_replica_record(name, shard, stream_id, mutations) when is_list(mutations) do + stream_table = replica_stream_meta_table(name, shard) + + head = + :ets.update_counter( + stream_table, + stream_id, + {2, 1}, + {stream_id, 0, 1, 0} + ) + + append_id = + :ets.update_counter( + replication_meta_table(name), + {:append_counter, shard}, + {2, 1}, + {{:append_counter, shard}, 0} + ) + + :ets.insert(replica_oplog_table(name, shard), {{stream_id, head}, append_id, mutations}) + :ets.insert(replica_oplog_order_table(name, shard), {append_id, stream_id, head}) + {head, mutations} + end + + def mark_local_replica_applied(name, shard, stream_id, seq) do + :ets.update_element(replica_stream_meta_table(name, shard), stream_id, {4, seq}) + :ok + end + + def local_replica_unapplied(name, shard) do + replica_stream_meta_table(name, shard) + |> :ets.tab2list() + |> Enum.flat_map(fn {stream_id, head, _floor, applied} -> + if applied < head do + replica_records(name, shard, stream_id, applied + 1, head - applied) + |> Enum.map(fn {seq, mutations} -> {stream_id, seq, mutations} end) + else + [] + end + end) + end + + def replica_stream_heads(name, shard) do + :ets.tab2list(replica_stream_meta_table(name, shard)) + |> Enum.map(fn {stream_id, head, floor, _applied} -> {stream_id, floor, head} end) + end + + def replica_stream_head(name, shard, stream_id) do + case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do + [{^stream_id, head, floor, applied}] -> {floor, head, applied} + [] -> {1, 0, 0} + end + end + + def replica_records(_name, _shard, _stream_id, _from_seq, limit) when limit <= 0, do: [] + + def replica_records(name, shard, stream_id, from_seq, limit) do + table = replica_oplog_table(name, shard) + + table + |> :ets.select( + [ + {{{stream_id, :"$1"}, :_, :"$2"}, [{:>=, :"$1", from_seq}], [{{:"$1", :"$2"}}]} + ], + limit + ) + |> case do + :"$end_of_table" -> [] + {records, _continuation} -> records + end + end + + def prune_replica_oplog(name, shard, max_entries) do + order_table = replica_oplog_order_table(name, shard) + do_prune_replica_oplog(name, shard, order_table, :ets.info(order_table, :size) - max_entries) + end + + defp do_prune_replica_oplog(_name, _shard, _order_table, remaining) when remaining <= 0, + do: :ok + + defp do_prune_replica_oplog(name, shard, order_table, remaining) do + case :ets.first(order_table) do + :"$end_of_table" -> + :ok + + append_id -> + [{^append_id, stream_id, seq}] = :ets.lookup(order_table, append_id) + {_floor, _head, applied} = replica_stream_head(name, shard, stream_id) + + if seq <= applied do + :ets.delete(order_table, append_id) + :ets.delete(replica_oplog_table(name, shard), {stream_id, seq}) + + case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do + [{^stream_id, head, floor, local_applied}] -> + :ets.insert( + replica_stream_meta_table(name, shard), + {stream_id, head, max(floor, seq + 1), local_applied} + ) + + [] -> + :ok + end + + do_prune_replica_oplog(name, shard, order_table, remaining - 1) + else + :ok + end + end + end + + def replica_cursor(name, shard, stream_id) do + case :ets.lookup(replica_cursor_table(name, shard), stream_id) do + [{^stream_id, seq}] -> seq + [] -> 0 + end + end + + def replica_cursor_streams_for_origin_cluster(name, shard, origin_node, cluster) do + :ets.select(replica_cursor_table(name, shard), [ + {{{name, origin_node, :"$1", shard, cluster, :"$2"}, :_}, [], + [{{name, origin_node, :"$1", shard, cluster, :"$2"}}]} + ]) + end + + def replica_cursor_streams_for_origin(name, shard, origin_node) do + :ets.select(replica_cursor_table(name, shard), [ + {{{name, origin_node, :"$1", shard, :"$2", :"$3"}, :_}, [], + [{{name, origin_node, :"$1", shard, :"$2", :"$3"}}]} + ]) + end + + def put_replica_cursor(name, shard, stream_id, seq) do + :ets.insert(replica_cursor_table(name, shard), {stream_id, seq}) + :ok + end + + def delete_replica_cursors_for_origin(name, shard, origin_node) do + :ets.select_delete(replica_cursor_table(name, shard), [ + {{{name, origin_node, :_, shard, :_, :_}, :_}, [], [true]} + ]) + + :ok + end + + def delete_replica_cursors_for_clusters(_name, _shard, []), do: :ok + + def delete_replica_cursors_for_clusters(name, shard, clusters) do + match_specs = + Enum.map(clusters, fn cluster -> + {{{name, :_, :_, shard, cluster, :_}, :_}, [], [true]} + end) + + :ets.select_delete(replica_cursor_table(name, shard), match_specs) + :ok + end + + def delete_replica_cursor(name, shard, stream_id) do + :ets.delete(replica_cursor_table(name, shard), stream_id) + :ok + end + + def drop_local_stream(name, shard, cluster, epoch) do + stream_id = + Group.Replica.Protocol.stream_id(name, node(), generation(name), shard, cluster, epoch) + + append_rows = + :ets.select(replica_oplog_table(name, shard), [ + {{{stream_id, :"$1"}, :"$2", :_}, [], [{{:"$1", :"$2"}}]} + ]) + + Enum.each(append_rows, fn {seq, append_id} -> + :ets.delete(replica_oplog_table(name, shard), {stream_id, seq}) + :ets.delete(replica_oplog_order_table(name, shard), append_id) + end) + + :ets.delete(replica_stream_meta_table(name, shard), stream_id) + + :ok + end + # ===================================================================== # Registry operations # ===================================================================== @@ -248,6 +614,224 @@ defmodule Group.Replica.Data do ]) end + # ===================================================================== + # Authoritative registry claims + # ===================================================================== + + def put_registry_claim(name, shard, stream_id, seq, key, pid, meta, time) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + claim_key = {cluster, key, origin_node, generation, epoch} + by_key = reg_claim_by_key_table(name, shard) + + case :ets.lookup(by_key, claim_key) do + [{^claim_key, _old_pid, _old_meta, _old_time, old_seq}] when old_seq >= seq -> + :ok + + [{^claim_key, old_pid, _old_meta, _old_time, _old_seq}] -> + :ets.delete( + reg_claim_by_pid_table(name, shard), + {old_pid, cluster, key, origin_node, generation, epoch} + ) + + insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) + + [] -> + insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) + end + end + + defp insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) do + {cluster, key, origin_node, generation, epoch} = claim_key + :ets.insert(reg_claim_by_key_table(name, shard), {claim_key, pid, meta, time, seq}) + + :ets.insert( + reg_claim_by_pid_table(name, shard), + {{pid, cluster, key, origin_node, generation, epoch}, meta, time, seq} + ) + + :ok + end + + def delete_registry_claim(name, shard, stream_id, seq, key, pid) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + claim_key = {cluster, key, origin_node, generation, epoch} + + case :ets.lookup(reg_claim_by_key_table(name, shard), claim_key) do + [{^claim_key, ^pid, _meta, _time, old_seq}] when old_seq <= seq -> + :ets.delete(reg_claim_by_key_table(name, shard), claim_key) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin_node, generation, epoch} + ) + + :ok + + _ -> + :ok + end + end + + def registry_claims(name, shard, cluster, key) do + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, key, :"$1", :"$2", :"$3"}, :"$4", :"$5", :"$6", :"$7"}, [], + [{{:"$4", :"$5", :"$6", :"$1", :"$2", :"$3", :"$7"}}]} + ]) + end + + def registry_claims_for_stream(name, shard, stream_id) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, :"$1", origin_node, generation, epoch}, :"$2", :"$3", :"$4", :_}, [], + [{{:"$1", :"$2", :"$3", :"$4"}}]} + ]) + end + + def replace_registry_claims_for_stream(name, shard, stream_id, snapshot_seq, claims) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + existing = registry_claims_for_stream(name, shard, stream_id) + + Enum.each(existing, fn {key, pid, _meta, _time} -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin_node, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin_node, generation, epoch} + ) + end) + + Enum.each(claims, fn {key, pid, meta, time} -> + put_registry_claim(name, shard, stream_id, snapshot_seq, key, pid, meta, time) + end) + + Enum.uniq(Enum.map(existing, &elem(&1, 0)) ++ Enum.map(claims, &elem(&1, 0))) + end + + def purge_registry_claims_for_origin(name, shard, origin_node) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{:"$1", :"$2", origin_node, :"$3", :"$4"}, :"$5", :"$6", :"$7", :_}, [], + [{{:"$1", :"$2", :"$5", :"$6", :"$7", :"$3", :"$4"}}]} + ]) + + Enum.each(claims, fn {cluster, key, pid, _meta, _time, generation, epoch} -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin_node, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin_node, generation, epoch} + ) + end) + + Enum.uniq(Enum.map(claims, fn {cluster, key, _, _, _, _, _} -> {cluster, key} end)) + end + + def purge_registry_claims_for_streams(_name, _shard, []), do: [] + + def purge_registry_claims_for_streams(name, shard, stream_ids) do + streams = + MapSet.new(stream_ids, fn stream_id -> + { + Group.Replica.Protocol.stream_cluster(stream_id), + Group.Replica.Protocol.stream_origin(stream_id), + Group.Replica.Protocol.stream_generation(stream_id), + Group.Replica.Protocol.stream_epoch(stream_id) + } + end) + + claims = + :ets.tab2list(reg_claim_by_key_table(name, shard)) + |> Enum.filter(fn {{cluster, _key, origin, generation, epoch}, _pid, _meta, _time, _seq} -> + MapSet.member?(streams, {cluster, origin, generation, epoch}) + end) + + Enum.each(claims, fn {{cluster, key, origin, generation, epoch}, pid, _meta, _time, _seq} -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin, generation, epoch} + ) + end) + + claims + |> Enum.map(fn {{cluster, key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + {cluster, key} + end) + |> Enum.uniq() + end + + def purge_registry_claims_for_cluster(name, shard, cluster, origin_node \\ :all) + + def purge_registry_claims_for_cluster(name, shard, cluster, :all) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, :"$1", :"$2", :"$3", :"$4"}, :"$5", :"$6", :"$7", :_}, [], + [{{:"$1", :"$5", :"$6", :"$7", :"$2", :"$3", :"$4"}}]} + ]) + + delete_registry_claim_rows(name, shard, cluster, claims) + end + + def purge_registry_claims_for_cluster(name, shard, cluster, origin_node) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, :"$1", origin_node, :"$2", :"$3"}, :"$4", :"$5", :"$6", :_}, [], + [{{:"$1", :"$4", :"$5", :"$6", origin_node, :"$2", :"$3"}}]} + ]) + + delete_registry_claim_rows(name, shard, cluster, claims) + end + + defp delete_registry_claim_rows(name, shard, cluster, claims) do + Enum.each(claims, fn {key, pid, _meta, _time, claim_origin, generation, epoch} -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, claim_origin, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, claim_origin, generation, epoch} + ) + end) + + Enum.uniq(Enum.map(claims, &elem(&1, 0))) + end + + def local_registry_claims_by_pids(name, shard, pids) do + local_node = node() + + Enum.flat_map(Enum.uniq(pids), fn pid -> + :ets.select(reg_claim_by_pid_table(name, shard), [ + {{{pid, :"$1", :"$2", local_node, :"$3", :"$4"}, :"$5", :"$6", :_}, [], + [{{pid, :"$1", :"$2", :"$5", :"$3", :"$4"}}]} + ]) + end) + end + # ===================================================================== # Process group operations # ===================================================================== @@ -423,6 +1007,17 @@ defmodule Group.Replica.Data do process DOWN cleanup. Returns lean `{pid, cluster, key, meta}` tuples for dispatch/event building. """ + def entries_for_pids(_name, _shard, []), do: {[], []} + + def entries_for_pids(name, shard, pids) do + pids = Enum.uniq(pids) + + { + select_entries_for_pids(reg_by_pid_table(name, shard), pids), + select_entries_for_pids(pg_by_pid_table(name, shard), pids) + } + end + def delete_all_for_pids(_name, _shard, []), do: {[], []} def delete_all_for_pids(name, shard, pids) do @@ -621,6 +1216,72 @@ defmodule Group.Replica.Data do {reg_by_cluster, pg_by_cluster} end + def pg_entries_for_origin(name, shard, cluster, origin_node) do + :ets.select(pg_by_key_table(name, shard), [ + {{{cluster, :"$1", :"$2"}, :"$3", :"$4", origin_node}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + ]) + end + + def delete_pg_for_origin_cluster(name, shard, cluster, origin_node) do + entries = pg_entries_for_origin(name, shard, cluster, origin_node) + + Enum.each(entries, fn {key, pid, _meta, _time} -> + :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) + :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + end) + + Enum.map(entries, fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) + end + + def delete_pg_for_origin_clusters(_name, _shard, [], _origin_node), do: [] + + def delete_pg_for_origin_clusters(name, shard, clusters, origin_node) do + cluster_set = MapSet.new(clusters) + + entries = + :ets.select(pg_by_key_table(name, shard), [ + {{{:"$1", :"$2", :"$3"}, :"$4", :"$5", origin_node}, [], + [{{:"$1", :"$2", :"$3", :"$4", :"$5"}}]} + ]) + |> Enum.filter(fn {cluster, _key, _pid, _meta, _time} -> + MapSet.member?(cluster_set, cluster) + end) + + Enum.each(entries, fn {cluster, key, pid, _meta, _time} -> + :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) + :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + end) + + entries + end + + def delete_registry_keys(name, shard, cluster, keys) do + Enum.flat_map(keys, fn key -> + case registry_lookup(name, shard, cluster, key) do + {pid, meta, time, _entry_node} -> + registry_delete(name, shard, cluster, key, pid) + [{cluster, key, pid, meta, time}] + + nil -> + [] + end + end) + end + + def delete_pg_cluster(name, shard, cluster) do + entries = + :ets.select(pg_by_key_table(name, shard), [ + {{{cluster, :"$1", :"$2"}, :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + ]) + + Enum.each(entries, fn {key, pid, _meta, _time} -> + :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) + :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + end) + + Enum.map(entries, fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) + end + def purge_node(name, shard, dead_node) do reg_table = reg_by_key_table(name, shard) reg_pid_table = reg_by_pid_table(name, shard) @@ -786,6 +1447,10 @@ defmodule Group.Replica.Data do GenServer.call(data_name(name), {:remove_cluster_node, clusters, node}, :infinity) end + def remove_clusters(name, clusters) when is_list(clusters) do + GenServer.call(data_name(name), {:remove_clusters, clusters}, :infinity) + end + def all_clusters(name) do table = cluster_nodes_table(name) :ets.select(table, [{{:"$1", :_}, [], [:"$1"]}]) |> Enum.uniq() @@ -796,6 +1461,10 @@ defmodule Group.Replica.Data do :ets.lookup(table, node()) |> Enum.map(&elem(&1, 1)) end + def clusters_for_node(name, target_node) do + :ets.lookup(node_clusters_table(name), target_node) |> Enum.map(&elem(&1, 1)) + end + def purge_cluster_node(name, dead_node) do GenServer.call(data_name(name), {:purge_cluster_node, dead_node}, :infinity) end @@ -841,11 +1510,21 @@ defmodule Group.Replica.Data do def reg_by_key_table(name, shard), do: :"#{name}_s#{shard}_reg_by_key" def reg_by_pid_table(name, shard), do: :"#{name}_s#{shard}_reg_by_pid" + def reg_claim_by_key_table(name, shard), do: :"#{name}_s#{shard}_reg_claim_by_key" + def reg_claim_by_pid_table(name, shard), do: :"#{name}_s#{shard}_reg_claim_by_pid" def pg_by_key_table(name, shard), do: :"#{name}_s#{shard}_pg_by_key" def pg_by_pid_table(name, shard), do: :"#{name}_s#{shard}_pg_by_pid" def cluster_nodes_table(name), do: :"#{name}_cluster_nodes" def node_clusters_table(name), do: :"#{name}_node_clusters" def cluster_leases_table(name), do: :"#{name}_cluster_leases" + def replication_meta_table(name), do: :"#{name}_replication_meta" + def local_cluster_epochs_table(name), do: :"#{name}_local_cluster_epochs" + def closed_local_cluster_epochs_table(name), do: :"#{name}_closed_local_cluster_epochs" + def remote_cluster_epochs_table(name), do: :"#{name}_remote_cluster_epochs" + def replica_stream_meta_table(name, shard), do: :"#{name}_s#{shard}_replica_stream_meta" + def replica_oplog_table(name, shard), do: :"#{name}_s#{shard}_replica_oplog" + def replica_oplog_order_table(name, shard), do: :"#{name}_s#{shard}_replica_oplog_order" + def replica_cursor_table(name, shard), do: :"#{name}_s#{shard}_replica_cursor" # ===================================================================== # GenServer callbacks @@ -867,6 +1546,22 @@ defmodule Group.Replica.Data do {:reply, :ok, state} end + def handle_call({:remove_clusters, clusters}, _from, state) do + Enum.each(clusters, fn cluster -> + nodes = cluster_nodes(state.name, cluster) + :ets.delete(cluster_nodes_table(state.name), cluster) + + Enum.each(nodes, fn cluster_node -> + :ets.delete_object( + node_clusters_table(state.name), + {cluster_node, cluster} + ) + end) + end) + + {:reply, :ok, state} + end + def handle_call({:purge_cluster_node, dead_node}, _from, state) do # Scan the forward index directly so this also repairs a one-sided row left # by an interrupted or older dual-index mutation. @@ -878,14 +1573,267 @@ defmodule Group.Replica.Data do {:reply, :ok, state} end + def handle_call({:activate_local_clusters, clusters}, _from, state) do + if clusters != [] do + :ets.update_counter( + replication_meta_table(state.name), + :cluster_epoch_revision, + {2, 1}, + {:cluster_epoch_revision, 0} + ) + end + + epochs = + Enum.map(clusters, fn cluster -> + epoch = + case :ets.lookup(local_cluster_epochs_table(state.name), cluster) do + [{^cluster, existing}] -> existing + [] -> make_ref() + end + + :ets.insert(local_cluster_epochs_table(state.name), {cluster, epoch}) + :ets.delete(closed_local_cluster_epochs_table(state.name), cluster) + {cluster, epoch} + end) + + {:reply, epochs, state} + end + + def handle_call(:local_replica_authority, _from, state) do + generation = generation(state.name) + revision = local_cluster_epoch_revision(state.name) + epochs = [{nil, generation} | :ets.tab2list(local_cluster_epochs_table(state.name))] + {:reply, {generation, revision, epochs}, state} + end + + def handle_call({:deactivate_local_clusters, clusters}, _from, state) do + if clusters != [] do + :ets.update_counter( + replication_meta_table(state.name), + :cluster_epoch_revision, + {2, 1}, + {:cluster_epoch_revision, 0} + ) + end + + epochs = + Enum.map(clusters, fn cluster -> + epoch = local_cluster_epoch(state.name, cluster) + :ets.delete(local_cluster_epochs_table(state.name), cluster) + if epoch, do: :ets.insert(closed_local_cluster_epochs_table(state.name), {cluster, epoch}) + {cluster, epoch} + end) + + {:reply, epochs, state} + end + + def handle_call( + {:put_remote_replica_info, shard, remote_node, generation, epoch_revision, epochs}, + _from, + state + ) do + # The epoch snapshot is node-wide authority, not shard-local replica data. + # Only shard 0 sends it, and Data serializes the one exact replacement for + # every local replica lane. Keeping the argument in the API makes the + # control-owner invariant explicit and catches accidental reintroduction of + # one full copy per shard. + 0 = shard + seen_generation = remote_generation(state.name, remote_node) + current_epochs = Map.new(epochs) + + stale_epochs = + if seen_generation == generation do + for {{^remote_node, cluster}, epoch} <- + :ets.match_object( + remote_cluster_epochs_table(state.name), + {{remote_node, :_}, :_} + ), + not is_nil(cluster), + Map.get(current_epochs, cluster) != epoch, + do: {cluster, epoch} + else + [] + end + + # A hello is a complete epoch snapshot. The replica handler fences older + # revisions before this call, so replace the shared view rather than merely + # adding rows; otherwise a dropped close control could leave a cluster epoch + # permanently valid after the heartbeat-driven repair. + :ets.select_delete(remote_cluster_epochs_table(state.name), [ + {{{remote_node, :_}, :_}, [], [true]} + ]) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_generation, remote_node}, generation} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, epoch_revision} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_exact, remote_node}, epoch_revision} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_observed, remote_node}, epoch_revision} + ) + + :ets.update_counter( + replication_meta_table(state.name), + {:remote_authority_installs, remote_node}, + {2, 1}, + {{:remote_authority_installs, remote_node}, 0} + ) + + for view_shard <- 0..(state.num_shards - 1) do + :ets.insert( + replication_meta_table(state.name), + {{:remote_view_info, view_shard, remote_node}, generation, epoch_revision, epoch_revision} + ) + end + + rows = + for {cluster, epoch} <- epochs, not is_nil(cluster), do: {{remote_node, cluster}, epoch} + + :ets.insert(remote_cluster_epochs_table(state.name), rows) + + {:reply, {seen_generation, stale_epochs}, state} + end + + def handle_call( + {:put_remote_view_info, shard, remote_node, generation, authoritative, observed}, + _from, + state + ) do + :ets.insert( + replication_meta_table(state.name), + {{:remote_view_info, shard, remote_node}, generation, authoritative, observed} + ) + + {:reply, :ok, state} + end + + def handle_call( + {:put_remote_cluster_epochs, shard, remote_node, revision, epochs}, + _from, + state + ) do + _ = shard + observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + + stale_epochs = + Enum.flat_map(epochs, fn {cluster, epoch} -> + case remote_cluster_epoch(state.name, remote_node, cluster) do + old_epoch when not is_nil(old_epoch) and old_epoch != epoch -> [{cluster, old_epoch}] + _ -> [] + end + end) + + rows = + for {cluster, epoch} <- epochs, + not is_nil(cluster), + do: {{remote_node, cluster}, epoch} + + :ets.insert(remote_cluster_epochs_table(state.name), rows) + + current_revision = remote_cluster_epoch_revision(state.name, remote_node) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, max(current_revision || revision, revision)} + ) + + {:reply, stale_epochs, state} + end + + def handle_call( + {:close_remote_cluster_epochs, shard, remote_node, revision, epochs}, + _from, + state + ) do + 0 = shard + observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + + closed = + Enum.filter(epochs, fn {cluster, epoch} -> + remote_cluster_epoch(state.name, remote_node, cluster) == epoch + end) + + Enum.each(epochs, fn {cluster, epoch} -> + case :ets.lookup(remote_cluster_epochs_table(state.name), {remote_node, cluster}) do + [{{^remote_node, ^cluster}, ^epoch}] -> + :ets.delete(remote_cluster_epochs_table(state.name), {remote_node, cluster}) + + _ -> + :ok + end + end) + + current_revision = remote_cluster_epoch_revision(state.name, remote_node) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, max(current_revision || revision, revision)} + ) + + {:reply, closed, state} + end + + def handle_call( + {:forget_remote_cluster_epochs, shard, remote_node, epochs}, + _from, + state + ) do + # Kept for the rolling-compatibility receive path. The node-wide authority + # table is intentionally not mutated by a shard-local purge. + _ = {shard, remote_node, epochs} + {:reply, :ok, state} + end + + def handle_call({:delete_remote_replica_info, shard, remote_node}, _from, state) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, shard, remote_node} + ) + + if shard == 0 do + :ets.delete(replication_meta_table(state.name), {:remote_generation, remote_node}) + :ets.delete(replication_meta_table(state.name), {:remote_epoch_revision, remote_node}) + :ets.delete(replication_meta_table(state.name), {:remote_epoch_exact, remote_node}) + :ets.delete(replication_meta_table(state.name), {:remote_epoch_observed, remote_node}) + :ets.delete(replication_meta_table(state.name), {:remote_authority_installs, remote_node}) + + if state.num_shards > 1 do + for view_shard <- 1..(state.num_shards - 1) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, view_shard, remote_node} + ) + end + end + + :ets.select_delete(remote_cluster_epochs_table(state.name), [ + {{{remote_node, :_}, :_}, [], [true]} + ]) + end + + {:reply, :ok, state} + end + @impl true def init({name, num_shards}) do # ETS performance options: # - read_concurrency: splits table into read-optimized segments (less lock contention) # - decentralized_counters: reduces contention on table size counter (OTP 23+) - # Note: write_concurrency is intentionally omitted — the sharded GenServer already - # serializes writes per shard, so ETS write locking is never contended. Adding - # write_concurrency adds overhead (~30-40% on serial benchmarks) without benefit. + # Per-shard tables omit write_concurrency because each shard GenServer serializes + # their writes. The shared replication metadata table is different: every shard + # atomically advances its own {:append_counter, shard} object there, so it needs + # concurrent writes without weakening ETS's single-object atomicity. set_opts = [ :set, :public, @@ -894,6 +1842,8 @@ defmodule Group.Replica.Data do decentralized_counters: true ] + shared_meta_opts = Keyword.put(set_opts, :write_concurrency, :auto) + ordered_set_opts = [ :ordered_set, :public, @@ -913,17 +1863,62 @@ defmodule Group.Replica.Data do for shard <- 0..(num_shards - 1) do :ets.new(reg_by_key_table(name, shard), set_opts) :ets.new(reg_by_pid_table(name, shard), ordered_set_opts) + :ets.new(reg_claim_by_key_table(name, shard), ordered_set_opts) + :ets.new(reg_claim_by_pid_table(name, shard), ordered_set_opts) :ets.new(pg_by_key_table(name, shard), ordered_set_opts) :ets.new(pg_by_pid_table(name, shard), ordered_set_opts) + :ets.new(replica_stream_meta_table(name, shard), set_opts) + :ets.new(replica_oplog_table(name, shard), ordered_set_opts) + :ets.new(replica_oplog_order_table(name, shard), ordered_set_opts) + :ets.new(replica_cursor_table(name, shard), set_opts) end :ets.new(cluster_nodes_table(name), bag_opts) :ets.new(node_clusters_table(name), bag_opts) :ets.new(cluster_leases_table(name), set_opts) + :ets.new(replication_meta_table(name), shared_meta_opts) + :ets.new(local_cluster_epochs_table(name), set_opts) + :ets.new(closed_local_cluster_epochs_table(name), set_opts) + :ets.new(remote_cluster_epochs_table(name), set_opts) + :ets.insert(replication_meta_table(name), {:generation, make_ref()}) + :ets.insert(replication_meta_table(name), {:cluster_epoch_revision, 0}) {:ok, %{name: name, num_shards: num_shards}} end + defp observe_remote_cluster_revision(name, remote_node, revision, num_shards) do + key = {:remote_epoch_observed, remote_node} + + case :ets.lookup(replication_meta_table(name), key) do + [{^key, current}] when current >= revision -> :ok + _ -> :ets.insert(replication_meta_table(name), {key, revision}) + end + + for shard <- 0..(num_shards - 1) do + view_key = {:remote_view_info, shard, remote_node} + + case :ets.lookup(replication_meta_table(name), view_key) do + [{^view_key, _generation, _authoritative, observed}] when observed >= revision -> + :ok + + [{^view_key, generation, authoritative, _observed}] -> + :ets.insert( + replication_meta_table(name), + {view_key, generation, authoritative, revision} + ) + + [] -> + :ets.insert( + replication_meta_table(name), + {view_key, remote_generation(name, remote_node), + remote_cluster_epoch_revision(name, remote_node), revision} + ) + end + end + + :ok + end + defp select(table, match_spec, :infinity), do: :ets.select(table, match_spec) defp select(_table, _match_spec, 0), do: [] diff --git a/lib/group/replica/protocol.ex b/lib/group/replica/protocol.ex new file mode 100644 index 0000000..fe88e24 --- /dev/null +++ b/lib/group/replica/protocol.ex @@ -0,0 +1,30 @@ +defmodule Group.Replica.Protocol do + @moduledoc false + + @version 1 + + def version, do: @version + + def stream_id(name, origin_node, origin_generation, shard, cluster, cluster_epoch) do + {name, origin_node, origin_generation, shard, cluster, cluster_epoch} + end + + def stream_name({name, _origin_node, _generation, _shard, _cluster, _epoch}), do: name + + def stream_origin({_name, origin_node, _generation, _shard, _cluster, _epoch}), + do: origin_node + + def stream_generation({_name, _origin_node, generation, _shard, _cluster, _epoch}), + do: generation + + def stream_shard({_name, _origin_node, _generation, shard, _cluster, _epoch}), do: shard + + def stream_cluster({_name, _origin_node, _generation, _shard, cluster, _epoch}), do: cluster + + def stream_epoch({_name, _origin_node, _generation, _shard, _cluster, epoch}), do: epoch + + def op_cluster({:register, cluster, _key, _pid, _meta, _time, _node}), do: cluster + def op_cluster({:unregister, cluster, _key, _pid, _meta, _reason}), do: cluster + def op_cluster({:join, cluster, _key, _pid, _meta, _time, _reason, _node}), do: cluster + def op_cluster({:leave, cluster, _key, _pid, _meta, _reason}), do: cluster +end diff --git a/lib/group/replica/transport.ex b/lib/group/replica/transport.ex new file mode 100644 index 0000000..8b23595 --- /dev/null +++ b/lib/group/replica/transport.ex @@ -0,0 +1,97 @@ +defmodule Group.Replica.Transport do + @moduledoc """ + Transport contract for Group replica data. + + Implementations must return promptly and must never wait for socket or remote + mailbox backpressure. This applies to `try_send/5` and the optional lifecycle + callbacks. Returning `:busy` or `:disconnected` is safe: replica anti-entropy + will retransmit the missing state. + + Erlang distribution remains Group's control plane and supplies the stable + node identity used here. A sideband adapter can use its `descriptor/2` in the + control hello to exchange endpoints, authenticate the connection as that + node, and pass inbound frames to `deliver/4`. + + Adapters do not need to preserve ordering. Group serializes writes per shard + and sequences each origin/generation/shard/cluster/epoch stream; receivers + discard duplicates and request gaps. Per-shard ordered delivery avoids repair + traffic and is therefore the preferred fast path. + """ + + @type frame :: term() + @type send_result :: :ok | :busy | :disconnected + + @callback id() :: term() + @callback descriptor(group :: atom(), opts :: keyword()) :: term() + @callback try_send( + group :: atom(), + target_node :: node(), + shard :: non_neg_integer(), + frame(), + opts :: keyword() + ) :: send_result() + + @callback child_spec(keyword()) :: Supervisor.child_spec() | :ignore + @callback peer_up(group :: atom(), node(), descriptor :: term(), opts :: keyword()) :: :ok + @callback peer_down(group :: atom(), node(), opts :: keyword()) :: :ok + + @optional_callbacks child_spec: 1, peer_up: 4, peer_down: 3 + + @doc """ + Delivers a frame received by a transport adapter to the local replica shard. + + `source_node` must come from the adapter's authenticated peer identity, never + from untrusted frame contents. Delivery is a local mailbox operation; stream + generation, epoch, group, shard, and origin are validated by the replica. + """ + def deliver(group, source_node, shard, frame) + when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 do + send(Group.Replica.shard_name(group, shard), {:group_replica_frame, source_node, frame}) + :ok + end + + def normalize(module) when is_atom(module), do: {module, []} + def normalize({module, opts}) when is_atom(module) and is_list(opts), do: {module, opts} + + def normalize(other) do + raise ArgumentError, + "expected :replica_transport to be a module or {module, opts}, got: #{inspect(other)}" + end + + def validate!({module, _opts} = transport) do + Code.ensure_loaded!(module) + + for {function, arity} <- [id: 0, descriptor: 2, try_send: 5] do + unless function_exported?(module, function, arity) do + raise ArgumentError, + "replica transport #{inspect(module)} must implement #{function}/#{arity}" + end + end + + transport + end +end + +defmodule Group.Replica.Transport.Distribution do + @moduledoc false + @behaviour Group.Replica.Transport + + alias Group.Replica + + @impl true + def id, do: :erlang_distribution + + @impl true + def descriptor(_group, _opts), do: :erlang_distribution + + @impl true + def try_send(group, target_node, shard, frame, _opts) do + destination = {Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, node(), frame} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end +end diff --git a/lib/group/supervisor.ex b/lib/group/supervisor.ex index e4e15b2..0bbd773 100644 --- a/lib/group/supervisor.ex +++ b/lib/group/supervisor.ex @@ -42,6 +42,27 @@ defmodule Group.Supervisor do replicated_pg_receiver_local_request_quota = positive_integer_opt(opts, :replicated_pg_receiver_local_request_quota, 8) + replica_transport = + opts + |> Keyword.get(:replica_transport, Group.Replica.Transport.Distribution) + |> Group.Replica.Transport.normalize() + |> Group.Replica.Transport.validate!() + + replicated_oplog_max_entries = + positive_integer_opt(opts, :replicated_oplog_max_entries, 65_536) + + replicated_anti_entropy_interval = + positive_integer_opt(opts, :replicated_anti_entropy_interval, 1_000) + + replicated_peer_lease_timeout = + positive_integer_opt(opts, :replicated_peer_lease_timeout, 15_000) + + if replicated_peer_lease_timeout <= replicated_anti_entropy_interval do + raise ArgumentError, + ":replicated_peer_lease_timeout must be greater than " <> + ":replicated_anti_entropy_interval" + end + # persistent_term config — must be set before children start (Replica reads it) config = %{ num_shards: num_shards, @@ -54,7 +75,11 @@ defmodule Group.Supervisor do replicated_sender_flush_interval: replicated_sender_flush_interval, busy_dist_retry_attempts: busy_dist_retry_attempts, busy_dist_retry_interval: busy_dist_retry_interval, - replicated_pg_receiver_local_request_quota: replicated_pg_receiver_local_request_quota + replicated_pg_receiver_local_request_quota: replicated_pg_receiver_local_request_quota, + replica_transport: replica_transport, + replicated_oplog_max_entries: replicated_oplog_max_entries, + replicated_anti_entropy_interval: replicated_anti_entropy_interval, + replicated_peer_lease_timeout: replicated_peer_lease_timeout } config = if extract_meta, do: Map.put(config, :extract_meta, extract_meta), else: config @@ -66,18 +91,33 @@ defmodule Group.Supervisor do :persistent_term.put({Group, name}, config) - children = [ - {Group.Replica.Data, name: name, num_shards: num_shards}, - { - Group.PeerReconnect, - name: name, - busy_dist_retry_attempts: busy_dist_retry_attempts, - busy_dist_retry_interval: busy_dist_retry_interval - }, - {Group.Replica.Supervisor, name: name, num_shards: num_shards}, - {Registry, keys: :duplicate, name: Group.registry_name(name)}, - {Group.ClusterLease, name: name, num_shards: num_shards} - ] + transport_children = + case replica_transport do + {module, transport_opts} -> + if function_exported?(module, :child_spec, 1) do + case module.child_spec([name: name, num_shards: num_shards] ++ transport_opts) do + :ignore -> [] + child_spec -> [child_spec] + end + else + [] + end + end + + children = + transport_children ++ + [ + {Group.Replica.Data, name: name, num_shards: num_shards}, + { + Group.PeerReconnect, + name: name, + busy_dist_retry_attempts: busy_dist_retry_attempts, + busy_dist_retry_interval: busy_dist_retry_interval + }, + {Group.Replica.Supervisor, name: name, num_shards: num_shards}, + {Registry, keys: :duplicate, name: Group.registry_name(name)}, + {Group.ClusterLease, name: name, num_shards: num_shards} + ] Supervisor.init(children, strategy: :rest_for_one) end diff --git a/priv/bench/README.md b/priv/bench/README.md index a51e3a8..6278aa5 100644 --- a/priv/bench/README.md +++ b/priv/bench/README.md @@ -7,7 +7,7 @@ separate BEAM VMs. ## Running ```bash -cd priv/group/priv/bench +cd priv/bench mix deps.get ``` @@ -30,10 +30,19 @@ Uses 3 separate BEAM VMs (coordinator + 2 replicas) as OS processes: The script compiles once, starts both replicas in the background, then launches the coordinator. Replicas are killed automatically on exit. +To isolate the 10,000-cluster lifecycle scenario: + +```bash +./run_distributed.sh --shards 4 \ + --coordinator-expr 'GroupBench.Distributed.run_many_clusters_only(shards: 4)' +``` + ## Local Scenarios All local benchmarks run for both the default (nil) cluster and a named cluster (`"game"`) to verify there's no performance difference between the two paths. +Each spawned process cohort is stopped after its measurement and before the +next case; cohort teardown is outside the measured interval. ### 1. Lookup throughput @@ -52,8 +61,10 @@ Slower than lookup because each call copies a 100-element list out of ETS. ### 3. Register throughput (shard scaling) Measures concurrent `Group.register/4` calls — each of 10K spawned processes -registers itself in parallel. Varies shard count (1, 2, 4, schedulers_online) -to show how write throughput scales with sharding. +registers itself in parallel. Uses the library default of 8 shards for the +non-scaling scenarios and a fixed 1, 2, 4, 8, 16, 32, 64 shard sweep. The +fixed sweep keeps results comparable across machines and avoids treating BEAM +scheduler count as a shard-count recommendation. ### 4. Register/unregister cycle @@ -84,7 +95,8 @@ The core distributed measurement. Registers a key on replica1, then spin-polls `Group.lookup` on replica2 until it appears. Repeats 1,000 times. Reports p50/p99/max latency covering the full path: GenServer call on replica1, -Erlang distribution message, GenServer cast on replica2, ETS insert. +write-ahead append, nonblocking replica transport, receiver application, and +ETS projection on replica2. ### 2. Bulk sync (new peer catches up) @@ -92,8 +104,10 @@ Measures how fast a new node catches up to an existing peer's state. Registers N keys on replica1 (1K and 10K), then starts Group on replica2 and polls until all N entries are visible. -Group sends all data in a single `cluster_state` message on peer discovery, so -this is bounded by serialization + network, not per-key round-trips. +Group advertises stream heads on peer discovery. A new peer requests the +missing range; if the bounded oplog no longer contains the prefix, Group sends +an exact per-origin snapshot. The measurement therefore covers the normal +catch-up decision as well as serialization and network transfer. ### 3. Concurrent cross-node writes @@ -115,8 +129,8 @@ compared to the default nil cluster. The critical distributed cleanup path. Registers 1K and 5K processes on replica1, kills them all, then measures how long until replica2 sees zero -entries. Exercises: local DOWN handler → `replicate_unregister` broadcast → -remote ETS cleanup. +entries. Exercises: local DOWN handler → authoritative sequenced unregister +records → nonblocking delta batch → remote ETS cleanup. This scenario catches O(N²) message amplification bugs where remote nodes redundantly monitor pids and re-broadcast cleanup messages. @@ -136,10 +150,42 @@ convergence on replica2 via the `replicate_leave` path. All members hash to the same shard (single key), making this the worst case for shard contention during bulk cleanup. +### 8. Many-cluster lifecycle + +Connects 10K named clusters, registers one process in each, forces peer +re-discovery, then disconnects and verifies cleanup. This exposes control-plane +message amplification and epoch-fence costs. + +The connect phase reports local `Group.connect/2` completion separately from +full remote control convergence. Full convergence checks the reverse cluster +index on both nodes for all 10K named clusters plus the default cluster; seeing +only the last submitted cluster is insufficient because controls may be +reordered or repaired asynchronously. On generation/epoch-aware builds the +barrier also requires every replica shard to hold the source's current control +revision and all 10K remote epoch rows. Registration starts only after this +barrier, so its result does not inherit unfinished connect work. + +Registration reports local completion, the remote count at that handoff, and +the remaining data-convergence tail separately. Re-discovery similarly splits +restart, local reconnect, control convergence, and data convergence; disconnect +splits local completion from remote cleanup. + +### 9. Busy application convergence + +Runs registry and PG churn across 50 clusters and 40K initial pids. Reports +application throughput and then verifies the replicas agree exactly. A fast +wall-clock result is not considered successful unless convergence completes. + +### 10. Local writes under replicated PG pressure + +Floods one receiver shard with remote membership updates while measuring local +register and join calls at increasing concurrency. This checks that bounded +replica turns preserve local control-plane progress. + ## Architecture ``` -priv/group/priv/bench/ +priv/bench/ ├── mix.exs # depends on :group via path: "../../" ├── run_distributed.sh # starts 3 VMs, cleans up on exit ├── README.md @@ -147,7 +193,7 @@ priv/group/priv/bench/ │ ├── group_bench.ex # CLI entry — dispatches local/distributed │ ├── group_bench/ │ │ ├── local.ex # 6 local benchmarks -│ │ ├── distributed.ex # coordinator: connects + drives 7 benchmarks +│ │ ├── distributed.ex # coordinator: connects + drives 10 benchmarks │ │ ├── replica.ex # helpers called by coordinator via :erpc │ │ └── helpers.ex # timing, formatting, percentile math ``` diff --git a/priv/bench/lib/group_bench/distributed.ex b/priv/bench/lib/group_bench/distributed.ex index 68369d2..66b601f 100644 --- a/priv/bench/lib/group_bench/distributed.ex +++ b/priv/bench/lib/group_bench/distributed.ex @@ -52,6 +52,21 @@ defmodule GroupBench.Distributed do IO.puts("\n Done.\n") end + def run_many_clusters_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 4) + Process.put(:bench_shards, shards) + + header("Distributed Many-Clusters Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + bench_many_clusters(@replicas) + + IO.puts("\n Done.\n") + end + # ── Connection ──────────────────────────────────────────────────────── defp connect_replicas do @@ -428,7 +443,7 @@ defmodule GroupBench.Distributed do start_group_on(r2) wait_for_peer_discovery(replicas) - {connect_us, _} = + {local_connect_us, _} = :timer.tc(fn -> t1 = Task.async(fn -> @@ -453,48 +468,84 @@ defmodule GroupBench.Distributed do end) Task.await_many([t1, t2], 120_000) - - # Wait for convergence — both nodes see each other in the last cluster - poll_until( - fn -> - n1 = :erpc.call(r1, Group, :nodes, [@name, "#{prefix}#{num_clusters}"]) - n2 = :erpc.call(r2, Group, :nodes, [@name, "#{prefix}#{num_clusters}"]) - length(n1) >= 1 and length(n2) >= 1 - end, - 60_000 - ) end) - IO.puts(" connect: #{format_number(div(connect_us, 1000))} ms") - IO.puts(" clusters/sec: #{format_number(round(num_clusters * 1_000_000 / connect_us))}") + # Local completion only means both nodes accepted all Group.connect calls. + # Controls may be reordered or repaired asynchronously, so seeing one + # sentinel cluster cannot prove that the other 9,999 have converged. + expected_cluster_count = num_clusters + 1 + + {control_convergence_us, _} = + try do + :timer.tc(fn -> + wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) + end) + rescue + error -> + r1_status = :erpc.call(r1, GroupBench.Replica, :cluster_control_status, [@name, r2]) + r2_status = :erpc.call(r2, GroupBench.Replica, :cluster_control_status, [@name, r1]) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; r1=#{inspect(r1_status)} " <> + "r2=#{inspect(r2_status)}" + ], + __STACKTRACE__ + end + + connect_total_us = local_connect_us + control_convergence_us + + IO.puts(" local calls: #{format_number(div(local_connect_us, 1000))} ms") + IO.puts(" control convergence: #{format_number(div(control_convergence_us, 1000))} ms") + IO.puts(" end-to-end connect: #{format_number(div(connect_total_us, 1000))} ms") + + IO.puts( + " local clusters/sec: #{format_number(round(num_clusters * 1_000_000 / local_connect_us))}" + ) + + IO.puts( + " converged clusters/sec: #{format_number(round(num_clusters * 1_000_000 / connect_total_us))}" + ) # -- 8b. Register 1 key per cluster -- subheader("register across #{format_number(num_clusters)} clusters") - {reg_us, pids} = + {local_reg_us, pids} = :timer.tc(fn -> - pids = - :erpc.call( - r1, - GroupBench.Replica, - :bulk_register_per_cluster, - [@name, num_clusters, prefix], - 120_000 - ) + :erpc.call( + r1, + GroupBench.Replica, + :bulk_register_per_cluster, + [@name, num_clusters, prefix], + 120_000 + ) + end) + + remote_count_at_handoff = + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) + {reg_convergence_us, _} = + :timer.tc(fn -> poll_until( fn -> :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters end, 60_000 ) - - pids end) - IO.puts(" register+converge: #{format_number(div(reg_us, 1000))} ms") - IO.puts(" ops/sec: #{format_number(round(num_clusters * 1_000_000 / reg_us))}") + reg_total_us = local_reg_us + reg_convergence_us + + IO.puts(" local registration: #{format_number(div(local_reg_us, 1000))} ms") + IO.puts(" remote at handoff: #{format_number(remote_count_at_handoff)} / 10,000") + IO.puts(" data convergence: #{format_number(div(reg_convergence_us, 1000))} ms") + IO.puts(" register end-to-end: #{format_number(div(reg_total_us, 1000))} ms") + + IO.puts( + " converged ops/sec: #{format_number(round(num_clusters * 1_000_000 / reg_total_us))}" + ) # -- 8c. Peer re-discovery with many clusters -- @@ -504,28 +555,75 @@ defmodule GroupBench.Distributed do stop_group_on(r2) Process.sleep(500) - {rediscovery_us, _} = + {restart_us, _} = :timer.tc(fn -> start_group_on(r2) wait_for_peer_discovery(replicas) + end) + {local_reconnect_us, _} = + :timer.tc(fn -> :erpc.call(r2, GroupBench.Replica, :bulk_connect, [@name, num_clusters, prefix], 120_000) + end) - poll_until( - fn -> - :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters - end, - 120_000 - ) + {reconnect_control_us, _} = + :timer.tc(fn -> + wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) end) - IO.puts(" re-discovery: #{format_number(div(rediscovery_us, 1000))} ms") + {rediscovery_data_us, _} = + try do + :timer.tc(fn -> + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters + end, + 120_000 + ) + end) + rescue + error -> + counts = + :erpc.call(r2, GroupBench.Replica, :registry_counts_by_shard, [@name]) + + diagnostics = + :erpc.call(r2, GroupBench.Replica, :replica_process_diagnostics, [@name]) + + r1_revision = + :erpc.call(r1, Group.Replica.Data, :local_cluster_epoch_revision, [@name]) + + r2_remote_revision = + :erpc.call(r2, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r1]) + + r1_remote_revision = + :erpc.call(r1, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r2]) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; rediscovery_counts=#{inspect(counts)} " <> + "r1_revision=#{r1_revision} " <> + "r2_remote_revision=#{inspect(r2_remote_revision)} " <> + "r1_remote_revision=#{inspect(r1_remote_revision)} " <> + "replica=#{inspect(diagnostics)}" + ], + __STACKTRACE__ + end + + rediscovery_total_us = + restart_us + local_reconnect_us + reconnect_control_us + rediscovery_data_us + + IO.puts(" restart + discovery: #{format_number(div(restart_us, 1000))} ms") + IO.puts(" local reconnect: #{format_number(div(local_reconnect_us, 1000))} ms") + IO.puts(" control convergence: #{format_number(div(reconnect_control_us, 1000))} ms") + IO.puts(" data convergence: #{format_number(div(rediscovery_data_us, 1000))} ms") + IO.puts(" re-discovery total: #{format_number(div(rediscovery_total_us, 1000))} ms") # -- 8d. Disconnect cleanup -- subheader("disconnect #{format_number(num_clusters)} clusters") - {disconnect_us, _} = + {local_disconnect_us, _} = :timer.tc(fn -> :erpc.call( r1, @@ -534,20 +632,58 @@ defmodule GroupBench.Distributed do [@name, num_clusters, prefix], 120_000 ) + end) + {cleanup_us, _} = + :timer.tc(fn -> # Wait for r2 to see r1's entries cleaned from all clusters - poll_until( - fn -> - :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) == 0 - end, - 60_000 - ) + try do + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) == 0 + end, + 60_000 + ) + rescue + error -> + count = :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) + sample = :erpc.call(r2, GroupBench.Replica, :registry_sample, [@name]) + + local_revision = + :erpc.call(r1, Group.Replica.Data, :local_cluster_epoch_revision, [@name]) + + remote_revision = + :erpc.call(r2, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r1]) + + observed_revision = + :erpc.call( + r2, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [@name, r1] + ) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; remaining=#{count} " <> + "local_revision=#{local_revision} " <> + "remote_revision=#{inspect(remote_revision)} " <> + "observed_revision=#{inspect(observed_revision)} " <> + "sample=#{inspect(sample)}" + ], + __STACKTRACE__ + end end) - IO.puts(" disconnect+cleanup: #{format_number(div(disconnect_us, 1000))} ms") + disconnect_total_us = local_disconnect_us + cleanup_us + + IO.puts(" local disconnect: #{format_number(div(local_disconnect_us, 1000))} ms") + IO.puts(" remote cleanup: #{format_number(div(cleanup_us, 1000))} ms") + IO.puts(" disconnect total: #{format_number(div(disconnect_total_us, 1000))} ms") IO.puts( - " clusters/sec: #{format_number(round(num_clusters * 1_000_000 / disconnect_us))}" + " converged clusters/sec: #{format_number(round(num_clusters * 1_000_000 / disconnect_total_us))}" ) # Kill leftover processes @@ -555,6 +691,37 @@ defmodule GroupBench.Distributed do stop_groups(replicas) end + defp wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) do + r1_revision = + :erpc.call(r1, GroupBench.Replica, :cluster_control_revision, [@name]) + + r2_revision = + :erpc.call(r2, GroupBench.Replica, :cluster_control_revision, [@name]) + + poll_until( + fn -> + r1_converged? = + :erpc.call( + r1, + GroupBench.Replica, + :cluster_control_converged?, + [@name, r2, expected_cluster_count, r2_revision] + ) + + r2_converged? = + :erpc.call( + r2, + GroupBench.Replica, + :cluster_control_converged?, + [@name, r1, expected_cluster_count, r1_revision] + ) + + r1_converged? and r2_converged? + end, + 120_000 + ) + end + # ── 9. Busy app simulation ────────────────────────────────────────── defp bench_busy_app([r1, r2] = replicas) do diff --git a/priv/bench/lib/group_bench/local.ex b/priv/bench/lib/group_bench/local.ex index dad6cf1..667bae3 100644 --- a/priv/bench/lib/group_bench/local.ex +++ b/priv/bench/lib/group_bench/local.ex @@ -6,11 +6,14 @@ defmodule GroupBench.Local do import GroupBench.Helpers @name :bench - @default_shards System.schedulers_online() + @default_shards 8 + @shard_counts [1, 2, 4, 8, 16, 32, 64] def run do header("Local Benchmarks") IO.puts(" schedulers_online: #{System.schedulers_online()}") + IO.puts(" default_shards: #{@default_shards}") + IO.puts(" shard_sweep: #{Enum.join(@shard_counts, ", ")}") bench_lookup() bench_members() @@ -36,21 +39,26 @@ defmodule GroupBench.Local do measure_count = 100_000 # Each process registers itself - register_from_spawned_processes(key_count, fn i -> - Group.register(@name, "key-#{i}", %{i: i}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(key_count, fn i -> + Group.register(@name, "key-#{i}", %{i: i}, cluster_opts(cluster_opt)) + end) - # warmup - warmup(1_000, fn -> Group.lookup(@name, "key-1", cluster_opts(cluster_opt)) end) + try do + # warmup + warmup(1_000, fn -> Group.lookup(@name, "key-1", cluster_opts(cluster_opt)) end) - # measure - samples = - collect_samples(measure_count, fn -> - i = :rand.uniform(key_count) - Group.lookup(@name, "key-#{i}", cluster_opts(cluster_opt)) - end) + # measure + samples = + collect_samples(measure_count, fn -> + i = :rand.uniform(key_count) + Group.lookup(@name, "key-#{i}", cluster_opts(cluster_opt)) + end) - report_latency("Group.lookup/3", samples) + report_latency("Group.lookup/3", samples) + after + stop_spawned_processes(pids) + end end) end end @@ -72,20 +80,25 @@ defmodule GroupBench.Local do total = group_count * members_per_group # Each process joins a group - register_from_spawned_processes(total, fn i -> - gi = rem(i - 1, group_count) + 1 - Group.join(@name, "group-#{gi}", %{}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(total, fn i -> + gi = rem(i - 1, group_count) + 1 + Group.join(@name, "group-#{gi}", %{}, cluster_opts(cluster_opt)) + end) - warmup(1_000, fn -> Group.members(@name, "group-1", cluster_opts(cluster_opt)) end) + try do + warmup(1_000, fn -> Group.members(@name, "group-1", cluster_opts(cluster_opt)) end) - samples = - collect_samples(measure_count, fn -> - gi = :rand.uniform(group_count) - Group.members(@name, "group-#{gi}", cluster_opts(cluster_opt)) - end) + samples = + collect_samples(measure_count, fn -> + gi = :rand.uniform(group_count) + Group.members(@name, "group-#{gi}", cluster_opts(cluster_opt)) + end) - report_latency("Group.members/3", samples) + report_latency("Group.members/3", samples) + after + stop_spawned_processes(pids) + end end) end end @@ -96,23 +109,26 @@ defmodule GroupBench.Local do header("3. Register Throughput (shard scaling)") n = 10_000 - shard_counts = Enum.uniq([1, 2, 4, @default_shards]) for {cluster_label, cluster_opt} <- clusters() do subheader("cluster: #{cluster_label}") - for shards <- shard_counts do + for shards <- @shard_counts do with_group([name: @name, shards: shards], fn -> maybe_connect_cluster(cluster_opt) - {wall_us, _} = + {wall_us, pids} = time_us(fn -> register_from_spawned_processes(n, fn i -> Group.register(@name, "reg-#{i}", %{}, cluster_opts(cluster_opt)) end) end) - report_throughput("shards=#{shards}", n, wall_us) + try do + report_throughput("shards=#{shards}", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -151,23 +167,26 @@ defmodule GroupBench.Local do header("5. Join Throughput (shard scaling)") n = 10_000 - shard_counts = Enum.uniq([1, 2, 4, @default_shards]) for {cluster_label, cluster_opt} <- clusters() do subheader("cluster: #{cluster_label}") - for shards <- shard_counts do + for shards <- @shard_counts do with_group([name: @name, shards: shards], fn -> maybe_connect_cluster(cluster_opt) - {wall_us, _} = + {wall_us, pids} = time_us(fn -> register_from_spawned_processes(n, fn i -> Group.join(@name, "join-group-#{rem(i, 100)}", %{}, cluster_opts(cluster_opt)) end) end) - report_throughput("shards=#{shards}", n, wall_us) + try do + report_throughput("shards=#{shards}", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -186,18 +205,25 @@ defmodule GroupBench.Local do with_group([name: @name, shards: @default_shards], fn -> maybe_connect_cluster(cluster_opt) :ok = Group.monitor(@name, :all, cluster_opts(cluster_opt)) + drain_stale_group_events() - {wall_us, _} = + {wall_us, pids} = time_us(fn -> - register_from_spawned_processes(n, fn i -> - Group.register(@name, "mon-#{i}", %{}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(n, fn i -> + Group.register(@name, "mon-#{i}", %{}, cluster_opts(cluster_opt)) + end) # drain all N events drain_events(n) + pids end) - report_throughput("events (register → receive)", n, wall_us) + try do + report_throughput("events (register → receive)", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -243,6 +269,19 @@ defmodule GroupBench.Local do pids end + defp stop_spawned_processes(pids) do + refs = Enum.map(pids, &{&1, Process.monitor(&1)}) + Enum.each(pids, &Process.exit(&1, :kill)) + + Enum.each(refs, fn {pid, ref} -> + receive do + {:DOWN, ^ref, :process, ^pid, _reason} -> :ok + after + 5_000 -> raise "Timed out stopping #{inspect(pid)}" + end + end) + end + defp drain_events(0), do: :ok defp drain_events(remaining) do @@ -254,4 +293,12 @@ defmodule GroupBench.Local do 5_000 -> IO.puts(" WARNING: timed out waiting for events, #{remaining} remaining") end end + + defp drain_stale_group_events do + receive do + {:group, _events, _info} -> drain_stale_group_events() + after + 0 -> :ok + end + end end diff --git a/priv/bench/lib/group_bench/replica.ex b/priv/bench/lib/group_bench/replica.ex index 22df3c0..0bace05 100644 --- a/priv/bench/lib/group_bench/replica.ex +++ b/priv/bench/lib/group_bench/replica.ex @@ -73,6 +73,25 @@ defmodule GroupBench.Replica do end) end + @doc false + def registry_counts_by_shard(name) do + for shard <- 0..(Group.get_config(name).num_shards - 1) do + {shard, :ets.info(Group.Replica.Data.reg_by_key_table(name, shard), :size)} + end + end + + def registry_sample(name, limit \\ 10) do + shards = Group.get_config(name).num_shards + + 0..(shards - 1) + |> Enum.flat_map(fn shard -> + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.take(limit) + end) + |> Enum.take(limit) + end + @doc """ Registers a single key from a spawned process. Returns after registration. """ @@ -577,23 +596,78 @@ defmodule GroupBench.Replica do pids = Enum.map(1..n, fn i -> spawn(fn -> - :ok = Group.register(name, "key", %{}, cluster: "#{prefix}#{i}") - send(parent, {:done, self()}) - Process.sleep(:infinity) + try do + :ok = Group.register(name, "key", %{}, cluster: "#{prefix}#{i}") + send(parent, {:done, self()}) + Process.sleep(:infinity) + catch + kind, reason -> + send( + parent, + {:failed, self(), kind, {reason, replica_process_diagnostics(name)}, + __STACKTRACE__} + ) + end end) end) Enum.each(pids, fn pid -> receive do - {:done, ^pid} -> :ok + {:done, ^pid} -> + :ok + + {:failed, ^pid, kind, reason, stacktrace} -> + :erlang.raise(kind, reason, stacktrace) after - 60_000 -> raise "Timed out waiting for bulk_register_per_cluster" + 60_000 -> + awaited_pid = + {pid, + Process.info(pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ])} + + registry_count = total_registry_count(name) + + raise "Timed out waiting for bulk_register_per_cluster: " <> + "replica=#{inspect(replica_process_diagnostics(name))} " <> + "awaited=#{inspect(awaited_pid)} registry_count=#{registry_count}" end end) pids end + @doc false + def replica_process_diagnostics(name) do + shards = + for shard <- 0..(Group.get_config(name).num_shards - 1) do + shard_pid = Process.whereis(Group.Replica.shard_name(name, shard)) + + {shard, + Process.info(shard_pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ])} + end + + data_pid = Process.whereis(Group.Replica.Data.data_name(name)) + + data = + Process.info(data_pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ]) + + %{shards: shards, data: data} + end + @doc """ Returns the number of clusters this node is a member of (via reverse index). """ @@ -601,6 +675,95 @@ defmodule GroupBench.Replica do length(Group.Replica.Data.my_clusters(name)) end + @doc """ + Returns the number of clusters currently associated with `target_node`. + + The many-cluster benchmark uses the reverse index instead of checking one + sentinel cluster: replica control messages may be reordered, so observing + the last submitted cluster does not prove that every earlier cluster has + converged. + """ + def cluster_count_for_node(name, target_node) do + name + |> Group.Replica.Data.node_clusters_table() + |> :ets.lookup(target_node) + |> length() + end + + @doc false + def cluster_control_revision(name) do + if function_exported?(Group.Replica.Data, :local_cluster_epoch_revision, 1) do + apply(Group.Replica.Data, :local_cluster_epoch_revision, [name]) + else + :legacy + end + end + + @doc false + def cluster_control_converged?(name, target_node, expected_cluster_count, source_revision) do + membership_converged? = + cluster_count_for_node(name, target_node) >= expected_cluster_count + + if source_revision == :legacy or + not function_exported?(Group.Replica.Data, :remote_view_generation, 3) do + membership_converged? + else + shards = Group.get_config(name).num_shards + data = Group.Replica.Data + generation = apply(data, :remote_generation, [name, target_node]) + revision = apply(data, :remote_cluster_epoch_revision, [name, target_node]) + epoch_table = apply(data, :remote_cluster_epochs_table, [name]) + + epoch_count = + :ets.select_count(epoch_table, [ + {{{target_node, :_}, :_}, [], [true]} + ]) + + shard_views_converged? = + Enum.all?(0..(shards - 1), fn shard -> + apply(data, :remote_view_generation, [name, shard, target_node]) == generation and + apply(data, :remote_view_cluster_epoch_revision, [name, shard, target_node]) == + source_revision and + apply(data, :remote_view_observed_revision, [name, shard, target_node]) == + source_revision + end) + + membership_converged? and not is_nil(generation) and revision == source_revision and + epoch_count >= expected_cluster_count - 1 and shard_views_converged? + end + end + + @doc false + def cluster_control_status(name, target_node) do + data = Group.Replica.Data + shards = Group.get_config(name).num_shards + epoch_table = data.remote_cluster_epochs_table(name) + + %{ + local_revision: data.local_cluster_epoch_revision(name), + local_epoch_count: :ets.info(data.local_cluster_epochs_table(name), :size), + membership_count: cluster_count_for_node(name, target_node), + remote_generation: data.remote_generation(name, target_node), + remote_revision: data.remote_cluster_epoch_revision(name, target_node), + remote_exact_revision: data.remote_cluster_epoch_exact_revision(name, target_node), + remote_observed: data.remote_cluster_epoch_observed_revision(name, target_node), + remote_epoch_count: + :ets.select_count(epoch_table, [ + {{{target_node, :_}, :_}, [], [true]} + ]), + authority_installs: data.remote_authority_install_count(name, target_node), + views: + for( + shard <- 0..(shards - 1), + do: + {shard, data.remote_view_generation(name, shard, target_node), + data.remote_view_cluster_epoch_revision(name, shard, target_node), + data.remote_view_observed_revision(name, shard, target_node)} + ), + replica: replica_process_diagnostics(name) + } + end + @doc """ Simulates a busy app worker: registers, joins groups, does lookups and dispatches, then some processes die and re-register. Returns final pids. diff --git a/test/README.md b/test/README.md index 58a6d51..6dc3dac 100644 --- a/test/README.md +++ b/test/README.md @@ -6,6 +6,7 @@ mix test # all tests mix test test/group_test.exs # local only mix test test/distributed_test.exs # distributed only +mix test test/replica_adversarial_test.exs # seeded transport chaos ``` ## Test files @@ -13,7 +14,8 @@ mix test test/distributed_test.exs # distributed only | File | What it tests | |------|---------------| | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | -| `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts | +| `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | +| `replica_adversarial_test.exs` | Reproducible mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | ## How distribution works @@ -194,6 +196,34 @@ TestCluster.start_group( The resolver uses "most recent wins" — keeps the registration with the higher timestamp. +### Replica transport fault injection + +`Group.TestReplicaTransport` implements the production transport behaviour but +can return `:busy`, drop selected frame types, duplicate or delay frames, and +capture frames for explicit stale-generation/epoch replay. Its `{:chaos, opts}` +mode is deterministic for a given frame, which makes failures reproducible. + +The distributed anti-entropy tests cover dropped creates and deletes, cursor +gaps, globally pruned multi-stream oplogs, exact snapshot fallback, malformed +authority, stale frame replay, lease expiry on a live VM, and multi-shard +generation recovery. They also restart a suspended data lane after deliberately +losing its cluster-close fence and require the lane to sweep the stale registry +and PG slices from shared authority. Authority topology tests suspend every +receiver shard and inspect the queued protocol: only shard 0 may receive/install +the full epoch snapshot, nonzero shards receive constant-size lane hellos, and +incremental opens stay on their matching shard. Separate tests suspend a +backlogged authority shard while other replica lanes continue converging and +deliver data before authority to prove rejection does not advance the cursor +and the same frame applies after authority repair. Concurrent snapshot tests +require every advertised revision to contain exactly that many unique named +epochs, and heartbeat tests prove observed revisions cannot advance the exact +authority marker. + +`Group.TestCluster.assert_replica_consistent/1` checks the +public dual indexes plus registry claim authority, oplog/order equivalence, and +contiguous retained stream ranges. Seeded tests additionally require every PID +retained as authority to still be alive after convergence. + ## Typical test patterns ### Basic replication test diff --git a/test/distributed_test.exs b/test/distributed_test.exs index e62a4d9..bf34406 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -555,10 +555,10 @@ defmodule Group.DistributedTest do timeout: 10_000 ) - # Custom resolvers own process lifecycle decisions. This resolver only - # picks a registry winner, so neither owner is terminated by Group. - assert TestCluster.rpc!(node_a, Process, :alive?, [pid_a]) + # The resolver selects the winner; each origin is responsible for + # retiring (and terminating) only its own losing owner. assert TestCluster.rpc!(node_b, Process, :alive?, [pid_b]) + refute TestCluster.rpc!(node_a, Process, :alive?, [pid_a]) end end @@ -1025,6 +1025,8 @@ defmodule Group.DistributedTest do cluster = "game" registry_key = "remote/registry" pg_key = "remote/pg" + retained_registry_key = "remote/registry/retained" + retained_pg_key = "remote/pg/retained" start_group_on_peers(peers, name: name, shards: 2) @@ -1045,13 +1047,47 @@ defmodule Group.DistributedTest do pg_pid = TestCluster.spawn_join(node_b, name, pg_key, %{from: :b}, cluster: cluster) + retained_registry_pid = + TestCluster.spawn_register_in_cluster( + node_b, + name, + retained_registry_key, + %{from: :b, retained: true}, + cluster + ) + + retained_pg_pid = + TestCluster.spawn_join( + node_b, + name, + retained_pg_key, + %{from: :b, retained: true}, + cluster: cluster + ) + TestCluster.assert_eventually(fn -> TestCluster.rpc!(node_a, Group, :lookup, [ name, registry_key, [cluster: cluster] ]) != nil and - TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) != [] + TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) != [] and + match?( + {^retained_registry_pid, _}, + TestCluster.rpc!(node_a, Group, :lookup, [ + name, + retained_registry_key, + [cluster: cluster] + ]) + ) and + match?( + [{^retained_pg_pid, _}], + TestCluster.rpc!(node_a, Group, :members, [ + name, + retained_pg_key, + [cluster: cluster] + ]) + ) end) assert :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, cluster]) @@ -1088,6 +1124,25 @@ defmodule Group.DistributedTest do ]) == nil assert TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) == [] + + TestCluster.assert_eventually(fn -> + match?( + {^retained_registry_pid, %{from: :b, retained: true}}, + TestCluster.rpc!(node_a, Group, :lookup, [ + name, + retained_registry_key, + [cluster: cluster] + ]) + ) and + match?( + [{^retained_pg_pid, %{from: :b, retained: true}}], + TestCluster.rpc!(node_a, Group, :members, [ + name, + retained_pg_key, + [cluster: cluster] + ]) + ) + end) end test "local join does not overtake an earlier remote cluster_disconnect after replicated PG flush" do @@ -1778,21 +1833,30 @@ defmodule Group.DistributedTest do messages = TestCluster.shard_messages(node_b, name, 0) case Enum.filter(messages, fn - {:replicate_registry_batch, _ops} -> true - {:cluster_disconnect, ["game"], _remote_pid} -> true - _ -> false + {:group_replica_frame, _source, {:delta_batch, 1, _runs}} -> + true + + {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} -> + true + + _ -> + false end) do [ - {:replicate_registry_batch, ops}, - {:cluster_disconnect, ["game"], _remote_pid} + {:group_replica_frame, _source, {:delta_batch, 1, runs}}, + {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} ] -> - Enum.any?(ops, fn - {:register, "game", ^key, reg_pid, %{v: 1}, _time, _entry_node} - when reg_pid == pid -> - true - - _ -> - false + Enum.any?(runs, fn {_stream_id, _first_seq, records, _head} -> + Enum.any?(records, fn {_seq, mutations} -> + Enum.any?(mutations, fn + {:register, "game", ^key, reg_pid, %{v: 1}, _time, _entry_node} + when reg_pid == pid -> + true + + _ -> + false + end) + end) end) _ -> @@ -2354,18 +2418,55 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_a, Group, :disconnect, [name, dropped_cluster]) # A's registrations in org/10 should be gone everywhere - TestCluster.assert_eventually( - fn -> - Enum.all?(nodes, fn check_node -> - TestCluster.rpc!(check_node, Group, :lookup, [ - name, - "user/0_1", - [cluster: dropped_cluster] - ]) == nil - end) - end, - timeout: 10_000 - ) + try do + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn check_node -> + TestCluster.rpc!(check_node, Group, :lookup, [ + name, + "user/0_1", + [cluster: dropped_cluster] + ]) == nil + end) + end, + timeout: 10_000 + ) + rescue + error -> + diagnostics = + for check_node <- nodes do + lookup = + TestCluster.rpc!(check_node, Group, :lookup, [ + name, + "user/0_1", + [cluster: dropped_cluster] + ]) + + protocol = + TestCluster.rpc!( + check_node, + Group.TestCluster, + :replica_protocol_state, + [name] + ) + |> Enum.map(fn %{shard: shard, cursors: cursors} -> + relevant = + Enum.filter(cursors, fn {stream_id, _seq} -> + Group.Replica.Protocol.stream_origin(stream_id) == node_a and + Group.Replica.Protocol.stream_cluster(stream_id) == dropped_cluster + end) + + {shard, relevant} + end) + + {check_node, lookup, protocol} + end + + flunk( + "cluster disconnect convergence failed: #{Exception.message(error)} " <> + "diagnostics=#{inspect(diagnostics, limit: :infinity)}" + ) + end # B and C still see each other's org/10 data TestCluster.assert_eventually(fn -> @@ -3834,6 +3935,1388 @@ defmodule Group.DistributedTest do end end + describe "replica transport anti-entropy" do + @tag timeout: 60_000 + test "dropped tails and dropped deletes converge without orphaning registry or PG rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_drop_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + reg_key = "anti-entropy/dropped-register" + pg_key = "anti-entropy/dropped-join" + reg_pid = TestCluster.spawn_register(node_a, name, reg_key, %{owner: :a}) + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{owner: :a}) + Process.sleep(100) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil + assert TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [pg_pid, :kill]) + TestCluster.flush_shards(node_a, name) + + assert match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) + assert match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + end) + end + + @tag timeout: 60_000 + test "busy sends recover from the advertised stream head" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_busy_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :busy]) + key = "anti-entropy/busy" + pid = TestCluster.spawn_register(node_a, name, key, %{}) + Process.sleep(100) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + @tag timeout: 60_000 + test "a pruned gap falls back to an exact origin snapshot and removes stale rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_snapshot_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 2 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stale_reg_key = "anti-entropy/snapshot/stale-reg" + stale_pg_key = "anti-entropy/snapshot/stale-pg" + stale_reg_pid = TestCluster.spawn_register(node_a, name, stale_reg_key, %{}) + stale_pg_pid = TestCluster.spawn_join(node_a, name, stale_pg_key, %{}) + + TestCluster.assert_eventually(fn -> + match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh_keys = for i <- 1..6, do: "anti-entropy/snapshot/fresh-#{i}" + fresh_pids = for key <- fresh_keys, do: TestCluster.spawn_register(node_a, name, key, %{}) + TestCluster.flush_shards(node_a, name) + + [{_stream_id, floor, head}] = + TestCluster.rpc!(node_a, Group.Replica.Data, :replica_stream_heads, [name, 0]) + + assert floor > 3 + assert head >= 10 + + assert match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) == [] and + Enum.zip(fresh_keys, fresh_pids) + |> Enum.all?(fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end) + end + + @tag timeout: 60_000 + test "the control lease removes a stopped Group on a still-connected node and probes recovery" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_lease_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + key = "anti-entropy/lease/stale" + stale_pid = TestCluster.spawn_register(node_a, name, key, %{}) + + TestCluster.assert_eventually(fn -> + match?({^stale_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + assert TestCluster.rpc!(node_b, Node, :ping, [node_a]) == :pong + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil and + node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end, + timeout: 5_000 + ) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + recovered_key = "anti-entropy/lease/recovered" + recovered_pid = TestCluster.spawn_register(node_a, name, recovered_key, %{}) + + TestCluster.assert_eventually(fn -> + match?( + {^recovered_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, recovered_key]) + ) + end) + end + + @tag timeout: 60_000 + test "a restarted data lane sweeps a cluster close missed while it was down" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_lane_restart_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + :ok = TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + reg_key = + Enum.find(1..1_000, fn suffix -> + Group.Replica.shard_index_for("game", "lane-reg-#{suffix}", 2) == 1 + end) + |> then(&"lane-reg-#{&1}") + + pg_key = + Enum.find(1..1_000, fn suffix -> + Group.Replica.shard_index_for("game", "lane-pg-#{suffix}", 2) == 1 + end) + |> then(&"lane-pg-#{&1}") + + reg_pid = TestCluster.spawn_register_in_cluster(node_a, name, reg_key, %{}, "game") + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{}, cluster: "game") + + TestCluster.assert_eventually(fn -> + match?( + {^reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key, [cluster: "game"]]) + ) and + match?( + [{^pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, pg_key, [cluster: "game"]]) + ) + end) + + old_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [old_lane]) + + :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + is_nil( + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + "game" + ]) + ) + end) + + control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + _state = TestCluster.rpc!(node_b, :sys, :get_state, [control]) + + messages = TestCluster.rpc!(node_b, Process, :info, [old_lane, :messages]) + + assert {:messages, queued} = messages + + assert Enum.any?(queued, fn + {:replica_cluster_close_control_local, ^node_a, _generation, _revision, + [{"game", _epoch}]} -> + true + + _ -> + false + end) + + true = TestCluster.rpc!(node_b, Process, :exit, [old_lane, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) do + lane when is_pid(lane) -> lane != old_lane + _ -> false + end + end) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key, [cluster: "game"]]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key, [cluster: "game"]]) == [] + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "full authority is installed once on shard zero while incremental control stays sharded" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_topology_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + b_lanes = + for shard <- 0..(shards - 1) do + {shard, TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, shard)])} + end + + Enum.each(b_lanes, fn {_shard, pid} -> + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [pid]) + end) + + installs_before = + TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) + + [_authority_epoch] = + TestCluster.rpc!(node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["authority-new"] + ]) + + remote_clusters = TestCluster.rpc!(node_b, Group.Replica.Data, :my_clusters, [name]) + + Enum.each(b_lanes, fn {shard, b_pid} -> + TestCluster.rpc!(node_a, :erlang, :send, [ + shard_name(name, shard), + {:peer_connect_ack, b_pid, shard, shards, remote_clusters} + ]) + end) + + TestCluster.assert_eventually(fn -> + Enum.all?(b_lanes, fn {shard, pid} -> + {:messages, messages} = TestCluster.rpc!(node_b, Process, :info, [pid, :messages]) + + if shard == 0 do + Enum.any?(messages, fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + else + Enum.any?(messages, fn + {:replica_lane_hello, remote_pid, _version, _generation, _revision, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + end + end) + end) + + mailboxes = + Map.new(b_lanes, fn {shard, pid} -> + {:messages, messages} = TestCluster.rpc!(node_b, Process, :info, [pid, :messages]) + {shard, messages} + end) + + full_authority_lanes = + for {shard, messages} <- mailboxes, + Enum.any?(messages, fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end), + do: shard + + assert full_authority_lanes == [0] + + assert Enum.any?(mailboxes[0], fn + {:replica_hello, remote_pid, _version, _generation, revision, epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a and + revision == Enum.count(epochs, &(not is_nil(elem(&1, 0)))) + + _ -> + false + end) + + for shard <- 1..(shards - 1) do + assert Enum.any?(mailboxes[shard], fn + {:replica_lane_hello, remote_pid, _version, _generation, _revision, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + end + + [epoch] = + TestCluster.rpc!(node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["lane-open"] + ]) + + TestCluster.rpc!(node_a, Group.Replica.Data, :add_cluster_node, [ + name, + ["lane-open"], + node_a + ]) + + assert :ok = + TestCluster.rpc!(node_a, Group.Replica, :local_request, [ + shard_name(name, 2), + {:cluster_connect, ["lane-open"], [epoch]}, + 5_000 + ]) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(node_b, Process, :info, [Map.fetch!(Map.new(b_lanes), 2), :messages]) + + Enum.any?(messages, fn + {:replica_cluster_open, remote_pid, _generation, _revision, [^epoch]} -> + node(remote_pid) == node_a + + _ -> + false + end) + end) + + {:messages, control_messages} = + TestCluster.rpc!(node_b, Process, :info, [Map.fetch!(Map.new(b_lanes), 0), :messages]) + + refute Enum.any?(control_messages, fn + {:replica_cluster_open, remote_pid, _generation, _revision, [^epoch]} -> + node(remote_pid) == node_a + + _ -> + false + end) + + {0, b_control} = List.keyfind!(b_lanes, 0, 0) + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_control]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) == installs_before + 1 + end) + + duplicate_full = + Enum.find(mailboxes[0], fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + + {:replica_hello, _remote_pid, _version, _generation, full_revision, _epochs, _transport, + _descriptor} = duplicate_full + + Enum.each(1..128, fn _ -> send(b_control, duplicate_full) end) + drain_ref = make_ref() + send(b_control, {:group_dispatch, [self()], {:authority_duplicates_drained, drain_ref}}) + assert_receive {:authority_duplicates_drained, ^drain_ref}, 5_000 + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) == installs_before + 1 + + Enum.each(b_lanes, fn + {0, _pid} -> :ok + {_shard, pid} -> :ok = TestCluster.rpc!(node_b, :sys, :resume, [pid]) + end) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + "lane-open" + ]) == elem(epoch, 1) + end) + + latest_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + a_lane = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 2)]) + + TestCluster.rpc!(node_b, :erlang, :send, [ + shard_name(name, 2), + {:replica_heartbeat, a_lane, Group.Replica.Protocol.version(), + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]), latest_revision} + ]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_view_observed_revision, [ + name, + 2, + node_a + ]) == latest_revision + end) + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, node_a] + ) == full_revision + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 2, node_a] + ) == full_revision + end + + @tag timeout: 60_000 + test "a backlogged authority shard cannot block independent replica lanes" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_backlog_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + b_control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + a_control = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_control]) + + TestCluster.rpc!(node_a, :erlang, :send, [ + shard_name(name, 0), + {:peer_connect_ack, b_control, 0, shards, [nil]} + ]) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(node_b, Process, :info, [b_control, :messages]) + + Enum.any?(messages, fn + {:replica_hello, ^a_control, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + true + + _ -> + false + end) + end) + + [reg_key] = keys_for_shard(nil, "authority-backlog/reg", shards, 1, 1) + [pg_key] = keys_for_shard(nil, "authority-backlog/pg", shards, 2, 1) + reg_pid = TestCluster.spawn_register(node_a, name, reg_key, %{lane: 1}) + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{lane: 2}) + + TestCluster.assert_eventually(fn -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_control]) + end + + @tag timeout: 60_000 + test "data arriving without authority is rejected without cursor advance and repairs later" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_before_data_#{System.unique_integer([:positive])}" + shards = 2 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + [key] = keys_for_shard(nil, "authority-before-data", shards, 1, 1) + pid = TestCluster.spawn_register(node_a, name, key, %{valid: true}) + TestCluster.flush_shards(node_a, name) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) != [] + end) + + {_target, 1, frame} = + node_a + |> TestCluster.rpc!(Group.TestReplicaTransport, :captured, [name]) + |> Enum.find(fn {_target, shard, _frame} -> shard == 1 end) + + stream_id = TestCluster.rpc!(node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Data, :delete_remote_replica_info, [ + name, + 0, + node_a + ]) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 1, + frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) == 0 + + a_lane = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 1)]) + + TestCluster.rpc!(node_b, :erlang, :send, [ + shard_name(name, 1), + {:replica_heartbeat, a_lane, Group.Replica.Protocol.version(), generation, revision} + ]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + generation + end) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 1, + frame + ]) + + TestCluster.assert_eventually(fn -> + match?({^pid, %{valid: true}}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "a delayed cluster close from an older epoch revision cannot undo a reconnect" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_epoch_fence_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + old_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + old_epoch = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch, [name, "game"]) + + TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + current_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert current_revision > old_revision + + key = "anti-entropy/epoch-fence/current" + pid = TestCluster.spawn_register_in_cluster(node_a, name, key, %{}, "game") + + TestCluster.assert_eventually(fn -> + match?( + {^pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: "game"]]) + ) + end) + + remote_pid = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + + TestCluster.rpc!(node_a, :erlang, :send, [ + {shard_name(name, 0), node_b}, + {:replica_cluster_close, remote_pid, generation, old_revision, [{"game", old_epoch}]} + ]) + + Process.sleep(100) + TestCluster.flush_shards(node_b, name) + + assert node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + + assert match?( + {^pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: "game"]]) + ) + end + + @tag timeout: 60_000 + test "duplicate and reordered replica frames are idempotent and emit one lifecycle event" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_duplicate_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + forwarder = TestCluster.spawn_monitor_forwarder(node_b, name, :all, self()) + assert_receive {:monitor_ready, ^forwarder}, 5_000 + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 0, duplicate_every: 2, max_delay: 75]} + ]) + + key = "anti-entropy/duplicate" + pid = TestCluster.spawn_register(node_a, name, key, %{version: 1}) + + assert_receive {:got_event, %Group.Event{type: :registered, key: ^key, pid: ^pid}}, 5_000 + + TestCluster.assert_eventually(fn -> + match?({^pid, %{version: 1}}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + Process.sleep(250) + + refute_received {:got_event, %Group.Event{type: :registered, key: ^key, pid: ^pid}} + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "captured data from old origin generations and cluster epochs cannot resurrect rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_stale_frames_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + generation_key = "anti-entropy/stale-generation" + old_generation_pid = TestCluster.spawn_register(node_a, name, generation_key, %{old: true}) + TestCluster.flush_shards(node_a, name) + + generation_frames = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + assert generation_frames != [] + assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) end, + timeout: 5_000 + ) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + Enum.each(generation_frames, fn {_target, shard, frame} -> + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + + new_generation_pid = + TestCluster.spawn_register(node_a, name, generation_key, %{old: false}) + + TestCluster.assert_eventually(fn -> + match?( + {^new_generation_pid, %{old: false}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) + ) + end) + + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :clear_captured, [name]) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + epoch_key = "anti-entropy/stale-epoch" + + old_epoch_pid = + TestCluster.spawn_register_in_cluster(node_a, name, epoch_key, %{old: true}, "game") + + TestCluster.flush_shards(node_a, name) + epoch_frames = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + assert epoch_frames != [] + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + Enum.each(epoch_frames, fn {_target, shard, frame} -> + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [ + name, + epoch_key, + [cluster: "game"] + ]) == nil + + new_epoch_pid = + TestCluster.spawn_register_in_cluster(node_a, name, epoch_key, %{old: false}, "game") + + TestCluster.assert_eventually(fn -> + match?( + {^new_epoch_pid, %{old: false}}, + TestCluster.rpc!(node_b, Group, :lookup, [ + name, + epoch_key, + [cluster: "game"] + ]) + ) + end) + + TestCluster.rpc!(node_a, Process, :exit, [old_generation_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [old_epoch_pid, :kill]) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "global shard pruning repairs a cold stream without sending or deleting other origins" do + peers = TestCluster.start_peers(3) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + name = :"anti_entropy_multistream_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 4 + ] + + start_group_on_peers(peers, opts) + + for target <- [node_a, node_b, node_c] do + TestCluster.assert_eventually(fn -> + length(TestCluster.rpc!(target, Group, :nodes, [name])) == 2 + end) + + TestCluster.rpc!(target, Group, :connect, [name, ["cold", "hot"]]) + end + + TestCluster.assert_eventually( + fn -> + length(TestCluster.rpc!(node_c, Group, :nodes, [name, "cold"])) == 3 and + length(TestCluster.rpc!(node_c, Group, :nodes, [name, "hot"])) == 3 + end, + timeout: 10_000 + ) + + b_key = "anti-entropy/other-origin" + b_group = "anti-entropy/other-origin-pg" + b_pid = TestCluster.spawn_register(node_b, name, b_key, %{origin: :b}) + b_pg_pid = TestCluster.spawn_join(node_b, name, b_group, %{origin: :b}) + + stale_key = "anti-entropy/cold/stale" + stale_group = "anti-entropy/cold/stale-pg" + + stale_pid = + TestCluster.spawn_register_in_cluster(node_a, name, stale_key, %{origin: :a}, "cold") + + stale_pg_pid = + TestCluster.spawn_join_in_cluster(node_a, name, stale_group, %{origin: :a}, "cold") + + TestCluster.assert_eventually(fn -> + match?({^b_pid, _}, TestCluster.rpc!(node_c, Group, :lookup, [name, b_key])) and + match?([{^b_pg_pid, _}], TestCluster.rpc!(node_c, Group, :members, [name, b_group])) and + match?( + {^stale_pid, _}, + TestCluster.rpc!(node_c, Group, :lookup, [name, stale_key, [cluster: "cold"]]) + ) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_c, Group, :members, [name, stale_group, [cluster: "cold"]]) + ) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh = + for i <- 1..12 do + key = "anti-entropy/hot/#{i}" + + {key, TestCluster.spawn_register_in_cluster(node_a, name, key, %{i: i}, "hot")} + end + + TestCluster.flush_shards(node_a, name) + + {_stream_id, floor, _head} = + node_a + |> TestCluster.rpc!(Group.Replica.Data, :replica_stream_heads, [name, 0]) + |> Enum.find(fn {stream_id, _floor, _head} -> + Group.Replica.Protocol.stream_cluster(stream_id) == "cold" + end) + + assert floor > 2 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_c, Group, :lookup, [name, stale_key, [cluster: "cold"]]) == + nil and + TestCluster.rpc!(node_c, Group, :members, [ + name, + stale_group, + [cluster: "cold"] + ]) == [] and + match?({^b_pid, _}, TestCluster.rpc!(node_c, Group, :lookup, [name, b_key])) and + match?( + [{^b_pg_pid, _}], + TestCluster.rpc!(node_c, Group, :members, [name, b_group]) + ) and + Enum.all?(fresh, fn {key, pid} -> + match?( + {^pid, _}, + TestCluster.rpc!(node_c, Group, :lookup, [name, key, [cluster: "hot"]]) + ) + end) + end, + timeout: 10_000 + ) + + assert :ok = + TestCluster.rpc!(node_c, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "invalid authority and a misattributed frame cannot poison a stream cursor" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stream_id = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + foreign_pid = + TestCluster.spawn_register(node_b, name, "anti-entropy/authority/foreign-owner", %{}) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + key = "anti-entropy/authority/legitimate" + legitimate_pid = TestCluster.spawn_register(node_a, name, key, %{valid: true}) + TestCluster.flush_shards(node_a, name) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) != [] + end) + + [{_target, 0, legitimate_frame} | _] = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + invalid_key = "anti-entropy/authority/forged" + + invalid_frame = + {:delta_batch, Group.Replica.Protocol.version(), + [ + {stream_id, 1, + [ + {1, + [ + {:register, nil, invalid_key, foreign_pid, %{forged: true}, System.system_time(), + node_b} + ]} + ], 1} + ]} + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 0, + invalid_frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, invalid_key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_id + ]) == 0 + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_b, + 0, + legitimate_frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_id + ]) == 0 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?( + {^legitimate_pid, %{valid: true}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) + ) + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "lease expiry purges every shard before accepting a fresh generation" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_multishard_lease_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stale = + for shard <- 0..(shards - 1) do + [reg_key] = keys_for_shard(nil, "anti-entropy/lease/reg/s#{shard}", shards, shard, 1) + [pg_key] = keys_for_shard(nil, "anti-entropy/lease/pg/s#{shard}", shards, shard, 1) + + { + reg_key, + TestCluster.spawn_register(node_a, name, reg_key, %{generation: :old, shard: shard}), + pg_key, + TestCluster.spawn_join(node_a, name, pg_key, %{generation: :old, shard: shard}) + } + end + + TestCluster.assert_eventually( + fn -> + Enum.all?(stale, fn {reg_key, reg_pid, pg_key, pg_pid} -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + end, + timeout: 10_000 + ) + + old_generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + assert TestCluster.rpc!(node_b, Node, :ping, [node_a]) == :pong + + TestCluster.assert_eventually( + fn -> + node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) and + Enum.all?(stale, fn {reg_key, _reg_pid, pg_key, _pg_pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + end) + end, + timeout: 5_000 + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + new_generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + refute new_generation == old_generation + + fresh = + for shard <- 0..(shards - 1) do + [reg_key] = keys_for_shard(nil, "anti-entropy/fresh/reg/s#{shard}", shards, shard, 1) + [pg_key] = keys_for_shard(nil, "anti-entropy/fresh/pg/s#{shard}", shards, shard, 1) + + { + reg_key, + TestCluster.spawn_register(node_a, name, reg_key, %{generation: :new, shard: shard}), + pg_key, + TestCluster.spawn_join(node_a, name, pg_key, %{generation: :new, shard: shard}) + } + end + + TestCluster.assert_eventually( + fn -> + Enum.all?(fresh, fn {reg_key, reg_pid, pg_key, pg_pid} -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + end, + timeout: 10_000 + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 120_000 + test "concurrent many-cluster controls converge every revision before replica writes" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_many_controls_#{System.unique_integer([:positive])}" + clusters = for i <- 1..512, do: "tenant/#{i}" + + opts = [ + name: name, + shards: 4, + replicated_sender_buffer_size: 8, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + tasks = + for node <- [node_a, node_b] do + Task.async(fn -> TestCluster.connect_many_concurrently(node, name, clusters) end) + end + + assert [:ok, :ok] = Task.await_many(tasks, 60_000) + + TestCluster.assert_eventually( + fn -> + Enum.all?([node_a, node_b], fn node -> + expected = MapSet.new([nil | clusters]) + + actual = + TestCluster.rpc!(node, Group.Replica.Data, :my_clusters, [name]) + |> MapSet.new() + + MapSet.subset?(expected, actual) + end) and + Enum.all?(clusters, fn cluster -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + end) + end, + timeout: 30_000, + interval: 100 + ) + + entries = + TestCluster.spawn_register_many_clusters( + node_a, + name, + clusters, + "anti-entropy/many-controls" + ) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_b, Group.TestCluster, :registry_entries_present?, [name, entries]) + end, + timeout: 30_000, + interval: 100 + ) + + for node <- [node_a, node_b] do + assert :ok = + TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) + end + end + end + # Helpers for event assertion tests defp flush_events do diff --git a/test/group_test.exs b/test/group_test.exs index 6bc6b80..7de75ae 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2288,7 +2288,12 @@ defmodule GroupTest do assert_receive {:replicated_registry_buffer_flushed, ^shard_name}, 1_000 :sys.replace_state(shard_pid, fn state -> - %{state | remote_shards: Map.put(state.remote_shards, node(), self())} + %{ + state + | remote_shards: Map.put(state.remote_shards, node(), self()), + peer_last_seen: + Map.put(state.peer_last_seen, node(), System.monotonic_time(:millisecond)) + } end) :erlang.trace(shard_pid, true, [:call]) @@ -2306,10 +2311,16 @@ defmodule GroupTest do {:erlang, :send_nosuspend, [ {^shard_name, ^local_node}, - {:replicate_process_down_batch, _reg_entries, _pg_entries}, + {:group_replica_frame, ^local_node, {:delta_batch, 1, runs}}, [:noconnect] ]}}, 1_000 + + assert Enum.any?(runs, fn {_stream_id, _first_seq, records, _head} -> + Enum.any?(records, fn {_seq, mutations} -> + {:unregister, nil, key, owner, %{}, :killed} in mutations + end) + end) end test "process death batches multiple :left events for same-shard keys", %{name: name} do @@ -2694,7 +2705,7 @@ defmodule GroupTest do Group.Replica.Data.registry_lookup_by_pid(name, 0, new_pid) end - test "custom conflict resolver controls the losing registry owner's lifecycle" do + test "custom conflict resolver selects winner and Group terminates only the local loser" do key = "replicated-registry/custom-loser/#{System.unique_integer([:positive])}" name = @@ -2739,8 +2750,9 @@ defmodule GroupTest do Group.lookup(name, key) == {remote_pid, %{owner: :remote}} end) - refute_receive {:DOWN, ^owner_ref, :process, ^local_owner, _reason}, 50 - assert Process.alive?(local_owner) + assert_receive {:DOWN, ^owner_ref, :process, ^local_owner, + {:group_registry_conflict, ^key, %{owner: :remote}}}, + 1_000 end test "batched remote conflict keeps the staged local winner when later unregister arrives" do @@ -2869,6 +2881,156 @@ defmodule GroupTest do end end + describe "replica write-ahead journal" do + test "concurrent shards retain independent append order", %{name: name} do + named_cluster = "journal/append-order" + operations_per_shard = 100 + :ok = Group.connect(name, named_cluster) + + parent = self() + + owners = + for shard <- 0..3 do + nil_keys = + keys_for_shard(nil, "journal/append-order/nil/#{shard}", 4, shard, 50) + + named_keys = + keys_for_shard( + named_cluster, + "journal/append-order/named/#{shard}", + 4, + shard, + 50 + ) + + spawn(fn -> + nil_keys + |> Enum.zip(named_keys) + |> Enum.each(fn {nil_key, named_key} -> + :ok = Group.register(name, nil_key, %{}) + :ok = Group.register(name, named_key, %{}, cluster: named_cluster) + end) + + send(parent, {:append_order_complete, shard, self()}) + Process.sleep(:infinity) + end) + end + + on_exit(fn -> Enum.each(owners, &kill_if_alive/1) end) + + Enum.with_index(owners) + |> Enum.each(fn {owner, shard} -> + assert_receive {:append_order_complete, ^shard, ^owner}, 10_000 + end) + + metadata = Group.Replica.Data.replication_meta_table(name) + assert :ets.info(metadata, :write_concurrency) == :auto + + for shard <- 0..3 do + assert [{{:append_counter, shard}, operations_per_shard}] == + :ets.lookup(metadata, {:append_counter, shard}) + + order_rows = + name + |> Group.Replica.Data.replica_oplog_order_table(shard) + |> :ets.tab2list() + |> Enum.sort() + + assert Enum.map(order_rows, &elem(&1, 0)) == + Enum.to_list(1..operations_per_shard) + + assert Enum.all?(order_rows, fn {_append_id, stream_id, _seq} -> + Group.Replica.Protocol.stream_shard(stream_id) == shard + end) + + oplog_rows = + name + |> Group.Replica.Data.replica_oplog_table(shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream_id, seq}, append_id, _mutations} -> + {append_id, stream_id, seq} + end) + + assert MapSet.new(order_rows) == oplog_rows + + order_rows + |> Enum.group_by(fn {_append_id, stream_id, _seq} -> stream_id end) + |> Enum.each(fn {_stream_id, rows} -> + assert Enum.map(rows, &elem(&1, 2)) == Enum.to_list(1..50) + end) + end + end + + test "a shard restart replays an appended mixed record and later cleans its owner" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + key = "journal/replay/#{System.unique_integer([:positive])}" + owner = spawn(fn -> Process.sleep(:infinity) end) + on_exit(fn -> kill_if_alive(owner) end) + + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + time = System.system_time() + + {seq, _mutations} = + Group.Replica.Data.append_replica_record(name, 0, stream_id, [ + {:register, nil, key, owner, %{kind: :registry}, time, node()}, + {:join, nil, key, owner, %{kind: :pg}, time, :join, node()} + ]) + + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + + is_pid(new_shard) and new_shard != old_shard and + Group.lookup(name, key) == {owner, %{kind: :registry}} and + Group.members(name, key) == [{owner, %{kind: :pg}}] + end) + + assert {_floor, ^seq, ^seq} = + Group.Replica.Data.replica_stream_head(name, 0, stream_id) + + Process.exit(owner, :kill) + + Group.TestCluster.assert_eventually(fn -> + Group.lookup(name, key) == nil and Group.members(name, key) == [] + end) + end + end + + describe "replica authority snapshots" do + test "revision and epoch rows remain coherent during concurrent activation", %{name: name} do + clusters = for i <- 1..1_000, do: "authority/#{i}" + + activators = + clusters + |> Enum.chunk_every(125) + |> Enum.map(fn chunk -> + Task.async(fn -> + Enum.each(chunk, fn cluster -> + [{^cluster, _epoch}] = + Group.Replica.Data.activate_local_clusters(name, [cluster]) + end) + end) + end) + + for _ <- 1..200 do + {generation, revision, epochs} = + Group.Replica.Data.local_replica_authority(name) + + assert {nil, generation} in epochs + assert revision == Enum.count(epochs, &(not is_nil(elem(&1, 0)))) + end + + Task.await_many(activators, 10_000) + + {generation, 1_000, epochs} = Group.Replica.Data.local_replica_authority(name) + assert {nil, generation} in epochs + assert length(epochs) == 1_001 + assert Map.new(epochs) |> map_size() == 1_001 + end + end + defp start_single_shard_group(opts \\ []) do name = :"test_timeout_group_#{System.unique_integer([:positive])}" opts = Keyword.merge([name: name, shards: 1, log: false], opts) @@ -2876,6 +3038,14 @@ defmodule GroupTest do name end + defp keys_for_shard(cluster, prefix, num_shards, shard, count) do + 1 + |> Stream.iterate(&(&1 + 1)) + |> Stream.map(&"#{prefix}/#{&1}") + |> Stream.filter(&(Group.Replica.shard_index_for(cluster, &1, num_shards) == shard)) + |> Enum.take(count) + end + defp suspend_only_shard(name) do shard = Group.Replica.shard_name(name, 0) :ok = :sys.suspend(shard) diff --git a/test/replica_adversarial_test.exs b/test/replica_adversarial_test.exs new file mode 100644 index 0000000..6b1437a --- /dev/null +++ b/test/replica_adversarial_test.exs @@ -0,0 +1,358 @@ +defmodule Group.ReplicaAdversarialTest do + use ExUnit.Case + + @moduletag :capture_log + @moduletag timeout: 120_000 + + alias Group.TestCluster + + @seeds [10_007, 20_011, 40_009] + @clusters ["red", "blue"] + + for seed <- @seeds do + @seed seed + @tag chaos_seed: seed + + test "seeded mixed-operation transport chaos converges without zombies (seed #{@seed})" do + seed = @seed + :rand.seed(:exsss, {seed, seed * 3 + 1, seed * 7 + 2}) + + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"replica_chaos_#{seed}_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 3, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000, + replicated_oplog_max_entries: 12 + ] + + for {_peer, node} <- peers do + {:ok, _pid} = TestCluster.start_group(node, opts) + :ok = TestCluster.rpc!(node, Group, :connect, [name, @clusters]) + end + + TestCluster.assert_eventually( + fn -> + Enum.all?([node_a, node_b], fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == 1 and + Enum.all?(@clusters, fn cluster -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 2 + end) + end) + end, + timeout: 10_000 + ) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]} + ]) + + :ok = + TestCluster.rpc!(node_b, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]} + ]) + + initial = %{ + active: %{node_a => MapSet.new(@clusters), node_b => MapSet.new(@clusters)}, + counter: 0, + pg_keys: MapSet.new(), + pids: [], + reg_keys: MapSet.new(), + trace: [] + } + + state = + Enum.reduce(1..72, initial, fn step, state -> + apply_random_operation(state, step, seed, name, node_a, node_b) + end) + + for node <- [node_a, node_b] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) + :ok = TestCluster.rpc!(node, Group, :connect, [name, @clusters]) + end + + assert_converges(name, node_a, node_b, state) + + # Let delayed frames from the chaos phase arrive, then prove they are + # duplicates/stale rather than a source of resurrection. + Process.sleep(150) + TestCluster.flush_shards(node_a, name) + TestCluster.flush_shards(node_b, name) + assert_converges(name, node_a, node_b, state) + + for node <- [node_a, node_b] do + assert :ok = + TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) + end + + TestCluster.assert_eventually( + fn -> retained_owners_alive?(name, [node_a, node_b]) end, + timeout: 15_000 + ) + end + end + + defp apply_random_operation(state, step, seed, name, node_a, node_b) do + nodes = [node_a, node_b] + + case :rand.uniform(12) do + choice when choice in 1..3 -> + add_registration(state, step, seed, name, random(nodes)) + + choice when choice in 4..6 -> + add_membership(state, step, seed, name, random(nodes)) + + 7 -> + kill_random_owner(state) + + 8 -> + add_registry_conflict(state, step, seed, name, node_a, node_b) + + 9 -> + toggle_cluster(state, step, name, random(nodes), random(@clusters)) + + 10 -> + change_transport_mode(state, step, name, random(nodes)) + + _ -> + Enum.reduce(1..3, state, fn offset, acc -> + add_registration(acc, step * 10 + offset, seed, name, random(nodes)) + end) + end + end + + defp add_registration(state, step, seed, name, origin) do + cluster = random([nil | MapSet.to_list(state.active[origin])]) + {key, state} = next_key(state, seed, "reg", step) + meta = %{seed: seed, step: step, origin: origin} + + pid = + if cluster do + TestCluster.spawn_register_in_cluster(origin, name, key, meta, cluster) + else + TestCluster.spawn_register(origin, name, key, meta) + end + + state + |> Map.update!(:pids, &[{origin, pid} | &1]) + |> Map.update!(:reg_keys, &MapSet.put(&1, {cluster, key})) + |> trace({:register, origin, cluster, key, pid}) + end + + defp add_membership(state, step, seed, name, origin) do + cluster = random([nil | MapSet.to_list(state.active[origin])]) + {key, state} = next_key(state, seed, "pg", step) + meta = %{seed: seed, step: step, origin: origin} + + pid = + if cluster do + TestCluster.spawn_join_in_cluster(origin, name, key, meta, cluster) + else + TestCluster.spawn_join(origin, name, key, meta) + end + + state + |> Map.update!(:pids, &[{origin, pid} | &1]) + |> Map.update!(:pg_keys, &MapSet.put(&1, {cluster, key})) + |> trace({:join, origin, cluster, key, pid}) + end + + defp kill_random_owner(%{pids: []} = state), do: trace(state, :kill_noop) + + defp kill_random_owner(state) do + {origin, pid} = random(state.pids) + TestCluster.rpc!(origin, Process, :exit, [pid, :kill]) + + state + |> Map.update!(:pids, &List.delete(&1, {origin, pid})) + |> trace({:kill, origin, pid}) + end + + defp add_registry_conflict(state, step, seed, name, node_a, node_b) do + {key, state} = next_key(state, seed, "conflict", step) + + # Establish the competing claims before either origin can observe the + # other. Drain any already-scheduled delayed frame first; otherwise this + # operation nondeterministically degenerates into a local :taken result. + for node <- [node_a, node_b] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + Process.sleep(75) + pid_a = TestCluster.spawn_register(node_a, name, key, %{side: :a, seed: seed}) + pid_b = TestCluster.spawn_register(node_b, name, key, %{side: :b, seed: seed}) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]} + ]) + + :ok = + TestCluster.rpc!(node_b, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]} + ]) + + state + |> Map.update!(:pids, &[{node_a, pid_a}, {node_b, pid_b} | &1]) + |> Map.update!(:reg_keys, &MapSet.put(&1, {nil, key})) + |> trace({:conflict, key, pid_a, pid_b}) + end + + defp toggle_cluster(state, step, name, origin, cluster) do + if MapSet.member?(state.active[origin], cluster) do + :ok = TestCluster.rpc!(origin, Group, :disconnect, [name, cluster]) + + state + |> put_in([:active, origin], MapSet.delete(state.active[origin], cluster)) + |> trace({:disconnect, step, origin, cluster}) + else + :ok = TestCluster.rpc!(origin, Group, :connect, [name, cluster]) + + state + |> put_in([:active, origin], MapSet.put(state.active[origin], cluster)) + |> trace({:connect, step, origin, cluster}) + end + end + + defp change_transport_mode(state, step, name, origin) do + mode = + random([ + :drop, + :busy, + {:chaos, [drop_every: 4, duplicate_every: 3, max_delay: 65]}, + {:chaos, [drop_every: 9, duplicate_every: 2, max_delay: 30]} + ]) + + :ok = TestCluster.rpc!(origin, Group.TestReplicaTransport, :set_mode, [name, mode]) + trace(state, {:transport, step, origin, mode}) + end + + defp next_key(state, seed, kind, step) do + counter = state.counter + 1 + {"chaos/#{seed}/#{kind}/#{step}/#{counter}", %{state | counter: counter}} + end + + defp assert_converges(name, node_a, node_b, state) do + TestCluster.assert_eventually( + fn -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name])) == 1 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name])) == 1 and + Enum.all?(@clusters, fn cluster -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + end) and + registry_equal?(name, node_a, node_b, state.reg_keys) and + memberships_equal?(name, node_a, node_b, state.pg_keys) + end, + timeout: 20_000, + interval: 75 + ) + rescue + error -> + flunk( + "chaos convergence failed: #{Exception.message(error)}\n" <> + "differences=#{inspect(convergence_differences(name, node_a, node_b, state), limit: :infinity)}\n" <> + "recent operations=#{inspect(Enum.take(state.trace, 20), limit: :infinity)}" + ) + end + + defp convergence_differences(name, node_a, node_b, state) do + registry = + state.reg_keys + |> Enum.flat_map(fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + value_a = TestCluster.rpc!(node_a, Group, :lookup, args) + value_b = TestCluster.rpc!(node_b, Group, :lookup, args) + if value_a == value_b, do: [], else: [{:registry, cluster, key, value_a, value_b}] + end) + |> Enum.take(10) + + pg = + state.pg_keys + |> Enum.flat_map(fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + value_a = TestCluster.rpc!(node_a, Group, :members, args) |> Enum.sort() + value_b = TestCluster.rpc!(node_b, Group, :members, args) |> Enum.sort() + if value_a == value_b, do: [], else: [{:pg, cluster, key, value_a, value_b}] + end) + |> Enum.take(10) + + nodes = + for cluster <- [nil | @clusters] do + value_a = group_nodes(node_a, name, cluster) + value_b = group_nodes(node_b, name, cluster) + {cluster, value_a, value_b} + end + + cluster_trace = + state.trace + |> Enum.filter(fn + {:connect, _step, _origin, _cluster} -> true + {:disconnect, _step, _origin, _cluster} -> true + _ -> false + end) + + protocol = + for node <- [node_a, node_b] do + {node, TestCluster.rpc!(node, Group.TestCluster, :replica_protocol_state, [name])} + end + + [ + nodes: nodes, + registry: registry, + pg: pg, + cluster_trace: cluster_trace, + protocol: protocol + ] + end + + defp group_nodes(node, name, nil), do: TestCluster.rpc!(node, Group, :nodes, [name]) + + defp group_nodes(node, name, cluster), + do: TestCluster.rpc!(node, Group, :nodes, [name, cluster]) + + defp registry_equal?(name, node_a, node_b, keys) do + Enum.all?(keys, fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + + TestCluster.rpc!(node_a, Group, :lookup, args) == + TestCluster.rpc!(node_b, Group, :lookup, args) + end) + end + + defp memberships_equal?(name, node_a, node_b, keys) do + Enum.all?(keys, fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + members_a = TestCluster.rpc!(node_a, Group, :members, args) |> Enum.sort() + members_b = TestCluster.rpc!(node_b, Group, :members, args) |> Enum.sort() + members_a == members_b + end) + end + + defp retained_owners_alive?(name, nodes) do + nodes + |> Enum.flat_map(fn node -> + TestCluster.rpc!(node, Group.TestCluster, :replica_owner_pids, [name]) + end) + |> Enum.uniq() + |> Enum.all?(fn pid -> TestCluster.rpc!(node(pid), Process, :alive?, [pid]) end) + end + + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + + defp trace(state, operation), do: Map.update!(state, :trace, &[operation | &1]) + defp random(values), do: Enum.at(values, :rand.uniform(length(values)) - 1) +end diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index f9b8c1e..ebffc86 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -299,6 +299,92 @@ defmodule Group.TestCluster do end) end + @doc "Spawn a process on a remote node that joins in a named cluster and sleeps." + def spawn_join_in_cluster(node, name, key, meta, cluster) do + :erpc.call(node, fn -> + parent = self() + + pid = + spawn(fn -> + :ok = Group.join(name, key, meta, cluster: cluster) + send(parent, {:joined, self()}) + Process.sleep(:infinity) + end) + + receive do + {:joined, ^pid} -> pid + after + 5000 -> raise "spawn_join_in_cluster timed out" + end + end) + end + + @doc "Connects every cluster through an independent concurrent caller." + def connect_many_concurrently(node, name, clusters) do + :erpc.call(node, __MODULE__, :do_connect_many_concurrently, [name, clusters], 60_000) + end + + @doc false + def do_connect_many_concurrently(name, clusters) do + clusters + |> Task.async_stream( + fn cluster -> Group.connect(name, cluster) end, + max_concurrency: 64, + ordered: false, + timeout: 30_000 + ) + |> Enum.each(fn {:ok, :ok} -> :ok end) + + :ok + end + + @doc "Spawns one long-lived registration owner in each named cluster." + def spawn_register_many_clusters(node, name, clusters, key_prefix) do + :erpc.call( + node, + __MODULE__, + :do_spawn_register_many_clusters, + [name, clusters, key_prefix], + 60_000 + ) + end + + @doc false + def do_spawn_register_many_clusters(name, clusters, key_prefix) do + parent = self() + + entries = + Enum.map(clusters, fn cluster -> + key = "#{key_prefix}/#{cluster}" + + pid = + spawn(fn -> + :ok = Group.register(name, key, %{cluster: cluster}, cluster: cluster) + send(parent, {:registered_many, self()}) + Process.sleep(:infinity) + end) + + {cluster, key, pid} + end) + + Enum.each(entries, fn {_cluster, _key, pid} -> + receive do + {:registered_many, ^pid} -> :ok + after + 30_000 -> raise "spawn_register_many_clusters timed out" + end + end) + + entries + end + + @doc false + def registry_entries_present?(name, entries) do + Enum.all?(entries, fn {cluster, key, pid} -> + match?({^pid, %{cluster: ^cluster}}, Group.lookup(name, key, cluster: cluster)) + end) + end + @doc "Monitor nodedown events from a remote node, forwarding to caller" def monitor_nodes_on(node, target_pid) do :erpc.call(node, fn -> @@ -553,6 +639,193 @@ defmodule Group.TestCluster do :ok end + @doc """ + Asserts the replica-only authority and journal invariants in addition to the + public dual-index invariants checked by `assert_ets_consistent/1`. + + This is intended for quiescent convergence points in adversarial tests. + """ + def assert_replica_consistent(name) do + :ok = assert_ets_consistent(name) + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + assert_registry_claim_indexes(name, shard) + assert_registry_projection_has_authority(name, shard) + assert_oplog_indexes(name, shard) + assert_replica_cursor_authority(name, shard) + end + + :ok + end + + @doc """ + Returns every PID currently retained as replica authority or visible PG state. + + Adversarial tests use this at a quiescent convergence point to prove that no + dead owner remains hidden behind otherwise-consistent dual indexes. + """ + def replica_owner_pids(name) do + num_shards = Group.get_config(name).num_shards + + 0..(num_shards - 1) + |> Enum.flat_map(fn shard -> + registry_pids = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.map(fn {{_cluster, _key, _origin, _generation, _epoch}, pid, _meta, _time, _seq} -> + pid + end) + + pg_pids = + Group.Replica.Data.pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.map(fn {{_cluster, _key, pid}, _meta, _time, _node} -> pid end) + + registry_pids ++ pg_pids + end) + |> Enum.uniq() + end + + @doc false + def replica_protocol_state(name) do + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + %{ + shard: shard, + heads: Group.Replica.Data.replica_stream_heads(name, shard), + cursors: + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.sort() + } + end + end + + defp assert_registry_claim_indexes(name, shard) do + by_key = Group.Replica.Data.reg_claim_by_key_table(name, shard) + by_pid = Group.Replica.Data.reg_claim_by_pid_table(name, shard) + + key_set = + :ets.tab2list(by_key) + |> MapSet.new(fn + {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + pid_set = + :ets.tab2list(by_pid) + |> MapSet.new(fn + {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + if key_set != pid_set do + raise "registry claim index inconsistency in #{name} shard #{shard}: " <> + "by_key_only=#{inspect(MapSet.difference(key_set, pid_set) |> MapSet.to_list())} " <> + "by_pid_only=#{inspect(MapSet.difference(pid_set, key_set) |> MapSet.to_list())}" + end + + case Enum.find(key_set, fn {_cluster, _key, pid, _meta, _time, origin, _gen, _epoch, _seq} -> + node(pid) != origin + end) do + nil -> :ok + invalid -> raise "registry claim has invalid origin authority: #{inspect(invalid)}" + end + end + + defp assert_registry_projection_has_authority(name, shard) do + claims = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> MapSet.new(fn + {{cluster, key, origin, _generation, _epoch}, pid, meta, time, _seq} -> + {cluster, key, pid, meta, time, origin} + end) + + visible = + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + missing_authority = MapSet.difference(visible, claims) + + if MapSet.size(missing_authority) > 0 do + raise "visible registry rows without an authoritative claim in #{name} shard #{shard}: " <> + inspect(MapSet.to_list(missing_authority)) + end + end + + defp assert_oplog_indexes(name, shard) do + oplog = + Group.Replica.Data.replica_oplog_table(name, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream_id, seq}, append_id, _mutations} -> + {append_id, stream_id, seq} + end) + + order = + Group.Replica.Data.replica_oplog_order_table(name, shard) + |> :ets.tab2list() + |> MapSet.new() + + if oplog != order do + raise "oplog/order index inconsistency in #{name} shard #{shard}: " <> + "oplog_only=#{inspect(MapSet.difference(oplog, order) |> MapSet.to_list())} " <> + "order_only=#{inspect(MapSet.difference(order, oplog) |> MapSet.to_list())}" + end + + Group.Replica.Data.replica_stream_meta_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream_id, head, floor, applied} -> + unless floor >= 1 and floor <= head + 1 and applied >= 0 and applied <= head do + raise "invalid stream bounds in #{name} shard #{shard}: " <> + inspect({stream_id, head, floor, applied}) + end + + retained = + oplog + |> Enum.filter(fn {_append_id, row_stream, _seq} -> row_stream == stream_id end) + |> Enum.map(&elem(&1, 2)) + |> Enum.sort() + + expected = if floor <= head, do: Enum.to_list(floor..head), else: [] + + if retained != expected do + raise "non-contiguous retained oplog in #{name} shard #{shard}: " <> + "stream=#{inspect(stream_id)} retained=#{inspect(retained)} " <> + "expected=#{inspect(expected)}" + end + end) + end + + defp assert_replica_cursor_authority(name, shard) do + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream_id, seq} -> + origin = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + + valid? = + Group.Replica.Protocol.stream_name(stream_id) == name and + Group.Replica.Protocol.stream_shard(stream_id) == shard and + origin != node() and + generation == Group.Replica.Data.remote_generation(name, origin) and + epoch == Group.Replica.Data.remote_cluster_epoch(name, origin, cluster) and + seq >= 0 + + unless valid? do + raise "replica cursor is not fenced by current authority in #{name} shard #{shard}: " <> + inspect({stream_id, seq}) + end + end) + end + @doc "Wait for a condition to become true, with retries" def assert_eventually(fun, opts \\ []) do timeout = Keyword.get(opts, :timeout, 2000) diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex new file mode 100644 index 0000000..5eef3d0 --- /dev/null +++ b/test/support/test_replica_transport.ex @@ -0,0 +1,140 @@ +defmodule Group.TestReplicaTransport do + @moduledoc false + @behaviour Group.Replica.Transport + + @impl true + def id, do: :group_test_transport + + @impl true + def descriptor(_group, _opts), do: :group_test_transport + + @simple_modes [:pass, :drop, :busy, :duplicate] + + def set_mode(group, mode) + when mode in @simple_modes or + (is_tuple(mode) and tuple_size(mode) == 2 and + elem(mode, 0) in [:drop_types, :duplicate_types, :capture_drop, :capture_pass]) or + (is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :delay_types) or + (is_tuple(mode) and tuple_size(mode) == 2 and elem(mode, 0) == :chaos) do + :persistent_term.put({__MODULE__, group}, mode) + :ok + end + + def captured(group) do + :persistent_term.get({__MODULE__, group, :captured}, []) |> Enum.reverse() + end + + def clear_captured(group) do + :persistent_term.erase({__MODULE__, group, :captured}) + :ok + end + + def clear(group) do + :persistent_term.erase({__MODULE__, group}) + :persistent_term.erase({__MODULE__, group, :captured}) + :ok + end + + @impl true + def try_send(group, target_node, shard, frame, _opts) do + case :persistent_term.get({__MODULE__, group}, :pass) do + :drop -> + :ok + + :busy -> + :busy + + :duplicate -> + deliver(group, target_node, shard, frame) + deliver(group, target_node, shard, frame) + + {:drop_types, types} -> + if frame_type(frame) in types, do: :ok, else: deliver(group, target_node, shard, frame) + + {:duplicate_types, types} -> + if frame_type(frame) in types do + deliver(group, target_node, shard, frame) + deliver(group, target_node, shard, frame) + else + deliver(group, target_node, shard, frame) + end + + {:delay_types, delays, default_delay} -> + delay = Map.get(delays, frame_type(frame), default_delay) + delayed_deliver(group, target_node, shard, frame, delay) + + {:capture_drop, types} -> + if frame_type(frame) in types, do: capture(group, target_node, shard, frame) + :ok + + {:capture_pass, types} -> + if frame_type(frame) in types, do: capture(group, target_node, shard, frame) + deliver(group, target_node, shard, frame) + + {:chaos, opts} -> + chaos_deliver(group, target_node, shard, frame, opts) + + :pass -> + deliver(group, target_node, shard, frame) + end + end + + defp chaos_deliver(group, target_node, shard, frame, opts) do + hash = :erlang.phash2({target_node, shard, frame}, 1_000_003) + drop_every = Keyword.get(opts, :drop_every, 0) + duplicate_every = Keyword.get(opts, :duplicate_every, 0) + max_delay = Keyword.get(opts, :max_delay, 0) + + cond do + drop_every > 0 and rem(hash, drop_every) == 0 -> + :ok + + duplicate_every > 0 and rem(hash, duplicate_every) == 0 -> + delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 + delayed_deliver(group, target_node, shard, frame, delay) + delayed_deliver(group, target_node, shard, frame, max(max_delay - delay, 0)) + + true -> + delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 + delayed_deliver(group, target_node, shard, frame, delay) + end + end + + defp delayed_deliver(group, target_node, shard, frame, delay) when delay <= 0, + do: deliver(group, target_node, shard, frame) + + defp delayed_deliver(group, target_node, shard, frame, delay) do + source_node = node() + + spawn(fn -> + receive do + after + delay -> deliver(group, target_node, shard, frame, source_node) + end + end) + + :ok + end + + defp capture(group, target_node, shard, frame) do + key = {__MODULE__, group, :captured} + captured = :persistent_term.get(key, []) + :persistent_term.put(key, [{target_node, shard, frame} | captured]) + end + + defp deliver(group, target_node, shard, frame), + do: deliver(group, target_node, shard, frame, node()) + + defp deliver(group, target_node, shard, frame, source_node) do + destination = {Group.Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, source_node, frame} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end + + defp frame_type(frame) when is_tuple(frame), do: elem(frame, 0) + defp frame_type(_frame), do: :unknown +end From d1773a0021ef46a3a56832c50aa287f0422e18e5 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Fri, 31 Jul 2026 07:54:44 +0000 Subject: [PATCH 3/7] Harden anti-entropy recovery and add TCP transport --- .gitignore | 3 + README.md | 47 +- lib/group.ex | 1 + lib/group/replica.ex | 121 +++-- lib/group/replica/data.ex | 287 ++++++++-- lib/group/replica/transport/tcp.ex | 448 ++++++++++++++++ mix.exs | 3 +- mix.lock | 1 + test/README.md | 44 +- test/distributed_test.exs | 495 ++++++++++++++++++ test/formal/GroupAntiEntropy.cfg | 17 + test/formal/GroupAntiEntropy.tla | 481 +++++++++++++++++ test/formal/README.md | 44 ++ test/formal/check.sh | 19 + test/group_test.exs | 185 +++++++ test/mutation/README.md | 24 + test/mutation/run.exs | 422 +++++++++++++++ test/replica_model_property_test.exs | 381 ++++++++++++++ test/support/controlled_replica_transport.ex | 49 ++ test/support/model_conflict_resolver.ex | 15 + test/support/replica_lifecycle_model.ex | 223 ++++++++ test/support/replica_model_scheduler.ex | 521 +++++++++++++++++++ test/support/test_cluster.ex | 84 ++- 23 files changed, 3829 insertions(+), 86 deletions(-) create mode 100644 lib/group/replica/transport/tcp.ex create mode 100644 test/formal/GroupAntiEntropy.cfg create mode 100644 test/formal/GroupAntiEntropy.tla create mode 100644 test/formal/README.md create mode 100755 test/formal/check.sh create mode 100644 test/mutation/README.md create mode 100644 test/mutation/run.exs create mode 100644 test/replica_model_property_test.exs create mode 100644 test/support/controlled_replica_transport.ex create mode 100644 test/support/model_conflict_resolver.ex create mode 100644 test/support/replica_lifecycle_model.ex create mode 100644 test/support/replica_model_scheduler.ex diff --git a/.gitignore b/.gitignore index caee45d..8328a5b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ doc/ # Temporary files, for example, from tests. tmp/ +# TLC's default state directory when the formal model is run by hand. +/states/ + # If the VM crashes, it generates a dump, let's ignore it too. erl_crash.dump diff --git a/README.md b/README.md index 0943994..c9d351e 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,9 @@ All operations are **eventually consistent**: `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, or `:disconnected`. Dropped and busy frames are repaired by anti-entropy. + `Group.Replica.Transport.TCP` is an included sideband adapter with bounded + per-peer writer queues; its socket owners are separate processes, so socket + backpressure cannot block a Group shard. - **`replicated_oplog_max_entries`** — maximum retained replica records per shard across all local streams. Defaults to 65,536. Pruning never waits for peer acknowledgements; a peer behind the retained floor receives an exact @@ -391,9 +394,13 @@ point-in-time value. The highest observed incremental revision is tracked separately and can never promote a partial view to exact authority. Discovery hints never mutate membership on their own. Authority installation fans a local fence to every lane, which sweeps only that lane's retained receive -streams. Because PG rows intentionally do not carry protocol epochs, a -superseded origin/cluster slice is cleared and its current cursor reset so the -next head reconstructs it from retained deltas or an exact snapshot. +streams. Shared authority may become visible before that fanout reaches a lane, +but the lane's constant-size view is not marked installed until its purge +finishes; data validation requires that marker. A heartbeat or lane hello can +confirm an installed view but cannot promote a pending one. Because PG rows +intentionally do not carry protocol epochs, a superseded origin/cluster slice +is cleared and its current cursor reset so the next head reconstructs it from +retained deltas or an exact snapshot. Replica state itself does not travel on the control plane. Once the hello is fenced, stream-head exchange on the replica transport catches the peer up. @@ -412,10 +419,13 @@ tail even when no later write occurs. If the requested sequence is older than the bounded oplog floor, the origin sends an exact snapshot of only its own registry claims and PG memberships; absence from that snapshot is a delete. -There are no leaders, quorum acknowledgements, tombstones, or known-membership -retention barriers. Oplog memory is bounded locally and independently of slow -peers. Deletes are normal ordered records while retained, and exact snapshots -close gaps after pruning. +There are no leaders, quorum acknowledgements, per-entry replicated tombstones, +or known-membership retention barriers. Oplog memory is bounded locally and +independently of slow peers. Deletes are normal ordered records while retained, +and exact snapshots close gaps after pruning. Named-cluster close uses only a +temporary local shard-completion barrier; the final shard removes it and all +routing rows, including after a caller timeout or shard restart. Reconnect +waits for that barrier so a prior close cannot erase newly accepted writes. The sender flush timer is mainly a fallback for idle periods. The unified outbound buffer also flushes immediately when it hits the configured size, when a new enqueue @@ -430,6 +440,26 @@ or reconnect, and generation fencing rejects data from a restarted origin. An alternative sideband adapter authenticates the peer as a dist-Erlang node and calls `Group.Replica.Transport.deliver/4` locally. +For example, replica data can use the included sideband TCP adapter while +authority and membership remain on dist Erlang: + +```elixir +replica_transport: + {Group.Replica.Transport.TCP, + [ + ip: {0, 0, 0, 0}, + advertised_ip: {10, 0, 1, 12}, + port: 44_321, + max_queue: 1_024 + ]} +``` + +Each node advertises its own reachable address. TCP frames are capability +authenticated by the dist-Erlang hello but are not encrypted, so use a trusted +network or place the connection behind TLS. The adapter deliberately has no +control/data ordering relationship; the generation/epoch lane barrier and +stream sequence checks supply correctness. + ### Named Cluster TTL Leases Named-cluster TTLs are a local way to reduce replication fanout to nodes that @@ -466,7 +496,8 @@ mix test ``` See [`test/README.md`](test/README.md) for details on the distributed test -infrastructure. +infrastructure, shrinkable StreamData lifecycle-model tests, and the bounded +TLA+ anti-entropy model. ## Benchmarks diff --git a/lib/group.ex b/lib/group.ex index f125efd..449e4fa 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -1107,6 +1107,7 @@ defmodule Group do @doc false def connect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do + timeout = Data.await_closed_local_clusters(name, clusters, timeout) _epochs = Data.activate_local_clusters(name, clusters) Data.add_cluster_node(name, clusters, node()) diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 95bb6b6..58ce632 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -248,9 +248,17 @@ defmodule Group.Replica do state = schedule_anti_entropy(state) - # Complete any write-ahead record left unapplied by a shard crash, then - # rebuild local process monitors from the surviving materialized tables. + # Repair any interrupted multi-table journal/index mutation, complete + # write-ahead records left unapplied by a shard crash, then rebuild local + # process monitors from the surviving materialized tables. + :ok = Data.repair_local_replica_journal(name, shard_index) state = replay_local_journal(state) + :ok = Data.repair_shard_indexes(name, shard_index) + + completed_clusters = + Data.mark_closed_cluster_shard(name, Data.closed_local_clusters(name), shard_index) + + if completed_clusters != [], do: Data.remove_clusters(name, completed_clusters) # Rebuild monitors from any surviving ETS data (after shard crash/restart) state = rebuild_monitors(state) @@ -452,27 +460,25 @@ defmodule Group.Replica do Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) } - if replica_authority_current?(state, remote_node, generation, epoch_revision) do - :ok = - Data.put_remote_view_info( - state.name, - state.shard_index, - remote_node, - generation, - Data.remote_cluster_epoch_exact_revision(state.name, remote_node), - epoch_revision - ) + cond do + replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) -> + state = + state + |> purge_remote_streams_outside_authority(remote_node) + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) - state = - state - |> purge_remote_streams_outside_authority(remote_node) - |> touch_replica_peer(remote_node) - |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) - |> send_replica_heads(remote_node) + {:noreply, state} - {:noreply, state} - else - {:noreply, request_replica_authority(state, remote_node)} + replica_authority_current?(state, remote_node, generation, epoch_revision) -> + # Shared authority arrived first; its local fanout is already the + # ordered marker that will purge and install this lane's view. + {:noreply, state} + + true -> + {:noreply, request_replica_authority(state, remote_node)} end else Logger.error( @@ -502,6 +508,8 @@ defmodule Group.Replica do state end + :ok = install_replica_view(state, remote_node, generation) + state = %{ state | cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), @@ -574,6 +582,7 @@ defmodule Group.Replica do |> purge_closed_remote_epochs(remote_node, stale) |> purge_superseded_remote_streams(remote_node, epochs) + :ok = install_replica_view(state, remote_node, generation) state = send_replica_heads(state, remote_node, Enum.map(shared, &elem(&1, 0))) {:noreply, take_one_local_request_turn(state)} @@ -606,10 +615,13 @@ defmodule Group.Replica do state = if replica_authority_current?(state, remote_node, generation, revision) do - state - |> purge_closed_remote_epochs(remote_node, stale) - |> purge_superseded_remote_streams(remote_node, epochs) - |> send_replica_heads(remote_node, shared) + state = + state + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + + :ok = install_replica_view(state, remote_node, generation) + send_replica_heads(state, remote_node, shared) else state end @@ -665,6 +677,7 @@ defmodule Group.Replica do |> mark_cluster_control_dirty(remote_node) |> purge_closed_remote_epochs(remote_node, closed) + :ok = install_replica_view(state, remote_node, generation) {:noreply, take_one_local_request_turn(state)} :stale -> @@ -690,7 +703,9 @@ defmodule Group.Replica do state = if replica_authority_current?(state, remote_node, generation, revision) do - purge_closed_remote_epochs(state, remote_node, closed) + state = purge_closed_remote_epochs(state, remote_node, closed) + :ok = install_replica_view(state, remote_node, generation) + state else state end @@ -714,23 +729,20 @@ defmodule Group.Replica do remote_node = node(remote_pid) state = - if version == Protocol.version() and - replica_authority_current?(state, remote_node, generation, epoch_revision) do - :ok = - Data.put_remote_view_info( - state.name, - state.shard_index, - remote_node, - generation, - Data.remote_cluster_epoch_exact_revision(state.name, remote_node), - epoch_revision - ) + cond do + version == Protocol.version() and + replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) -> + state + |> put_remote_shard(remote_node, remote_pid) + |> touch_replica_peer(remote_node) - state - |> put_remote_shard(remote_node, remote_pid) - |> touch_replica_peer(remote_node) - else - request_replica_authority(state, remote_node) + version == Protocol.version() and + replica_authority_current?(state, remote_node, generation, epoch_revision) -> + state + + true -> + request_replica_authority(state, remote_node) end {:noreply, state} @@ -2061,6 +2073,8 @@ defmodule Group.Replica do :ok end) + completed_clusters = Data.mark_closed_cluster_shard(name, clusters, shard) + if completed_clusters != [], do: Data.remove_clusters(name, completed_clusters) notify_monitors(name, events) {:ok, state} end @@ -2849,6 +2863,8 @@ defmodule Group.Replica do state end + :ok = install_replica_view(state, remote_node, generation) + state = %{ state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), @@ -2916,6 +2932,26 @@ defmodule Group.Replica do Data.remote_cluster_epoch_observed_revision(state.name, remote_node) == epoch_revision end + defp replica_view_current?(state, remote_node) do + Data.remote_view_generation(state.name, state.shard_index, remote_node) == + Data.remote_generation(state.name, remote_node) and + Data.remote_view_cluster_epoch_revision(state.name, state.shard_index, remote_node) == + Data.remote_cluster_epoch_exact_revision(state.name, remote_node) and + Data.remote_view_observed_revision(state.name, state.shard_index, remote_node) == + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + end + + defp install_replica_view(state, remote_node, generation) do + Data.put_remote_view_info( + state.name, + state.shard_index, + remote_node, + generation, + Data.remote_cluster_epoch_exact_revision(state.name, remote_node), + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + ) + end + defp schedule_anti_entropy(state) do ref = make_ref() @@ -3178,6 +3214,7 @@ defmodule Group.Replica do Protocol.stream_name(stream_id) == state.name and Protocol.stream_origin(stream_id) == source_node and Protocol.stream_shard(stream_id) == state.shard_index and + replica_view_current?(state, source_node) and Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and Protocol.stream_epoch(stream_id) == Data.remote_cluster_epoch(state.name, source_node, cluster) and diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index e6e4c81..94f803a 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -2,6 +2,8 @@ defmodule Group.Replica.Data do @moduledoc false use GenServer + alias Group.Replica.Protocol + _archdoc = """ GenServer that owns ETS tables for all shards. @@ -183,11 +185,23 @@ defmodule Group.Replica.Data do def closed_local_cluster_epoch(name, cluster) do case :ets.lookup(closed_local_cluster_epochs_table(name), cluster) do - [{^cluster, epoch}] -> epoch + [{^cluster, epoch, _pending_shards}] -> epoch [] -> nil end end + def closed_local_clusters(name) do + closed_local_cluster_epochs_table(name) + |> :ets.tab2list() + |> Enum.map(&elem(&1, 0)) + end + + def await_closed_local_clusters(name, clusters, timeout) + when is_list(clusters) and is_integer(timeout) and timeout >= 0 do + started_at = System.monotonic_time(:millisecond) + await_closed_local_clusters(name, clusters, timeout, started_at) + end + def remote_generation(name, remote_node) do case :ets.lookup(replication_meta_table(name), {:remote_generation, remote_node}) do [{{:remote_generation, ^remote_node}, generation}] -> generation @@ -319,6 +333,10 @@ defmodule Group.Replica.Data do GenServer.call(data_name(name), {:deactivate_local_clusters, clusters}, :infinity) end + def mark_closed_cluster_shard(name, clusters, shard) do + GenServer.call(data_name(name), {:mark_closed_cluster_shard, clusters, shard}, :infinity) + end + def local_stream_id(name, shard, cluster) do case local_cluster_epoch(name, cluster) do nil -> @@ -378,11 +396,209 @@ defmodule Group.Replica.Data do end) end + @doc false + def repair_local_replica_journal(name, shard) do + stream_table = replica_stream_meta_table(name, shard) + oplog_table = replica_oplog_table(name, shard) + order_table = replica_oplog_order_table(name, shard) + + stream_table + |> :ets.tab2list() + |> Enum.each(fn {stream_id, head, floor, applied} -> + if current_local_stream?(name, shard, stream_id) do + present = + oplog_table + |> :ets.select([ + {{{stream_id, :"$1"}, :_, :_}, [], [:"$1"]} + ]) + |> MapSet.new() + + repaired_floor = + floor + |> missing_applied_sequences(applied, present) + |> case do + [] -> floor + missing -> Enum.max(missing) + 1 + end + + repaired_head = contiguous_unapplied_head(applied, head, present) + + if repaired_head < head do + :ets.select_delete(oplog_table, [ + {{{stream_id, :"$1"}, :_, :_}, [{:>, :"$1", repaired_head}], [true]} + ]) + end + + repaired_floor = min(repaired_floor, repaired_head + 1) + repaired_applied = min(applied, repaired_head) + + :ets.insert( + stream_table, + {stream_id, repaired_head, repaired_floor, repaired_applied} + ) + else + drop_local_stream( + name, + shard, + Protocol.stream_cluster(stream_id), + Protocol.stream_epoch(stream_id) + ) + end + end) + + :ets.delete_all_objects(order_table) + + oplog_table + |> :ets.tab2list() + |> Enum.each(fn {{stream_id, seq}, append_id, _mutations} -> + :ets.insert(order_table, {append_id, stream_id, seq}) + end) + + :ok + end + + @doc false + def repair_shard_indexes(name, shard) do + purge_inactive_cluster_rows(name, shard) + rebuild_registry_reverse_index(name, shard) + rebuild_registry_claim_reverse_index(name, shard) + rebuild_pg_reverse_index(name, shard) + :ok + end + def replica_stream_heads(name, shard) do :ets.tab2list(replica_stream_meta_table(name, shard)) |> Enum.map(fn {stream_id, head, floor, _applied} -> {stream_id, floor, head} end) end + defp missing_applied_sequences(floor, applied, _present) when floor > applied, do: [] + + defp missing_applied_sequences(floor, applied, present) do + Enum.reject(floor..applied, &MapSet.member?(present, &1)) + end + + defp contiguous_unapplied_head(applied, head, _present) when applied >= head, do: head + + defp contiguous_unapplied_head(applied, head, present) do + Enum.reduce_while((applied + 1)..head, applied, fn seq, _last -> + if MapSet.member?(present, seq), do: {:cont, seq}, else: {:halt, seq - 1} + end) + end + + defp current_local_stream?(name, shard, stream_id) do + cluster = Protocol.stream_cluster(stream_id) + + Protocol.stream_name(stream_id) == name and + Protocol.stream_origin(stream_id) == node() and + Protocol.stream_generation(stream_id) == generation(name) and + Protocol.stream_shard(stream_id) == shard and + Protocol.stream_epoch(stream_id) == local_cluster_epoch(name, cluster) + end + + defp await_closed_local_clusters(name, clusters, timeout, started_at) do + pending? = + Enum.any?(clusters, fn cluster -> + not is_nil(closed_local_cluster_epoch(name, cluster)) + end) + + elapsed = System.monotonic_time(:millisecond) - started_at + + cond do + not pending? -> + max(timeout - elapsed, 0) + + elapsed >= timeout -> + exit( + {:timeout, + {GenServer, :call, + [data_name(name), {:await_closed_local_clusters, clusters}, timeout]}} + ) + + true -> + receive do + after + min(10, timeout - elapsed) -> + await_closed_local_clusters(name, clusters, timeout, started_at) + end + end + end + + defp rebuild_registry_reverse_index(name, shard) do + reverse = reg_by_pid_table(name, shard) + :ets.delete_all_objects(reverse) + + reg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {{cluster, key}, pid, meta, time, entry_node} -> + :ets.insert(reverse, {{pid, cluster, key}, meta, time, entry_node}) + end) + end + + defp rebuild_registry_claim_reverse_index(name, shard) do + reverse = reg_claim_by_pid_table(name, shard) + :ets.delete_all_objects(reverse) + + reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> + :ets.insert( + reverse, + {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} + ) + end) + end + + defp rebuild_pg_reverse_index(name, shard) do + reverse = pg_by_pid_table(name, shard) + :ets.delete_all_objects(reverse) + + pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {{cluster, key, pid}, meta, time, entry_node} -> + :ets.insert(reverse, {{pid, cluster, key}, meta, time, entry_node}) + end) + end + + defp purge_inactive_cluster_rows(name, shard) do + clusters = + Enum.concat([ + Enum.map(:ets.tab2list(reg_by_key_table(name, shard)), fn + {{cluster, _key}, _pid, _meta, _time, _entry_node} -> cluster + end), + Enum.map(:ets.tab2list(reg_claim_by_key_table(name, shard)), fn + {{cluster, _key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + cluster + end), + Enum.map(:ets.tab2list(pg_by_key_table(name, shard)), fn + {{cluster, _key, _pid}, _meta, _time, _entry_node} -> cluster + end), + Enum.map(:ets.tab2list(replica_cursor_table(name, shard)), fn {stream_id, _seq} -> + Protocol.stream_cluster(stream_id) + end) + ]) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> Enum.filter(&is_nil(local_cluster_epoch(name, &1))) + + Enum.each(clusters, fn cluster -> + :ets.select_delete(reg_by_key_table(name, shard), [ + {{{cluster, :_}, :_, :_, :_, :_}, [], [true]} + ]) + + :ets.select_delete(reg_claim_by_key_table(name, shard), [ + {{{cluster, :_, :_, :_, :_}, :_, :_, :_, :_}, [], [true]} + ]) + + :ets.select_delete(pg_by_key_table(name, shard), [ + {{{cluster, :_, :_}, :_, :_, :_}, [], [true]} + ]) + end) + + :ok = delete_replica_cursors_for_clusters(name, shard, clusters) + if clusters != [], do: remove_clusters(name, clusters) + :ok + end + def replica_stream_head(name, shard, stream_id) do case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do [{^stream_id, head, floor, applied}] -> {floor, head, applied} @@ -1620,13 +1836,49 @@ defmodule Group.Replica.Data do Enum.map(clusters, fn cluster -> epoch = local_cluster_epoch(state.name, cluster) :ets.delete(local_cluster_epochs_table(state.name), cluster) - if epoch, do: :ets.insert(closed_local_cluster_epochs_table(state.name), {cluster, epoch}) + + if epoch do + pending_shards = MapSet.new(0..(state.num_shards - 1)) + + :ets.insert( + closed_local_cluster_epochs_table(state.name), + {cluster, epoch, pending_shards} + ) + end + {cluster, epoch} end) {:reply, epochs, state} end + def handle_call({:mark_closed_cluster_shard, clusters, shard}, _from, state) do + completed = + Enum.reduce(clusters, [], fn cluster, acc -> + case :ets.lookup(closed_local_cluster_epochs_table(state.name), cluster) do + [{^cluster, epoch, pending_shards}] -> + pending_shards = MapSet.delete(pending_shards, shard) + + if MapSet.size(pending_shards) == 0 do + :ets.delete(closed_local_cluster_epochs_table(state.name), cluster) + [cluster | acc] + else + :ets.insert( + closed_local_cluster_epochs_table(state.name), + {cluster, epoch, pending_shards} + ) + + acc + end + + [] -> + acc + end + end) + + {:reply, Enum.reverse(completed), state} + end + def handle_call( {:put_remote_replica_info, shard, remote_node, generation, epoch_revision, epochs}, _from, @@ -1690,13 +1942,6 @@ defmodule Group.Replica.Data do {{:remote_authority_installs, remote_node}, 0} ) - for view_shard <- 0..(state.num_shards - 1) do - :ets.insert( - replication_meta_table(state.name), - {{:remote_view_info, view_shard, remote_node}, generation, epoch_revision, epoch_revision} - ) - end - rows = for {cluster, epoch} <- epochs, not is_nil(cluster), do: {{remote_node, cluster}, epoch} @@ -1886,7 +2131,7 @@ defmodule Group.Replica.Data do {:ok, %{name: name, num_shards: num_shards}} end - defp observe_remote_cluster_revision(name, remote_node, revision, num_shards) do + defp observe_remote_cluster_revision(name, remote_node, revision, _num_shards) do key = {:remote_epoch_observed, remote_node} case :ets.lookup(replication_meta_table(name), key) do @@ -1894,28 +2139,6 @@ defmodule Group.Replica.Data do _ -> :ets.insert(replication_meta_table(name), {key, revision}) end - for shard <- 0..(num_shards - 1) do - view_key = {:remote_view_info, shard, remote_node} - - case :ets.lookup(replication_meta_table(name), view_key) do - [{^view_key, _generation, _authoritative, observed}] when observed >= revision -> - :ok - - [{^view_key, generation, authoritative, _observed}] -> - :ets.insert( - replication_meta_table(name), - {view_key, generation, authoritative, revision} - ) - - [] -> - :ets.insert( - replication_meta_table(name), - {view_key, remote_generation(name, remote_node), - remote_cluster_epoch_revision(name, remote_node), revision} - ) - end - end - :ok end diff --git a/lib/group/replica/transport/tcp.ex b/lib/group/replica/transport/tcp.ex new file mode 100644 index 0000000..1914ac6 --- /dev/null +++ b/lib/group/replica/transport/tcp.ex @@ -0,0 +1,448 @@ +defmodule Group.Replica.Transport.TCP do + @moduledoc """ + Sideband TCP transport for replica data. + + Erlang distribution still carries Group discovery and authority controls. + Replica frames use independent TCP connections, so there is no ordering + relationship between a control message and its data lane. + + `try_send/5` never writes a socket. It reserves one slot in a bounded + per-peer queue and sends to a dedicated writer process. The writer may block + up to `:send_timeout` without blocking a Group shard. A full queue returns + `:busy`; a missing connection returns `:disconnected`. + + The endpoint capability in the dist-Erlang hello prevents an unrelated + socket client from injecting frames. This transport is intended for trusted + cluster networks; it does not encrypt traffic. Put it behind a private + network or a TLS/WebSocket tunnel when confidentiality is required. + + ## Options + + * `:ip` - listen address, default `{127, 0, 0, 1}` + * `:advertised_ip` - address placed in the hello, defaults to `:ip` + * `:port` - listen port, default `0` (ephemeral) + * `:max_queue` - maximum queued frames per peer, default `1_024` + * `:connect_timeout` - outbound connect timeout in milliseconds, default `1_000` + * `:send_timeout` - writer socket send timeout in milliseconds, default `1_000` + * `:reconnect_interval` - retry delay in milliseconds, default `50` + """ + + use GenServer + + @behaviour Group.Replica.Transport + + @impl true + def id, do: :group_sideband_tcp_v1 + + @impl true + def child_spec(opts) do + name = Keyword.fetch!(opts, :name) + + %{ + id: {__MODULE__, name}, + start: {__MODULE__, :start_link, [opts]}, + type: :worker, + restart: :permanent, + shutdown: 5_000 + } + end + + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, opts, name: server_name(name)) + end + + @impl true + def descriptor(group, _opts) do + :persistent_term.get({__MODULE__, group, :descriptor}) + end + + @impl true + def try_send(group, target_node, shard, frame, _opts) do + case :ets.lookup(route_table(group), target_node) do + [{^target_node, writer, queued, max_queue}] -> + if :atomics.add_get(queued, 1, 1) <= max_queue do + if :erlang.send_nosuspend(writer, {:replica_frame, shard, frame}) do + :ok + else + :atomics.sub(queued, 1, 1) + :busy + end + else + :atomics.sub(queued, 1, 1) + :busy + end + + [] -> + :disconnected + end + end + + @impl true + def peer_up(group, remote_node, descriptor, _opts) do + send_manager(group, {:peer_up, remote_node, descriptor}) + end + + @impl true + def peer_down(group, remote_node, _opts) do + send_manager(group, {:peer_down, remote_node}) + end + + @doc false + def disconnect_peer(group, remote_node) do + GenServer.call(server_name(group), {:disable_peer, remote_node}) + end + + @doc false + def reconnect_peer(group, remote_node) do + GenServer.call(server_name(group), {:enable_peer, remote_node}) + end + + @doc false + def connected?(group, remote_node) do + :ets.member(route_table(group), remote_node) + end + + @doc false + def status(group) do + GenServer.call(server_name(group), :status) + end + + @impl true + def init(opts) do + Process.flag(:trap_exit, true) + group = Keyword.fetch!(opts, :name) + ip = Keyword.get(opts, :ip, {127, 0, 0, 1}) + advertised_ip = Keyword.get(opts, :advertised_ip, ip) + port = Keyword.get(opts, :port, 0) + + {:ok, listener} = + :gen_tcp.listen(port, [ + :binary, + packet: 4, + active: false, + reuseaddr: true, + ip: ip + ]) + + {:ok, {_listen_ip, listen_port}} = :inet.sockname(listener) + capability = :erlang.term_to_binary({node(), make_ref(), System.unique_integer()}) + descriptor = {:group_sideband_tcp_v1, advertised_ip, listen_port, capability} + :persistent_term.put({__MODULE__, group, :descriptor}, descriptor) + + :ets.new(route_table(group), [ + :named_table, + :public, + :set, + read_concurrency: true, + write_concurrency: true + ]) + + manager = self() + acceptor = spawn_link(fn -> accept_loop(listener, manager, group, capability) end) + + {:ok, + %{ + group: group, + listener: listener, + acceptor: acceptor, + peers: %{}, + writers: %{}, + inbound: %{}, + disabled: MapSet.new(), + max_queue: Keyword.get(opts, :max_queue, 1_024), + connect_timeout: Keyword.get(opts, :connect_timeout, 1_000), + send_timeout: Keyword.get(opts, :send_timeout, 1_000), + reconnect_interval: Keyword.get(opts, :reconnect_interval, 50) + }} + end + + @impl true + def handle_info({:peer_up, remote_node, descriptor}, state) do + state = %{state | peers: Map.put(state.peers, remote_node, descriptor)} + + state = + if MapSet.member?(state.disabled, remote_node) do + state + else + ensure_writer(state, remote_node) + end + + {:noreply, state} + end + + def handle_info({:peer_down, remote_node}, state) do + {:noreply, drop_peer(state, remote_node, true)} + end + + def handle_info({:writer_ready, remote_node, writer, queued}, state) do + if Map.get(state.writers, remote_node) == writer and + not MapSet.member?(state.disabled, remote_node) do + :ets.insert( + route_table(state.group), + {remote_node, writer, queued, state.max_queue} + ) + end + + {:noreply, state} + end + + def handle_info({:writer_failed, remote_node, writer}, state) do + {:noreply, writer_failed(state, remote_node, writer)} + end + + def handle_info({:reader_ready, source_node, reader}, state) do + {:noreply, %{state | inbound: Map.put(state.inbound, source_node, reader)}} + end + + def handle_info({:reconnect, remote_node}, state) do + {:noreply, ensure_writer(state, remote_node)} + end + + def handle_info({:EXIT, pid, _reason}, %{acceptor: pid} = state) do + {:stop, :acceptor_stopped, state} + end + + def handle_info({:EXIT, writer, _reason}, state) do + case Enum.find(state.writers, fn {_node, pid} -> pid == writer end) do + {remote_node, ^writer} -> + {:noreply, writer_failed(state, remote_node, writer)} + + nil -> + {:noreply, state} + end + end + + @impl true + def handle_call({:disable_peer, remote_node}, _from, state) do + state = %{state | disabled: MapSet.put(state.disabled, remote_node)} + {:reply, :ok, drop_writer(state, remote_node)} + end + + def handle_call({:enable_peer, remote_node}, _from, state) do + state = %{state | disabled: MapSet.delete(state.disabled, remote_node)} + {:reply, :ok, ensure_writer(state, remote_node)} + end + + def handle_call(:status, _from, state) do + {:reply, + %{ + peers: Map.keys(state.peers), + writers: Map.keys(state.writers), + connected: :ets.tab2list(route_table(state.group)) |> Enum.map(&elem(&1, 0)), + inbound: state.inbound, + disabled: MapSet.to_list(state.disabled) + }, state} + end + + @impl true + def terminate(_reason, state) do + :persistent_term.erase({__MODULE__, state.group, :descriptor}) + :gen_tcp.close(state.listener) + :ok + end + + defp send_manager(group, message) do + case Process.whereis(server_name(group)) do + nil -> + :ok + + pid -> + _ = :erlang.send_nosuspend(pid, message) + :ok + end + end + + defp ensure_writer(state, remote_node) do + cond do + MapSet.member?(state.disabled, remote_node) -> + state + + Map.has_key?(state.writers, remote_node) -> + state + + descriptor = Map.get(state.peers, remote_node) -> + manager = self() + + writer = + spawn_link(fn -> + writer_connect( + manager, + state.group, + remote_node, + descriptor, + state.connect_timeout, + state.send_timeout + ) + end) + + %{state | writers: Map.put(state.writers, remote_node, writer)} + + true -> + state + end + end + + defp writer_failed(state, remote_node, writer) do + if Map.get(state.writers, remote_node) == writer do + :ets.delete(route_table(state.group), remote_node) + state = %{state | writers: Map.delete(state.writers, remote_node)} + + if Map.has_key?(state.peers, remote_node) and + not MapSet.member?(state.disabled, remote_node) do + Process.send_after(self(), {:reconnect, remote_node}, state.reconnect_interval) + end + + state + else + state + end + end + + defp drop_peer(state, remote_node, remove_descriptor?) do + state = drop_writer(state, remote_node) + + if remove_descriptor? do + %{state | peers: Map.delete(state.peers, remote_node)} + else + state + end + end + + defp drop_writer(state, remote_node) do + :ets.delete(route_table(state.group), remote_node) + + case Map.pop(state.writers, remote_node) do + {nil, writers} -> + %{state | writers: writers} + + {writer, writers} -> + Process.exit(writer, :shutdown) + %{state | writers: writers} + end + end + + defp writer_connect( + manager, + group, + remote_node, + {:group_sideband_tcp_v1, host, port, capability}, + connect_timeout, + send_timeout + ) do + opts = [ + :binary, + packet: 4, + active: false, + send_timeout: send_timeout, + send_timeout_close: true + ] + + case :gen_tcp.connect(host, port, opts, connect_timeout) do + {:ok, socket} -> + case :gen_tcp.send(socket, :erlang.term_to_binary({:hello, group, node(), capability})) do + :ok -> + queued = :atomics.new(1, signed: false) + send(manager, {:writer_ready, remote_node, self(), queued}) + writer_loop(socket, manager, remote_node, queued) + + {:error, _reason} -> + :gen_tcp.close(socket) + send(manager, {:writer_failed, remote_node, self()}) + end + + {:error, _reason} -> + send(manager, {:writer_failed, remote_node, self()}) + end + end + + defp writer_connect(manager, _group, remote_node, _descriptor, _connect_timeout, _send_timeout) do + send(manager, {:writer_failed, remote_node, self()}) + end + + defp writer_loop(socket, manager, remote_node, queued) do + receive do + {:replica_frame, shard, frame} -> + result = :gen_tcp.send(socket, :erlang.term_to_binary({shard, frame})) + :atomics.sub(queued, 1, 1) + + case result do + :ok -> + writer_loop(socket, manager, remote_node, queued) + + {:error, _reason} -> + :gen_tcp.close(socket) + send(manager, {:writer_failed, remote_node, self()}) + end + end + end + + defp accept_loop(listener, manager, group, capability) do + case :gen_tcp.accept(listener) do + {:ok, socket} -> + reader = + spawn(fn -> + receive do + {:accepted_socket, accepted} -> + reader_handshake(accepted, manager, group, capability) + end + end) + + :ok = :gen_tcp.controlling_process(socket, reader) + send(reader, {:accepted_socket, socket}) + accept_loop(listener, manager, group, capability) + + {:error, :closed} -> + :ok + + {:error, _reason} -> + send(manager, {:EXIT, self(), :accept_failed}) + end + end + + defp reader_handshake(socket, manager, group, capability) do + with {:ok, payload} <- :gen_tcp.recv(socket, 0), + {:ok, {:hello, ^group, source_node, ^capability}} <- decode(payload), + true <- is_atom(source_node) do + send(manager, {:reader_ready, source_node, self()}) + reader_loop(socket, group, source_node) + else + _ -> :gen_tcp.close(socket) + end + end + + defp reader_loop(socket, group, source_node) do + case :gen_tcp.recv(socket, 0) do + {:ok, payload} -> + case decode_authenticated_frame(payload) do + {:ok, {shard, frame}} when is_integer(shard) and shard >= 0 -> + :ok = Group.Replica.Transport.deliver(group, source_node, shard, frame) + reader_loop(socket, group, source_node) + + _ -> + :gen_tcp.close(socket) + end + + {:error, _reason} -> + :ok + end + end + + defp decode(payload) do + {:ok, :erlang.binary_to_term(payload, [:safe])} + rescue + ArgumentError -> :error + end + + # The capability handshake above establishes the same trusted-cluster + # boundary as Erlang distribution. Replica metadata is an arbitrary BEAM + # term and may legitimately contain atoms not yet loaded on this node. + defp decode_authenticated_frame(payload) do + {:ok, :erlang.binary_to_term(payload)} + rescue + ArgumentError -> :error + end + + defp server_name(group), do: :"#{group}_replica_tcp_transport" + defp route_table(group), do: :"#{group}_replica_tcp_routes" +end diff --git a/mix.exs b/mix.exs index c618678..44fb164 100644 --- a/mix.exs +++ b/mix.exs @@ -34,7 +34,8 @@ defmodule Group.MixProject do defp deps do [ - {:ex_doc, "~> 0.30", only: :dev, runtime: false} + {:ex_doc, "~> 0.30", only: :dev, runtime: false}, + {:stream_data, "~> 1.4", only: :test} ] end diff --git a/mix.lock b/mix.lock index 0c19687..90b786f 100644 --- a/mix.lock +++ b/mix.lock @@ -5,4 +5,5 @@ "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, } diff --git a/test/README.md b/test/README.md index 6dc3dac..65c12aa 100644 --- a/test/README.md +++ b/test/README.md @@ -7,6 +7,7 @@ mix test # all tests mix test test/group_test.exs # local only mix test test/distributed_test.exs # distributed only mix test test/replica_adversarial_test.exs # seeded transport chaos +mix test test/replica_model_property_test.exs # shrinkable model-based histories ``` ## Test files @@ -16,6 +17,32 @@ mix test test/replica_adversarial_test.exs # seeded transport chaos | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | | `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | | `replica_adversarial_test.exs` | Reproducible mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | +| `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | + +## Model-based and formal checks + +`replica_model_property_test.exs` runs real Group instances on three peer VMs. +The controlled transport queues each replica frame so generated commands can +deliver, duplicate, drop, reorder, or strand it. After the bounded-fault +prefix, the test enables fair delivery and compares every tracked registry and +PG key against an independent application-level lifecycle oracle. It also +requires internal replica indexes to be consistent, every retained owner to be +alive, and registry conflict losers to be dead. Restart, pruning, and named +cluster histories retain independent C-owned state while A recovers, so repair +cannot pass merely by making one origin and one receiver agree. + +StreamData reports the ExUnit seed and shrinks a failure to its smallest command +history. Local defaults are intentionally quick. Increase the budgets without +changing the generator: + +```bash +GROUP_MODEL_RUNS=1000 GROUP_MODEL_COMMANDS=100 \ + mix test test/replica_model_property_test.exs +``` + +The independent TLA+ model and TLC configuration live in `test/formal/`. +See [`formal/README.md`](formal/README.md) for its checked invariants, finite +model bounds, and run command. ## How distribution works @@ -203,6 +230,12 @@ can return `:busy`, drop selected frame types, duplicate or delay frames, and capture frames for explicit stale-generation/epoch replay. Its `{:chaos, opts}` mode is deterministic for a given frame, which makes failures reproducible. +`Group.ControlledReplicaTransport` is the model-test transport. It queues frames +at the test process without scheduling timers; `Group.ReplicaModelScheduler` +then owns the exact delivery schedule. These roles are separate so the existing +timing-oriented regressions retain their original mechanics while property +failures can be replayed and shrunk exactly. + The distributed anti-entropy tests cover dropped creates and deletes, cursor gaps, globally pruned multi-stream oplogs, exact snapshot fallback, malformed authority, stale frame replay, lease expiry on a live VM, and multi-shard @@ -217,13 +250,22 @@ deliver data before authority to prove rejection does not advance the cursor and the same frame applies after authority repair. Concurrent snapshot tests require every advertised revision to contain exactly that many unique named epochs, and heartbeat tests prove observed revisions cannot advance the exact -authority marker. +authority marker. Crash-window tests interrupt journal, dual-index, receive +cursor, and named-cluster close updates, then require startup repair to remove +every invisible row and temporary close barrier. A three-node sideband TCP test +disconnects one origin's real socket, prunes its oplog, reconnects it, and +requires snapshot recovery without changing the third node's independent +registry or PG state. `Group.TestCluster.assert_replica_consistent/1` checks the public dual indexes plus registry claim authority, oplog/order equivalence, and contiguous retained stream ranges. Seeded tests additionally require every PID retained as authority to still be alive after convergence. +The isolated mutation runner in `test/mutation/` disables individual protocol +guards and repair steps only in copied checkouts. See +[`mutation/README.md`](mutation/README.md) for the command and artifact format. + ## Typical test patterns ### Basic replication test diff --git a/test/distributed_test.exs b/test/distributed_test.exs index bf34406..e43d1c4 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -5208,6 +5208,14 @@ defmodule Group.DistributedTest do assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + assert :ok = + TestCluster.rpc!( + node_b, + Group.TestCluster, + :assert_replica_origin_purged, + [name, node_a] + ) + {:ok, _pid} = TestCluster.start_group(node_a, opts) TestCluster.assert_eventually(fn -> @@ -5315,6 +5323,493 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) end end + + @tag timeout: 60_000 + test "mixed-generation data is rejected even when its epoch matches current authority" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_generation_guard_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + key = "anti-entropy/mixed-generation" + meta = %{forged: true} + pid = TestCluster.spawn_register(node_a, name, key, meta) + TestCluster.flush_shards(node_a, name) + + current_generation = + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + stream_id = + Group.Replica.Protocol.stream_id( + name, + node_a, + make_ref(), + 0, + nil, + current_generation + ) + + mutation = {:register, nil, key, pid, meta, System.monotonic_time(), node_a} + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 0, + {:delta_batch, Group.Replica.Protocol.version(), [{stream_id, 1, [{1, [mutation]}], 1}]} + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [name, 0, stream_id]) == + 0 + end + + @tag timeout: 60_000 + test "an out-of-order delta cannot advance the cursor across a missing sequence" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_gap_guard_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + first_key = "anti-entropy/gap/first" + second_key = "anti-entropy/gap/second" + group_key = "anti-entropy/gap/group" + first_pid = TestCluster.spawn_register(node_a, name, first_key, %{seq: 1}) + second_pid = TestCluster.spawn_register(node_a, name, second_key, %{seq: 2}) + member_pid = TestCluster.spawn_join(node_a, name, group_key, %{seq: 3}) + TestCluster.flush_shards(node_a, name) + + captured = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + frames_by_first_seq = + Map.new(captured, fn + {_target, shard, + {:delta_batch, _version, [{_stream_id, first_seq, _records, _head}]} = frame} -> + {first_seq, {shard, frame}} + end) + + {shard, {:delta_batch, _version, [{stream_id, 2, _records, _head}]} = second_frame} = + Map.fetch!(frames_by_first_seq, 2) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + second_frame + ]) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + shard, + stream_id + ]) == 0 + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) == nil + + {^shard, first_frame} = Map.fetch!(frames_by_first_seq, 1) + {^shard, third_frame} = Map.fetch!(frames_by_first_seq, 3) + + for frame <- [first_frame, second_frame, third_frame] do + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + frame + ]) + end + + TestCluster.flush_shards(node_b, name) + + assert match?( + {^first_pid, %{seq: 1}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, first_key]) + ) + + assert match?( + {^second_pid, %{seq: 2}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) + ) + + assert match?( + [{^member_pid, %{seq: 3}}], + TestCluster.rpc!(node_b, Group, :members, [name, group_key]) + ) + + # Model a receiver crash after all materialized ETS writes but before + # its cursor write. Replaying the accepted prefix must be exactly + # idempotent for both registry claims and PG membership. + :ok = + TestCluster.rpc!(node_b, Group.Replica.Data, :put_replica_cursor, [ + name, + shard, + stream_id, + 0 + ]) + + for frame <- [first_frame, second_frame, third_frame] do + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + frame + ]) + end + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + shard, + stream_id + ]) == 3 + + assert match?( + {^first_pid, %{seq: 1}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, first_key]) + ) + + assert match?( + {^second_pid, %{seq: 2}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) + ) + + assert match?( + [{^member_pid, %{seq: 3}}], + TestCluster.rpc!(node_b, Group, :members, [name, group_key]) + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "a new full authority generation purges nonzero shards before any lane-down signal" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_fanout_#{System.unique_integer([:positive])}" + shards = 2 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + key = + Enum.find(Stream.iterate(0, &(&1 + 1)), fn suffix -> + :erlang.phash2({nil, "anti-entropy/authority-fanout/#{suffix}"}, shards) == 1 + end) + |> then(&"anti-entropy/authority-fanout/#{&1}") + + pid = TestCluster.spawn_register(node_a, name, key, %{generation: :old}) + + TestCluster.assert_eventually(fn -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + forwarder = TestCluster.spawn_monitor_forwarder(node_b, name, :all, self()) + assert_receive {:monitor_ready, ^forwarder}, 5_000 + + a_control = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + b_control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + b_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) + new_generation = make_ref() + + new_key = + Enum.find(Stream.iterate(0, &(&1 + 1)), fn suffix -> + :erlang.phash2({nil, "anti-entropy/authority-fanout/new/#{suffix}"}, shards) == 1 + end) + |> then(&"anti-entropy/authority-fanout/new/#{&1}") + + stream_id = + Group.Replica.Protocol.stream_id( + name, + node_a, + new_generation, + 1, + nil, + new_generation + ) + + new_mutation = + {:register, nil, new_key, pid, %{generation: :new}, System.system_time(), node_a} + + new_frame = + {:delta_batch, Group.Replica.Protocol.version(), + [{stream_id, 1, [{1, [new_mutation]}], 1}]} + + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_lane]) + + # Queue new-generation lane data before the control shard can enqueue + # its local authority marker. Shared authority will be current by the + # time this frame runs, but shard 1 must still reject it until its own + # old-generation purge has completed. + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 1, + new_frame + ]) + + send( + b_control, + {:replica_hello, a_control, Group.Replica.Protocol.version(), new_generation, 0, + [{nil, new_generation}], Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + new_generation + end) + + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_lane]) + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + assert TestCluster.rpc!(node_b, Group, :lookup, [name, new_key]) == nil + refute_receive {:got_event, %Group.Event{key: ^new_key}}, 100 + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) == 0 + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 1, + new_frame + ]) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, new_key]) == + {pid, %{generation: :new}} + + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "three nodes recover a pruned origin over sideband TCP without disturbing another origin" do + peers = TestCluster.start_peers(3) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + name = :"anti_entropy_sideband_tcp_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replica_transport: + {Group.Replica.Transport.TCP, + [ + max_queue: 16, + connect_timeout: 250, + send_timeout: 250, + reconnect_interval: 10 + ]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 2 + ] + + start_group_on_peers(peers, opts) + + nodes = [node_a, node_b, node_c] + + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn source -> + Enum.all?(nodes -- [source], fn target -> + TestCluster.rpc!( + source, + Group.Replica.Transport.TCP, + :connected?, + [name, target] + ) + end) + end) + end, + timeout: 10_000 + ) + + stale_key = "sideband/a/stale" + fresh_key = "sideband/a/fresh" + c_key = "sideband/c/independent" + c_group = "sideband/c/group" + + stale_pid = TestCluster.spawn_register(node_a, name, stale_key, %{origin: :a}) + + c_pid = + TestCluster.spawn_register_and_join( + node_c, + name, + c_key, + %{origin: :c}, + c_group, + %{origin: :c} + ) + + TestCluster.assert_eventually(fn -> + Enum.all?(nodes, fn receiver -> + match?( + {^stale_pid, %{origin: :a}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, stale_key]) + ) and + match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, c_key]) + ) and + match?( + [{^c_pid, %{origin: :c}}], + TestCluster.rpc!(receiver, Group, :members, [name, c_group]) + ) + end) + end) + + :ok = + TestCluster.rpc!(node_a, Group.Replica.Transport.TCP, :disconnect_peer, [name, node_b]) + + old_reader = + TestCluster.rpc!(node_b, Group.Replica.Transport.TCP, :status, [name]) + |> get_in([:inbound, node_a]) + + refute TestCluster.rpc!( + node_a, + Group.Replica.Transport.TCP, + :connected?, + [name, node_b] + ) + + true = TestCluster.rpc!(node_a, Process, :exit, [stale_pid, :kill]) + + for i <- 1..6 do + churn_key = "sideband/a/churn/#{i}" + churn_pid = TestCluster.spawn_register(node_a, name, churn_key, %{i: i}) + true = TestCluster.rpc!(node_a, Process, :exit, [churn_pid, :kill]) + end + + fresh_pid = TestCluster.spawn_register(node_a, name, fresh_key, %{origin: :a, fresh: true}) + TestCluster.flush_shards(node_a, name) + + assert match?( + {^stale_pid, %{origin: :a}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_key]) + ) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, fresh_key]) == nil + + assert match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, c_key]) + ) + + :ok = + TestCluster.rpc!(node_a, Group.Replica.Transport.TCP, :reconnect_peer, [name, node_b]) + + TestCluster.assert_eventually(fn -> + new_reader = + TestCluster.rpc!(node_b, Group.Replica.Transport.TCP, :status, [name]) + |> get_in([:inbound, node_a]) + + is_pid(new_reader) and new_reader != old_reader + end) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!( + node_a, + Group.Replica.Transport.TCP, + :connected?, + [name, node_b] + ) and + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_key]) == nil and + match?( + {^fresh_pid, %{origin: :a, fresh: true}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, fresh_key]) + ) and + match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, c_key]) + ) and + match?( + [{^c_pid, %{origin: :c}}], + TestCluster.rpc!(node_b, Group, :members, [name, c_group]) + ) + end, + timeout: 10_000 + ) + + for receiver <- nodes do + assert :ok = + TestCluster.rpc!( + receiver, + Group.TestCluster, + :assert_replica_consistent, + [name] + ) + end + end end # Helpers for event assertion tests diff --git a/test/formal/GroupAntiEntropy.cfg b/test/formal/GroupAntiEntropy.cfg new file mode 100644 index 0000000..566fae5 --- /dev/null +++ b/test/formal/GroupAntiEntropy.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec + +CONSTANTS + Nodes = {n1, n2, n3} + Origins = {n1} + Keys = {k1} + MaxSeq = 2 + OplogBound = 1 + MaxMessages = 1 + +INVARIANTS + TypeOK + BoundedJournal + CurrentReplicaIsAStreamPrefix + +PROPERTY + HealedConvergence diff --git a/test/formal/GroupAntiEntropy.tla b/test/formal/GroupAntiEntropy.tla new file mode 100644 index 0000000..96a3238 --- /dev/null +++ b/test/formal/GroupAntiEntropy.tla @@ -0,0 +1,481 @@ +------------------------- MODULE GroupAntiEntropy ------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +An abstract model of Group's per-origin anti-entropy stream. + +The model intentionally does not duplicate the Elixir data structures. It +models the protocol contract: exact generation/epoch authority, sequenced +deltas, bounded retained prefixes, exact snapshot fallback, arbitrary finite +loss/reordering/duplication, and fair repair after the network heals. +*) + +CONSTANTS Nodes, Origins, Keys, MaxSeq, OplogBound, MaxMessages + +ASSUME /\ IsFiniteSet(Nodes) + /\ Cardinality(Nodes) >= 2 + /\ Origins \subseteq Nodes + /\ Cardinality(Origins) >= 1 + /\ IsFiniteSet(Keys) + /\ Cardinality(Keys) >= 1 + /\ MaxSeq >= 1 + /\ OplogBound >= 1 + /\ MaxMessages >= 1 + +Seq == 1..MaxSeq +Generations == 0..2 +Revisions == 0..4 +Epochs == 0..4 +BoolMap == [Keys -> BOOLEAN] +EmptyView == [key \in Keys |-> FALSE] +EmptyRecord == [key |-> CHOOSE key \in Keys : TRUE, value |-> FALSE] + +HelloMessages == + [kind : {"hello"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + wireActive : BOOLEAN] + +DeltaMessages == + [kind : {"delta"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + seq : Seq, + key : Keys, + value : BOOLEAN] + +SnapshotMessages == + [kind : {"snapshot"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + seq : 0..MaxSeq, + state : BoolMap] + +Message == HelloMessages \union DeltaMessages \union SnapshotMessages + +VARIABLES phase, + generation, + revision, + epoch, + active, + truth, + head, + floor, + history, + authorityGeneration, + authorityRevision, + authorityEpoch, + authorityActive, + cursor, + replica, + messages + +vars == + <> + +RECURSIVE Replay(_, _) +Replay(records, n) == + IF n = 0 + THEN EmptyView + ELSE [Replay(records, n - 1) EXCEPT + ![records[n].key] = records[n].value] + +CurrentAuthority(receiver, origin) == + /\ authorityGeneration[receiver][origin] = generation[origin] + /\ authorityRevision[receiver][origin] = revision[origin] + /\ authorityEpoch[receiver][origin] = epoch[origin] + /\ authorityActive[receiver][origin] = active[origin] + +PairConverged(receiver, origin) == + IF receiver = origin + THEN TRUE + ELSE + /\ CurrentAuthority(receiver, origin) + /\ IF active[origin] + THEN /\ cursor[receiver][origin] = head[origin] + /\ replica[receiver][origin] = truth[origin] + ELSE /\ cursor[receiver][origin] = 0 + /\ replica[receiver][origin] = EmptyView + +Converged == + \A receiver \in Nodes : + \A origin \in Origins : + PairConverged(receiver, origin) + +Init == + /\ phase = "faulting" + /\ generation = [node \in Nodes |-> 1] + /\ revision = [node \in Nodes |-> 0] + /\ epoch = [node \in Nodes |-> 0] + /\ active = [node \in Nodes |-> FALSE] + /\ truth = [node \in Nodes |-> EmptyView] + /\ head = [node \in Nodes |-> 0] + /\ floor = [node \in Nodes |-> 1] + /\ history = [node \in Nodes |-> + [seq \in Seq |-> EmptyRecord]] + /\ authorityGeneration = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityRevision = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityEpoch = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityActive = + [receiver \in Nodes |-> [origin \in Nodes |-> FALSE]] + /\ cursor = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ replica = + [receiver \in Nodes |-> [origin \in Nodes |-> EmptyView]] + /\ messages = {} + +Open(origin) == + /\ phase = "faulting" + /\ ~active[origin] + /\ revision[origin] < 4 + /\ epoch[origin] < 4 + /\ active' = [active EXCEPT ![origin] = TRUE] + /\ revision' = [revision EXCEPT ![origin] = @ + 1] + /\ epoch' = [epoch EXCEPT ![origin] = @ + 1] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Close(origin) == + /\ phase = "faulting" + /\ active[origin] + /\ revision[origin] < 4 + /\ active' = [active EXCEPT ![origin] = FALSE] + /\ revision' = [revision EXCEPT ![origin] = @ + 1] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Restart(origin) == + /\ phase = "faulting" + /\ generation[origin] < 2 + /\ generation' = [generation EXCEPT ![origin] = @ + 1] + /\ revision' = [revision EXCEPT ![origin] = 0] + /\ epoch' = [epoch EXCEPT ![origin] = 0] + /\ active' = [active EXCEPT ![origin] = FALSE] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Mutate(origin, key, value) == + /\ phase = "faulting" + /\ active[origin] + /\ head[origin] < MaxSeq + /\ LET next == head[origin] + 1 + nextFloor == IF next - floor[origin] + 1 > OplogBound + THEN floor[origin] + 1 + ELSE floor[origin] + IN /\ history' = + [history EXCEPT + ![origin][next] = [key |-> key, value |-> value]] + /\ truth' = [truth EXCEPT ![origin][key] = value] + /\ head' = [head EXCEPT ![origin] = next] + /\ floor' = [floor EXCEPT ![origin] = nextFloor] + /\ UNCHANGED <> + +SendHello(origin, receiver) == + /\ origin # receiver + /\ Cardinality(messages) < MaxMessages + /\ messages' = + messages \union + {[kind |-> "hello", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + wireActive |-> active[origin]]} + /\ UNCHANGED <> + +SendDelta(origin, receiver, seq) == + /\ origin # receiver + /\ active[origin] + /\ seq \in floor[origin]..head[origin] + /\ Cardinality(messages) < MaxMessages + /\ LET record == history[origin][seq] + IN messages' = + messages \union + {[kind |-> "delta", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + seq |-> seq, + key |-> record.key, + value |-> record.value]} + /\ UNCHANGED <> + +SendSnapshot(origin, receiver) == + /\ origin # receiver + /\ active[origin] + /\ Cardinality(messages) < MaxMessages + /\ messages' = + messages \union + {[kind |-> "snapshot", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + seq |-> head[origin], + state |-> truth[origin]]} + /\ UNCHANGED <> + +FreshHello(message) == + \/ message.wireGeneration > authorityGeneration[message.to][message.from] + \/ /\ message.wireGeneration = + authorityGeneration[message.to][message.from] + /\ message.wireRevision >= authorityRevision[message.to][message.from] + +DeliverHello(message) == + /\ message.kind = "hello" + /\ LET changed == + \/ message.wireGeneration # + authorityGeneration[message.to][message.from] + \/ message.wireRevision # + authorityRevision[message.to][message.from] + \/ message.wireEpoch # + authorityEpoch[message.to][message.from] + \/ message.wireActive # + authorityActive[message.to][message.from] + install == FreshHello(message) + IN /\ authorityGeneration' = + IF install + THEN [authorityGeneration EXCEPT + ![message.to][message.from] = message.wireGeneration] + ELSE authorityGeneration + /\ authorityRevision' = + IF install + THEN [authorityRevision EXCEPT + ![message.to][message.from] = message.wireRevision] + ELSE authorityRevision + /\ authorityEpoch' = + IF install + THEN [authorityEpoch EXCEPT + ![message.to][message.from] = message.wireEpoch] + ELSE authorityEpoch + /\ authorityActive' = + IF install + THEN [authorityActive EXCEPT + ![message.to][message.from] = message.wireActive] + ELSE authorityActive + /\ cursor' = + IF install /\ (changed \/ ~message.wireActive) + THEN [cursor EXCEPT ![message.to][message.from] = 0] + ELSE cursor + /\ replica' = + IF install /\ (changed \/ ~message.wireActive) + THEN [replica EXCEPT + ![message.to][message.from] = EmptyView] + ELSE replica + /\ UNCHANGED <> + +ValidData(message) == + /\ authorityGeneration[message.to][message.from] = message.wireGeneration + /\ authorityRevision[message.to][message.from] = message.wireRevision + /\ authorityEpoch[message.to][message.from] = message.wireEpoch + /\ authorityActive[message.to][message.from] + +DeliverDelta(message) == + /\ message.kind = "delta" + /\ IF ValidData(message) /\ + message.seq = cursor[message.to][message.from] + 1 + THEN /\ cursor' = + [cursor EXCEPT + ![message.to][message.from] = message.seq] + /\ replica' = + [replica EXCEPT + ![message.to][message.from][message.key] = message.value] + ELSE /\ UNCHANGED cursor + /\ UNCHANGED replica + /\ UNCHANGED <> + +DeliverSnapshot(message) == + /\ message.kind = "snapshot" + /\ IF ValidData(message) /\ + message.seq >= cursor[message.to][message.from] + THEN /\ cursor' = + [cursor EXCEPT + ![message.to][message.from] = message.seq] + /\ replica' = + [replica EXCEPT + ![message.to][message.from] = message.state] + ELSE /\ UNCHANGED cursor + /\ UNCHANGED replica + /\ UNCHANGED <> + +Deliver(message) == + /\ message \in messages + /\ \/ DeliverHello(message) + \/ DeliverDelta(message) + \/ DeliverSnapshot(message) + +Drop(message) == + /\ phase = "faulting" + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED <> + +Heal == + /\ phase = "faulting" + /\ phase' = "healed" + /\ UNCHANGED <> + +(* +After healing, Repair represents one fair successful hello/head/need response. +If the retained prefix no longer contains the next sequence, it performs an +exact snapshot replacement. Otherwise it applies exactly the next delta. +*) +Repair(receiver, origin) == + /\ phase = "healed" + /\ receiver # origin + /\ ~PairConverged(receiver, origin) + /\ IF ~CurrentAuthority(receiver, origin) + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT + ![receiver][origin] = generation[origin]] + /\ authorityRevision' = + [authorityRevision EXCEPT + ![receiver][origin] = revision[origin]] + /\ authorityEpoch' = + [authorityEpoch EXCEPT + ![receiver][origin] = epoch[origin]] + /\ authorityActive' = + [authorityActive EXCEPT + ![receiver][origin] = active[origin]] + /\ cursor' = [cursor EXCEPT ![receiver][origin] = 0] + /\ replica' = + [replica EXCEPT ![receiver][origin] = EmptyView] + ELSE IF ~active[origin] + THEN /\ UNCHANGED <> + /\ cursor' = [cursor EXCEPT ![receiver][origin] = 0] + /\ replica' = + [replica EXCEPT ![receiver][origin] = EmptyView] + ELSE IF cursor[receiver][origin] + 1 < floor[origin] + THEN /\ UNCHANGED <> + /\ cursor' = + [cursor EXCEPT ![receiver][origin] = head[origin]] + /\ replica' = + [replica EXCEPT ![receiver][origin] = truth[origin]] + ELSE LET next == cursor[receiver][origin] + 1 + record == history[origin][next] + IN /\ UNCHANGED <> + /\ cursor' = [cursor EXCEPT ![receiver][origin] = next] + /\ replica' = + [replica EXCEPT + ![receiver][origin][record.key] = record.value] + /\ UNCHANGED <> + +Next == + \/ \E origin \in Origins : Open(origin) + \/ \E origin \in Origins : Close(origin) + \/ \E origin \in Origins : Restart(origin) + \/ \E origin \in Origins, key \in Keys, value \in BOOLEAN : + Mutate(origin, key, value) + \/ \E origin \in Origins, receiver \in Nodes : + SendHello(origin, receiver) + \/ \E origin \in Origins, receiver \in Nodes, seq \in Seq : + SendDelta(origin, receiver, seq) + \/ \E origin \in Origins, receiver \in Nodes : + SendSnapshot(origin, receiver) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ Heal + \/ \E receiver \in Nodes, origin \in Origins : + Repair(receiver, origin) + +TypeOK == + /\ phase \in {"faulting", "healed"} + /\ generation \in [Nodes -> Generations] + /\ revision \in [Nodes -> Revisions] + /\ epoch \in [Nodes -> Epochs] + /\ active \in [Nodes -> BOOLEAN] + /\ truth \in [Nodes -> BoolMap] + /\ head \in [Nodes -> 0..MaxSeq] + /\ floor \in [Nodes -> 1..(MaxSeq + 1)] + /\ history \in [Nodes -> [Seq -> [key : Keys, value : BOOLEAN]]] + /\ authorityGeneration \in [Nodes -> [Nodes -> Generations]] + /\ authorityRevision \in [Nodes -> [Nodes -> Revisions]] + /\ authorityEpoch \in [Nodes -> [Nodes -> Epochs]] + /\ authorityActive \in [Nodes -> [Nodes -> BOOLEAN]] + /\ cursor \in [Nodes -> [Nodes -> 0..MaxSeq]] + /\ replica \in [Nodes -> [Nodes -> BoolMap]] + /\ messages \subseteq Message + +BoundedJournal == + \A origin \in Origins : + /\ floor[origin] <= head[origin] + 1 + /\ head[origin] - floor[origin] + 1 <= OplogBound + +CurrentReplicaIsAStreamPrefix == + \A receiver \in Nodes : + \A origin \in Origins : + IF receiver # origin /\ + CurrentAuthority(receiver, origin) /\ + active[origin] + THEN /\ cursor[receiver][origin] <= head[origin] + /\ replica[receiver][origin] = + Replay(history[origin], cursor[receiver][origin]) + ELSE TRUE + +HealedConvergence == + phase = "healed" ~> Converged + +Spec == + /\ Init + /\ [][Next]_vars + /\ \A receiver \in Nodes : + \A origin \in Origins : + WF_vars(Repair(receiver, origin)) + +============================================================================= diff --git a/test/formal/README.md b/test/formal/README.md new file mode 100644 index 0000000..05c2c10 --- /dev/null +++ b/test/formal/README.md @@ -0,0 +1,44 @@ +# Group anti-entropy formal model + +`GroupAntiEntropy.tla` is an independent finite-state model of the replica +contract. It covers: + +- generation and named-cluster epoch fencing; +- arbitrary finite frame loss, duplication, and reordering; +- contiguous sequence application; +- bounded oplog pruning; +- exact per-origin snapshot fallback; and +- fair convergence after healing. + +The default TLC configuration uses three nodes: one origin and two independent +receivers. The origin has one key, a two-record stream, a one-record oplog, and +the system retains one arbitrary network frame. This forces delta repair, +snapshot fallback, stale-frame fencing, and independent recovery at both +receivers. The retained frame may be redelivered for duplication, while +nondeterministic sequence selection and delivery model out-of-order arrival +without paying the state-space cost of every two-frame set. + +The TLA+ protocol state is deliberately factored per origin: no transition for +one origin reads or writes another origin's stream. Checking multiple origins +in this model therefore forms a Cartesian product of the same state machine +rather than adding an interaction. Concurrent A/C authority, registry conflict +projection, and preservation of C-owned state while A recovers are instead +driven against three real BEAM nodes by `replica_model_property_test.exs`. + +Run it with Java 17 or later and a current `tla2tools.jar`: + +```bash +TLA_JAR=/path/to/tla2tools.jar test/formal/check.sh +``` + +`TLC_WORKERS` controls worker concurrency and defaults to 4. `TLA_CONFIG` can +point at an alternate finite configuration. + +TLC proves the listed invariants and liveness property for the configured +finite instance, not for arbitrary unbounded node and key sets. Larger models +should be run periodically by increasing `Nodes`, `Origins`, `Keys`, `MaxSeq`, +`OplogBound`, and `MaxMessages`. + +The checked three-node default explores 1,835,826 states, finds 490,236 +distinct states to a depth of 30, and completes in roughly 1 minute 40 seconds +on the development machine used for the validation run. diff --git a/test/formal/check.sh b/test/formal/check.sh new file mode 100755 index 0000000..1336a08 --- /dev/null +++ b/test/formal/check.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "${TLA_JAR:-}" ]]; then + echo "TLA_JAR must point to tla2tools.jar" >&2 + exit 2 +fi + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +metadir="${repo_root}/tmp/tlc" +config="${TLA_CONFIG:-${repo_root}/test/formal/GroupAntiEntropy.cfg}" +mkdir -p "${metadir}" + +exec java -XX:+UseParallelGC -cp "${TLA_JAR}" tlc2.TLC \ + -cleanup \ + -metadir "${metadir}" \ + -workers "${TLC_WORKERS:-4}" \ + -config "${config}" \ + "${repo_root}/test/formal/GroupAntiEntropy.tla" diff --git a/test/group_test.exs b/test/group_test.exs index 7de75ae..572eb46 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -1344,6 +1344,9 @@ defmodule GroupTest do assert :ok = Group.join(name, key, %{}, cluster: cluster) assert Group.members(name, key, cluster: cluster) == [{self(), %{}}] + remote_route = :"disconnect-timeout@remote" + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) + shard_zero = Group.Replica.shard_name(name, 0) :ok = :sys.suspend(shard_zero) @@ -1358,6 +1361,48 @@ defmodule GroupTest do :sys.get_state(shard_zero) refute Group.connected?(name, cluster) assert Group.members(name, key, cluster: cluster) == [] + + wait_until(fn -> + Group.Replica.Data.closed_local_clusters(name) == [] and + Group.Replica.Data.cluster_nodes(name, cluster) == [] + end) + end + + test "reconnect waits for every old-epoch shard cleanup before admitting new writes" do + name = :"test_reconnect_barrier_#{System.unique_integer([:positive])}" + cluster = "reconnect_barrier" + key = "reconnect/barrier/#{System.unique_integer([:positive])}" + start_supervised!({Group, name: name, shards: 2, log: false}) + + assert :ok = Group.connect(name, cluster) + shard_zero = Group.Replica.shard_name(name, 0) + :ok = :sys.suspend(shard_zero) + + reconnect_caller = + try do + assert_genserver_call_timeout(fn -> + Group.disconnect(name, cluster, timeout: 10) + end) + + caller = + spawn_requester( + fn -> Group.connect(name, cluster) end, + :reconnect_barrier_result + ) + + refute_receive {:reconnect_barrier_result, ^caller, _result}, 50 + caller + after + resume_shard_if_alive(shard_zero) + end + + on_exit(fn -> kill_if_alive(reconnect_caller) end) + assert_receive {:reconnect_barrier_result, ^reconnect_caller, :ok}, 1_000 + + assert Group.Replica.Data.closed_local_clusters(name) == [] + assert :ok = Group.join(name, key, %{epoch: :new}, cluster: cluster) + assert Group.members(name, key, cluster: cluster) == [{self(), %{epoch: :new}}] + assert :ok = Group.TestCluster.assert_replica_consistent(name) end end @@ -2996,6 +3041,146 @@ defmodule GroupTest do Group.lookup(name, key) == nil and Group.members(name, key) == [] end) end + + test "a shard restart repairs an interrupted append tail and interrupted prune floor" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + first_key = "journal/crash-window/first/#{System.unique_integer([:positive])}" + second_key = "journal/crash-window/second/#{System.unique_integer([:positive])}" + third_key = "journal/crash-window/third/#{System.unique_integer([:positive])}" + + :ok = Group.register(name, first_key, %{seq: 1}) + :ok = Group.register(name, second_key, %{seq: 2}) + + oplog = Group.Replica.Data.replica_oplog_table(name, 0) + order = Group.Replica.Data.replica_oplog_order_table(name, 0) + stream_meta = Group.Replica.Data.replica_stream_meta_table(name, 0) + + [{{^stream_id, 1}, first_append_id, _mutations}] = :ets.lookup(oplog, {stream_id, 1}) + + # Pruning removes order -> record -> advances floor. Model a kill after + # the first two writes but before the floor update. + :ets.delete(order, first_append_id) + :ets.delete(oplog, {stream_id, 1}) + + # Appending advances head -> append counter -> record -> order. Model a + # kill after the head update but before the record exists. + assert 3 = :ets.update_counter(stream_meta, stream_id, {2, 1}) + + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + is_pid(new_shard) and new_shard != old_shard + end) + + :sys.get_state(Group.Replica.shard_name(name, 0)) + assert {2, 2, 2} = Group.Replica.Data.replica_stream_head(name, 0, stream_id) + assert :ok = Group.TestCluster.assert_replica_consistent(name) + + :ok = Group.register(name, third_key, %{seq: 3}) + assert {2, 3, 3} = Group.Replica.Data.replica_stream_head(name, 0, stream_id) + + assert Group.lookup(name, first_key) == {self(), %{seq: 1}} + assert Group.lookup(name, second_key) == {self(), %{seq: 2}} + assert Group.lookup(name, third_key) == {self(), %{seq: 3}} + assert :ok = Group.TestCluster.assert_replica_consistent(name) + end + + test "a shard restart rebuilds every one-sided materialized index" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + reg_key = "indexes/crash-window/reg/#{System.unique_integer([:positive])}" + pg_key = "indexes/crash-window/pg/#{System.unique_integer([:positive])}" + + :ok = Group.register(name, reg_key, %{kind: :registry}) + :ok = Group.join(name, pg_key, %{kind: :pg}) + + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + + :ets.delete(Group.Replica.Data.reg_by_pid_table(name, 0), {self(), nil, reg_key}) + :ets.delete(Group.Replica.Data.pg_by_pid_table(name, 0), {self(), nil, pg_key}) + + :ets.delete( + Group.Replica.Data.reg_claim_by_pid_table(name, 0), + {self(), nil, reg_key, node(), generation, epoch} + ) + + orphan = spawn(fn -> Process.sleep(:infinity) end) + on_exit(fn -> kill_if_alive(orphan) end) + + :ets.insert( + Group.Replica.Data.reg_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/reg"}, %{}, 0, node()} + ) + + :ets.insert( + Group.Replica.Data.pg_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/pg"}, %{}, 0, node()} + ) + + :ets.insert( + Group.Replica.Data.reg_claim_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/claim", node(), generation, epoch}, %{}, 0, 1} + ) + + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + is_pid(new_shard) and new_shard != old_shard + end) + + :sys.get_state(Group.Replica.shard_name(name, 0)) + assert Group.lookup(name, reg_key) == {self(), %{kind: :registry}} + assert Group.members(name, pg_key) == [{self(), %{kind: :pg}}] + assert Group.Replica.Data.registry_lookup_by_pid(name, 0, orphan) == [] + assert Group.Replica.Data.entries_by_pid(name, 0, orphan) == [] + assert :ok = Group.TestCluster.assert_replica_consistent(name) + end + + test "a shard restart completes an interrupted named-cluster close without retained rows" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + cluster = "close/crash-window/#{System.unique_integer([:positive])}" + reg_key = "close/crash-window/reg/#{System.unique_integer([:positive])}" + pg_key = "close/crash-window/pg/#{System.unique_integer([:positive])}" + remote_route = :"close-crash-window@remote" + + :ok = Group.connect(name, cluster) + :ok = Group.register(name, reg_key, %{kind: :registry}, cluster: cluster) + :ok = Group.join(name, pg_key, %{kind: :pg}, cluster: cluster) + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) + + stream_id = Group.Replica.Data.local_stream_id(name, 0, cluster) + old_epoch = Group.Replica.Protocol.stream_epoch(stream_id) + + # Group.disconnect/3 closes authority and routing before its request + # reaches every shard. Model a shard kill in that exact window. + assert [{^cluster, ^old_epoch}] = + Group.Replica.Data.deactivate_local_clusters(name, [cluster]) + + :ok = Group.Replica.Data.remove_cluster_node(name, [cluster], node()) + assert Group.Replica.Data.closed_local_clusters(name) == [cluster] + + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + + is_pid(new_shard) and new_shard != old_shard and + Group.lookup(name, reg_key, cluster: cluster) == nil and + Group.members(name, pg_key, cluster: cluster) == [] and + Group.Replica.Data.closed_local_clusters(name) == [] and + Group.Replica.Data.cluster_nodes(name, cluster) == [] + end) + + assert :ets.lookup(Group.Replica.Data.replica_stream_meta_table(name, 0), stream_id) == [] + assert :ok = Group.TestCluster.assert_replica_consistent(name) + end end describe "replica authority snapshots" do diff --git a/test/mutation/README.md b/test/mutation/README.md new file mode 100644 index 0000000..269edf8 --- /dev/null +++ b/test/mutation/README.md @@ -0,0 +1,24 @@ +# Replica mutation campaign + +This campaign calibrates the anti-entropy tests against deliberate failures of +catastrophic protocol obligations. It covers generation and epoch fencing, +contiguous sequence application, exact registry and PG snapshots, below-floor +repair, process-down sequencing, conflict-loser retirement, authority fanout, +per-lane authority installation, periodic head advertisement, interrupted +journal/index repair, and named-cluster close completion. + +The runner first verifies every unmodified regression target. It then copies +the current checkout once per mutant, changes only that copy, recompiles it, +and runs the designated real multi-node regression. A compiling mutant is +`killed` only when the regression fails. Any surviving or non-compiling mutant +makes the campaign fail. + +```bash +mix run --no-start test/mutation/run.exs + +# List or run selected mutations +mix run --no-start test/mutation/run.exs --list +mix run --no-start test/mutation/run.exs disable_below_floor_snapshot +``` + +Artifacts and complete logs are written below `tmp/mutation/`. diff --git a/test/mutation/run.exs b/test/mutation/run.exs new file mode 100644 index 0000000..60ecf6f --- /dev/null +++ b/test/mutation/run.exs @@ -0,0 +1,422 @@ +defmodule Group.MutationCampaign do + @moduledoc """ + Runs protocol mutations in isolated repository copies. + + A mutant is useful only when it compiles and its designated regression test + fails. The source checkout is never edited. + """ + + @repo Path.expand("../..", __DIR__) + @timeout_seconds "120" + + # Each entry replaces one correct production fragment with an intentionally + # faulty fragment, but only inside an isolated campaign checkout. + @mutations [ + %{ + name: "accept_old_generation", + file: "lib/group/replica.ex", + correct_source: + "Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and", + faulty_source: "true and", + test: ["test/distributed_test.exs:5328"] + }, + %{ + name: "accept_old_epoch", + file: "lib/group/replica.ex", + correct_source: """ + Protocol.stream_epoch(stream_id) == + Data.remote_cluster_epoch(state.name, source_node, cluster) and + """, + faulty_source: """ + true and + """, + test: ["test/distributed_test.exs:4774"] + }, + %{ + name: "advance_cursor_across_gap", + file: "lib/group/replica.ex", + correct_source: """ + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + request_replica_need(state, source_node, stream_id, cursor + 1) + """, + faulty_source: """ + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + :ok = + Data.put_replica_cursor( + state.name, + state.shard_index, + stream_id, + first_seq - 1 + ) + + apply_replica_delta_run( + state, + source_node, + stream_id, + records, + advertised_head + ) + """, + test: ["test/distributed_test.exs:5386"] + }, + %{ + name: "registry_snapshot_is_additive", + file: "lib/group/replica/data.ex", + correct_source: "existing = registry_claims_for_stream(name, shard, stream_id)", + faulty_source: "existing = []", + test: ["test/distributed_test.exs:4030"] + }, + %{ + name: "pg_snapshot_is_additive", + file: "lib/group/replica.ex", + correct_source: """ + current = + state.name + |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) + |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + """, + faulty_source: """ + current = %{} + """, + test: ["test/distributed_test.exs:4030"] + }, + %{ + name: "disable_below_floor_snapshot", + file: "lib/group/replica.ex", + correct_source: """ + true -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + """, + faulty_source: """ + true -> + {:state, state} + """, + test: ["test/replica_model_property_test.exs:180"] + }, + %{ + name: "do_not_sequence_process_down", + file: "lib/group/replica.ex", + correct_source: """ + sequenced_downs = + append_process_down_records(state, reason_by_pid, pending_reg, pending_pg) + """, + faulty_source: + " sequenced_downs =\n if false,\n do: append_process_down_records(state, reason_by_pid, pending_reg, pending_pg),\n else: []\n", + test: ["test/distributed_test.exs:3940"] + }, + %{ + name: "do_not_exit_conflict_loser", + file: "lib/group/replica.ex", + correct_source: """ + winner_meta = if winner, do: elem(winner, 1), else: nil + exit_local_conflict_loser(pid, key, winner_meta) + acc + """, + faulty_source: """ + _winner_meta = if winner, do: elem(winner, 1), else: nil + _ = Process.alive?(pid) + acc + """, + test: ["test/replica_model_property_test.exs:77"] + }, + %{ + name: "heartbeat_promotes_observed_authority", + file: "lib/group/replica.ex", + correct_source: + " replica_view_current?(state, remote_node) ->\n" <> + " state\n" <> + " |> put_remote_shard(remote_node, remote_pid)\n" <> + " |> touch_replica_peer(remote_node)", + faulty_source: + " replica_view_current?(state, remote_node) ->\n" <> + " :ok =\n" <> + " Data.put_remote_view_info(\n" <> + " state.name,\n" <> + " state.shard_index,\n" <> + " remote_node,\n" <> + " generation,\n" <> + " epoch_revision,\n" <> + " epoch_revision\n" <> + " )\n\n" <> + " state\n" <> + " |> put_remote_shard(remote_node, remote_pid)\n" <> + " |> touch_replica_peer(remote_node)", + test: ["test/distributed_test.exs:4247"] + }, + %{ + name: "skip_authority_fanout", + file: "lib/group/replica.ex", + correct_source: """ + fan_out_to_siblings( + state, + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs} + ) + """, + faulty_source: """ + :ok + """, + test: ["test/distributed_test.exs:5531"] + }, + %{ + name: "skip_generation_purge", + file: "lib/group/replica.ex", + correct_source: """ + defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do + {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) + + affected = + Data.purge_registry_claims_for_origin( + state.name, + state.shard_index, + remote_node + ) + + {state, events} = + Enum.reduce(affected, {state, []}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + + notify_monitors(state.name, events) + Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) + state + end + """, + faulty_source: """ + defp maybe_purge_remote_generation(state, _remote_node, _old_generation, _generation), + do: state + """, + test: ["test/distributed_test.exs:5531"] + }, + %{ + name: "disable_periodic_heads", + file: "lib/group/replica.ex", + correct_source: """ + defp broadcast_replica_heads(state) do + Enum.reduce(state.peer_last_seen, state, fn {target_node, _last_seen}, acc -> + send_replica_heads(acc, target_node) + end) + end + """, + faulty_source: """ + defp broadcast_replica_heads(state), do: state + """, + test: ["test/distributed_test.exs:3940"] + }, + %{ + name: "skip_journal_crash_repair", + file: "lib/group/replica.ex", + correct_source: ":ok = Data.repair_local_replica_journal(name, shard_index)", + faulty_source: ":ok", + test: ["test/group_test.exs:3045"] + }, + %{ + name: "skip_index_crash_repair", + file: "lib/group/replica.ex", + correct_source: ":ok = Data.repair_shard_indexes(name, shard_index)", + faulty_source: ":ok", + test: ["test/group_test.exs:3091"] + }, + %{ + name: "skip_inactive_cluster_repair", + file: "lib/group/replica/data.ex", + correct_source: """ + def repair_shard_indexes(name, shard) do + purge_inactive_cluster_rows(name, shard) + """, + faulty_source: """ + def repair_shard_indexes(name, shard) do + if false, do: purge_inactive_cluster_rows(name, shard) + """, + test: ["test/group_test.exs:3145"] + }, + %{ + name: "skip_closed_cluster_completion", + file: "lib/group/replica.ex", + correct_source: """ + completed_clusters = + Data.mark_closed_cluster_shard(name, Data.closed_local_clusters(name), shard_index) + """, + faulty_source: """ + completed_clusters = [] + """, + test: ["test/group_test.exs:3145"] + }, + %{ + name: "accept_shared_authority_before_lane_install", + file: "lib/group/replica.ex", + correct_source: """ + Protocol.stream_shard(stream_id) == state.shard_index and + replica_view_current?(state, source_node) and + """, + faulty_source: """ + Protocol.stream_shard(stream_id) == state.shard_index and + true and + """, + test: ["test/distributed_test.exs:5531"] + } + ] + + def run(args) do + selected = select_mutations(args) + campaign_dir = campaign_dir() + File.mkdir_p!(campaign_dir) + + IO.puts("mutation artifacts: #{campaign_dir}") + verify_baselines!(selected, campaign_dir) + + results = Enum.map(selected, &run_mutant(&1, campaign_dir)) + print_summary(results) + + if Enum.any?(results, fn {_name, status, _path} -> status != :killed end) do + System.halt(1) + end + end + + defp select_mutations(["--list"]) do + Enum.each(@mutations, &IO.puts(&1.name)) + System.halt(0) + end + + defp select_mutations([]), do: @mutations + + defp select_mutations(names) do + by_name = Map.new(@mutations, &{&1.name, &1}) + unknown = names -- Map.keys(by_name) + + if unknown != [] do + raise "unknown mutations: #{Enum.join(unknown, ", ")}" + end + + Enum.map(names, &Map.fetch!(by_name, &1)) + end + + defp verify_baselines!(mutations, campaign_dir) do + mutations + |> Enum.map(& &1.test) + |> Enum.uniq() + |> Enum.each(fn test -> + label = test |> hd() |> String.replace(~r/[^A-Za-z0-9_.-]/, "_") + log = Path.join(campaign_dir, "baseline-#{label}.log") + IO.write("baseline #{Enum.join(test, " ")} ... ") + {output, status} = run_test(@repo, test) + File.write!(log, output) + + if status == 0 do + IO.puts("pass") + else + IO.puts("FAIL") + raise "baseline failed; see #{log}" + end + end) + end + + defp run_mutant(mutation, campaign_dir) do + work = Path.join(campaign_dir, mutation.name) + File.mkdir_p!(work) + copy_checkout!(work) + + source_path = Path.join(work, mutation.file) + source = File.read!(source_path) + matches = :binary.matches(source, mutation.correct_source) + + if length(matches) != 1 do + log = Path.join(work, "mutation-error.log") + File.write!(log, "expected one match, found #{length(matches)}\n") + IO.puts("#{mutation.name}: INVALID (replacement matched #{length(matches)} times)") + {mutation.name, :invalid, work} + else + File.write!( + source_path, + String.replace(source, mutation.correct_source, mutation.faulty_source) + ) + + compile_log = Path.join(work, "compile.log") + {compile_output, compile_status} = run_mix(work, ["compile", "--warnings-as-errors"]) + File.write!(compile_log, compile_output) + + if compile_status != 0 do + IO.puts("#{mutation.name}: INVALID (does not compile)") + {mutation.name, :invalid, work} + else + test_log = Path.join(work, "test.log") + {test_output, test_status} = run_test(work, mutation.test) + File.write!(test_log, test_output) + + case test_status do + 0 -> + IO.puts("#{mutation.name}: SURVIVED") + {mutation.name, :survived, work} + + 124 -> + IO.puts("#{mutation.name}: killed (timeout)") + {mutation.name, :killed, work} + + _ -> + IO.puts("#{mutation.name}: killed") + {mutation.name, :killed, work} + end + end + end + end + + defp copy_checkout!(target) do + rsync = System.find_executable("rsync") || raise "rsync is required" + + {_output, 0} = + System.cmd( + rsync, + [ + "-a", + "--exclude=.git", + "--exclude=deps", + "--exclude=tmp", + "#{@repo}/", + "#{target}/" + ], + stderr_to_stdout: true + ) + + File.ln_s!(Path.join(@repo, "deps"), Path.join(target, "deps")) + end + + defp run_test(directory, test) do + run_with_timeout(directory, ["mix", "test" | test], + env: [{"GROUP_MODEL_RUNS", "1"}, {"GROUP_MODEL_COMMANDS", "8"}] + ) + end + + defp run_mix(directory, args) do + System.cmd("mix", args, + cd: directory, + env: [{"MIX_ENV", "test"}], + stderr_to_stdout: true + ) + end + + defp run_with_timeout(directory, command, opts) do + timeout = System.find_executable("timeout") || raise "timeout is required" + env = Keyword.fetch!(opts, :env) + + System.cmd(timeout, [@timeout_seconds | command], + cd: directory, + env: [{"MIX_ENV", "test"} | env], + stderr_to_stdout: true + ) + end + + defp campaign_dir do + stamp = Calendar.strftime(DateTime.utc_now(), "%Y%m%dT%H%M%S") + Path.join([@repo, "tmp", "mutation", "#{stamp}-#{System.unique_integer([:positive])}"]) + end + + defp print_summary(results) do + IO.puts("\nmutation summary") + + Enum.each(results, fn {name, status, path} -> + IO.puts(" #{String.pad_trailing(name, 38)} #{status} #{path}") + end) + end +end + +Group.MutationCampaign.run(System.argv()) diff --git a/test/replica_model_property_test.exs b/test/replica_model_property_test.exs new file mode 100644 index 0000000..4999c3a --- /dev/null +++ b/test/replica_model_property_test.exs @@ -0,0 +1,381 @@ +defmodule Group.ReplicaModelPropertyTest do + use ExUnit.Case, async: false + use ExUnitProperties + + alias Group.{ + ControlledReplicaTransport, + ModelConflictResolver, + ReplicaModelScheduler, + TestCluster + } + + @moduletag :capture_log + @moduletag timeout: 180_000 + + @model_runs System.get_env("GROUP_MODEL_RUNS", "12") |> String.to_integer() + @max_commands System.get_env("GROUP_MODEL_COMMANDS", "30") |> String.to_integer() + + setup_all do + peers = TestCluster.start_peers(3, schedulers: 4) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + {:ok, nodes: %{a: node_a, b: node_b, c: node_c}} + end + + property "accepted owner lifecycles converge without permanent orphans or zombies", %{ + nodes: nodes + } do + check all( + commands <- command_history(), + max_runs: @model_runs, + max_shrinking_steps: 100 + ) do + name = :"replica_model_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: self()}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_oplog_max_entries: 4 + ] + + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + commands + |> Enum.reduce(ReplicaModelScheduler.sync(scheduler), fn command, state -> + ReplicaModelScheduler.execute(state, command) + end) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "concurrent claims converge to one live winner and permanently retire the loser", %{ + nodes: nodes + } do + check all( + key_slot <- integer(0..2), + winner <- member_of([:a, :b]), + schedule <- list_of(network_command(), min_length: 0, max_length: 20), + max_runs: max(div(@model_runs, 2), 1), + max_shrinking_steps: 100 + ) do + name = :"replica_conflict_model_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: self()}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_oplog_max_entries: 2 + ] + + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + {rank_a, rank_b} = if winner == :a, do: {2, 1}, else: {1, 2} + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:register, 12, :c, key_slot + 10, 1}) + |> ReplicaModelScheduler.execute({:join, 12, :c, key_slot + 10, 1}) + |> ReplicaModelScheduler.execute({:claim, 10, :a, key_slot, rank_a}) + |> ReplicaModelScheduler.execute({:claim, 11, :b, key_slot, rank_b}) + + scheduler = + schedule + |> Enum.reduce(scheduler, fn command, state -> + ReplicaModelScheduler.execute(state, command) + end) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "old frames cannot survive a real Group restart and generation change", %{nodes: nodes} do + check all( + before_restart <- list_of(network_command(), max_length: 8), + after_restart <- list_of(network_command(), max_length: 12), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_restart_model_#{System.unique_integer([:positive])}" + opts = model_opts(name, self(), shards: 2, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:transport, :a, :pass}) + |> ReplicaModelScheduler.execute({:transport, :b, :pass}) + |> ReplicaModelScheduler.execute({:transport, :c, :pass}) + |> ReplicaModelScheduler.execute({:register, 30, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:join, 30, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:register, 32, :c, 1, 1}) + |> ReplicaModelScheduler.execute({:join, 32, :c, 1, 1}) + |> ReplicaModelScheduler.stabilize_and_assert!() + |> ReplicaModelScheduler.execute({:transport, :a, :capture}) + |> ReplicaModelScheduler.execute({:transport, :b, :capture}) + |> ReplicaModelScheduler.execute({:transport, :c, :capture}) + |> ReplicaModelScheduler.execute({:register, 30, :a, 0, 2}) + |> ReplicaModelScheduler.execute({:join, 30, :a, 0, 2}) + |> run_schedule(before_restart) + |> ReplicaModelScheduler.execute({:restart, :a}) + |> ReplicaModelScheduler.execute(:deliver_all) + |> ReplicaModelScheduler.execute({:register, 31, :a, 1, 2}) + |> ReplicaModelScheduler.execute({:join, 31, :a, 1, 2}) + |> run_schedule(after_restart) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "a receiver below the real oplog floor converges through an exact snapshot", %{ + nodes: nodes + } do + check all( + schedule <- list_of(network_command(), max_length: 10), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_pruning_model_#{System.unique_integer([:positive])}" + opts = model_opts(name, self(), shards: 1, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:transport, :a, :pass}) + |> ReplicaModelScheduler.execute({:transport, :b, :pass}) + |> ReplicaModelScheduler.execute({:transport, :c, :pass}) + |> ReplicaModelScheduler.execute({:register, 40, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:join, 40, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:register, 47, :c, 1, 1}) + |> ReplicaModelScheduler.execute({:join, 47, :c, 1, 1}) + |> ReplicaModelScheduler.stabilize_and_assert!() + |> ReplicaModelScheduler.execute({:transport, :a, :drop}) + |> ReplicaModelScheduler.execute({:kill, 40}) + + scheduler = + Enum.reduce(41..46, scheduler, fn owner_id, state -> + state + |> ReplicaModelScheduler.execute({:register, owner_id, :a, 0, owner_id}) + |> ReplicaModelScheduler.execute({:unregister, owner_id, 0}) + end) + + scheduler = + scheduler + |> run_schedule(schedule) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "named-cluster close and reopen fences stale epoch frames on real shards", %{ + nodes: nodes + } do + check all( + cluster_slot <- integer(0..2), + key_slot <- integer(0..2), + before_reopen <- list_of(network_command(), max_length: 8), + after_reopen <- list_of(network_command(), max_length: 12), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_authority_model_#{System.unique_integer([:positive])}" + cluster = "model_cluster_#{cluster_slot}" + opts = model_opts(name, self(), shards: 2, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:connect, :a, cluster}) + |> ReplicaModelScheduler.execute({:connect, :b, cluster}) + |> ReplicaModelScheduler.execute({:connect, :c, cluster}) + |> ReplicaModelScheduler.execute({:register_cluster, 50, :a, cluster, key_slot, 1}) + |> ReplicaModelScheduler.execute({:join_cluster, 50, :a, cluster, key_slot, 1}) + |> ReplicaModelScheduler.execute({ + :register_cluster, + 52, + :c, + cluster, + key_slot + 10, + 1 + }) + |> ReplicaModelScheduler.execute({ + :join_cluster, + 52, + :c, + cluster, + key_slot + 10, + 1 + }) + |> run_schedule(before_reopen) + |> ReplicaModelScheduler.execute({:disconnect, :a, cluster}) + |> ReplicaModelScheduler.execute({:connect, :a, cluster}) + |> ReplicaModelScheduler.execute(:deliver_all) + |> ReplicaModelScheduler.execute({:register_cluster, 51, :a, cluster, key_slot, 2}) + |> ReplicaModelScheduler.execute({:join_cluster, 51, :a, cluster, key_slot, 2}) + |> run_schedule(after_reopen) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + defp command_history do + list_of(command(), min_length: 1, max_length: @max_commands) + end + + defp command do + one_of([ + owner_command(:register), + owner_command(:join), + owner_key_command(:unregister), + owner_key_command(:leave), + map(owner_id(), &{:kill, &1}), + transport_mode_command(), + map(non_negative_integer(), &{:deliver, &1}), + map(non_negative_integer(), &{:duplicate, &1}), + map(non_negative_integer(), &{:drop, &1}), + constant(:deliver_all), + constant(:anti_entropy), + constant(:flush) + ]) + end + + defp network_command do + one_of([ + transport_mode_command(), + map(non_negative_integer(), &{:deliver, &1}), + map(non_negative_integer(), &{:duplicate, &1}), + map(non_negative_integer(), &{:drop, &1}), + constant(:anti_entropy), + constant(:flush) + ]) + end + + defp owner_command(operation) do + gen all( + owner_id <- owner_id(), + node_id <- member_of([:a, :b, :c]), + slot <- integer(0..1), + revision <- integer(0..3) + ) do + {operation, owner_id, node_id, slot, revision} + end + end + + defp owner_key_command(operation) do + gen all( + owner_id <- owner_id(), + slot <- integer(0..1) + ) do + {operation, owner_id, slot} + end + end + + defp transport_mode_command do + gen all( + node_id <- member_of([:a, :b, :c]), + mode <- member_of([:capture, :busy, :drop]) + ) do + {:transport, node_id, mode} + end + end + + defp owner_id, do: integer(0..5) + + defp transition_runs, do: max(div(@model_runs, 3), 1) + + defp run_schedule(state, commands) do + Enum.reduce(commands, state, fn command, acc -> + ReplicaModelScheduler.execute(acc, command) + end) + end + + defp model_opts(name, controller, overrides) do + [ + name: name, + shards: Keyword.fetch!(overrides, :shards), + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: controller}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_oplog_max_entries: Keyword.fetch!(overrides, :oplog) + ] + end + + defp start_groups(nodes, opts) do + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + end + + defp await_discovery(nodes, name) do + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn {_id, node} -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == map_size(nodes) - 1 + end) + end, + timeout: 10_000 + ) + end +end diff --git a/test/support/controlled_replica_transport.ex b/test/support/controlled_replica_transport.ex new file mode 100644 index 0000000..70d7d48 --- /dev/null +++ b/test/support/controlled_replica_transport.ex @@ -0,0 +1,49 @@ +defmodule Group.ControlledReplicaTransport do + @moduledoc false + @behaviour Group.Replica.Transport + + @impl true + def id, do: :group_controlled_replica_transport + + @impl true + def descriptor(_group, _opts), do: :group_controlled_replica_transport + + def set_mode(group, mode) when mode in [:capture, :pass, :busy, :drop] do + :persistent_term.put({__MODULE__, group, :mode}, mode) + :ok + end + + def clear(group) do + :persistent_term.erase({__MODULE__, group, :mode}) + :ok + end + + @impl true + def try_send(group, target_node, shard, frame, opts) do + case :persistent_term.get({__MODULE__, group, :mode}, :capture) do + :capture -> + controller = Keyword.fetch!(opts, :controller) + send(controller, {__MODULE__, :frame, group, node(), target_node, shard, frame}) + :ok + + :pass -> + deliver(group, target_node, shard, frame) + + :busy -> + :busy + + :drop -> + :ok + end + end + + defp deliver(group, target_node, shard, frame) do + destination = {Group.Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, node(), frame} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end +end diff --git a/test/support/model_conflict_resolver.ex b/test/support/model_conflict_resolver.ex new file mode 100644 index 0000000..6ac3502 --- /dev/null +++ b/test/support/model_conflict_resolver.ex @@ -0,0 +1,15 @@ +defmodule Group.ModelConflictResolver do + @moduledoc false + + def resolve(_name, _key, {pid1, meta1, _time1}, {pid2, meta2, _time2}) do + rank1 = Map.fetch!(meta1, :rank) + rank2 = Map.fetch!(meta2, :rank) + + cond do + rank1 > rank2 -> pid1 + rank2 > rank1 -> pid2 + pid1 > pid2 -> pid1 + true -> pid2 + end + end +end diff --git a/test/support/replica_lifecycle_model.ex b/test/support/replica_lifecycle_model.ex new file mode 100644 index 0000000..abb0dc1 --- /dev/null +++ b/test/support/replica_lifecycle_model.ex @@ -0,0 +1,223 @@ +defmodule Group.ReplicaLifecycleModel do + @moduledoc """ + Independent application-level oracle for replica convergence tests. + + This model deliberately knows nothing about Group's oplog, receive cursors, + batching, authority tables, or wire frames. It records only operations that + the public API accepted and the owner lifecycle consequences that follow. + """ + + defstruct owners: %{}, + registrations: %{}, + memberships: MapSet.new(), + seen_registration_keys: MapSet.new(), + seen_membership_keys: MapSet.new() + + @type owner_id :: non_neg_integer() + @type cluster :: term() + @type key :: binary() + @type scoped_key :: {cluster(), key()} + @type owner :: %{node: atom(), alive?: boolean()} + @type t :: %__MODULE__{ + owners: %{optional(owner_id()) => owner()}, + registrations: %{optional(scoped_key()) => %{optional(owner_id()) => map()}}, + memberships: MapSet.t({cluster(), key(), owner_id(), map()}), + seen_registration_keys: MapSet.t(scoped_key()), + seen_membership_keys: MapSet.t(scoped_key()) + } + + def new, do: %__MODULE__{} + + def owner(%__MODULE__{} = model, owner_id), do: Map.get(model.owners, owner_id) + + def put_owner(%__MODULE__{} = model, owner_id, node) do + case owner(model, owner_id) do + nil -> + put_in(model.owners[owner_id], %{node: node, alive?: true}) + + %{node: ^node} -> + model + + %{node: other_node} -> + raise ArgumentError, + "logical owner #{inspect(owner_id)} moved from #{inspect(other_node)} to #{inspect(node)}" + end + end + + def record_register(%__MODULE__{} = model, owner_id, key, meta, :ok) do + model + |> ensure_alive!(owner_id) + |> Map.update!(:seen_registration_keys, &MapSet.put(&1, key)) + |> Map.update!(:registrations, fn registrations -> + Map.update(registrations, key, %{owner_id => meta}, &Map.put(&1, owner_id, meta)) + end) + end + + def record_register(%__MODULE__{} = model, _owner_id, key, _meta, {:error, :taken}) do + Map.update!(model, :seen_registration_keys, &MapSet.put(&1, key)) + end + + def record_unregister(%__MODULE__{} = model, owner_id, key, :ok) do + registrations = + model.registrations + |> Map.update(key, %{}, &Map.delete(&1, owner_id)) + |> drop_empty_claim_sets() + + %{model | registrations: registrations} + end + + def record_unregister(%__MODULE__{} = model, _owner_id, _key, {:error, _reason}), do: model + + def record_join(%__MODULE__{} = model, owner_id, {cluster, key} = scope, meta, :ok) do + model + |> ensure_alive!(owner_id) + |> Map.update!(:seen_membership_keys, &MapSet.put(&1, scope)) + |> Map.update!(:memberships, fn memberships -> + memberships + |> Enum.reject(fn {member_cluster, member_key, member_owner, _old_meta} -> + member_cluster == cluster and member_key == key and member_owner == owner_id + end) + |> MapSet.new() + |> MapSet.put({cluster, key, owner_id, meta}) + end) + end + + def record_leave(%__MODULE__{} = model, owner_id, {cluster, key}, :ok) do + memberships = + model.memberships + |> Enum.reject(fn {member_cluster, member_key, member_owner, _meta} -> + member_cluster == cluster and member_key == key and member_owner == owner_id + end) + |> MapSet.new() + + %{model | memberships: memberships} + end + + def record_leave(%__MODULE__{} = model, _owner_id, _key, {:error, _reason}), do: model + + def kill(%__MODULE__{} = model, owner_id) do + case owner(model, owner_id) do + nil -> + model + + owner -> + registrations = + model.registrations + |> Map.new(fn {key, claims} -> {key, Map.delete(claims, owner_id)} end) + |> drop_empty_claim_sets() + + memberships = + model.memberships + |> Enum.reject(fn {_cluster, _key, member_owner, _meta} -> + member_owner == owner_id + end) + |> MapSet.new() + + %{ + model + | owners: Map.put(model.owners, owner_id, %{owner | alive?: false}), + registrations: registrations, + memberships: memberships + } + end + end + + def restart_node(%__MODULE__{} = model, node) do + remove_owned_scope(model, fn owner -> owner.node == node end, fn _scope -> true end) + end + + def disconnect_cluster(%__MODULE__{} = model, node, cluster) do + remove_owned_scope( + model, + fn owner -> owner.node == node end, + fn {entry_cluster, _key} -> entry_cluster == cluster end + ) + end + + def expected_registrations(%__MODULE__{} = model) do + Map.new(model.registrations, fn {key, claims} -> + {owner_id, meta} = + Enum.max_by(claims, fn {owner_id, meta} -> + {Map.get(meta, :rank, owner_id), owner_id} + end) + + {key, {owner_id, meta}} + end) + end + + def expected_memberships(%__MODULE__{} = model) do + model.memberships + |> Enum.group_by( + fn {cluster, key, _owner_id, _meta} -> {cluster, key} end, + fn {_cluster, _key, owner_id, meta} -> {owner_id, meta} end + ) + |> Map.new(fn {key, members} -> {key, Enum.sort(members)} end) + end + + def resolve_registry_conflicts(%__MODULE__{} = model) do + losing_owners = + model.registrations + |> Enum.flat_map(fn {_key, claims} -> + if map_size(claims) > 1 do + {winner, _meta} = + Enum.max_by(claims, fn {owner_id, meta} -> + {Map.get(meta, :rank, owner_id), owner_id} + end) + + Map.keys(claims) -- [winner] + else + [] + end + end) + |> MapSet.new() + + if MapSet.size(losing_owners) == 0 do + model + else + losing_owners + |> Enum.reduce(model, &kill(&2, &1)) + |> resolve_registry_conflicts() + end + end + + defp ensure_alive!(model, owner_id) do + case owner(model, owner_id) do + %{alive?: true} -> model + other -> raise ArgumentError, "owner #{inspect(owner_id)} is not alive: #{inspect(other)}" + end + end + + defp drop_empty_claim_sets(registrations) do + Map.reject(registrations, fn {_key, claims} -> map_size(claims) == 0 end) + end + + defp remove_owned_scope(model, owner_filter, scope_filter) do + owner_ids = + model.owners + |> Enum.filter(fn {_owner_id, owner} -> owner_filter.(owner) end) + |> MapSet.new(&elem(&1, 0)) + + registrations = + model.registrations + |> Map.new(fn {scope, claims} -> + claims = + if scope_filter.(scope) do + Map.reject(claims, fn {owner_id, _meta} -> MapSet.member?(owner_ids, owner_id) end) + else + claims + end + + {scope, claims} + end) + |> drop_empty_claim_sets() + + memberships = + model.memberships + |> Enum.reject(fn {cluster, key, owner_id, _meta} -> + scope_filter.({cluster, key}) and MapSet.member?(owner_ids, owner_id) + end) + |> MapSet.new() + + %{model | registrations: registrations, memberships: memberships} + end +end diff --git a/test/support/replica_model_scheduler.ex b/test/support/replica_model_scheduler.ex new file mode 100644 index 0000000..b698324 --- /dev/null +++ b/test/support/replica_model_scheduler.ex @@ -0,0 +1,521 @@ +defmodule Group.ReplicaModelScheduler do + @moduledoc false + + alias Group.{ControlledReplicaTransport, ReplicaLifecycleModel, TestCluster} + + defmodule Envelope do + @moduledoc false + defstruct [:id, :source, :target, :shard, :frame] + end + + defstruct [:name, :nodes, :model, :group_opts, owners: %{}, queue: [], next_frame_id: 1] + + def new(name, nodes, group_opts \\ []) do + %__MODULE__{ + name: name, + nodes: Map.new(nodes), + model: ReplicaLifecycleModel.new(), + group_opts: group_opts + } + end + + def execute(%__MODULE__{} = state, {:register, owner_id, node_id, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = registration_key(owner_id, slot) + meta = %{owner: owner_id, rank: owner_id, revision: revision} + result = owner_call(state, owner_id, {:register, state.name, key, meta, []}) + + model = + ReplicaLifecycleModel.record_register(state.model, owner_id, {nil, key}, meta, result) + + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:claim, owner_id, node_id, key_slot, rank}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = "model/conflict/#{key_slot}" + meta = %{owner: owner_id, rank: rank} + result = owner_call(state, owner_id, {:register, state.name, key, meta, []}) + + model = + ReplicaLifecycleModel.record_register(state.model, owner_id, {nil, key}, meta, result) + + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:unregister, owner_id, slot}) do + with_existing_owner(state, owner_id, fn state, _pid -> + key = registration_key(owner_id, slot) + result = owner_call(state, owner_id, {:unregister, state.name, key, []}) + model = ReplicaLifecycleModel.record_unregister(state.model, owner_id, {nil, key}, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:join, owner_id, node_id, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = membership_key(owner_id, slot) + meta = %{owner: owner_id, revision: revision} + result = owner_call(state, owner_id, {:join, state.name, key, meta, []}) + model = ReplicaLifecycleModel.record_join(state.model, owner_id, {nil, key}, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:leave, owner_id, slot}) do + with_existing_owner(state, owner_id, fn state, _pid -> + key = membership_key(owner_id, slot) + result = owner_call(state, owner_id, {:leave, state.name, key, []}) + model = ReplicaLifecycleModel.record_leave(state.model, owner_id, {nil, key}, result) + sync(%{state | model: model}) + end) + end + + def execute( + %__MODULE__{} = state, + {:register_cluster, owner_id, node_id, cluster, slot, revision} + ) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = cluster_registration_key(owner_id, slot) + meta = %{owner: owner_id, rank: owner_id, revision: revision, cluster: cluster} + opts = [cluster: cluster] + result = owner_call(state, owner_id, {:register, state.name, key, meta, opts}) + scope = {cluster, key} + model = ReplicaLifecycleModel.record_register(state.model, owner_id, scope, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:join_cluster, owner_id, node_id, cluster, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = cluster_membership_key(owner_id, slot) + meta = %{owner: owner_id, revision: revision, cluster: cluster} + opts = [cluster: cluster] + result = owner_call(state, owner_id, {:join, state.name, key, meta, opts}) + scope = {cluster, key} + model = ReplicaLifecycleModel.record_join(state.model, owner_id, scope, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:connect, node_id, cluster}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, Group, :connect, [state.name, cluster]) + sync(state) + end + + def execute(%__MODULE__{} = state, {:disconnect, node_id, cluster}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, Group, :disconnect, [state.name, cluster]) + model = ReplicaLifecycleModel.disconnect_cluster(state.model, node, cluster) + sync(%{state | model: model}) + end + + def execute(%__MODULE__{} = state, {:kill, owner_id}) do + case Map.get(state.owners, owner_id) do + nil -> + state + + %{pid: pid, node: node} -> + if remote_alive?(node, pid) do + true = TestCluster.rpc!(node, Process, :exit, [pid, :kill]) + end + + state + |> Map.update!(:model, &ReplicaLifecycleModel.kill(&1, owner_id)) + |> sync() + end + end + + def execute(%__MODULE__{} = state, {:transport, node_id, mode}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, ControlledReplicaTransport, :set_mode, [state.name, mode]) + state + end + + def execute(%__MODULE__{} = state, {:deliver, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, envelope -> + deliver_envelope(state, envelope, 1) + end) + end + + def execute(%__MODULE__{} = state, {:duplicate, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, envelope -> + deliver_envelope(state, envelope, 2) + end) + end + + def execute(%__MODULE__{} = state, {:drop, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, _envelope -> state end) + end + + def execute(%__MODULE__{} = state, :deliver_all) do + state = sync(state) + envelopes = state.queue + + Enum.reduce(envelopes, %{state | queue: []}, fn envelope, acc -> + deliver_envelope(acc, envelope, 1) + end) + end + + def execute(%__MODULE__{} = state, {:restart, node_id}) do + state = sync(state) + node = Map.fetch!(state.nodes, node_id) + :ok = stop_group_local(node, state.name) + model = ReplicaLifecycleModel.restart_node(state.model, node) + {:ok, _pid} = TestCluster.start_group(node, state.group_opts) + state = %{state | model: model} + + TestCluster.assert_eventually( + fn -> + Enum.all?(state.nodes, fn {_id, peer} -> + expected = map_size(state.nodes) - 1 + length(TestCluster.rpc!(peer, Group, :nodes, [state.name])) == expected + end) + end, + timeout: 10_000, + interval: 25 + ) + + sync(state) + end + + def execute(%__MODULE__{} = state, :anti_entropy) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :trigger_anti_entropy_local, [state.name]) + end) + + sync(state) + end + + def execute(%__MODULE__{} = state, :flush), do: sync(state) + + def stabilize_and_assert!(%__MODULE__{} = state) do + state = sync(state) + + Enum.each(state.nodes, fn {_id, node} -> + :ok = TestCluster.rpc!(node, ControlledReplicaTransport, :set_mode, [state.name, :pass]) + end) + + expected = ReplicaLifecycleModel.resolve_registry_conflicts(state.model) + state = %{state | model: expected, queue: []} + + TestCluster.assert_eventually( + fn -> + pump_anti_entropy(state) + converged?(state) + end, + timeout: 15_000, + interval: 25 + ) + + Enum.each(state.nodes, fn {_id, node} -> + :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [state.name]) + end) + + assert_expected_owner_lifecycle!(state) + assert_no_dead_retained_owners!(state) + state + end + + def cleanup(%__MODULE__{} = state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :cleanup_owners_local, [state.name]) + TestCluster.rpc!(node, ControlledReplicaTransport, :clear, [state.name]) + stop_group_local(node, state.name) + end) + + drain_transport_messages(state.name) + :ok + end + + def sync(%__MODULE__{} = state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.flush_shards(node, state.name) + end) + + drain(state, 2) + end + + def drain(%__MODULE__{} = state, wait_ms \\ 0) do + receive do + {ControlledReplicaTransport, :frame, group, source, target, shard, frame} + when group == state.name -> + envelope = %Envelope{ + id: state.next_frame_id, + source: source, + target: target, + shard: shard, + frame: frame + } + + drain( + %{state | queue: state.queue ++ [envelope], next_frame_id: state.next_frame_id + 1}, + wait_ms + ) + after + wait_ms -> state + end + end + + def trigger_anti_entropy_local(name) do + num_shards = Group.get_config(name).num_shards + + Enum.each(0..(num_shards - 1), fn shard -> + shard_name = Group.Replica.shard_name(name, shard) + state = :sys.get_state(shard_name) + send(shard_name, {:group_replica_anti_entropy, state.anti_entropy_ref}) + end) + + :ok + end + + def spawn_owner(name) do + pid = spawn(fn -> owner_loop() end) + key = {__MODULE__, :owners, name} + owners = :persistent_term.get(key, []) + :persistent_term.put(key, [pid | owners]) + pid + end + + def cleanup_owners_local(name) do + key = {__MODULE__, :owners, name} + + key + |> :persistent_term.get([]) + |> Enum.each(fn pid -> + if Process.alive?(pid), do: Process.exit(pid, :kill) + end) + + :persistent_term.erase(key) + :ok + end + + def call_owner(pid, operation) do + if Process.alive?(pid) do + ref = make_ref() + send(pid, {__MODULE__, :call, self(), ref, operation}) + + receive do + {__MODULE__, :reply, ^ref, result} -> result + after + 5_000 -> raise "model owner #{inspect(pid)} did not answer #{inspect(operation)}" + end + else + {:error, :owner_dead} + end + end + + defp owner_loop do + receive do + {__MODULE__, :call, caller, ref, operation} -> + result = apply_owner_operation(operation) + send(caller, {__MODULE__, :reply, ref, result}) + owner_loop() + end + end + + defp apply_owner_operation({:register, name, key, meta, opts}), + do: Group.register(name, key, meta, opts) + + defp apply_owner_operation({:unregister, name, key, opts}), + do: Group.unregister(name, key, opts) + + defp apply_owner_operation({:join, name, key, meta, opts}), + do: Group.join(name, key, meta, opts) + + defp apply_owner_operation({:leave, name, key, opts}), + do: Group.leave(name, key, opts) + + defp with_owner(state, owner_id, node_id, fun) do + case Map.get(state.owners, owner_id) do + nil -> + node = Map.fetch!(state.nodes, node_id) + pid = TestCluster.rpc!(node, __MODULE__, :spawn_owner, [state.name]) + model = ReplicaLifecycleModel.put_owner(state.model, owner_id, node) + + state = %{ + state + | owners: Map.put(state.owners, owner_id, %{node: node, pid: pid}), + model: model + } + + fun.(state, pid) + + %{node: node, pid: pid} -> + if remote_alive?(node, pid), do: fun.(state, pid), else: state + end + end + + defp with_existing_owner(state, owner_id, fun) do + case Map.get(state.owners, owner_id) do + nil -> + state + + %{node: node, pid: pid} -> + if remote_alive?(node, pid), do: fun.(state, pid), else: state + end + end + + defp owner_call(state, owner_id, operation) do + %{node: node, pid: pid} = Map.fetch!(state.owners, owner_id) + + case TestCluster.rpc!(node, __MODULE__, :call_owner, [pid, operation]) do + {:error, :owner_dead} -> + raise "model owner #{owner_id} died outside an expected lifecycle transition" + + result -> + result + end + end + + defp take_envelope(%{queue: []} = state, _selector, _fun), do: state + + defp take_envelope(state, selector, fun) do + index = rem(selector, length(state.queue)) + {envelope, queue} = List.pop_at(state.queue, index) + fun.(%{state | queue: queue}, envelope) + end + + defp deliver_envelope(state, envelope, times) do + Enum.each(1..times, fn _ -> + :ok = + TestCluster.rpc!( + envelope.target, + Group.Replica.Transport, + :deliver, + [state.name, envelope.source, envelope.shard, envelope.frame] + ) + + TestCluster.flush_shards(envelope.target, state.name) + end) + + drain(state, 2) + end + + defp pump_anti_entropy(state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :trigger_anti_entropy_local, [state.name]) + end) + + Enum.each(1..2, fn _ -> + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.flush_shards(node, state.name) + end) + end) + end + + defp converged?(state) do + expected_registrations = ReplicaLifecycleModel.expected_registrations(state.model) + expected_memberships = ReplicaLifecycleModel.expected_memberships(state.model) + pid_to_owner = Map.new(state.owners, fn {owner_id, %{pid: pid}} -> {pid, owner_id} end) + + Enum.all?(state.nodes, fn {_node_id, node} -> + registrations_match?( + node, + state.name, + state.model.seen_registration_keys, + expected_registrations, + pid_to_owner + ) and + memberships_match?( + node, + state.name, + state.model.seen_membership_keys, + expected_memberships, + pid_to_owner + ) + end) + end + + defp registrations_match?(node, name, keys, expected, pid_to_owner) do + Enum.all?(keys, fn {cluster, key} = scope -> + actual = + case TestCluster.rpc!(node, Group, :lookup, [name, key, cluster_opts(cluster)]) do + nil -> nil + {pid, meta} -> {Map.get(pid_to_owner, pid, {:unknown_pid, pid}), meta} + end + + actual == Map.get(expected, scope) + end) + end + + defp memberships_match?(node, name, keys, expected, pid_to_owner) do + Enum.all?(keys, fn {cluster, key} = scope -> + actual = + node + |> TestCluster.rpc!(Group, :members, [name, key, cluster_opts(cluster)]) + |> Enum.map(fn {pid, meta} -> {Map.get(pid_to_owner, pid, {:unknown_pid, pid}), meta} end) + |> Enum.sort() + + actual == Map.get(expected, scope, []) + end) + end + + defp assert_no_dead_retained_owners!(state) do + retained = + state.nodes + |> Enum.flat_map(fn {_id, node} -> + TestCluster.rpc!(node, TestCluster, :replica_owner_pids, [state.name]) + end) + |> Enum.uniq() + + dead = + Enum.reject(retained, fn pid -> TestCluster.rpc!(node(pid), Process, :alive?, [pid]) end) + + if dead != [], do: raise("dead owners retained after convergence: #{inspect(dead)}") + end + + defp assert_expected_owner_lifecycle!(state) do + mismatches = + Enum.flat_map(state.model.owners, fn {owner_id, %{alive?: expected_alive?}} -> + %{node: node, pid: pid} = Map.fetch!(state.owners, owner_id) + actual_alive? = remote_alive?(node, pid) + + if expected_alive? == actual_alive? do + [] + else + [{owner_id, expected_alive?, actual_alive?, pid}] + end + end) + + if mismatches != [] do + raise "owner lifecycle diverged from model: #{inspect(mismatches)}" + end + end + + defp remote_alive?(node, pid), do: TestCluster.rpc!(node, Process, :alive?, [pid]) + + defp stop_group_local(node, name) do + case TestCluster.rpc!(node, Process, :whereis, [:"#{name}_group_sup"]) do + nil -> :ok + pid -> TestCluster.rpc!(node, Supervisor, :stop, [pid, :normal, 5_000]) + end + catch + :exit, _ -> :ok + end + + defp drain_transport_messages(name) do + receive do + {ControlledReplicaTransport, :frame, ^name, _source, _target, _shard, _frame} -> + drain_transport_messages(name) + after + 0 -> :ok + end + end + + defp registration_key(owner_id, slot), do: "model/reg/#{owner_id}/#{slot}" + defp membership_key(owner_id, slot), do: "model/pg/#{owner_id}/#{slot}" + defp cluster_registration_key(owner_id, slot), do: "model/cluster/reg/#{owner_id}/#{slot}" + defp cluster_membership_key(owner_id, slot), do: "model/cluster/pg/#{owner_id}/#{slot}" + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] +end diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index ebffc86..8f97eb7 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -5,16 +5,36 @@ defmodule Group.TestCluster do def start_peers(count, opts \\ []) do cookie = Keyword.get(opts, :cookie, Node.get_cookie()) code_paths = :code.get_path() + schedulers = Keyword.get(opts, :schedulers) + + scheduler_args = + if schedulers, do: [~c"+S", ~c"#{schedulers}:#{schedulers}"], else: [] args = - [~c"-setcookie", ~c"#{cookie}", ~c"-kernel", ~c"prevent_overlapping_partitions", ~c"false"] ++ + scheduler_args ++ + [ + ~c"-setcookie", + ~c"#{cookie}", + ~c"-kernel", + ~c"prevent_overlapping_partitions", + ~c"false" + ] ++ Enum.flat_map(code_paths, fn p -> [~c"-pa", p] end) for _i <- 1..count do name = :"peer#{System.unique_integer([:positive])}" + # A fixed inet_dist_listen_min/max inherited through ERL_AFLAGS makes + # every child contend for the parent VM's distribution port. Peer args + # above carry every setting the test nodes require explicitly. {:ok, pid, node} = - :peer.start(%{name: name, host: ~c"127.0.0.1", longnames: true, args: args}) + :peer.start(%{ + name: name, + host: ~c"127.0.0.1", + longnames: true, + args: args, + env: [{~c"ERL_AFLAGS", ~c""}] + }) {:ok, _} = :rpc.call(node, :application, :ensure_all_started, [:elixir]) {:ok, _} = :rpc.call(node, :application, :ensure_all_started, [:group]) @@ -659,6 +679,59 @@ defmodule Group.TestCluster do :ok end + @doc false + def assert_replica_origin_purged(name, origin) do + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + retained_claims = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {{_cluster, _key, row_origin, _generation, _epoch}, _pid, _meta, _time, + _seq} -> + row_origin == origin + end) + + retained_registry = + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _pid, _meta, _time, entry_node} -> entry_node == origin end) + + retained_pg = + Group.Replica.Data.pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _meta, _time, entry_node} -> entry_node == origin end) + + retained_cursors = + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {stream_id, _seq} -> + Group.Replica.Protocol.stream_origin(stream_id) == origin + end) + + retained_view = Group.Replica.Data.remote_view_generation(name, shard, origin) + + unless retained_claims == [] and retained_registry == [] and retained_pg == [] and + retained_cursors == [] and is_nil(retained_view) do + raise "replica origin was not fully purged from #{name} shard #{shard}: " <> + inspect(%{ + claims: retained_claims, + registry: retained_registry, + pg: retained_pg, + cursors: retained_cursors, + view_generation: retained_view + }) + end + end + + unless is_nil(Group.Replica.Data.remote_generation(name, origin)) and + Group.Replica.Data.clusters_for_node(name, origin) == [] do + raise "replica origin retained shared authority after purge: #{inspect(origin)}" + end + + :ok + end + @doc """ Returns every PID currently retained as replica authority or visible PG state. @@ -778,6 +851,13 @@ defmodule Group.TestCluster do "order_only=#{inspect(MapSet.difference(order, oplog) |> MapSet.to_list())}" end + max_entries = Group.get_config(name).replicated_oplog_max_entries + + if MapSet.size(order) > max_entries do + raise "oplog bound exceeded in #{name} shard #{shard}: " <> + "size=#{MapSet.size(order)} max=#{max_entries}" + end + Group.Replica.Data.replica_stream_meta_table(name, shard) |> :ets.tab2list() |> Enum.each(fn {stream_id, head, floor, applied} -> From 976bdb5b9c15ab27d9566e9ab72c10bbb6531bef Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Sun, 9 Aug 2026 17:52:07 +0000 Subject: [PATCH 4/7] Chunk exact snapshots and harden transport boundaries --- CHANGELOG.md | 9 + README.md | 50 +- lib/group.ex | 5 + lib/group/replica.ex | 494 +++++++++++++++-- lib/group/replica/data.ex | 43 ++ lib/group/replica/protocol.ex | 2 +- lib/group/replica/snapshot.ex | 144 +++++ lib/group/replica/transport.ex | 23 + lib/group/replica/transport/outbox.ex | 318 +++++++++++ lib/group/replica/transport/tcp.ex | 123 +++-- lib/group/supervisor.ex | 4 + priv/bench/README.md | 8 + priv/bench/lib/group_bench/distributed.ex | 59 ++ test/README.md | 11 +- test/distributed_test.exs | 4 +- test/formal/README.md | 15 + test/formal/SnapshotAssembly.cfg | 7 + test/formal/SnapshotAssembly.tla | 181 ++++++ test/formal/check.sh | 3 +- test/group_test.exs | 2 +- test/mutation/README.md | 4 +- test/mutation/run.exs | 141 ++++- test/replica_model_property_test.exs | 17 +- test/replica_snapshot_distributed_test.exs | 608 +++++++++++++++++++++ test/replica_snapshot_test.exs | 75 +++ test/replica_transport_outbox_test.exs | 155 ++++++ test/support/test_cluster.ex | 22 + 27 files changed, 2413 insertions(+), 114 deletions(-) create mode 100644 lib/group/replica/snapshot.ex create mode 100644 lib/group/replica/transport/outbox.ex create mode 100644 test/formal/SnapshotAssembly.cfg create mode 100644 test/formal/SnapshotAssembly.tla create mode 100644 test/replica_snapshot_distributed_test.exs create mode 100644 test/replica_snapshot_test.exs create mode 100644 test/replica_transport_outbox_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 22721e4..671cf9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,13 @@ ## Unreleased +- **Breaking**: replica protocol v2 splits exact snapshots into + transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage + chunks in shard-owned private ETS and advance the stream cursor only after an + exact, authority-fenced assembly is complete; loss, duplication, reordering, + supersession, expiry, and shard crashes remain repairable by anti-entropy. + Single-chunk snapshots retain a direct fast path. Sideband transports can use + per-shard local outboxes for bounded batching without adding a hop to the + default dist-Erlang adapter. Late-starting replica lanes now rebuild their + view from shared exact authority when startup fanout races registration. - Replace replica state sends/snapshots with per-origin, generation- and cluster-epoch-fenced streams: sequenced deltas repair gaps from a bounded oplog and fall back to exact origin snapshots after pruning. Replica data now diff --git a/README.md b/README.md index c9d351e..00ea74c 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ All operations are **eventually consistent**: replicated_pg_receiver_local_request_quota: 8, replica_transport: Group.Replica.Transport.Distribution, replicated_oplog_max_entries: 65_536, + replicated_snapshot_chunk_target_bytes: 1_048_576, replicated_anti_entropy_interval: 1_000, replicated_peer_lease_timeout: 15_000 } @@ -293,13 +294,18 @@ All operations are **eventually consistent**: `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, or `:disconnected`. Dropped and busy frames are repaired by anti-entropy. - `Group.Replica.Transport.TCP` is an included sideband adapter with bounded - per-peer writer queues; its socket owners are separate processes, so socket - backpressure cannot block a Group shard. + `Group.Replica.Transport.TCP` is an included sideband adapter with local + per-shard batching and bounded per-peer writer queues; its socket owners are + separate processes, so socket backpressure cannot block a Group shard. - **`replicated_oplog_max_entries`** — maximum retained replica records per shard across all local streams. Defaults to 65,536. Pruning never waits for peer acknowledgements; a peer behind the retained floor receives an exact snapshot. +- **`replicated_snapshot_chunk_target_bytes`** — target maximum encoded size + of each exact-snapshot frame. Defaults to 1 MiB and applies above every + transport, including dist Erlang. A single row larger than the target is + sent alone. Receivers stage chunks in shard-owned private ETS and replace + visible state only after the complete exact slice is present. - **`replicated_anti_entropy_interval`** — interval in milliseconds for stream head advertisements and nonblocking control heartbeats. Defaults to 1,000. - **`replicated_peer_lease_timeout`** — time without a dist-Erlang control @@ -311,7 +317,7 @@ All operations are **eventually consistent**: ``` Group.Supervisor (:"my_app_group_sup") -├── optional transport child — sideband adapter listener/pool +├── optional transport child — sideband manager and per-shard outboxes ├── Group.Replica.Data — owns ETS, journal, generations, and epochs ├── Group.PeerReconnect — bounded recovery after busy remote dispatch ├── Group.Replica.Supervisor — supervises N shard GenServers @@ -422,10 +428,14 @@ registry claims and PG memberships; absence from that snapshot is a delete. There are no leaders, quorum acknowledgements, per-entry replicated tombstones, or known-membership retention barriers. Oplog memory is bounded locally and independently of slow peers. Deletes are normal ordered records while retained, -and exact snapshots close gaps after pruning. Named-cluster close uses only a -temporary local shard-completion barrier; the final shard removes it and all -routing rows, including after a caller timeout or shard restart. Reconnect -waits for that barrier so a prior close cannot erase newly accepted writes. +and exact snapshots close gaps after pruning. Exact snapshots are split into +transport-neutral byte-bounded frames; loss, duplication, or reordering leaves +the old visible slice and cursor untouched until all chunks arrive. Incomplete +staging expires after a peer-lease interval without progress and is destroyed +automatically with its owning shard. Named-cluster close uses only a temporary +local shard-completion barrier; the final shard removes it and all routing rows, +including after a caller timeout or shard restart. Reconnect waits for that +barrier so a prior close cannot erase newly accepted writes. The sender flush timer is mainly a fallback for idle periods. The unified outbound buffer also flushes immediately when it hits the configured size, when a new enqueue @@ -450,7 +460,11 @@ replica_transport: ip: {0, 0, 0, 0}, advertised_ip: {10, 0, 1, 12}, port: 44_321, - max_queue: 1_024 + max_queue: 1_024, + outbox_batch_size: 64, + outbox_batch_bytes: 1_048_576, + outbox_flush_interval: 1, + outbox_deadline: 100 ]} ``` @@ -460,6 +474,24 @@ network or place the connection behind TLS. The adapter deliberately has no control/data ordering relationship; the generation/epoch lane barrier and stream sequence checks supply correctness. +The default distribution adapter still sends directly to the remote shard and +does not pay for a local outbox. Sideband adapters can delegate `try_send/5` to +`Group.Replica.Transport.Outbox.try_send/5` and supervise one outbox per shard +with `Group.Replica.Transport.Outbox.child_spec/1`. An outbox groups frames by +target and invokes the adapter's `send_batch/4` callback. Calls that expire or +return `:busy`/`:disconnected` are dropped without a local retry; the next +anti-entropy exchange repairs them. + +A message-oriented backend fits this callback shape by obtaining a connection +once from `init_outbox/3`, then sending each `send_batch/4` result to a +registered ingress name on the target node. Queue pressure maps to `:busy` and +a missing session maps to `:disconnected`. Ingress must attach the authenticated +connection's source node; an adapter must never trust a source node supplied +inside the payload. Exact snapshots are already bounded by Group. A transport +with a smaller maximum frame may additionally segment an encoded batch, but it +must completely reassemble that batch before calling +`Group.Replica.Transport.deliver_batch/4`. + ### Named Cluster TTL Leases Named-cluster TTLs are a local way to reduce replication fanout to nodes that diff --git a/lib/group.ex b/lib/group.ex index 449e4fa..e3dd976 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -229,9 +229,14 @@ defmodule Group do - `:replica_transport` — replica data transport module or `{module, opts}` tuple. Defaults to `Group.Replica.Transport.Distribution`. The transport must be nonblocking and may return `:busy`; anti-entropy repairs dropped frames. + Sideband transports can use `Group.Replica.Transport.Outbox` for lossy, + batched, per-shard isolation without adding a hop to the default transport. - `:replicated_oplog_max_entries` — maximum retained replica records per shard before old prefixes are pruned and lagging peers require a snapshot (default: `65_536`) + - `:replicated_snapshot_chunk_target_bytes` — target maximum encoded size of + each transport-neutral exact-snapshot chunk (default: `1_048_576`). A + single registry or membership row larger than the target remains one chunk. - `:replicated_anti_entropy_interval` — milliseconds between repeated stream head advertisements (default: `1_000`) - `:replicated_peer_lease_timeout` — milliseconds without a dist-Erlang diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 58ce632..d0f888d 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -60,14 +60,17 @@ defmodule Group.Replica do - heads advertises {stream, retained_floor, head}. - delta_batch carries one or more contiguous stream runs. - need requests the receiver's next missing sequence. - - snapshot exactly replaces one origin's registry claims and PG slice when - the requested prefix has already been pruned. + - snapshot_chunk carries a byte-bounded part of one exact origin slice when + the requested prefix has already been pruned. Receivers stage chunks in a + private ETS table and expose nothing until every chunk is present. Every stream field is validated against the authenticated source node and current generation/epoch. An old generation, a closed epoch, a wrong shard, or a transitive claim for another node's pid is rejected. Control/data reordering is safe: early frames are ignored and repeated heads repair them; - late frames fail their generation or epoch fence. + late frames fail their generation or epoch fence. Snapshot chunks may be + lost, duplicated, reordered, or mixed across retransmissions at the same + stream head; exact row counts and set insertion prevent partial commits. ## Bounded recovery @@ -117,13 +120,13 @@ defmodule Group.Replica do yielding. FIFO is preserved within the local request lane, while protocol and cluster barriers flush earlier buffered state first. - Receive-only handlers for the previous direct batch/snapshot messages remain - for rolling compatibility and tests; protocol v1 never emits them. + Snapshot staging is owned by the receiving shard, expires after a peer lease + without progress, and disappears automatically if the shard crashes. """ require Logger - alias Group.Replica.{Data, Protocol} + alias Group.Replica.{Data, Protocol, Snapshot} defstruct [ :name, @@ -137,6 +140,7 @@ defmodule Group.Replica do :replicated_sender_flush_interval, :replicated_pg_receiver_local_request_quota, :replicated_oplog_max_entries, + :replicated_snapshot_chunk_target_bytes, :replicated_anti_entropy_interval, :replicated_peer_lease_timeout, :replica_transport, @@ -159,7 +163,8 @@ defmodule Group.Replica do cluster_control_dirty: %{}, authority_dirty_notified: MapSet.new(), monitors: %{}, - peer_transports: %{} + peer_transports: %{}, + snapshot_transfers: %{} ] def start_link(opts) do @@ -240,6 +245,7 @@ defmodule Group.Replica do replicated_pg_receiver_local_request_quota: config.replicated_pg_receiver_local_request_quota, replicated_oplog_max_entries: config.replicated_oplog_max_entries, + replicated_snapshot_chunk_target_bytes: config.replicated_snapshot_chunk_target_bytes, replicated_anti_entropy_interval: config.replicated_anti_entropy_interval, replicated_peer_lease_timeout: config.replicated_peer_lease_timeout, replica_transport: elem(config.replica_transport, 0), @@ -473,9 +479,10 @@ defmodule Group.Replica do {:noreply, state} replica_authority_current?(state, remote_node, generation, epoch_revision) -> - # Shared authority arrived first; its local fanout is already the - # ordered marker that will purge and install this lane's view. - {:noreply, state} + # The shared authority can arrive before this sibling is registered, + # so shard-zero fanout is intentionally lossy at startup. Rebuild + # this lane directly from the exact shared authority. + {:noreply, install_current_replica_lane(state, remote_node, generation)} true -> {:noreply, request_replica_authority(state, remote_node)} @@ -768,10 +775,17 @@ defmodule Group.Replica do {:noreply, take_priority_turn(state)} end + def handle_info({:group_replica_batch, remote_node, frames}, state) + when is_atom(remote_node) and is_list(frames) do + state = Enum.reduce(frames, state, &handle_replica_frame(&2, remote_node, &1)) + {:noreply, take_priority_turn(state)} + end + def handle_info({@anti_entropy_timer, ref}, state) do state = if state.anti_entropy_ref == ref do state + |> expire_stale_snapshot_transfers() |> expire_stale_replica_peers() |> probe_replica_peers() |> request_quiet_cluster_hellos() @@ -2952,6 +2966,20 @@ defmodule Group.Replica do ) end + defp install_current_replica_lane(state, remote_node, generation) do + old_generation = + Data.remote_view_generation(state.name, state.shard_index, remote_node) + + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = purge_remote_streams_outside_authority(state, remote_node) + :ok = install_replica_view(state, remote_node, generation) + + state + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + end + defp schedule_anti_entropy(state) do ref = make_ref() @@ -3116,7 +3144,45 @@ defmodule Group.Replica do end) end + defp expire_stale_snapshot_transfers(%{snapshot_transfers: transfers} = state) + when map_size(transfers) == 0, + do: state + + defp expire_stale_snapshot_transfers(state) do + now = monotonic_millis() + + Enum.reduce(state.snapshot_transfers, state, fn {key, transfer}, acc -> + if now - transfer.last_progress > acc.replicated_peer_lease_timeout do + discard_snapshot_transfer(acc, key) + else + acc + end + end) + end + + defp discard_snapshot_transfer(state, key) do + case Map.pop(state.snapshot_transfers, key) do + {nil, transfers} -> + %{state | snapshot_transfers: transfers} + + {transfer, transfers} -> + :ok = Snapshot.delete_staging_table(transfer.table) + %{state | snapshot_transfers: transfers} + end + end + + defp discard_snapshot_transfers_for_source(state, source_node) do + Enum.reduce(state.snapshot_transfers, state, fn + {{^source_node, _stream_id} = key, _transfer}, acc -> + discard_snapshot_transfer(acc, key) + + {_key, _transfer}, acc -> + acc + end) + end + defp expire_replica_peer(state, remote_node) do + state = discard_snapshot_transfers_for_source(state, remote_node) %{name: name, shard_index: shard} = state if shard == 0 do @@ -3268,44 +3334,279 @@ defmodule Group.Replica do defp handle_replica_frame( state, source_node, - {:snapshot, version, stream_id, snapshot_seq, reg_data, pg_data} + {:snapshot_chunk, version, stream_id, snapshot_seq, chunk_index, chunk_count, + registry_count, pg_count, reg_data, pg_data} ) - when version == @protocol_version do - if valid_remote_stream?(state, source_node, stream_id) and - snapshot_seq >= Data.replica_cursor(state.name, state.shard_index, stream_id) do - state = flush_pending_replicated_barrier(state) - cluster = Protocol.stream_cluster(stream_id) + when version == @protocol_version and is_integer(snapshot_seq) and snapshot_seq >= 0 and + is_integer(chunk_index) and is_integer(chunk_count) and + is_integer(registry_count) and is_integer(pg_count) and is_list(reg_data) and + is_list(pg_data) do + if valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) and + valid_snapshot_manifest?( + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) and + valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do + if chunk_count == 1 and registry_count == length(reg_data) and + pg_count == length(pg_data) do + apply_complete_snapshot_rows( + state, + source_node, + stream_id, + snapshot_seq, + reg_data, + pg_data + ) + else + stage_replica_snapshot_chunk( + state, + source_node, + stream_id, + snapshot_seq, + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) + end + else + state + end + end + + defp handle_replica_frame(state, _source_node, _frame), do: state + + defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do + valid_remote_stream?(state, source_node, stream_id) and + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end - reg_data = - Enum.filter(reg_data, fn {key, pid, _meta, _time} -> + defp valid_snapshot_manifest?( + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) do + total_count = registry_count + pg_count + chunk_row_count = length(reg_data) + length(pg_data) + + chunk_count > 0 and chunk_index > 0 and chunk_index <= chunk_count and + registry_count >= 0 and pg_count >= 0 and + chunk_count <= max(total_count, 1) and + ((total_count == 0 and chunk_count == 1 and chunk_row_count == 0) or + (total_count > 0 and chunk_row_count > 0)) + end + + defp valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do + cluster = Protocol.stream_cluster(stream_id) + + Enum.all?(reg_data, fn + {key, pid, _meta, _time} when is_pid(pid) -> + node(pid) == source_node and + shard_index_for(cluster, key, state.num_shards) == state.shard_index + + _other -> + false + end) and + Enum.all?(pg_data, fn + {key, pid, _meta, _time} when is_pid(pid) -> node(pid) == source_node and shard_index_for(cluster, key, state.num_shards) == state.shard_index - end) - affected_registry_keys = - Data.replace_registry_claims_for_stream( - state.name, - state.shard_index, - stream_id, - snapshot_seq, - reg_data - ) + _other -> + false + end) + end - {state, events} = - Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> - reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) - end) + defp stage_replica_snapshot_chunk( + state, + source_node, + stream_id, + snapshot_seq, + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) do + key = {source_node, stream_id} + manifest = {chunk_count, registry_count, pg_count} - events = replace_remote_pg_snapshot(state, source_node, cluster, pg_data, events) - :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, snapshot_seq) - notify_monitors(state.name, events) - state + case snapshot_transfer(state, key, snapshot_seq, manifest) do + {:ignore, state} -> + state + + {:ok, state, transfer} -> + cond do + MapSet.member?(transfer.received, chunk_index) -> + state + + transfer.registry_seen + length(reg_data) > registry_count or + transfer.pg_seen + length(pg_data) > pg_count -> + discard_snapshot_transfer(state, key) + + true -> + case Snapshot.stage_rows(transfer.table, chunk_index, reg_data, pg_data) do + :ok -> + transfer = %{ + transfer + | received: MapSet.put(transfer.received, chunk_index), + registry_seen: transfer.registry_seen + length(reg_data), + pg_seen: transfer.pg_seen + length(pg_data), + last_progress: monotonic_millis() + } + + state = put_snapshot_transfer(state, key, transfer) + maybe_commit_snapshot_transfer(state, key, source_node, stream_id) + + {:error, :duplicate_row} -> + discard_snapshot_transfer(state, key) + end + end + end + end + + defp snapshot_transfer(state, key, snapshot_seq, manifest) do + case Map.get(state.snapshot_transfers, key) do + nil -> + {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + + %{snapshot_seq: existing_seq} when existing_seq > snapshot_seq -> + {:ignore, state} + + %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> + state = discard_snapshot_transfer(state, key) + {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + + %{manifest: ^manifest} = transfer -> + {:ok, state, transfer} + + _conflicting_transfer -> + {:ignore, discard_snapshot_transfer(state, key)} + end + end + + defp new_snapshot_transfer(snapshot_seq, {chunk_count, registry_count, pg_count} = manifest) do + %{ + snapshot_seq: snapshot_seq, + manifest: manifest, + chunk_count: chunk_count, + registry_count: registry_count, + pg_count: pg_count, + registry_seen: 0, + pg_seen: 0, + received: MapSet.new(), + last_progress: monotonic_millis(), + table: Snapshot.new_staging_table() + } + end + + defp put_snapshot_transfer(state, key, transfer) do + %{state | snapshot_transfers: Map.put(state.snapshot_transfers, key, transfer)} + end + + defp maybe_commit_snapshot_transfer(state, key, source_node, stream_id) do + transfer = Map.fetch!(state.snapshot_transfers, key) + + if MapSet.size(transfer.received) == transfer.chunk_count do + if transfer.registry_seen == transfer.registry_count and + transfer.pg_seen == transfer.pg_count do + commit_snapshot_transfer(state, key, source_node, stream_id, transfer) + else + discard_snapshot_transfer(state, key) + end else state end end - defp handle_replica_frame(state, _source_node, _frame), do: state + defp commit_snapshot_transfer(state, key, source_node, stream_id, transfer) do + state = + if valid_snapshot_stream?(state, source_node, stream_id, transfer.snapshot_seq) do + state = flush_pending_replicated_barrier(state) + cluster = Protocol.stream_cluster(stream_id) + + affected_registry_keys = + Data.replace_registry_claims_for_stream_from_staging( + state.name, + state.shard_index, + stream_id, + transfer.snapshot_seq, + transfer.table, + transfer.chunk_count + ) + + {state, events} = + Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) + end) + + events = + replace_remote_pg_snapshot_from_staging( + state, + source_node, + cluster, + transfer.table, + transfer.chunk_count, + events + ) + + :ok = + Data.put_replica_cursor( + state.name, + state.shard_index, + stream_id, + transfer.snapshot_seq + ) + + notify_monitors(state.name, events) + state + else + state + end + + discard_snapshot_transfer(state, key) + end + + defp apply_complete_snapshot_rows( + state, + source_node, + stream_id, + snapshot_seq, + reg_data, + pg_data + ) do + state = flush_pending_replicated_barrier(state) + cluster = Protocol.stream_cluster(stream_id) + + affected_registry_keys = + Data.replace_registry_claims_for_stream( + state.name, + state.shard_index, + stream_id, + snapshot_seq, + reg_data + ) + + {state, events} = + Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) + end) + + events = replace_remote_pg_snapshot_rows(state, source_node, cluster, pg_data, events) + :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, snapshot_seq) + notify_monitors(state.name, events) + state + end defp apply_replica_delta_run(state, source_node, stream_id, records, advertised_head) do if valid_remote_stream?(state, source_node, stream_id) do @@ -3624,33 +3925,117 @@ defmodule Group.Replica do defp send_replica_snapshot(state, target_node, stream_id, head) do cluster = Protocol.stream_cluster(stream_id) + reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) + pg_data = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, node()) - {_reg_by_cluster, pg_by_cluster} = - Data.local_data_by_cluster(state.name, state.shard_index, [cluster]) + envelope_bytes = + Snapshot.frame_envelope_bytes(stream_id, head, length(reg_data), length(pg_data)) - reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) + snapshot = + Snapshot.chunk_rows( + reg_data, + pg_data, + state.replicated_snapshot_chunk_target_bytes, + envelope_bytes + ) - try_send_replica_frame( - state, - target_node, - {:snapshot, Protocol.version(), stream_id, head, reg_data, - Map.get(pg_by_cluster, cluster, [])} - ) + chunk_count = length(snapshot.chunks) + + snapshot.chunks + |> Enum.with_index(1) + |> Enum.reduce(state, fn {{reg_chunk, pg_chunk}, chunk_index}, acc -> + try_send_replica_frame( + acc, + target_node, + {:snapshot_chunk, Protocol.version(), stream_id, head, chunk_index, chunk_count, + snapshot.registry_count, snapshot.pg_count, reg_chunk, pg_chunk} + ) + end) + end + + defp replace_remote_pg_snapshot_from_staging( + state, + source_node, + cluster, + staging_table, + chunk_count, + events + ) do + current = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, source_node) + + events = + Snapshot.fold_pg(staging_table, chunk_count, events, fn {key, pid, meta, time}, acc -> + case Data.pg_lookup(state.name, state.shard_index, cluster, key, pid) do + nil -> + :ok = + Data.pg_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + source_node + ) + + [build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}) | acc] + + {^meta, ^time, ^source_node} -> + acc + + {old_meta, _old_time, ^source_node} -> + :ok = + Data.pg_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + source_node + ) + + if old_meta == meta do + acc + else + [ + build_event(state.name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + | acc + ] + end + end + end) + + Enum.reduce(current, events, fn {key, pid, old_meta, _old_time}, acc -> + if Snapshot.member_pg?(staging_table, key, pid) do + acc + else + :ok = Data.pg_delete(state.name, state.shard_index, cluster, key, pid) + + event = + build_event(state.name, :left, key, pid, old_meta, %{ + reason: :reconcile, + cluster: cluster + }) + + [event | acc] + end + end) end - defp replace_remote_pg_snapshot(state, source_node, cluster, pg_data, events) do + defp replace_remote_pg_snapshot_rows(state, source_node, cluster, pg_data, events) do current = state.name |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) desired = - pg_data - |> Enum.filter(fn {key, pid, _meta, _time} -> - node(pid) == source_node and - shard_index_for(cluster, key, state.num_shards) == state.shard_index - end) - |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + Map.new(pg_data, fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) {inserts, deletes, events} = current @@ -4029,7 +4414,10 @@ defmodule Group.Replica do %{name: name, shard_index: shard_index, num_shards: num_shards} = state for i <- 0..(num_shards - 1), i != shard_index do - send(shard_name(name, i), message) + case Process.whereis(shard_name(name, i)) do + pid when is_pid(pid) -> send(pid, message) + nil -> :ok + end end end diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 94f803a..7511fc7 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -939,6 +939,49 @@ defmodule Group.Replica.Data do Enum.uniq(Enum.map(existing, &elem(&1, 0)) ++ Enum.map(claims, &elem(&1, 0))) end + def replace_registry_claims_for_stream_from_staging( + name, + shard, + stream_id, + snapshot_seq, + staging_table, + chunk_count + ) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + existing = registry_claims_for_stream(name, shard, stream_id) + + keys = + Enum.reduce(existing, MapSet.new(), fn {key, pid, _meta, _time}, keys -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin_node, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin_node, generation, epoch} + ) + + MapSet.put(keys, key) + end) + + keys = + Group.Replica.Snapshot.fold_registry( + staging_table, + chunk_count, + keys, + fn {key, pid, meta, time}, keys -> + put_registry_claim(name, shard, stream_id, snapshot_seq, key, pid, meta, time) + MapSet.put(keys, key) + end + ) + + MapSet.to_list(keys) + end + def purge_registry_claims_for_origin(name, shard, origin_node) do claims = :ets.select(reg_claim_by_key_table(name, shard), [ diff --git a/lib/group/replica/protocol.ex b/lib/group/replica/protocol.ex index fe88e24..a79d983 100644 --- a/lib/group/replica/protocol.ex +++ b/lib/group/replica/protocol.ex @@ -1,7 +1,7 @@ defmodule Group.Replica.Protocol do @moduledoc false - @version 1 + @version 2 def version, do: @version diff --git a/lib/group/replica/snapshot.ex b/lib/group/replica/snapshot.ex new file mode 100644 index 0000000..13dc29a --- /dev/null +++ b/lib/group/replica/snapshot.ex @@ -0,0 +1,144 @@ +defmodule Group.Replica.Snapshot do + @moduledoc false + + # The target is for the complete snapshot frame, not just its rows. Per-row + # external sizes conservatively include an extra ETF version byte, and this + # reserve covers the frame tuple, stream identity, both list headers, and + # integer fields. A single entry larger than the target remains one chunk. + @default_envelope_reserve 512 + @max_compact_chunk_count 2_147_483_647 + + def chunk_rows(registry_rows, pg_rows, target_bytes) + when is_list(registry_rows) and is_list(pg_rows) and is_integer(target_bytes) and + target_bytes > 0 do + chunk_rows(registry_rows, pg_rows, target_bytes, @default_envelope_reserve) + end + + def chunk_rows(registry_rows, pg_rows, target_bytes, envelope_bytes) + when is_list(registry_rows) and is_list(pg_rows) and is_integer(target_bytes) and + target_bytes > 0 and is_integer(envelope_bytes) and envelope_bytes >= 0 do + registry_rows = Enum.sort_by(registry_rows, fn {key, _pid, _meta, _time} -> key end) + pg_rows = Enum.sort_by(pg_rows, fn {key, pid, _meta, _time} -> {key, pid} end) + payload_target = max(target_bytes - envelope_bytes, 1) + + acc = %{chunks: [], registry: [], pg: [], bytes: 0, count: 0} + acc = Enum.reduce(registry_rows, acc, &add_row(&2, :registry, &1, payload_target)) + acc = Enum.reduce(pg_rows, acc, &add_row(&2, :pg, &1, payload_target)) + + chunks = + acc + |> flush_chunk() + |> Map.fetch!(:chunks) + |> Enum.reverse() + |> case do + [] -> [{[], []}] + chunks -> chunks + end + + %{ + registry_count: length(registry_rows), + pg_count: length(pg_rows), + chunks: chunks + } + end + + def frame_envelope_bytes(stream_id, snapshot_seq, registry_count, pg_count) do + # A non-empty ETF list adds a five-byte LIST_EXT header relative to an + # empty list. Reserve that once for each domain. The large chunk integers + # ensure every practical index/count uses no more space than this envelope. + :erlang.external_size( + {:snapshot_chunk, Group.Replica.Protocol.version(), stream_id, snapshot_seq, + @max_compact_chunk_count, @max_compact_chunk_count, registry_count, pg_count, [], []} + ) + 10 + end + + def new_staging_table do + :ets.new(__MODULE__, [:set, :private]) + end + + def delete_staging_table(table) do + try do + :ets.delete(table) + rescue + ArgumentError -> true + end + + :ok + end + + def stage_rows(table, chunk_index, registry_rows, pg_rows) do + objects = + Enum.map(registry_rows, &staging_object(:registry, &1)) ++ + Enum.map(pg_rows, &staging_object(:pg, &1)) + + size_before = :ets.info(table, :size) + + if :ets.insert_new(table, objects) and + :ets.info(table, :size) - size_before == length(objects) do + true = :ets.insert_new(table, {{:chunk, chunk_index}, registry_rows, pg_rows}) + :ok + else + {:error, :duplicate_row} + end + end + + def fold_registry(table, chunk_count, acc, fun) when is_function(fun, 2) do + Enum.reduce(1..chunk_count, acc, fn chunk_index, inner -> + {registry_rows, _pg_rows} = fetch_chunk(table, chunk_index) + Enum.reduce(registry_rows, inner, fun) + end) + end + + def fold_pg(table, chunk_count, acc, fun) when is_function(fun, 2) do + Enum.reduce(1..chunk_count, acc, fn chunk_index, inner -> + {_registry_rows, pg_rows} = fetch_chunk(table, chunk_index) + Enum.reduce(pg_rows, inner, fun) + end) + end + + def member_pg?(table, key, pid), do: :ets.member(table, {:pg, key, pid}) + + defp add_row(%{count: count, bytes: bytes} = acc, domain, row, target) do + row_bytes = :erlang.external_size(row) + + acc = + if count > 0 and bytes + row_bytes > target do + flush_chunk(acc) + else + acc + end + + case domain do + :registry -> + %{ + acc + | registry: [row | acc.registry], + bytes: acc.bytes + row_bytes, + count: acc.count + 1 + } + + :pg -> + %{acc | pg: [row | acc.pg], bytes: acc.bytes + row_bytes, count: acc.count + 1} + end + end + + defp flush_chunk(%{count: 0} = acc), do: acc + + defp flush_chunk(acc) do + chunk = {Enum.reverse(acc.registry), Enum.reverse(acc.pg)} + + %{acc | chunks: [chunk | acc.chunks], registry: [], pg: [], bytes: 0, count: 0} + end + + defp staging_object(:registry, {key, _pid, _meta, _time}), + do: {{:registry, key}} + + defp staging_object(:pg, {key, pid, _meta, _time}), + do: {{:pg, key, pid}} + + defp fetch_chunk(table, chunk_index) do + case :ets.lookup(table, {:chunk, chunk_index}) do + [{{:chunk, ^chunk_index}, registry_rows, pg_rows}] -> {registry_rows, pg_rows} + end + end +end diff --git a/lib/group/replica/transport.ex b/lib/group/replica/transport.ex index 8b23595..033acd6 100644 --- a/lib/group/replica/transport.ex +++ b/lib/group/replica/transport.ex @@ -16,6 +16,11 @@ defmodule Group.Replica.Transport do and sequences each origin/generation/shard/cluster/epoch stream; receivers discard duplicates and request gaps. Per-shard ordered delivery avoids repair traffic and is therefore the preferred fast path. + + A sideband implementation can delegate `try_send/5` to + `Group.Replica.Transport.Outbox.try_send/5`. That adds one local send only for + the configured sideband transport; the default distribution adapter retains + its direct remote `:erlang.send_nosuspend/3` path. """ @type frame :: term() @@ -50,6 +55,24 @@ defmodule Group.Replica.Transport do :ok end + @doc """ + Delivers a complete batch received from one authenticated peer. + + A finite-frame transport may segment the encoded batch on the wire, but it + must authenticate the peer and reassemble every segment before calling this + function. Group never observes or applies a partial batch. + """ + def deliver_batch(group, source_node, shard, frames) + when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 and + is_list(frames) do + send( + Group.Replica.shard_name(group, shard), + {:group_replica_batch, source_node, frames} + ) + + :ok + end + def normalize(module) when is_atom(module), do: {module, []} def normalize({module, opts}) when is_atom(module) and is_list(opts), do: {module, opts} diff --git a/lib/group/replica/transport/outbox.ex b/lib/group/replica/transport/outbox.ex new file mode 100644 index 0000000..aee2ba6 --- /dev/null +++ b/lib/group/replica/transport/outbox.ex @@ -0,0 +1,318 @@ +defmodule Group.Replica.Transport.Outbox do + @moduledoc """ + Lossy per-shard outboxes for sideband replica transports. + + This module is an implementation helper, not a replacement for + `Group.Replica.Transport`. Distribution can continue sending directly with + `:erlang.send_nosuspend/3`. A sideband adapter delegates `try_send/5` to + `try_send/5`, which performs only a local `send/2` to the matching shard + outbox. + + Each outbox batches frames by target node outside the Group shard. Expired + frames and batches rejected by the backend are deliberately dropped; + anti-entropy repairs them. Backends may perform bounded blocking work in + `send_batch/4` because they run in the outbox rather than a Group process. + + A backend using this helper implements: + + @behaviour Group.Replica.Transport.Outbox + + def init_outbox(group, shard, opts), do: {:ok, backend_state} + + def send_batch(target_node, frames, deadline, backend_state) do + # Return promptly once `deadline` has passed. It is safe to drop. + {:ok, backend_state} + end + + The backend is responsible for authenticated ingress and must pass only + complete logical frames to `Group.Replica.Transport.deliver_batch/4`. + + ## Options + + * `:outbox_batch_size` - maximum logical frames collected per flush, + default `64` + * `:outbox_batch_bytes` - approximate external-term bytes collected per + flush, default `1_048_576` + * `:outbox_flush_interval` - maximum batching delay in milliseconds, + default `1` + * `:outbox_deadline` - maximum useful residence time for an outbound frame + in milliseconds, default `100` + + The deadline bounds stale work, not mailbox memory. A backend must also put a + finite bound on every socket enqueue or write it performs. Exact snapshot + frames are independently bounded by `:replicated_snapshot_chunk_target_bytes`. + Other logical frames or a whole batch may still exceed `:outbox_batch_bytes`; + a transport with a smaller finite frame size must segment and completely + reassemble those batches before local delivery. + """ + + @type frame :: Group.Replica.Transport.frame() + @type send_result :: Group.Replica.Transport.send_result() + @type backend_state :: term() + + @callback init_outbox(group :: atom(), shard :: non_neg_integer(), opts :: keyword()) :: + {:ok, backend_state()} + + @callback send_batch( + target_node :: node(), + frames :: [frame()], + deadline :: integer(), + backend_state() + ) :: {send_result(), backend_state()} + + @default_deadline 100 + + @doc """ + Returns a supervisor child specification for one outbox per Group shard. + + `:name`, `:num_shards`, and `:backend` are required. All options are passed + unchanged to `backend.init_outbox/3`. + """ + def child_spec(opts) do + group = Keyword.fetch!(opts, :name) + + %{ + id: {__MODULE__, group}, + start: {Group.Replica.Transport.Outbox.Supervisor, :start_link, [opts]}, + type: :supervisor, + restart: :permanent, + shutdown: :infinity + } + end + + @doc """ + Enqueues a frame into its local shard outbox. + + This performs no backend or socket operation. `:ok` only means the message + was sent to the current local outbox PID; the outbox may later drop it on + expiry or backpressure. A concurrently terminating outbox can also lose an + accepted message, which anti-entropy repairs. + """ + def try_send(group, target_node, shard, frame, opts) + when is_atom(group) and is_atom(target_node) and is_integer(shard) and shard >= 0 and + is_list(opts) do + case Process.whereis(name(group, shard)) do + pid when is_pid(pid) -> + deadline = monotonic_ms() + deadline(opts) + send(pid, {:group_replica_outbox_send, target_node, deadline, frame}) + :ok + + nil -> + :disconnected + end + end + + @doc false + def name(group, shard), do: :"#{group}_replica_transport_outbox_#{shard}" + + @doc false + def monotonic_ms, do: System.monotonic_time(:millisecond) + + defp deadline(opts) do + case Keyword.get(opts, :outbox_deadline, @default_deadline) do + value when is_integer(value) and value > 0 -> + value + + other -> + raise ArgumentError, "expected :outbox_deadline to be positive, got: #{inspect(other)}" + end + end +end + +defmodule Group.Replica.Transport.Outbox.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + group = Keyword.fetch!(opts, :name) + num_shards = Keyword.fetch!(opts, :num_shards) + backend = Keyword.fetch!(opts, :backend) + + Code.ensure_loaded!(backend) + + for {function, arity} <- [init_outbox: 3, send_batch: 4] do + unless function_exported?(backend, function, arity) do + raise ArgumentError, + "outbox backend #{inspect(backend)} must implement #{function}/#{arity}" + end + end + + children = + for shard <- 0..(num_shards - 1) do + %{ + id: {Group.Replica.Transport.Outbox.Worker, group, shard}, + start: {Group.Replica.Transport.Outbox.Worker, :start_link, [opts, shard]}, + restart: :permanent, + shutdown: 5_000 + } + end + + Supervisor.init(children, strategy: :one_for_one) + end +end + +defmodule Group.Replica.Transport.Outbox.Worker do + @moduledoc false + use GenServer + + alias Group.Replica.Transport.Outbox + + @default_batch_size 64 + @default_batch_bytes 1_048_576 + @default_flush_interval 1 + + def start_link(opts, shard) do + group = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, {opts, shard}, name: Outbox.name(group, shard)) + end + + @impl true + def init({opts, shard}) do + group = Keyword.fetch!(opts, :name) + backend = Keyword.fetch!(opts, :backend) + {:ok, backend_state} = backend.init_outbox(group, shard, opts) + + {:ok, + %{ + group: group, + shard: shard, + backend: backend, + backend_state: backend_state, + batch_size: positive_opt(opts, :outbox_batch_size, @default_batch_size), + batch_bytes: positive_opt(opts, :outbox_batch_bytes, @default_batch_bytes), + flush_interval: non_negative_opt(opts, :outbox_flush_interval, @default_flush_interval), + pending: [], + pending_count: 0, + pending_bytes: 0, + flush_ref: nil + }} + end + + @impl true + def handle_info({:group_replica_outbox_send, target_node, deadline, frame}, state) + when is_atom(target_node) and is_integer(deadline) do + if deadline <= Outbox.monotonic_ms() do + {:noreply, state} + else + bytes = :erlang.external_size({target_node, frame}) + + state = + if state.pending_count > 0 and + (state.pending_count + 1 > state.batch_size or + state.pending_bytes + bytes > state.batch_bytes) do + flush(state) + else + state + end + + state = enqueue(state, target_node, deadline, frame, bytes) + + if state.pending_count >= state.batch_size or state.pending_bytes >= state.batch_bytes do + {:noreply, flush(state)} + else + {:noreply, schedule_flush(state)} + end + end + end + + def handle_info({:group_replica_outbox_flush, ref}, %{flush_ref: ref} = state) do + {:noreply, flush(%{state | flush_ref: nil})} + end + + def handle_info({:group_replica_outbox_flush, _stale_ref}, state), do: {:noreply, state} + def handle_info(_message, state), do: {:noreply, state} + + defp enqueue(state, target_node, deadline, frame, bytes) do + entry = {target_node, deadline, frame} + + %{ + state + | pending: [entry | state.pending], + pending_count: state.pending_count + 1, + pending_bytes: state.pending_bytes + bytes + } + end + + defp schedule_flush(%{pending_count: 0} = state), do: state + defp schedule_flush(%{flush_ref: ref} = state) when is_reference(ref), do: state + + defp schedule_flush(state) do + ref = make_ref() + Process.send_after(self(), {:group_replica_outbox_flush, ref}, state.flush_interval) + %{state | flush_ref: ref} + end + + defp flush(%{pending_count: 0} = state), do: cancel_flush(state) + + defp flush(state) do + state = cancel_flush(state) + now = Outbox.monotonic_ms() + + batches = + state.pending + |> Enum.reverse() + |> Enum.reject(fn {_target_node, deadline, _frame} -> deadline <= now end) + |> Enum.group_by(fn {target_node, _deadline, _frame} -> target_node end) + + backend_state = + Enum.reduce(batches, state.backend_state, fn {target_node, entries}, backend_state -> + frames = Enum.map(entries, fn {_target_node, _deadline, frame} -> frame end) + + deadline = + entries + |> Enum.map(fn {_target_node, deadline, _frame} -> deadline end) + |> Enum.min() + + if deadline <= Outbox.monotonic_ms() do + backend_state + else + case state.backend.send_batch(target_node, frames, deadline, backend_state) do + {result, next_backend_state} when result in [:ok, :busy, :disconnected] -> + next_backend_state + + other -> + raise "invalid #{inspect(state.backend)}.send_batch/4 return: #{inspect(other)}" + end + end + end) + + %{ + state + | backend_state: backend_state, + pending: [], + pending_count: 0, + pending_bytes: 0 + } + end + + defp cancel_flush(%{flush_ref: nil} = state), do: state + + defp cancel_flush(state) do + Process.cancel_timer(state.flush_ref) + %{state | flush_ref: nil} + end + + defp positive_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value > 0 -> + value + + other -> + raise ArgumentError, "expected #{inspect(key)} to be positive, got: #{inspect(other)}" + end + end + + defp non_negative_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value >= 0 -> + value + + other -> + raise ArgumentError, + "expected #{inspect(key)} to be non-negative, got: #{inspect(other)}" + end + end +end diff --git a/lib/group/replica/transport/tcp.ex b/lib/group/replica/transport/tcp.ex index 1914ac6..0cedaa8 100644 --- a/lib/group/replica/transport/tcp.ex +++ b/lib/group/replica/transport/tcp.ex @@ -6,10 +6,11 @@ defmodule Group.Replica.Transport.TCP do Replica frames use independent TCP connections, so there is no ordering relationship between a control message and its data lane. - `try_send/5` never writes a socket. It reserves one slot in a bounded - per-peer queue and sends to a dedicated writer process. The writer may block - up to `:send_timeout` without blocking a Group shard. A full queue returns - `:busy`; a missing connection returns `:disconnected`. + `try_send/5` only sends to a local per-shard outbox. The outbox batches + frames and forwards each target batch to a bounded per-peer writer queue. + The writer may block up to `:send_timeout` without blocking a Group shard. + Expired, busy, and disconnected batches are dropped and repaired by + anti-entropy. The endpoint capability in the dist-Erlang hello prevents an unrelated socket client from injecting frames. This transport is intended for trusted @@ -21,18 +22,23 @@ defmodule Group.Replica.Transport.TCP do * `:ip` - listen address, default `{127, 0, 0, 1}` * `:advertised_ip` - address placed in the hello, defaults to `:ip` * `:port` - listen port, default `0` (ephemeral) - * `:max_queue` - maximum queued frames per peer, default `1_024` + * `:max_queue` - maximum queued batches per peer, default `1_024` * `:connect_timeout` - outbound connect timeout in milliseconds, default `1_000` * `:send_timeout` - writer socket send timeout in milliseconds, default `1_000` * `:reconnect_interval` - retry delay in milliseconds, default `50` + + See `Group.Replica.Transport.Outbox` for batching and deadline options. """ use GenServer @behaviour Group.Replica.Transport + @behaviour Group.Replica.Transport.Outbox + + alias Group.Replica.Transport.Outbox @impl true - def id, do: :group_sideband_tcp_v1 + def id, do: :group_sideband_tcp_v2 @impl true def child_spec(opts) do @@ -40,10 +46,10 @@ defmodule Group.Replica.Transport.TCP do %{ id: {__MODULE__, name}, - start: {__MODULE__, :start_link, [opts]}, - type: :worker, + start: {Group.Replica.Transport.TCP.Supervisor, :start_link, [opts]}, + type: :supervisor, restart: :permanent, - shutdown: 5_000 + shutdown: :infinity } end @@ -58,24 +64,34 @@ defmodule Group.Replica.Transport.TCP do end @impl true - def try_send(group, target_node, shard, frame, _opts) do - case :ets.lookup(route_table(group), target_node) do - [{^target_node, writer, queued, max_queue}] -> - if :atomics.add_get(queued, 1, 1) <= max_queue do - if :erlang.send_nosuspend(writer, {:replica_frame, shard, frame}) do - :ok - else - :atomics.sub(queued, 1, 1) - :busy - end - else - :atomics.sub(queued, 1, 1) - :busy + def try_send(group, target_node, shard, frame, opts), + do: Outbox.try_send(group, target_node, shard, frame, opts) + + @impl Group.Replica.Transport.Outbox + def init_outbox(group, shard, _opts), do: {:ok, %{group: group, shard: shard}} + + @impl Group.Replica.Transport.Outbox + def send_batch(target_node, frames, deadline, %{group: group, shard: shard} = state) do + result = + try do + case :ets.lookup(route_table(group), target_node) do + [{^target_node, writer, queued, max_queue}] -> + if :atomics.add_get(queued, 1, 1) <= max_queue do + send(writer, {:replica_batch, deadline, shard, frames}) + :ok + else + :atomics.sub(queued, 1, 1) + :busy + end + + [] -> + :disconnected end + rescue + ArgumentError -> :disconnected + end - [] -> - :disconnected - end + {result, state} end @impl true @@ -126,8 +142,9 @@ defmodule Group.Replica.Transport.TCP do ]) {:ok, {_listen_ip, listen_port}} = :inet.sockname(listener) + capability = :erlang.term_to_binary({node(), make_ref(), System.unique_integer()}) - descriptor = {:group_sideband_tcp_v1, advertised_ip, listen_port, capability} + descriptor = {:group_sideband_tcp_v2, advertised_ip, listen_port, capability} :persistent_term.put({__MODULE__, group, :descriptor}, descriptor) :ets.new(route_table(group), [ @@ -248,7 +265,7 @@ defmodule Group.Replica.Transport.TCP do :ok pid -> - _ = :erlang.send_nosuspend(pid, message) + send(pid, message) :ok end end @@ -326,7 +343,7 @@ defmodule Group.Replica.Transport.TCP do manager, group, remote_node, - {:group_sideband_tcp_v1, host, port, capability}, + {:group_sideband_tcp_v2, host, port, capability}, connect_timeout, send_timeout ) do @@ -362,12 +379,18 @@ defmodule Group.Replica.Transport.TCP do defp writer_loop(socket, manager, remote_node, queued) do receive do - {:replica_frame, shard, frame} -> - result = :gen_tcp.send(socket, :erlang.term_to_binary({shard, frame})) + {:replica_batch, deadline, shard, frames} -> + result = + if deadline <= Outbox.monotonic_ms() do + :expired + else + :gen_tcp.send(socket, :erlang.term_to_binary({:batch, shard, frames})) + end + :atomics.sub(queued, 1, 1) case result do - :ok -> + result when result in [:ok, :expired] -> writer_loop(socket, manager, remote_node, queued) {:error, _reason} -> @@ -415,8 +438,9 @@ defmodule Group.Replica.Transport.TCP do case :gen_tcp.recv(socket, 0) do {:ok, payload} -> case decode_authenticated_frame(payload) do - {:ok, {shard, frame}} when is_integer(shard) and shard >= 0 -> - :ok = Group.Replica.Transport.deliver(group, source_node, shard, frame) + {:ok, {:batch, shard, frames}} + when is_integer(shard) and shard >= 0 and is_list(frames) -> + :ok = Group.Replica.Transport.deliver_batch(group, source_node, shard, frames) reader_loop(socket, group, source_node) _ -> @@ -446,3 +470,36 @@ defmodule Group.Replica.Transport.TCP do defp server_name(group), do: :"#{group}_replica_tcp_transport" defp route_table(group), do: :"#{group}_replica_tcp_routes" end + +defmodule Group.Replica.Transport.TCP.Supervisor do + @moduledoc false + use Supervisor + + alias Group.Replica.Transport.{Outbox, TCP} + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + group = Keyword.fetch!(opts, :name) + + manager = %{ + id: {TCP, group, :manager}, + start: {TCP, :start_link, [opts]}, + type: :worker, + restart: :transient, + shutdown: 5_000, + significant: true + } + + outboxes = + opts + |> Keyword.put(:backend, TCP) + |> Outbox.child_spec() + + Supervisor.init([manager, outboxes], + strategy: :rest_for_one, + auto_shutdown: :any_significant + ) + end +end diff --git a/lib/group/supervisor.ex b/lib/group/supervisor.ex index 0bbd773..df2e97d 100644 --- a/lib/group/supervisor.ex +++ b/lib/group/supervisor.ex @@ -51,6 +51,9 @@ defmodule Group.Supervisor do replicated_oplog_max_entries = positive_integer_opt(opts, :replicated_oplog_max_entries, 65_536) + replicated_snapshot_chunk_target_bytes = + positive_integer_opt(opts, :replicated_snapshot_chunk_target_bytes, 1_048_576) + replicated_anti_entropy_interval = positive_integer_opt(opts, :replicated_anti_entropy_interval, 1_000) @@ -78,6 +81,7 @@ defmodule Group.Supervisor do replicated_pg_receiver_local_request_quota: replicated_pg_receiver_local_request_quota, replica_transport: replica_transport, replicated_oplog_max_entries: replicated_oplog_max_entries, + replicated_snapshot_chunk_target_bytes: replicated_snapshot_chunk_target_bytes, replicated_anti_entropy_interval: replicated_anti_entropy_interval, replicated_peer_lease_timeout: replicated_peer_lease_timeout } diff --git a/priv/bench/README.md b/priv/bench/README.md index 6278aa5..679b01e 100644 --- a/priv/bench/README.md +++ b/priv/bench/README.md @@ -37,6 +37,14 @@ To isolate the 10,000-cluster lifecycle scenario: --coordinator-expr 'GroupBench.Distributed.run_many_clusters_only(shards: 4)' ``` +To measure the exact-snapshot fallback independently (one shard models one +busy lane of a much larger sharded deployment): + +```bash +./run_distributed.sh --shards 1 \ + --coordinator-expr 'GroupBench.Distributed.run_snapshot_sync_only(shards: 1, entries: 50000)' +``` + ## Local Scenarios All local benchmarks run for both the default (nil) cluster and a named cluster diff --git a/priv/bench/lib/group_bench/distributed.ex b/priv/bench/lib/group_bench/distributed.ex index 66b601f..9d3b34f 100644 --- a/priv/bench/lib/group_bench/distributed.ex +++ b/priv/bench/lib/group_bench/distributed.ex @@ -67,6 +67,23 @@ defmodule GroupBench.Distributed do IO.puts("\n Done.\n") end + def run_snapshot_sync_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 1) + entries = Keyword.get(opts, :entries, 50_000) + Process.put(:bench_shards, shards) + + header("Distributed Exact-Snapshot Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" entries: #{format_number(entries)}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + bench_snapshot_sync(@replicas, entries) + + IO.puts("\n Done.\n") + end + # ── Connection ──────────────────────────────────────────────────────── defp connect_replicas do @@ -208,6 +225,48 @@ defmodule GroupBench.Distributed do end end + defp bench_snapshot_sync([r1, r2] = replicas, key_count) do + header("Exact Snapshot (receiver below oplog floor)") + subheader("#{format_number(key_count)} keys") + + start_group_on(r1, + replicated_oplog_max_entries: 64, + replicated_anti_entropy_interval: 100, + replicated_peer_lease_timeout: 15_000 + ) + + :erpc.call( + r1, + GroupBench.Replica, + :bulk_register, + [@name, key_count, "snapshot-"], + 180_000 + ) + + {sync_us, _} = + :timer.tc(fn -> + start_group_on(r2, + replicated_oplog_max_entries: 64, + replicated_anti_entropy_interval: 100, + replicated_peer_lease_timeout: 15_000 + ) + + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= key_count + end, + 120_000 + ) + end) + + rate = if sync_us > 0, do: round(key_count * 1_000_000 / sync_us), else: 0 + + IO.puts(" sync time: #{format_number(div(sync_us, 1000))} ms") + IO.puts(" keys/sec: #{format_number(rate)}") + + stop_groups(replicas) + end + # ── 3. Concurrent cross-node writes ────────────────────────────────── defp bench_concurrent_cross_node([r1, r2] = replicas) do diff --git a/test/README.md b/test/README.md index 65c12aa..16ce634 100644 --- a/test/README.md +++ b/test/README.md @@ -18,6 +18,8 @@ mix test test/replica_model_property_test.exs # shrinkable model-based histories | `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | | `replica_adversarial_test.exs` | Reproducible mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | | `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | +| `replica_snapshot_test.exs` | Pure byte partitioning and set-valued private-ETS snapshot staging | +| `replica_snapshot_distributed_test.exs` | Real-node exact-snapshot loss, reorder, duplicate, conflicting retransmission, supersession, authority fencing, expiry, and shard-crash recovery | ## Model-based and formal checks @@ -29,7 +31,9 @@ PG key against an independent application-level lifecycle oracle. It also requires internal replica indexes to be consistent, every retained owner to be alive, and registry conflict losers to be dead. Restart, pruning, and named cluster histories retain independent C-owned state while A recovers, so repair -cannot pass merely by making one origin and one receiver agree. +cannot pass merely by making one origin and one receiver agree. Model groups +use a deliberately tiny snapshot target so pruning recovery traverses the real +multi-chunk assembly path. StreamData reports the ExUnit seed and shrinks a failure to its smallest command history. Local defaults are intentionally quick. Increase the budgets without @@ -257,6 +261,11 @@ disconnects one origin's real socket, prunes its oplog, reconnects it, and requires snapshot recovery without changing the third node's independent registry or PG state. +`replica_transport_outbox_test.exs` proves that a blocked sideband backend +cannot delay the Group-facing local send, frames expire behind that backend, +busy batches are not retried locally, and batching preserves per-target order. +The real three-node TCP recovery test runs through the same outbox path. + `Group.TestCluster.assert_replica_consistent/1` checks the public dual indexes plus registry claim authority, oplog/order equivalence, and contiguous retained stream ranges. Seeded tests additionally require every PID diff --git a/test/distributed_test.exs b/test/distributed_test.exs index e43d1c4..3a7c2d7 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -1833,7 +1833,7 @@ defmodule Group.DistributedTest do messages = TestCluster.shard_messages(node_b, name, 0) case Enum.filter(messages, fn - {:group_replica_frame, _source, {:delta_batch, 1, _runs}} -> + {:group_replica_frame, _source, {:delta_batch, _version, _runs}} -> true {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} -> @@ -1843,7 +1843,7 @@ defmodule Group.DistributedTest do false end) do [ - {:group_replica_frame, _source, {:delta_batch, 1, runs}}, + {:group_replica_frame, _source, {:delta_batch, _version, runs}}, {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} ] -> Enum.any?(runs, fn {_stream_id, _first_seq, records, _head} -> diff --git a/test/formal/README.md b/test/formal/README.md index 05c2c10..7cb743e 100644 --- a/test/formal/README.md +++ b/test/formal/README.md @@ -10,6 +10,13 @@ contract. It covers: - exact per-origin snapshot fallback; and - fair convergence after healing. +`SnapshotAssembly.tla` separately models the non-atomic wire delivery of an +exact snapshot. It explores arbitrary chunk loss, duplication, reordering, +newer-snapshot supersession, authority epoch changes, staging expiry, and +receiver crashes. Its invariants require visible data and the cursor to remain +at a previously committed exact state until every chunk of one valid snapshot +is present; stale or mixed partial state can never become visible. + The default TLC configuration uses three nodes: one origin and two independent receivers. The origin has one key, a two-record stream, a one-record oplog, and the system retains one arbitrary network frame. This forces delta repair, @@ -29,6 +36,11 @@ Run it with Java 17 or later and a current `tla2tools.jar`: ```bash TLA_JAR=/path/to/tla2tools.jar test/formal/check.sh + +TLA_JAR=/path/to/tla2tools.jar \ + TLA_SPEC="$PWD/test/formal/SnapshotAssembly.tla" \ + TLA_CONFIG="$PWD/test/formal/SnapshotAssembly.cfg" \ + test/formal/check.sh ``` `TLC_WORKERS` controls worker concurrency and defaults to 4. `TLA_CONFIG` can @@ -42,3 +54,6 @@ should be run periodically by increasing `Nodes`, `Origins`, `Keys`, `MaxSeq`, The checked three-node default explores 1,835,826 states, finds 490,236 distinct states to a depth of 30, and completes in roughly 1 minute 40 seconds on the development machine used for the validation run. + +The snapshot-assembly model explores 15,681 states, finds 1,088 distinct states +to a depth of 13, and completes in under a second on the same class of machine. diff --git a/test/formal/SnapshotAssembly.cfg b/test/formal/SnapshotAssembly.cfg new file mode 100644 index 0000000..38a4dba --- /dev/null +++ b/test/formal/SnapshotAssembly.cfg @@ -0,0 +1,7 @@ +SPECIFICATION Spec + +INVARIANTS + TypeOK + VisibleIsAnExactCommittedSnapshot + StagingNeverLeaksIntoVisible + StagingBelongsToOneSnapshot diff --git a/test/formal/SnapshotAssembly.tla b/test/formal/SnapshotAssembly.tla new file mode 100644 index 0000000..3d6c18d --- /dev/null +++ b/test/formal/SnapshotAssembly.tla @@ -0,0 +1,181 @@ +------------------------- MODULE SnapshotAssembly ------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +Finite model of the exact-snapshot chunk assembly boundary. It deliberately +models two snapshots in one authority epoch plus a new-epoch snapshot so TLC +can explore loss, duplication, reordering, supersession, stale final chunks, +expiry, and receiver crashes independently of the larger anti-entropy model. +*) + +Snapshots == {1, 2, 3} +Chunks == {1, 2} +Rows == {"a", "b", "c", "d"} + +SnapshotEpoch(snapshot) == + CASE snapshot = 0 -> 0 + [] snapshot = 1 -> 1 + [] snapshot = 2 -> 1 + [] snapshot = 3 -> 2 + +SnapshotSeq(snapshot) == + CASE snapshot = 0 -> 0 + [] snapshot = 1 -> 1 + [] snapshot = 2 -> 2 + [] snapshot = 3 -> 1 + +SnapshotRows(snapshot) == + CASE snapshot = 1 -> {"a", "b"} + [] snapshot = 2 -> {"c", "d"} + [] snapshot = 3 -> {"a", "d"} + +ChunkRows(snapshot, chunk) == + CASE snapshot = 1 /\ chunk = 1 -> {"a"} + [] snapshot = 1 /\ chunk = 2 -> {"b"} + [] snapshot = 2 /\ chunk = 1 -> {"c"} + [] snapshot = 2 /\ chunk = 2 -> {"d"} + [] snapshot = 3 /\ chunk = 1 -> {"a"} + [] snapshot = 3 /\ chunk = 2 -> {"d"} + +Message == [snapshot : Snapshots, chunk : Chunks] + +VARIABLES authorityEpoch, + cursor, + visible, + stagedSnapshot, + stagedChunks, + stagedRows, + messages + +vars == + <> + +Init == + /\ authorityEpoch = 1 + /\ cursor = 0 + /\ visible = {} + /\ stagedSnapshot = 0 + /\ stagedChunks = {} + /\ stagedRows = {} + /\ messages = {} + +Send(snapshot, chunk) == + /\ messages' = messages \union + {[snapshot |-> snapshot, chunk |-> chunk]} + /\ UNCHANGED <> + +Valid(message) == + /\ SnapshotEpoch(message.snapshot) = authorityEpoch + /\ SnapshotSeq(message.snapshot) > cursor + +StartsNewAssembly(message) == + /\ Valid(message) + /\ \/ stagedSnapshot = 0 + \/ SnapshotEpoch(stagedSnapshot) # authorityEpoch + \/ SnapshotSeq(message.snapshot) > SnapshotSeq(stagedSnapshot) + +StartAssembly(message) == + /\ StartsNewAssembly(message) + /\ stagedSnapshot' = message.snapshot + /\ stagedChunks' = {message.chunk} + /\ stagedRows' = ChunkRows(message.snapshot, message.chunk) + /\ UNCHANGED <> + +ContinueAssembly(message) == + /\ Valid(message) + /\ stagedSnapshot = message.snapshot + /\ LET nextChunks == stagedChunks \union {message.chunk} + nextRows == stagedRows \union + ChunkRows(message.snapshot, message.chunk) + IN IF nextChunks = Chunks + THEN /\ cursor' = SnapshotSeq(message.snapshot) + /\ visible' = SnapshotRows(message.snapshot) + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + ELSE /\ UNCHANGED <> + /\ stagedChunks' = nextChunks + /\ stagedRows' = nextRows + /\ UNCHANGED <> + +IgnoreChunk(message) == + /\ ~StartsNewAssembly(message) + /\ ~(/\ Valid(message) + /\ stagedSnapshot = message.snapshot) + /\ UNCHANGED vars + +Deliver(message) == + /\ message \in messages + /\ \/ StartAssembly(message) + \/ ContinueAssembly(message) + \/ IgnoreChunk(message) + +Drop(message) == + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED <> + +InstallNewAuthority == + /\ authorityEpoch = 1 + /\ authorityEpoch' = 2 + /\ cursor' = 0 + /\ visible' = {} + (* The implementation may retain invisible old staging until expiry. *) + /\ UNCHANGED <> + +ExpireStaging == + /\ stagedSnapshot # 0 + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ UNCHANGED <> + +CrashReceiver == + /\ stagedSnapshot # 0 + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ UNCHANGED <> + +Next == + \/ \E snapshot \in Snapshots, chunk \in Chunks : Send(snapshot, chunk) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ InstallNewAuthority + \/ ExpireStaging + \/ CrashReceiver + +TypeOK == + /\ authorityEpoch \in {1, 2} + /\ cursor \in 0..2 + /\ visible \subseteq Rows + /\ stagedSnapshot \in {0} \union Snapshots + /\ stagedChunks \subseteq Chunks + /\ stagedRows \subseteq Rows + /\ messages \subseteq Message + +VisibleIsAnExactCommittedSnapshot == + \/ /\ authorityEpoch = 1 + /\ \/ /\ cursor = 0 /\ visible = {} + \/ /\ cursor = 1 /\ visible = SnapshotRows(1) + \/ /\ cursor = 2 /\ visible = SnapshotRows(2) + \/ /\ authorityEpoch = 2 + /\ \/ /\ cursor = 0 /\ visible = {} + \/ /\ cursor = 1 /\ visible = SnapshotRows(3) + +StagingNeverLeaksIntoVisible == + stagedSnapshot # 0 /\ stagedChunks # Chunks => + VisibleIsAnExactCommittedSnapshot + +StagingBelongsToOneSnapshot == + stagedSnapshot # 0 => + /\ stagedRows = + UNION {ChunkRows(stagedSnapshot, chunk) : chunk \in stagedChunks} + /\ stagedChunks # Chunks + +Spec == Init /\ [][Next]_vars + +============================================================================= diff --git a/test/formal/check.sh b/test/formal/check.sh index 1336a08..1f5b1c5 100755 --- a/test/formal/check.sh +++ b/test/formal/check.sh @@ -9,6 +9,7 @@ fi repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" metadir="${repo_root}/tmp/tlc" config="${TLA_CONFIG:-${repo_root}/test/formal/GroupAntiEntropy.cfg}" +spec="${TLA_SPEC:-${repo_root}/test/formal/GroupAntiEntropy.tla}" mkdir -p "${metadir}" exec java -XX:+UseParallelGC -cp "${TLA_JAR}" tlc2.TLC \ @@ -16,4 +17,4 @@ exec java -XX:+UseParallelGC -cp "${TLA_JAR}" tlc2.TLC \ -metadir "${metadir}" \ -workers "${TLC_WORKERS:-4}" \ -config "${config}" \ - "${repo_root}/test/formal/GroupAntiEntropy.tla" + "${spec}" diff --git a/test/group_test.exs b/test/group_test.exs index 572eb46..f6356a5 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2356,7 +2356,7 @@ defmodule GroupTest do {:erlang, :send_nosuspend, [ {^shard_name, ^local_node}, - {:group_replica_frame, ^local_node, {:delta_batch, 1, runs}}, + {:group_replica_frame, ^local_node, {:delta_batch, _version, runs}}, [:noconnect] ]}}, 1_000 diff --git a/test/mutation/README.md b/test/mutation/README.md index 269edf8..fa9e50e 100644 --- a/test/mutation/README.md +++ b/test/mutation/README.md @@ -5,7 +5,9 @@ catastrophic protocol obligations. It covers generation and epoch fencing, contiguous sequence application, exact registry and PG snapshots, below-floor repair, process-down sequencing, conflict-loser retirement, authority fanout, per-lane authority installation, periodic head advertisement, interrupted -journal/index repair, and named-cluster close completion. +journal/index repair, and named-cluster close completion. Snapshot calibration +also covers incomplete commit, conflicting retransmission rows, newer-snapshot +supersession, stale-authority fencing, and staging expiry. The runner first verifies every unmodified regression target. It then copies the current checkout once per mutant, changes only that copy, recompiles it, diff --git a/test/mutation/run.exs b/test/mutation/run.exs index 60ecf6f..6efefbe 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -62,23 +62,120 @@ defmodule Group.MutationCampaign do %{ name: "registry_snapshot_is_additive", file: "lib/group/replica/data.ex", - correct_source: "existing = registry_claims_for_stream(name, shard, stream_id)", - faulty_source: "existing = []", - test: ["test/distributed_test.exs:4030"] + correct_source: "Enum.reduce(existing, MapSet.new(), fn {key, pid, _meta, _time}, keys ->", + faulty_source: + "Enum.reduce(Enum.take(existing, 0), MapSet.new(), fn {key, pid, _meta, _time}, keys ->", + test: ["test/replica_snapshot_distributed_test.exs:16"] }, %{ name: "pg_snapshot_is_additive", file: "lib/group/replica.ex", + correct_source: "Enum.reduce(current, events, fn {key, pid, old_meta, _old_time}, acc ->", + faulty_source: + "Enum.reduce(Enum.take(current, 0), events, fn {key, pid, old_meta, _old_time}, acc ->", + test: ["test/replica_snapshot_distributed_test.exs:16"] + }, + %{ + name: "single_chunk_registry_snapshot_is_additive", + file: "lib/group/replica/data.ex", + correct_source: "Enum.each(existing, fn {key, pid, _meta, _time} ->", + faulty_source: "Enum.each(Enum.take(existing, 0), fn {key, pid, _meta, _time} ->", + test: ["test/distributed_test.exs:4030"] + }, + %{ + name: "single_chunk_pg_snapshot_is_additive", + file: "lib/group/replica.ex", + correct_source: " current\n |> Map.keys()\n", + faulty_source: " %{}\n |> Map.keys()\n", + test: ["test/distributed_test.exs:4030"] + }, + %{ + name: "commit_incomplete_snapshot", + file: "lib/group/replica.ex", correct_source: """ - current = - state.name - |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) - |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + if MapSet.size(transfer.received) == transfer.chunk_count do + if transfer.registry_seen == transfer.registry_count and + transfer.pg_seen == transfer.pg_count do + commit_snapshot_transfer(state, key, source_node, stream_id, transfer) + else + discard_snapshot_transfer(state, key) + end + else + state + end """, faulty_source: """ - current = %{} + if MapSet.size(transfer.received) >= 1 do + commit_snapshot_transfer(state, key, source_node, stream_id, transfer) + else + state + end """, - test: ["test/distributed_test.exs:4030"] + test: ["test/replica_snapshot_distributed_test.exs:16"] + }, + %{ + name: "allow_duplicate_snapshot_rows", + file: "lib/group/replica/snapshot.ex", + correct_source: """ + if :ets.insert_new(table, objects) and + :ets.info(table, :size) - size_before == length(objects) do + """, + faulty_source: """ + if :ets.insert(table, objects) and size_before >= 0 do + """, + test: ["test/replica_snapshot_distributed_test.exs:164"] + }, + %{ + name: "do_not_supersede_partial_snapshot", + file: "lib/group/replica.ex", + correct_source: """ + %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> + state = discard_snapshot_transfer(state, key) + {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + """, + faulty_source: """ + %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> + _ = existing_seq + {:ignore, state} + """, + test: ["test/replica_snapshot_distributed_test.exs:107"] + }, + %{ + name: "accept_stale_snapshot_authority", + file: "lib/group/replica.ex", + correct_source: """ + defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do + valid_remote_stream?(state, source_node, stream_id) and + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end + """, + faulty_source: """ + defp valid_snapshot_stream?(state, _source_node, stream_id, snapshot_seq) do + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end + """, + test: ["test/replica_snapshot_distributed_test.exs:202"] + }, + %{ + name: "disable_snapshot_staging_expiry", + file: "lib/group/replica.ex", + correct_source: """ + if now - transfer.last_progress > acc.replicated_peer_lease_timeout do + discard_snapshot_transfer(acc, key) + else + acc + end + """, + faulty_source: """ + _ = {now, transfer} + + if Process.get(:force_snapshot_staging_expiry, false) do + discard_snapshot_transfer(acc, key) + else + acc + end + """, + test: ["test/replica_snapshot_distributed_test.exs:202"] }, %{ name: "disable_below_floor_snapshot", @@ -158,6 +255,32 @@ defmodule Group.MutationCampaign do """, test: ["test/distributed_test.exs:5531"] }, + %{ + name: "assume_authority_fanout_reaches_late_lane", + file: "lib/group/replica.ex", + correct_source: """ + defp install_current_replica_lane(state, remote_node, generation) do + old_generation = + Data.remote_view_generation(state.name, state.shard_index, remote_node) + + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = purge_remote_streams_outside_authority(state, remote_node) + :ok = install_replica_view(state, remote_node, generation) + + state + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + end + """, + faulty_source: """ + defp install_current_replica_lane(state, remote_node, generation) do + _ = {remote_node, generation} + state + end + """, + test: ["test/replica_snapshot_distributed_test.exs:329"] + }, %{ name: "skip_generation_purge", file: "lib/group/replica.ex", diff --git a/test/replica_model_property_test.exs b/test/replica_model_property_test.exs index 4999c3a..e4175d2 100644 --- a/test/replica_model_property_test.exs +++ b/test/replica_model_property_test.exs @@ -41,6 +41,7 @@ defmodule Group.ReplicaModelPropertyTest do replicated_sender_buffer_size: 1, replicated_anti_entropy_interval: 60_000, replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, replicated_oplog_max_entries: 4 ] @@ -87,6 +88,7 @@ defmodule Group.ReplicaModelPropertyTest do replicated_sender_buffer_size: 1, replicated_anti_entropy_interval: 60_000, replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, replicated_oplog_max_entries: 2 ] @@ -201,9 +203,17 @@ defmodule Group.ReplicaModelPropertyTest do scheduler = Enum.reduce(41..46, scheduler, fn owner_id, state -> - state - |> ReplicaModelScheduler.execute({:register, owner_id, :a, 0, owner_id}) - |> ReplicaModelScheduler.execute({:unregister, owner_id, 0}) + state = + ReplicaModelScheduler.execute( + state, + {:register, owner_id, :a, 0, owner_id} + ) + + if rem(owner_id, 2) == 0 do + ReplicaModelScheduler.execute(state, {:unregister, owner_id, 0}) + else + state + end end) scheduler = @@ -358,6 +368,7 @@ defmodule Group.ReplicaModelPropertyTest do replicated_sender_buffer_size: 1, replicated_anti_entropy_interval: 60_000, replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, replicated_oplog_max_entries: Keyword.fetch!(overrides, :oplog) ] end diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs new file mode 100644 index 0000000..a517770 --- /dev/null +++ b/test/replica_snapshot_distributed_test.exs @@ -0,0 +1,608 @@ +defmodule Group.ReplicaSnapshotDistributedTest do + use ExUnit.Case, async: false + + @moduletag :capture_log + @moduletag timeout: 120_000 + + alias Group.TestCluster + + setup_all do + peers = TestCluster.start_peers(2, schedulers: 4) + on_exit(fn -> TestCluster.stop_peers(peers) end) + [{_, node_a}, {_, node_b}] = peers + {:ok, node_a: node_a, node_b: node_b} + end + + test "loss, reordering, and duplication expose nothing until exact commit", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + + stale_reg_key = "snapshot/stale-reg" + stale_pg_key = "snapshot/stale-pg" + stale_reg_pid = TestCluster.spawn_register(node_a, name, stale_reg_key, %{stale: true}) + stale_pg_pid = TestCluster.spawn_join(node_a, name, stale_pg_key, %{stale: true}) + + TestCluster.assert_eventually(fn -> + match?({^stale_reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key])) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + end) + + stream_id = local_stream(node_a, name, nil) + old_cursor = replica_cursor(node_b, name, stream_id) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh_reg = + for index <- 1..8 do + key = "snapshot/fresh-reg/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("r", 160) + })} + end + + fresh_pg_key = "snapshot/fresh-pg" + + fresh_pg = + for _index <- 1..8 do + TestCluster.spawn_join(node_a, name, fresh_pg_key, %{ + payload: String.duplicate("p", 160) + }) + end + + TestCluster.flush_shards(node_a, name) + frames = capture_snapshot(node_a, node_b, name, stream_id, old_cursor + 1) + assert length(frames) > 2 + [missing | delivered] = frames + + deliver_frames(node_b, node_a, name, Enum.reverse(delivered)) + deliver_frames(node_b, node_a, name, [List.last(delivered)]) + TestCluster.flush_shards(node_b, name) + + assert replica_cursor(node_b, name, stream_id) == old_cursor + + assert match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) + + assert match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + + assert Enum.all?(fresh_reg, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + assert TestCluster.rpc!(node_b, Group, :members, [name, fresh_pg_key]) == [] + + deliver_frames(node_b, node_a, name, [missing]) + TestCluster.flush_shards(node_b, name) + snapshot_seq = elem(missing, 3) + + assert replica_cursor(node_b, name, stream_id) == snapshot_seq + assert TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) == nil + assert TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) == [] + + assert Enum.all?(fresh_reg, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert MapSet.new( + Enum.map( + TestCluster.rpc!(node_b, Group, :members, [name, fresh_pg_key]), + &elem(&1, 0) + ) + ) == + MapSet.new(fresh_pg) + + assert snapshot_transfer_count(node_b, name) == 0 + end + + test "a newer exact snapshot supersedes an incomplete older one", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/superseded/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("o", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + older = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(older) > 1 + + Enum.each(old_entries, fn {_key, pid} -> + TestCluster.rpc!(node_a, Process, :exit, [pid, :kill]) + end) + + fresh_entries = + for index <- 1..8 do + key = "snapshot/replacement/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("n", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + newer = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert elem(hd(newer), 3) > elem(hd(older), 3) + + deliver_frames(node_b, node_a, name, [hd(older)]) + assert snapshot_transfer_count(node_b, name) == 1 + + deliver_frames(node_b, node_a, name, Enum.reverse(newer)) + deliver_frames(node_b, node_a, name, tl(older)) + TestCluster.flush_shards(node_b, name) + + assert replica_cursor(node_b, name, stream_id) == elem(hd(newer), 3) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + assert Enum.all?(fresh_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert snapshot_transfer_count(node_b, name) == 0 + end + + test "conflicting retransmission chunks cannot manufacture an exact snapshot", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/conflicting/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("m", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + [first, second | rest] = frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + [first_row | _] = elem(first, 8) + [_second_row | second_tail] = elem(second, 8) + conflicting_second = put_elem(second, 8, [first_row | second_tail]) + + deliver_frames(node_b, node_a, name, [first, conflicting_second | rest]) + + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + deliver_frames(node_b, node_a, name, frames) + + assert replica_cursor(node_b, name, stream_id) == elem(first, 3) + + assert Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + test "an authority epoch change fences partial chunks and their staging expires", context do + cluster = "snapshot-epoch" + + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 250 + ) + + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, cluster]) + :ok = TestCluster.rpc!(node_b, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/epoch/#{index}" + + {key, + TestCluster.spawn_register_in_cluster( + node_a, + name, + key, + %{payload: String.duplicate("e", 160)}, + cluster + )} + end + + TestCluster.flush_shards(node_a, name) + old_stream = local_stream(node_a, name, cluster) + old_epoch = Group.Replica.Protocol.stream_epoch(old_stream) + frames = capture_snapshot(node_a, node_b, name, old_stream, 1) + assert length(frames) > 1 + {partial, [last]} = Enum.split(frames, -1) + deliver_frames(node_b, node_a, name, partial) + assert snapshot_transfer_count(node_b, name) == 1 + + :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, cluster]) + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + epoch = + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + cluster + ]) + + not is_nil(epoch) and epoch != old_epoch + end) + + deliver_frames(node_b, node_a, name, [last]) + TestCluster.flush_shards(node_b, name) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: cluster]]) == nil + end) + + assert replica_cursor(node_b, name, old_stream) == 0 + + TestCluster.assert_eventually( + fn -> snapshot_transfer_count(node_b, name) == 0 end, + timeout: 2_000, + interval: 25 + ) + end + + test "an origin restart fences a partial old generation and commits the new exact snapshot", + context do + %{name: name, node_a: node_a, node_b: node_b, opts: opts} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/restart/old/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("o", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + old_stream = local_stream(node_a, name, nil) + old_frames = capture_snapshot(node_a, node_b, name, old_stream, 1) + assert length(old_frames) > 1 + + {old_partial, _old_tail} = Enum.split(old_frames, -1) + deliver_frames(node_b, node_a, name, old_partial) + assert snapshot_transfer_count(node_b, name) == 1 + assert replica_cursor(node_b, name, old_stream) == 0 + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + new_stream = local_stream(node_a, name, nil) + refute new_stream == old_stream + new_generation = Group.Replica.Protocol.stream_generation(new_stream) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + new_generation + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + new_entries = + for index <- 1..8 do + key = "snapshot/restart/new/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("n", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + new_frames = capture_snapshot(node_a, node_b, name, new_stream, 1) + assert length(new_frames) > 1 + + deliver_frames(node_b, node_a, name, Enum.reverse(new_frames)) + new_snapshot_seq = new_frames |> hd() |> elem(3) + + assert replica_cursor(node_b, name, new_stream) == new_snapshot_seq + + assert Enum.all?(new_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + # Replaying every old-generation chunk after the new exact commit must not + # resurrect the old slice or advance its cursor. + deliver_frames(node_b, node_a, name, old_frames) + assert replica_cursor(node_b, name, old_stream) == 0 + assert replica_cursor(node_b, name, new_stream) == new_snapshot_seq + + assert Enum.all?(new_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + TestCluster.assert_eventually( + fn -> snapshot_transfer_count(node_b, name) == 0 end, + timeout: 5_000, + interval: 25 + ) + end + + test "a receiver shard crash destroys partial staging and anti-entropy rebuilds exactly", + context do + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/crash/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("c", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(frames) > 1 + deliver_frames(node_b, node_a, name, [hd(frames)]) + + {old_shard, staging_info_after_crash} = + TestCluster.rpc!(node_b, TestCluster, :kill_shard_with_snapshot_staging, [name, 0]) + + assert staging_info_after_crash == :undefined + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) do + pid when is_pid(pid) -> pid != old_shard + nil -> false + end + end) + + assert snapshot_transfer_count(node_b, name) == 0 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> + Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end, + timeout: 10_000, + interval: 25 + ) + + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + test "authority fanout tolerates a sibling that is not registered", context do + name = :"authority_startup_fanout_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + for node <- [context.node_a, context.node_b] do + {:ok, _pid} = TestCluster.start_group(node, opts) + end + + TestCluster.assert_eventually(fn -> + context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) + end) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, Group.TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + sibling = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + sibling_monitor = Process.monitor(sibling) + TestCluster.rpc!(context.node_b, Process, :exit, [sibling, :kill]) + assert_receive {:DOWN, ^sibling_monitor, :process, ^sibling, :killed}, 5_000 + + assert TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) == + nil + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, "late-lane-authority"]) + + {generation, revision, epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + control_monitor = Process.monitor(target_control) + + send( + target_control, + {:replica_hello, source_control, Group.Replica.Protocol.version(), generation, revision, + epochs, Group.Replica.Transport.Distribution.id(), + Group.Replica.Transport.Distribution.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + refute_receive {:DOWN, ^control_monitor, :process, ^target_control, _reason}, 250 + assert TestCluster.rpc!(context.node_b, Process, :alive?, [target_control]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + sibling = + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) + + view_generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) + + view_exact_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 1, context.node_a] + ) + + exact_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + is_pid(sibling) and view_generation == generation and + view_exact_revision == exact_revision + end) + end + + defp start_pair(context, extra_opts \\ []) do + name = :"snapshot_chunks_#{System.unique_integer([:positive])}" + + opts = + Keyword.merge( + [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 2, + replicated_snapshot_chunk_target_bytes: 700, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ], + extra_opts + ) + + for node <- [context.node_a, context.node_b] do + {:ok, _pid} = TestCluster.start_group(node, opts) + end + + TestCluster.assert_eventually(fn -> + context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) and + context.node_a in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) + end) + + %{name: name, node_a: context.node_a, node_b: context.node_b, opts: opts} + end + + defp capture_snapshot(node_a, node_b, name, stream_id, next_seq) do + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :clear_captured, [name]) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:snapshot_chunk]} + ]) + + :ok = + TestCluster.rpc!(node_a, Group.Replica.Transport, :deliver, [ + name, + node_b, + 0, + {:needs, Group.Replica.Protocol.version(), [{stream_id, next_seq}]} + ]) + + TestCluster.flush_shards(node_a, name) + + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + |> Enum.flat_map(fn + {^node_b, 0, + {:snapshot_chunk, _version, ^stream_id, _seq, _index, _count, _reg_count, _pg_count, _reg, + _pg} = frame} -> + [frame] + + _other -> + [] + end) + |> Enum.sort_by(&elem(&1, 4)) + end + + defp deliver_frames(node_b, node_a, name, frames) do + Enum.each(frames, fn frame -> + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 0, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + end + + defp local_stream(node, name, cluster) do + TestCluster.rpc!(node, Group.Replica.Data, :local_stream_id, [name, 0, cluster]) + end + + defp replica_cursor(node, name, stream_id) do + TestCluster.rpc!(node, Group.Replica.Data, :replica_cursor, [name, 0, stream_id]) + end + + defp snapshot_transfer_count(node, name) do + TestCluster.rpc!(node, :erlang, :map_size, [ + TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]).snapshot_transfers + ]) + end +end diff --git a/test/replica_snapshot_test.exs b/test/replica_snapshot_test.exs new file mode 100644 index 0000000..d87700f --- /dev/null +++ b/test/replica_snapshot_test.exs @@ -0,0 +1,75 @@ +defmodule Group.ReplicaSnapshotTest do + use ExUnit.Case, async: true + + alias Group.Replica.Snapshot + + test "partitions a complete exact slice into byte-bounded deterministic chunks" do + pid = self() + metadata = %{payload: String.duplicate("x", 96)} + + registry_rows = + for index <- 1..80 do + {"registry/#{index}", pid, metadata, index} + end + + pg_rows = + for index <- 1..80 do + {"pg/#{index}", pid, metadata, index} + end + + target = 2_048 + stream_id = {:group, node(), make_ref(), 0, nil, make_ref()} + envelope = Snapshot.frame_envelope_bytes(stream_id, 123, 80, 80) + + snapshot = + Snapshot.chunk_rows(Enum.reverse(registry_rows), Enum.reverse(pg_rows), target, envelope) + + assert snapshot.registry_count == 80 + assert snapshot.pg_count == 80 + assert length(snapshot.chunks) > 1 + + assert snapshot.chunks == + Snapshot.chunk_rows(registry_rows, pg_rows, target, envelope).chunks + + assert snapshot.chunks |> Enum.flat_map(&elem(&1, 0)) |> MapSet.new() == + MapSet.new(registry_rows) + + assert snapshot.chunks |> Enum.flat_map(&elem(&1, 1)) |> MapSet.new() == + MapSet.new(pg_rows) + + chunk_count = length(snapshot.chunks) + + Enum.with_index(snapshot.chunks, 1) + |> Enum.each(fn {{registry, pg}, index} -> + frame = + {:snapshot_chunk, Group.Replica.Protocol.version(), stream_id, 123, index, chunk_count, + snapshot.registry_count, snapshot.pg_count, registry, pg} + + assert :erlang.external_size(frame) <= target + end) + end + + test "represents an empty exact slice and permits one intrinsically oversized row" do + assert Snapshot.chunk_rows([], [], 1_024).chunks == [{[], []}] + + row = {"large", self(), String.duplicate("x", 4_096), 1} + snapshot = Snapshot.chunk_rows([row], [], 1_024) + assert snapshot.chunks == [{[row], []}] + end + + test "staging is set-valued across chunks and remains private to its owner" do + table = Snapshot.new_staging_table() + registry = {"registry", self(), %{v: 1}, 1} + pg = {"pg", self(), %{v: 2}, 2} + + assert :ok = Snapshot.stage_rows(table, 1, [registry], [pg]) + assert {:error, :duplicate_row} = Snapshot.stage_rows(table, 2, [registry], []) + + assert Snapshot.fold_registry(table, 1, [], fn row, acc -> [row | acc] end) == [registry] + assert Snapshot.fold_pg(table, 1, [], fn row, acc -> [row | acc] end) == [pg] + assert Snapshot.member_pg?(table, "pg", self()) + + assert :ok = Snapshot.delete_staging_table(table) + assert :ets.info(table) == :undefined + end +end diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs new file mode 100644 index 0000000..329ed96 --- /dev/null +++ b/test/replica_transport_outbox_test.exs @@ -0,0 +1,155 @@ +defmodule Group.ReplicaTransportOutboxTest do + use ExUnit.Case, async: true + + alias Group.Replica.Transport.Outbox + + defmodule Backend do + @behaviour Outbox + + @impl true + def init_outbox(group, shard, opts) do + {:ok, + %{ + controller: Keyword.fetch!(opts, :controller), + group: group, + shard: shard, + result: Keyword.get(opts, :backend_result, :ok), + sleep: Keyword.get(opts, :backend_sleep, 0) + }} + end + + @impl true + def send_batch(target_node, frames, deadline, state) do + send( + state.controller, + {:outbox_batch, state.group, state.shard, target_node, frames, deadline} + ) + + if state.sleep > 0, do: Process.sleep(state.sleep) + {state.result, state} + end + end + + test "batches frames per target while preserving per-target order" do + group = unique_group(:batch) + target_a = :"outbox-a@test" + target_b = :"outbox-b@test" + + start_outboxes(group, + outbox_batch_size: 3, + outbox_flush_interval: 1_000 + ) + + assert :ok = Outbox.try_send(group, target_a, 0, {:frame, 1}, outbox_deadline: 1_000) + assert :ok = Outbox.try_send(group, target_b, 0, {:frame, 2}, outbox_deadline: 1_000) + assert :ok = Outbox.try_send(group, target_a, 0, {:frame, 3}, outbox_deadline: 1_000) + + batches = + for _ <- 1..2, into: %{} do + assert_receive {:outbox_batch, ^group, 0, target, frames, deadline}, 1_000 + assert deadline > Outbox.monotonic_ms() + {target, frames} + end + + assert batches == %{ + target_a => [{:frame, 1}, {:frame, 3}], + target_b => [{:frame, 2}] + } + end + + test "a blocked backend never blocks the Group-facing local send" do + group = unique_group(:blocked) + target = :"outbox-blocked@test" + + start_outboxes(group, + outbox_batch_size: 1, + backend_sleep: 200 + ) + + assert :ok = Outbox.try_send(group, target, 0, :first, outbox_deadline: 1_000) + assert_receive {:outbox_batch, ^group, 0, ^target, [:first], _deadline}, 1_000 + + caller = self() + + spawn(fn -> + result = Outbox.try_send(group, target, 0, :expires_behind_backend, outbox_deadline: 10) + send(caller, {:try_send_returned, result}) + end) + + assert_receive {:try_send_returned, :ok}, 100 + refute_receive {:outbox_batch, ^group, 0, ^target, [:expires_behind_backend], _deadline}, 300 + end + + test "expired frames and backend backpressure are dropped without local retries" do + expired_group = unique_group(:expired) + target = :"outbox-expired@test" + + start_outboxes(expired_group, + outbox_flush_interval: 50 + ) + + assert :ok = + Outbox.try_send(expired_group, target, 0, :expired, outbox_deadline: 5) + + refute_receive {:outbox_batch, ^expired_group, 0, ^target, [:expired], _deadline}, 100 + + busy_group = unique_group(:busy) + + start_outboxes(busy_group, + outbox_batch_size: 1, + backend_result: :busy + ) + + assert :ok = Outbox.try_send(busy_group, target, 0, :busy, outbox_deadline: 1_000) + assert_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 1_000 + refute_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 100 + end + + test "complete inbound batches use one authenticated local delivery" do + group = unique_group(:deliver) + source_node = :"outbox-source@test" + parent = self() + shard_name = Group.Replica.shard_name(group, 0) + + receiver = + spawn(fn -> + Process.register(self(), shard_name) + send(parent, :receiver_ready) + + receive do + message -> send(parent, {:receiver_message, message}) + end + end) + + assert_receive :receiver_ready + + assert :ok = + Group.Replica.Transport.deliver_batch( + group, + source_node, + 0, + [{:heads, 1, []}, {:needs, 1, []}] + ) + + assert_receive {:receiver_message, + {:group_replica_batch, ^source_node, [{:heads, 1, []}, {:needs, 1, []}]}} + + refute Process.alive?(receiver) + end + + defp start_outboxes(group, opts) do + base = [ + name: group, + num_shards: 1, + backend: Backend, + controller: self(), + outbox_deadline: 100 + ] + + start_supervised!(Outbox.child_spec(Keyword.merge(base, opts))) + end + + defp unique_group(suffix) do + :"outbox_#{suffix}_#{System.unique_integer([:positive])}" + end +end diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index 8f97eb7..80cd37e 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -54,6 +54,12 @@ defmodule Group.TestCluster do end) end + @doc false + def resume_if_alive(pid) when is_pid(pid) do + if Process.alive?(pid), do: :sys.resume(pid) + :ok + end + @doc "Call a function on a remote node, raise on badrpc" def rpc!(node, mod, fun, args) do case :rpc.call(node, mod, fun, args) do @@ -503,6 +509,22 @@ defmodule Group.TestCluster do end) end + @doc false + def kill_shard_with_snapshot_staging(name, shard_index) do + shard = Process.whereis(Group.Replica.shard_name(name, shard_index)) + state = :sys.get_state(shard) + {_key, transfer} = Enum.at(state.snapshot_transfers, 0) + monitor = Process.monitor(shard) + Process.exit(shard, :kill) + + receive do + {:DOWN, ^monitor, :process, ^shard, :killed} -> + {shard, :ets.info(transfer.table)} + after + 5_000 -> raise "snapshot staging owner did not terminate" + end + end + @doc "Returns the current message_queue_len for a shard on a remote node." def shard_message_queue_len(node, name, shard) do :erpc.call(node, __MODULE__, :do_shard_message_queue_len, [name, shard]) From bc5bf48f1ada2c57696ef84e25806b7d2537d371 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Tue, 11 Aug 2026 04:22:31 +0000 Subject: [PATCH 5/7] test: add anti-entropy release qualification --- CHANGELOG.md | 6 + CLAUDE.md | 391 ++--- README.md | 53 +- lib/group.ex | 12 +- lib/group/replica/data.ex | 46 +- lib/group/replica/transport.ex | 10 +- mix.exs | 16 + test/README.md | 25 +- test/formal/GroupAntiEntropyExtended.cfg | 17 + test/formal/PeerEviction.cfg | 15 + test/formal/PeerEviction.tla | 207 +++ test/formal/README.md | 26 +- test/formal/check_matrix.sh | 22 + test/jepsen/.gitignore | 5 + test/jepsen/Dockerfile.node | 22 + test/jepsen/README.md | 149 ++ test/jepsen/campaign.sh | 67 + test/jepsen/checker.sh | 6 + test/jepsen/docker-compose.yml | 49 + test/jepsen/entrypoint.sh | 10 + test/jepsen/lein.sh | 20 + test/jepsen/node.exs | 1376 ++++++++++++++++++ test/jepsen/project.clj | 8 + test/jepsen/qualify.sh | 55 + test/jepsen/run.sh | 41 + test/jepsen/src/group/jepsen/client.clj | 132 ++ test/jepsen/src/group/jepsen/core.clj | 187 +++ test/jepsen/src/group/jepsen/db.clj | 28 + test/jepsen/src/group/jepsen/docker.clj | 121 ++ test/jepsen/src/group/jepsen/model.clj | 219 +++ test/jepsen/src/group/jepsen/nemesis.clj | 166 +++ test/jepsen/test/group/jepsen/model_test.clj | 208 +++ test/replica_adversarial_test.exs | 114 +- 33 files changed, 3578 insertions(+), 251 deletions(-) create mode 100644 test/formal/GroupAntiEntropyExtended.cfg create mode 100644 test/formal/PeerEviction.cfg create mode 100644 test/formal/PeerEviction.tla create mode 100755 test/formal/check_matrix.sh create mode 100644 test/jepsen/.gitignore create mode 100644 test/jepsen/Dockerfile.node create mode 100644 test/jepsen/README.md create mode 100755 test/jepsen/campaign.sh create mode 100755 test/jepsen/checker.sh create mode 100644 test/jepsen/docker-compose.yml create mode 100755 test/jepsen/entrypoint.sh create mode 100755 test/jepsen/lein.sh create mode 100644 test/jepsen/node.exs create mode 100644 test/jepsen/project.clj create mode 100755 test/jepsen/qualify.sh create mode 100755 test/jepsen/run.sh create mode 100644 test/jepsen/src/group/jepsen/client.clj create mode 100644 test/jepsen/src/group/jepsen/core.clj create mode 100644 test/jepsen/src/group/jepsen/db.clj create mode 100644 test/jepsen/src/group/jepsen/docker.clj create mode 100644 test/jepsen/src/group/jepsen/model.clj create mode 100644 test/jepsen/src/group/jepsen/nemesis.clj create mode 100644 test/jepsen/test/group/jepsen/model_test.clj diff --git a/CHANGELOG.md b/CHANGELOG.md index 671cf9d..2f41d3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ ## Unreleased +- Add layered anti-entropy qualification: three-node StreamData lifecycle + models, seeded adversarial transport histories, TLA+ models for convergence, + chunk assembly, and permanent peer eviction, plus a Docker-backed Jepsen + oracle across distribution, sideband TCP, and lossy/reordering transports. + `mix test` is the every-PR ExUnit/property/checker gate and `mix test.soak` + runs the six-profile nightly/release campaign. - **Breaking**: replica protocol v2 splits exact snapshots into transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage chunks in shard-owned private ETS and advance the stream cursor only after an diff --git a/CLAUDE.md b/CLAUDE.md index 29fb3e3..9c326c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,208 +1,261 @@ -# Group — CLAUDE.md +# Group — Maintainer Architecture Guide -## What is Group +## What Group Is -Distributed process registry + process groups + lifecycle monitoring + isolated subclusters. +Group is an eventually consistent distributed process registry, process-group +service, lifecycle monitor, and named-subcluster layer. Erlang distribution is +the membership and authority control plane. Replica state moves over a +configurable, nonblocking data transport and converges through sequenced +anti-entropy streams. ## Project Structure ``` lib/ - group.ex — Public API: register, join, members, monitor, dispatch, connect/disconnect - group/event.ex — %Group.Event{} struct - group/supervisor.ex — Top-level supervisor (rest_for_one: Data → PeerReconnect → Replica.Supervisor → Registry → ClusterLease) - group/cluster_lease.ex — Local named-cluster TTL sweeper - group/peer_reconnect.ex — Bounded reconnect loop after busy distribution links - group/replica/ - data.ex — GenServer that owns ETS tables and serializes shared membership mutations. - supervisor.ex — one_for_one supervisor for Replica shards - group/replica.ex — Sharded GenServer: replication, peer discovery, conflict resolution, monitoring - group/application.ex — Empty app supervisor (Group instances are started by consumers) + group.ex — public API and configuration docs + group/supervisor.ex — rest_for_one instance supervisor + group/cluster_lease.ex — local named-cluster TTL policy + group/peer_reconnect.ex — bounded retry for busy dispatch links + group/replica.ex — sharded writes, control, AE, projection + group/replica/data.ex — ETS owner, journal, authority, indexes + group/replica/protocol.ex — stream identity and mutation helpers + group/replica/snapshot.ex — byte-targeted snapshot chunks/staging + group/replica/transport.ex — replica transport contract + dist adapter + group/replica/transport/outbox.ex — optional lossy sideband outboxes + group/replica/transport/tcp.ex — included sideband TCP adapter test/ - test_helper.exs — Ensures EPMD/distribution are running; disables prevent_overlapping_partitions - group_test.exs — Local tests (async: true) - distributed_test.exs — Multi-node tests using OTP :peer - support/ - test_cluster.ex — Peer node helpers (start_peers, spawn_register, spawn_join, etc.) - test_conflict_resolver.ex — Custom resolver for tests -priv/bench/ — Benchmarks (run_local.sh, run_distributed.sh) + replica_model_property_test.exs — shrinkable real-node lifecycle model + replica_adversarial_test.exs — seeded three-node transport chaos + replica_snapshot_* — chunk/assembly failure coverage + formal/ — TLC protocol, assembly, eviction models + jepsen/ — independent-VM lifecycle oracle/campaign +priv/bench/ — local and distributed benchmark project ``` -## Running Tests +## Required Gates ```bash -mix test # all tests -mix test test/group_test.exs # local only -mix test test/distributed_test.exs # distributed only -``` - -Tests require `elixirc_paths(:test)` includes `test/support/`. Distributed tests use OTP `:peer` module for real Erlang nodes. +mix test # every-PR ExUnit/property/chaos/pure-checker gate +mix test.soak # nightly/release six-profile Jepsen campaign -## Running Benchmarks - -```bash -cd priv/bench && ./run_local.sh -cd priv/bench && ./run_distributed.sh +# Focused development +mix test test/group_test.exs +mix test test/distributed_test.exs +mix test test/replica_model_property_test.exs ``` -## Architecture +`mix test` accepts normal Mix test paths/options and then runs the pure Jepsen +checker qualification. `mix test.soak` runs that gate first, then twenty +five-minute histories for distribution/TCP/chaos × mixed/permanent scenarios. +See `test/README.md`, `test/formal/README.md`, and `test/jepsen/README.md`. -### Supervision Tree +## Supervision and Ownership ``` Group.Supervisor (rest_for_one) -├── Group.Replica.Data — owns all ETS tables; serializes membership mutations -├── Group.PeerReconnect — bounded reconnects after busy-link disconnects -├── Group.Replica.Supervisor — one_for_one, N shard GenServers -│ ├── Replica shard 0 -│ ├── Replica shard 1 +├── optional sideband transport manager + per-shard outboxes +├── Group.Replica.Data +├── Group.PeerReconnect +├── Group.Replica.Supervisor +│ ├── Group.Replica shard 0 +│ ├── Group.Replica shard 1 │ └── ... -├── Registry (Elixir) — :duplicate, for monitor subscriptions -└── Group.ClusterLease — local named-cluster TTL sweeper +├── Registry — local monitor subscriptions +└── Group.ClusterLease — local named-cluster TTL sweeper ``` -`rest_for_one` means: if Data dies, every later child restarts. If PeerReconnect dies, the Replica supervisor and monitor Registry restart. If Replica.Supervisor dies, Registry and ClusterLease restart. Replica shards rebuild local process monitors from surviving ETS after restart. - -### Sharding - -`phash2({cluster, key}, num_shards)` routes to shard. Default 8 shards. Must match across all nodes (validated on peer_connect). Including cluster in hash avoids false contention between default and named cluster operations. - -### ETS Tables (per shard × 4 + 3 shared) - -| Table | Type | Key | Tuple | -|-------|------|-----|-------| -| `reg_by_key` | `:set` | `{cluster, key}` | `{{cluster, key}, pid, meta, time, node}` | -| `reg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | `{{pid, cluster, key}, meta, time, node}` | -| `pg_by_key` | `:ordered_set` | `{cluster, key, pid}` | `{{cluster, key, pid}, meta, time, node}` | -| `pg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | `{{pid, cluster, key}, meta, time, node}` | -| `cluster_nodes` | `:bag` | cluster | `{cluster, node}` | -| `node_clusters` | `:bag` | node | `{node, cluster}` | -| `cluster_leases` | `:set` | cluster | `{cluster, ttl_ms, expires_at}` | - -**Why ordered_set for by_pid tables**: Contiguous range scans for `entries_by_pid` and `delete_all_for_pid` (process death cleanup). Also enables efficient existence checks with `select(..., 1)`. - -**Why ordered_set for pg_by_key**: `pg_members/4` scans `{cluster, key, *}` as a contiguous range — O(members in group), not O(table). - -**Why set for reg_by_key**: Only needs direct lookup/delete by `{cluster, key}` — O(1). - -All tables: `:public`, `read_concurrency: true`, `decentralized_counters: true`. Per-key registry/PG writes serialize through the owning Replica shard; shared cluster-membership dual-index mutations serialize through `Group.Replica.Data`. No table enables `write_concurrency`. - -### Reads vs Writes - -- **Reads** (`lookup`, `members`, `local_registry_count`) go directly to ETS — no GenServer involved -- **Writes** (`register`, `join`, `leave`, `unregister`) go through the shard's - local request lane (`send` + monitor + tagged reply), not `GenServer.call` -- **Replication** arrives as `handle_info` messages on shard GenServers - -### Config - -Stored in `persistent_term` keyed by `{Group, name}`. Includes shard/buffering/reconnect settings plus `log`, and optionally `extract_meta` and `resolve_registry_conflict`. +`Group.Replica.Data` owns every ETS table. A shard restart therefore preserves +the tables, repairs interrupted index/journal work, replays appended-but-not- +applied local mutations, and rebuilds monitors for locally owned processes. +The optional transport precedes Data so losing its manager restarts the whole +instance and cannot leave stale transport sessions attached to retained state. -## Key Protocols +Reads use public ETS directly. Writes route through +`:erlang.phash2({cluster, key}, num_shards)` to a local shard request lane. +Shard counts must match across peers. -### Peer Discovery (per-shard, on nodeup/init) +## ETS State -1. Each shard sends `{:peer_connect, pid, shard, num_shards, clusters}` to its counterpart -2. Receiver adds sender to nil cluster ETS, computes shared clusters, sends `{:peer_connect_ack, ...}` -3. Both sides send `{:cluster_state, cluster, reg_data, pg_data}` for each shared cluster -4. `merge_remote_cluster_data` applies data: new entries insert, conflicts go through `resolve_conflict` +Per shard: -### Replication (steady state) +| Table | Key | Role | +|---|---|---| +| `reg_by_key` | `{cluster, key}` | visible registry winner | +| `reg_by_pid` | `{pid, cluster, key}` | visible reverse index | +| `reg_claim_by_key` | `{cluster, key, origin, generation, epoch}` | authoritative claims | +| `reg_claim_by_pid` | `{pid, cluster, key, origin, generation, epoch}` | claim reverse index | +| `pg_by_key` | `{cluster, key, pid}` | visible PG membership | +| `pg_by_pid` | `{pid, cluster, key}` | PG reverse index | +| `replica_stream_meta` | stream id | head, retained floor, journal position | +| `replica_oplog` | `{stream, sequence}` | retained mutation record | +| `replica_oplog_order` | append id | shard-wide pruning order | +| `replica_cursor` | stream id | highest contiguous sequence applied | -After discovery, writes replicate in two stages: -- nil cluster: uses `state.remote_shards` map -- Named clusters: uses `cluster_nodes` ETS table -- Sender batches: `replicate_registry_batch`, `replicate_pg_batch` -- Receiver buffers registry and PG lanes separately, bulk-applies ETS writes, - then takes a bounded fairness turn for local work -- Remote shard sends use `send_nosuspend(..., [:noconnect])`; a `false` result - force-disconnects that node and enters bounded reconnect retries for that - peer only +Shared tables hold cluster/node indexes, local TTL leases, local and remote +cluster epochs, closed-cluster barriers, origin generations, exact/observed +authority revisions, installed lane views, and journal metadata. -### Conflict Resolution +Per-shard tables omit `write_concurrency` because one shard serializes their +writes. The shared replication metadata table uses +`write_concurrency: :auto`: every shard atomically updates only its own +`{:append_counter, shard}` object. Cross-shard arrival order has no semantic +meaning; an append id exists only to bound that shard's oplog across streams. -`resolve_conflict/5` and the batched equivalent handle all registry key conflicts (live replicated registry ops and partition-heal `merge_remote_cluster_data`). +## Authority and Stream Identity -- Default resolver: most recent timestamp wins; pid ordering tiebreaker on equal timestamps -- **Tiebreaker MUST be deterministic across all nodes**: `time2 > time1 or (time2 == time1 and pid2 > pid1)`. Using `>=` causes mutual kill. -- Custom: `resolve_registry_conflict: {mod, func, extra_args}` option; the callback owns any process exits -- The default resolver kills the loser with `{:group_registry_conflict, key, winner_meta}` -- Runs synchronously inside shard GenServer — must return quickly +Every mutation belongs to: -### Named Cluster Connect/Disconnect - -- `connect/2`: adds to ETS, picks random shard S, S notifies remote S, remote acks with bundled data + fans out to siblings -- `disconnect/2`: removes local membership, calls ALL local shards to purge the complete replicated cluster view, and has shard 0 broadcast to remotes -- `connect(..., ttl: ms)`: still checks `cluster_nodes` first, so an already-connected - cluster stays an ETS-fast noop and does not refresh the TTL -- TTL rows are local policy only; they do not change `cluster_nodes` / - `node_clusters` semantics -- On TTL expiry, `Group.ClusterLease` disconnects only if the local node has no - cluster-scoped monitors, no local registrations, and no local PG memberships - in that cluster. Otherwise it extends the lease by one TTL interval. - -### Nodedown / Process Death - -- `nodedown`: every shard calls `purge_cluster_node` (unconditional — not gated on shard 0) then `purge_node` -- Process DOWN: `delete_all_for_pids` + one non-suspending `replicate_process_down_batch` per target peer -- Remote shard DOWN (monitored pid): treated like nodedown for that node - -### Monitor Events - -Events delivered as `{:group, [%Group.Event{}, ...], %{name: name}}`. Batched per handler turn: -- Single ops: one event per message -- Bulk ops (nodedown, process DOWN, cluster_state merge): all events in one message - -Patterns: `:all`, `{:exact, key}`, `{:prefix, "prefix/"}` - -### Prefix Queries - -`Group.members(name, "prefix/")` scans ALL shards (can't hash prefix to one shard). Uses ETS range guards: -```elixir -{:andalso, {:>=, :"$1", prefix}, {:<, :"$1", next_binary_prefix(prefix)}} ``` -where `next_binary_prefix` increments the last byte of the prefix string. - -Keys ending with `"/"` are rejected by `validate_key!/1` in register/unregister/join/leave — trailing slash is reserved for prefix queries. - -### Dispatch - -`dispatch/4` sends to all members (registry + PG). Groups remote PG members by node and sends one non-suspending `{:group_dispatch, pids, message}` per remote node (O(nodes), not O(members)). The target shard is chosen by `phash2(self(), num_shards)` for per-sender ordering. - -`dispatch_local/4` skips cross-node messaging. - -## ETS Match Spec Patterns - -- Use `{:==, :"$N", value}` for runtime variables in guards (e.g., filtering by node) -- **NOT** `{:const, value}` — it's invalid in ETS match specs -- Literal Elixir variables interpolate directly into match pattern tuple positions as exact-match filters -- For result bodies, runtime values can't be embedded directly — use `Enum.map` post-select - -## Distributed Test Patterns +{group, origin_node, origin_generation, shard, cluster, cluster_epoch} +``` -- `Group.TestCluster.start_peers(N)` starts N real Erlang nodes via OTP `:peer` -- All helpers (`spawn_register`, `spawn_join`, `spawn_monitor_forwarder`) use `:erpc.call` with compiled modules from `test/support/` -- `Node.spawn` with anonymous functions won't work — remote node needs the defining module's beam file -- `assert_eventually/2` polls with retries for async replication -- `flush_shards/2` sends a mailbox barrier through each shard so buffered sender - and receiver replication work is flushed too -- `assert_ets_consistent/1` verifies dual-index tables match -- Partition tests use 3 nodes (isolate 1 from other 2). 2-node partitions are unreliable because the test node bridges them. -- `Supervisor.start_link` links to caller — in RPC context, must `Process.unlink(pid)` or supervisor dies on RPC return -- `test_helper.exs`: starts EPMD when needed, calls `Node.start/2`, sets the cookie, and disables `prevent_overlapping_partitions` +The origin appends a strictly increasing sequence before applying the +materialized change. Generation fences a restarted Group instance. A +named-cluster epoch fences close/reopen. The nil cluster uses the origin +generation as its epoch. + +Shard 0 installs one exact node-wide authority snapshot: + +- origin generation; +- complete active cluster→epoch set; +- exact authority revision; and +- authenticated transport descriptor. + +Other shards exchange constant-size lane hellos. Shared authority is not enough +to accept data: each shard records an installed lane view only after it has +purged streams outside that authority. Exact and incrementally observed +revisions are distinct; heartbeats and partial cluster-control bursts can never +promote an incomplete epoch set to exact authority. + +`peer_connect` and `peer_connect_ack` are discovery hints, not authority. +The old `cluster_state` handler is receive-only rolling compatibility; new +replica recovery must use heads/deltas/exact snapshots and must not add new +dependencies on additive full-state merge. + +## Anti-Entropy + +Replica data frames are: + +- `heads`: stream, retained floor, and head; +- `delta_batch`: one or more contiguous stream runs; +- `need`: the receiver's next missing sequence; and +- `snapshot_chunk`: one byte-targeted part of an exact origin slice. + +A receiver advances its cursor only through a contiguous prefix. Duplicates are +idempotent; gaps request the missing suffix. Periodic heads recover a dropped +tail even if no later write occurs. If the requested sequence is below the +bounded oplog floor, the origin sends an exact snapshot containing only its own +claims and memberships for that shard/cluster stream. Absence is deletion. + +The oplog is bounded per shard and never waits for acknowledgements. There are +no leaders, quorums, replicated per-entry tombstones, known-member lists, or +retention barriers. A slow peer cannot pin memory. + +Snapshots are transport-neutral and byte-targeted (1 MiB by default). A single +oversized row is one chunk. Multi-chunk receivers stage rows in shard-owned +private ETS and preserve the old visible slice/cursor until the complete, +authority-valid manifest is present. Loss, duplication, reordering, +supersession, stale authority, expiry, and shard crash must leave no partial +visible state. Staging expires after one peer-lease interval without progress. + +## Nonblocking Transport + +All cross-node Group control sends use +`:erlang.send_nosuspend(..., [:noconnect])`. The default distribution replica +adapter sends directly the same way and adds no local hop. `:busy` and +`:disconnected` mean “drop this frame”; periodic anti-entropy repairs it. + +A sideband adapter may use one local `Group.Replica.Transport.Outbox` per +shard. Outboxes batch by peer, impose deadlines, and run bounded socket work +outside Group shards. Queue overflow, expiry, or socket backpressure drops the +batch. The included TCP adapter adds bounded per-peer writer queues and +capability-authenticated ingress while distribution still authenticates node +identity and carries authority. TCP is not encrypted. + +Transport ordering is not required for correctness. Per-shard ordered delivery +is a fast path; stream sequences reject duplicate/out-of-order data, and +generation/epoch/lane fences handle control/data reordering. Ingress must derive +`source_node` from the authenticated connection, never from payload data, and +must reassemble any transport segmentation before `deliver_batch/4`. + +## Registry Projection and Process Ownership + +Registry claims remain authoritative per origin even when hidden by another +origin's visible winner. Conflict resolution folds claims deterministically. +The configured callback chooses a pid (or neither), but Group owns lifecycle +effects: each losing origin appends its own authoritative unregister and exits +only its local process with +`{:group_registry_conflict, key, winner_meta}`. + +A node never monitors or exits another node's member processes. Local shards +monitor locally owned registration/PG pids. Local `DOWN` appends ordered +unregister/leave mutations before deletion and replication. Remote owner death +arrives through those records; `nodedown` or lease expiry is the fallback for +an origin that cannot emit them. + +## Peer and Cluster Lifecycle + +- Dist-Erlang `nodedown` immediately purges that node's visible rows, claims, + cursors, authority, cluster routing, and sideband session on every shard. +- Constant-size control heartbeats cover the case where the Erlang node remains + connected but its Group instance disappears. Peer-lease expiry performs the + same complete purge. +- Discovery probes continue after expiry. A returning current/new generation + is fenced, installs authority per lane, and reconstructs through deltas or an + exact snapshot. +- Incremental named-cluster open/close controls are generation fenced and + batched. A quiet exact hello repairs dropped/reordered control messages. +- Local cluster close uses a temporary all-shard completion barrier. The last + shard removes routing/epoch rows; restart repair completes abandoned closes, + and reconnect waits so an old close cannot erase new writes. + +TTL leases are local policy only. On expiry, Group disconnects a named cluster +only when no local registrations, PG memberships, or cluster monitors remain. + +## Batching and Fairness + +Registry and PG mutations share one outbound sender buffer so local mailbox +order is retained. It flushes on size, age, idle timer, and before control or +routing barriers. Receivers apply contiguous runs in bulk and emit lifecycle +events in operation batches. After replicated work, each shard takes a bounded +FIFO local-request turn to prevent replica pressure from starving callers. + +## Distributed Test Rules + +- Use three peers for partition/recovery tests. Two nodes cannot cover an + independent survivor while an origin and receiver disagree. +- Remote helpers must be compiled under `test/support/`; call them with MFA + through `:erpc`. +- Unlink supervisors started inside RPC helpers. +- `flush_shards/2` is only a mailbox/barrier aid; convergence assertions must + still wait for AE and inspect exact public/internal state. +- Always assert dead owners are absent, retained owners are alive, claims and + projections agree, cursors are contiguous, no partial snapshot remains, and + retired origins have no rows. ## Critical Invariants -1. **purge_cluster_node is unconditional** — every shard calls it on nodedown/DOWN, not just shard 0. Late peer_connect on non-zero shard can re-add dead node after shard 0 cleaned it. -2. **Dispatch :unregistered for evicted local pid** in "remote wins" branch of resolve_conflict — monitors need to see the eviction. -3. **merge_remote_cluster_data uses Enum.reduce** (not `for`) to thread `{state, events}` through, since resolve_conflict modifies `state.monitors`. -4. **Additive merge only** — cluster_state merge inserts but never deletes. Local named-cluster disconnect therefore purges the complete local cluster view, and replicated batches are membership-gated at apply time. -5. **PG tables have no overwrite conflicts** — `pg_by_key` key includes pid: `{cluster, key, pid}`. -6. **Named-cluster data is membership-gated** — both `cluster_state` and buffered replicated operations reject data for clusters the local node has left. +1. A cursor never advances across a gap or before a full exact snapshot commits. +2. Exact snapshots replace one origin slice; they are never additive merges. +3. Authority requires generation, exact epoch revision, and installed lane + readiness. Observed heartbeats/controls are not exact authority. +4. A stale generation, epoch, lane, shard, transitive pid, or unauthenticated + source is rejected before applying replica data. +5. Registry claims are retained per origin until that origin deletes them or is + retired; the visible winner is reconstructible from remaining claims. +6. Only an owner node monitors, retires, or exits its member processes. +7. Oplog pruning is local and bounded; lagging peers use exact snapshot repair. +8. `nodedown` and peer-lease expiry purge every public and internal reference + to the retired origin. Remote shard death cannot leave state permanently; + lease expiry or fenced rediscovery completes cleanup/recovery. +9. Snapshot staging is private, all-or-nothing, authority fenced, and expiring. +10. Local append/journal repair makes an appended mutation either replayable or + durably applied after a shard crash. +11. Cross-node control and replica calls never block a Group shard. +12. Cluster close completion survives caller timeout and shard restart. ## Logging -- `log:` option: `:info` (default), `false` (routine logs disabled), `:verbose` (all shards) -- `log/2` (normal), `log_verbose/2` (verbose only), `log_once/2` (shard 0 only) use `Logger.info`; `:verbose` is Group's own flag, not a Logger level -- Registry conflicts are unconditional `Logger.error` events; busy distribution links are unconditional `Logger.warning` events -- Runtime change: `Group.log_level(name, level)` +`log: :info | :verbose | false` controls routine Group logs and can be changed +with `Group.log_level/2`. Registry conflicts remain errors and busy dispatch +links remain warnings regardless of the routine level. diff --git a/README.md b/README.md index 00ea74c..8a00c32 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ lifecycle monitoring, and isolated subclusters for Elixir. No external dependenc where only connected nodes participate. - **Sharded writes** — writes fan out across N GenServer shards to reduce contention. Reads go directly to ETS. +- **Nonblocking anti-entropy** — replica sends never wait on a remote socket; + sequenced deltas, bounded oplogs, and exact snapshots repair dropped work. ## Installation @@ -218,6 +220,9 @@ All operations are **eventually consistent**: - When connectivity returns, per-origin stream heads repair missing sequence ranges from a bounded oplog; a lag beyond the retained prefix falls back to an exact snapshot of that origin's shard/cluster slice. +- A dist-Erlang `nodedown` removes that node's view immediately. If the Erlang + node remains connected but its Group instance stops responding, a bounded + control-plane lease removes the same state and later discovery can rebuild it. - Registry conflicts (same key registered on two nodes during a partition) can be resolved with a configurable `resolve_registry_conflict` callback. The callback selects a winner; each origin retires and terminates only its own @@ -349,8 +354,14 @@ Each shard has materialized read indexes plus authority/recovery indexes: |---|---|---|---| | `reg_by_key` | `:set` | `{cluster, key}` | Registry lookup — O(1) | | `reg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | Reverse index for death cleanup | +| `reg_claim_by_key` | `:ordered_set` | `{cluster, key, origin, generation, epoch}` | One authoritative registry claim per origin | +| `reg_claim_by_pid` | `:ordered_set` | `{pid, cluster, key, origin, generation, epoch}` | Reverse claim index for owner death and repair | | `pg_by_key` | `:ordered_set` | `{cluster, key, pid}` | Group membership lookup | | `pg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | Reverse index for death cleanup | +| `replica_stream_meta` | `:set` | `stream_id` | Local stream head, retained floor, and applied journal position | +| `replica_oplog` | `:ordered_set` | `{stream_id, sequence}` | Bounded sequenced mutation records | +| `replica_oplog_order` | `:ordered_set` | `append_id` | Shard-wide pruning order across streams | +| `replica_cursor` | `:set` | `stream_id` | Highest contiguous remote sequence applied | Registry claim tables retain one authoritative claim per origin independently of the visible winner. Stream metadata, oplog, append-order, and receive-cursor @@ -358,15 +369,23 @@ tables support crash replay and gap repair. Keeping claims separate from the single visible `reg_by_key` projection prevents a losing-but-still-live remote claim from being forgotten before its owner emits an authoritative delete. -Plus 3 shared tables: +The node also has shared control/authority tables: - `cluster_nodes` (`:bag`, cluster→nodes) - `node_clusters` (`:bag`, node→clusters) - `cluster_leases` (`:set`, cluster→`{ttl_ms, expires_at}`) for local `connect(..., ttl: ms)` policy - -`cluster_nodes` / `node_clusters` remain the authoritative cluster-membership -tables. `cluster_leases` is only local lease metadata used by the sweeper. +- `replication_meta` (`:set`) for the local generation, authority revisions, + per-lane installed views, journal metadata, and one atomic append counter per + shard +- `local_cluster_epochs` and `closed_local_cluster_epochs` (`:set`) for active + and closing local named-cluster lifetimes +- `remote_cluster_epochs` (`:set`) for exact generation-fenced remote authority + +`cluster_nodes` / `node_clusters` are the routing projection read by APIs and +replication fanout. Generation-fenced local/remote epoch tables are the +authority used to install that projection. `cluster_leases` is only local +policy metadata used by the sweeper. `Group.Replica.Data` owns all tables and is supervised with `rest_for_one` so tables survive shard crashes. @@ -438,9 +457,10 @@ including after a caller timeout or shard restart. Reconnect waits for that barrier so a prior close cannot erase newly accepted writes. The sender flush timer is mainly a fallback for idle periods. The unified -outbound buffer also flushes immediately when it hits the configured size, when a new enqueue -finds the buffer already past its flush interval, and before control or -routing work such as cluster connect/disconnect or peer-protocol handling. +outbound buffer also flushes immediately when it hits the configured size, +when a new enqueue finds the buffer already past its flush interval, and before +control or routing work such as cluster connect/disconnect or peer-protocol +handling. Transport ordering is not required for correctness: each shard serializes writes, each stream numbers them, and receivers reject gaps and duplicates. @@ -509,27 +529,34 @@ no longer care about a cluster. ### Process Death Cleanup -Shards monitor all registered/joined processes. On `DOWN`, the shard: +Shards monitor only locally owned registered/joined processes. A node never +monitors or exits another node's member processes. On a local owner `DOWN`, the +shard: 1. Removes entries from both the primary and reverse-index ETS tables. 2. Appends authoritative unregister/leave mutations before deleting the rows, then sends one non-suspending sequenced delta batch per peer. 3. Fires `:unregistered` / `:left` events to local monitors. -### Node Disconnect +### Peer Removal and Recovery On `nodedown`, each shard purges all entries owned by the disconnected node -from its ETS tables and fires events for each removed entry. +from its ETS tables, claims, cursors, and authority indexes and fires events for +each removed entry. If dist Erlang stays connected but a Group instance or its +control lane disappears, heartbeat lease expiry performs the same complete +purge. Discovery probes continue after expiry; a returning instance announces +a new or current generation and anti-entropy reconstructs its live state. ## Testing ```bash mix test +mix test.soak # nightly/release qualification ``` -See [`test/README.md`](test/README.md) for details on the distributed test -infrastructure, shrinkable StreamData lifecycle-model tests, and the bounded -TLA+ anti-entropy model. +See [`test/README.md`](test/README.md) for the every-PR gate, shrinkable +StreamData lifecycle-model tests, bounded TLA+ models, and the nightly +three-node Jepsen transport/lifecycle campaign. ## Benchmarks diff --git a/lib/group.ex b/lib/group.ex index e3dd976..0e949e8 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -17,9 +17,11 @@ defmodule Group do - Writes (register, join, etc.) return immediately after local update - Other nodes receive updates asynchronously over the replica transport - During network partitions, nodes may have divergent views - - When partitions heal, conflicts are resolved. The built-in resolver kills - each losing origin records an authoritative delete and terminates only its - own local process with `{:group_registry_conflict, key, winner_meta}` + - When connectivity heals, stream gaps are repaired from a bounded oplog or + an exact per-origin snapshot; dropped replica sends are therefore safe + - Registry conflicts resolve deterministically. Each losing origin records an + authoritative delete and terminates only its own local process with + `{:group_registry_conflict, key, winner_meta}` ## Clusters @@ -170,6 +172,10 @@ defmodule Group do - **Memberships** are stored in replicated, sharded ETS indexes and are automatically cleaned up when member processes die. + + - **Process ownership is local**: a shard monitors and exits only processes + owned by its own node. Remote lifecycle changes arrive as sequenced replica + records or are removed by `nodedown`/peer-lease expiry. """ alias Group.Replica diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 7511fc7..43e4d47 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -56,6 +56,24 @@ defmodule Group.Replica.Data do `entries_by_pid` can select all entries for a pid as a contiguous range scan. Also used by `maybe_demonitor` to check if a pid has any remaining entries (select with limit 1). + ### reg_claim_by_key / reg_claim_by_pid — authoritative registry claims + + {{cluster, key, origin, generation, epoch}, pid, meta, time, sequence} + {{pid, cluster, key, origin, generation, epoch}, meta, time, sequence} + + The visible `reg_by_key` row is only a deterministic projection. These tables retain one + independently versioned claim per origin so a losing-but-live claim is not forgotten. + Only its owner stream can delete it; conflict resolution then recomputes the visible winner. + + ### replica_stream_meta / replica_oplog / replica_oplog_order / replica_cursor + + Local streams are keyed by `{group, origin, generation, shard, cluster, epoch}`. + `replica_stream_meta` records each stream's head, retained floor, and applied journal + position. `replica_oplog` stores `{stream, sequence}` mutation records while + `replica_oplog_order` gives them one shard-wide append order for bounded pruning. + `replica_cursor` records only the highest contiguous sequence applied from each remote + stream. A gap below the retained floor is repaired by exact per-origin snapshot replacement. + ### cluster_nodes — `:bag`, keyed by cluster name {cluster, node} @@ -74,9 +92,9 @@ defmodule Group.Replica.Data do targeted deletes from both tables — O(clusters for node) instead of O(total entries). Both tables are shared across all shards. Used for the default cluster (nil) and named - clusters. The nil cluster is maintained by the peer_connect protocol — nodes are added on - peer discovery and removed on nodedown/shard death. `Group.nodes/1` reads nil cluster - from cluster_nodes. + clusters. Peer-connect messages are discovery hints; shard 0's generation-fenced exact + authority installs membership. `nodedown`, shard death, or peer-lease expiry removes it. + `Group.nodes/1` reads the nil cluster from cluster_nodes. ### cluster_leases — `:set`, keyed by cluster name @@ -94,6 +112,15 @@ defmodule Group.Replica.Data do Keeping leases separate avoids adding policy state to the hot cluster-membership lookups used by `Group.connect/3`, peer discovery, and replication fanout. + ### replication_meta and epoch tables + + `replication_meta` holds the local origin generation, exact and observed authority + revisions, per-shard installed remote views, journal metadata, and one append counter per + shard. `local_cluster_epochs` and `closed_local_cluster_epochs` fence local named-cluster + lifetimes; `remote_cluster_epochs` is the exact node-wide authority installed by shard 0. + Exact and merely observed revisions are separate so a partial control burst cannot be + promoted to authoritative membership. + ## Match Spec Patterns All match specs use `{:==, :"$N", value}` guards to filter on runtime values (e.g. node @@ -104,10 +131,12 @@ defmodule Group.Replica.Data do ## Bulk Operations & Their Costs - `purge_node/3`: Full table scan via `ets.select` filtering by node, then individual - deletes. O(table size) for the scan, but this only runs on nodedown — rare path. + deletes. O(table size) for the scan, but this only runs on nodedown, remote shard death, + or peer-lease expiry — rare paths. - `local_data_by_cluster/3`: Full table scan filtering by `node() == local_node`, - grouped by cluster. Only runs during discovery/sync protocol. + grouped by cluster. Retained only for the legacy receive-only cluster-state + compatibility path; current recovery uses anti-entropy streams. - `registry_count`, `pg_count`, `pg_count_by_prefix`, `local_registry_count`, `local_pg_count`, `local_registry_present?`, `local_pg_present?`: Uses @@ -127,8 +156,11 @@ defmodule Group.Replica.Data do local processes that registered before the crash would be orphaned — nobody would monitor them, and their ETS entries would persist forever if they later died. - Remote data doesn't need this protection — the discovery protocol re-syncs everything - from remote nodes on restart. Only local process entries need the ETS scan. + Only locally owned member processes are monitored. Remote owner death arrives as a + sequenced delete from that owner; nodedown or peer-lease expiry removes the complete + remote origin if it cannot report the delete. Periodic anti-entropy then rebuilds a + returning origin from retained deltas or an exact snapshot. No node monitors or exits + another node's member processes. The `state.monitors` map also deduplicates: a pid registered under multiple keys in the same shard gets one monitor, not one per key. diff --git a/lib/group/replica/transport.ex b/lib/group/replica/transport.ex index 033acd6..b7f46c1 100644 --- a/lib/group/replica/transport.ex +++ b/lib/group/replica/transport.ex @@ -96,7 +96,15 @@ defmodule Group.Replica.Transport do end defmodule Group.Replica.Transport.Distribution do - @moduledoc false + @moduledoc """ + Default nonblocking replica transport over Erlang distribution. + + Frames are sent directly to the matching remote shard with + `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a + busy distribution socket and never initiates a connection. A busy or absent + link returns `:busy`; Group drops that frame and repairs it through periodic + anti-entropy. + """ @behaviour Group.Replica.Transport alias Group.Replica diff --git a/mix.exs b/mix.exs index 44fb164..6adb847 100644 --- a/mix.exs +++ b/mix.exs @@ -10,8 +10,10 @@ defmodule Group.MixProject do version: @version, elixir: "~> 1.19", elixirc_paths: elixirc_paths(Mix.env()), + test_ignore_filters: [~r"^test/jepsen/", ~r"^test/mutation/"], start_permanent: Mix.env() == :prod, deps: deps(), + aliases: aliases(), package: package(), docs: docs(), name: "Group", @@ -32,6 +34,10 @@ defmodule Group.MixProject do ] end + def cli do + [preferred_envs: ["test.soak": :test]] + end + defp deps do [ {:ex_doc, "~> 0.30", only: :dev, runtime: false}, @@ -55,4 +61,14 @@ defmodule Group.MixProject do source_ref: "v#{@version}" ] end + + defp aliases do + [ + test: ["test", "cmd test/jepsen/checker.sh"], + "test.soak": [ + "test", + "cmd env GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/campaign.sh" + ] + ] + end end diff --git a/test/README.md b/test/README.md index 16ce634..224f250 100644 --- a/test/README.md +++ b/test/README.md @@ -3,20 +3,29 @@ ## Running tests ```bash -mix test # all tests +mix test # every-PR ExUnit/property/chaos/checker gate +mix test.soak # nightly six-profile Jepsen campaign mix test test/group_test.exs # local only mix test test/distributed_test.exs # distributed only mix test test/replica_adversarial_test.exs # seeded transport chaos mix test test/replica_model_property_test.exs # shrinkable model-based histories +test/jepsen/run.sh # one OS-partition/restart Jepsen model test ``` +`mix test` preserves normal Mix test arguments while always running the pure +Jepsen lifecycle-checker qualification after ExUnit. It does not require +Docker. `mix test.soak` first runs that complete PR gate, then runs the +distribution/TCP/chaos × mixed/permanent Jepsen campaign. The soak defaults to +20 five-minute fault histories per combination and is intended for nightly and +release qualification rather than individual edits. + ## Test files | File | What it tests | |------|---------------| | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | | `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | -| `replica_adversarial_test.exs` | Reproducible mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | +| `replica_adversarial_test.exs` | Reproducible three-node mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | | `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | | `replica_snapshot_test.exs` | Pure byte partitioning and set-valued private-ETS snapshot staging | | `replica_snapshot_distributed_test.exs` | Real-node exact-snapshot loss, reorder, duplicate, conflicting retransmission, supersession, authority fencing, expiry, and shard-crash recovery | @@ -48,6 +57,18 @@ The independent TLA+ model and TLC configuration live in `test/formal/`. See [`formal/README.md`](formal/README.md) for its checked invariants, finite model bounds, and run command. +The Docker-backed Jepsen harness lives in [`jepsen/`](jepsen/). It drives +three independent BEAM containers through concurrent, multi-entry owner +lifecycles, named-cluster epoch churn, directed/full partitions, transport +session resets, and VM restarts. The same workload runs over distribution, +real sideband TCP, and a lossy/duplicating/reordering transport. After healing, +its independent oracle checks exact public views and the internal registry, +PG, claim, cluster, cursor, oplog, snapshot-staging, and retired-origin +invariants. Its permanent-retirement scenario proves eviction even when a peer +never returns. `test/jepsen/campaign.sh` runs the full profile/scenario matrix; +`test/jepsen/qualify.sh` mutation-tests the implementation and proves that the +live checker rejects injected faults. + ## How distribution works The test node starts as a named Erlang node in `test_helper.exs`: diff --git a/test/formal/GroupAntiEntropyExtended.cfg b/test/formal/GroupAntiEntropyExtended.cfg new file mode 100644 index 0000000..25d4ff5 --- /dev/null +++ b/test/formal/GroupAntiEntropyExtended.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec + +CONSTANTS + Nodes = {n1, n2, n3} + Origins = {n1} + Keys = {k1, k2} + MaxSeq = 3 + OplogBound = 2 + MaxMessages = 1 + +INVARIANTS + TypeOK + BoundedJournal + CurrentReplicaIsAStreamPrefix + +PROPERTY + HealedConvergence diff --git a/test/formal/PeerEviction.cfg b/test/formal/PeerEviction.cfg new file mode 100644 index 0000000..3a3017a --- /dev/null +++ b/test/formal/PeerEviction.cfg @@ -0,0 +1,15 @@ +SPECIFICATION Spec + +CONSTANTS + Receivers = {r1, r2} + Keys = {k1, k2} + MaxGeneration = 2 + MaxMessages = 2 + +INVARIANTS + TypeOK + VisibleRowsMatchInstalledGeneration + NoRowsWithoutAuthority + +PROPERTY + HealedConvergence diff --git a/test/formal/PeerEviction.tla b/test/formal/PeerEviction.tla new file mode 100644 index 0000000..35b98ad --- /dev/null +++ b/test/formal/PeerEviction.tla @@ -0,0 +1,207 @@ +--------------------------- MODULE PeerEviction --------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +Finite model of the peer lease and reincarnation boundary. An origin may +disappear permanently or restart at a larger generation while arbitrary old +hello and snapshot messages remain in the network. A receiver may expire its +lease at any point. Once the faulting prefix ends, fair repair must either +install the current generation exactly or erase the permanently absent peer. +*) + +CONSTANTS Receivers, Keys, MaxGeneration, MaxMessages + +ASSUME /\ IsFiniteSet(Receivers) + /\ Cardinality(Receivers) >= 1 + /\ IsFiniteSet(Keys) + /\ Cardinality(Keys) >= 1 + /\ MaxGeneration >= 2 + /\ MaxMessages >= 1 + +Generations == 1..MaxGeneration +Token == Generations \X Keys + +HelloMessage == + [kind : {"hello"}, to : Receivers, wireGeneration : Generations, + wireUp : BOOLEAN] + +SnapshotMessage == + [kind : {"snapshot"}, to : Receivers, wireGeneration : Generations, + rows : SUBSET Token] + +Message == HelloMessage \union SnapshotMessage + +VARIABLES phase, up, generation, truth, authorityGeneration, authorityUp, view, messages + +vars == + <> + +Init == + /\ phase = "faulting" + /\ up = TRUE + /\ generation = 1 + /\ truth = {} + /\ authorityGeneration = [receiver \in Receivers |-> 0] + /\ authorityUp = [receiver \in Receivers |-> FALSE] + /\ view = [receiver \in Receivers |-> {}] + /\ messages = {} + +Write(key, present) == + /\ phase = "faulting" + /\ up + /\ IF present + THEN truth' = truth \union {<>} + ELSE truth' = truth \ {<>} + /\ UNCHANGED + <> + +Crash == + /\ phase = "faulting" + /\ up + /\ up' = FALSE + /\ truth' = {} + /\ UNCHANGED + <> + +Restart == + /\ phase = "faulting" + /\ ~up + /\ generation < MaxGeneration + /\ up' = TRUE + /\ generation' = generation + 1 + /\ truth' = {} + /\ UNCHANGED <> + +SendHello(receiver) == + /\ phase = "faulting" + /\ Cardinality(messages) < MaxMessages + /\ messages' = messages \union + {[kind |-> "hello", to |-> receiver, + wireGeneration |-> generation, wireUp |-> up]} + /\ UNCHANGED + <> + +SendSnapshot(receiver) == + /\ phase = "faulting" + /\ up + /\ Cardinality(messages) < MaxMessages + /\ messages' = messages \union + {[kind |-> "snapshot", to |-> receiver, + wireGeneration |-> generation, rows |-> truth]} + /\ UNCHANGED + <> + +DeliverHello(message) == + /\ message.kind = "hello" + /\ IF message.wireGeneration >= authorityGeneration[message.to] + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT ![message.to] = message.wireGeneration] + /\ authorityUp' = [authorityUp EXCEPT ![message.to] = message.wireUp] + /\ view' = + IF message.wireGeneration # authorityGeneration[message.to] + \/ ~message.wireUp + THEN [view EXCEPT ![message.to] = {}] + ELSE view + ELSE /\ UNCHANGED authorityGeneration + /\ UNCHANGED authorityUp + /\ UNCHANGED view + /\ UNCHANGED <> + +DeliverSnapshot(message) == + /\ message.kind = "snapshot" + /\ IF message.wireGeneration = authorityGeneration[message.to] + /\ authorityUp[message.to] + THEN view' = [view EXCEPT ![message.to] = message.rows] + ELSE UNCHANGED view + /\ UNCHANGED + <> + +Deliver(message) == + /\ phase = "faulting" + /\ message \in messages + /\ \/ DeliverHello(message) + \/ DeliverSnapshot(message) + +Drop(message) == + /\ phase = "faulting" + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED + <> + +ExpireLease(receiver) == + /\ phase = "faulting" + /\ authorityGeneration' = [authorityGeneration EXCEPT ![receiver] = 0] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = FALSE] + /\ view' = [view EXCEPT ![receiver] = {}] + /\ UNCHANGED <> + +Heal == + /\ phase = "faulting" + /\ phase' = "healed" + /\ UNCHANGED + <> + +Repair(receiver) == + /\ phase = "healed" + /\ IF up + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT ![receiver] = generation] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = TRUE] + /\ view' = [view EXCEPT ![receiver] = truth] + ELSE /\ authorityGeneration' = + [authorityGeneration EXCEPT ![receiver] = 0] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = FALSE] + /\ view' = [view EXCEPT ![receiver] = {}] + /\ UNCHANGED <> + +Next == + \/ \E key \in Keys, present \in BOOLEAN : Write(key, present) + \/ Crash + \/ Restart + \/ \E receiver \in Receivers : SendHello(receiver) + \/ \E receiver \in Receivers : SendSnapshot(receiver) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ \E receiver \in Receivers : ExpireLease(receiver) + \/ Heal + \/ \E receiver \in Receivers : Repair(receiver) + +TypeOK == + /\ phase \in {"faulting", "healed"} + /\ up \in BOOLEAN + /\ generation \in Generations + /\ truth \subseteq Token + /\ authorityGeneration \in [Receivers -> 0..MaxGeneration] + /\ authorityUp \in [Receivers -> BOOLEAN] + /\ view \in [Receivers -> SUBSET Token] + /\ messages \subseteq Message + +VisibleRowsMatchInstalledGeneration == + \A receiver \in Receivers : + \A token \in view[receiver] : token[1] = authorityGeneration[receiver] + +NoRowsWithoutAuthority == + \A receiver \in Receivers : + (authorityGeneration[receiver] = 0 \/ ~authorityUp[receiver]) + => view[receiver] = {} + +Converged == + \A receiver \in Receivers : + IF up + THEN /\ authorityGeneration[receiver] = generation + /\ authorityUp[receiver] + /\ view[receiver] = truth + ELSE /\ authorityGeneration[receiver] = 0 + /\ ~authorityUp[receiver] + /\ view[receiver] = {} + +HealedConvergence == phase = "healed" ~> Converged + +Spec == + /\ Init + /\ [][Next]_vars + /\ \A receiver \in Receivers : WF_vars(Repair(receiver)) + +============================================================================= diff --git a/test/formal/README.md b/test/formal/README.md index 7cb743e..d34b5ea 100644 --- a/test/formal/README.md +++ b/test/formal/README.md @@ -17,6 +17,14 @@ receiver crashes. Its invariants require visible data and the cursor to remain at a previously committed exact state until every chunk of one valid snapshot is present; stale or mixed partial state can never become visible. +`PeerEviction.tla` isolates the lifecycle boundary for a peer which never +returns and for a later process using the same node name with a fresh +generation. During its finite faulty prefix it retains and reorders stale +hello and snapshot messages while leases expire. Authority consists of both a +generation and an active bit, so an inactive hello fences even a same-epoch +snapshot. After healing, fair repair must either install only the current +generation or erase every row and authority reference for the absent peer. + The default TLC configuration uses three nodes: one origin and two independent receivers. The origin has one key, a two-record stream, a one-record oplog, and the system retains one arbitrary network frame. This forces delta repair, @@ -41,6 +49,12 @@ TLA_JAR=/path/to/tla2tools.jar \ TLA_SPEC="$PWD/test/formal/SnapshotAssembly.tla" \ TLA_CONFIG="$PWD/test/formal/SnapshotAssembly.cfg" \ test/formal/check.sh + +# Run all default models +TLA_JAR=/path/to/tla2tools.jar test/formal/check_matrix.sh + +# Also run the larger two-key, three-sequence anti-entropy state space +TLA_JAR=/path/to/tla2tools.jar TLA_EXTENDED=1 test/formal/check_matrix.sh ``` `TLC_WORKERS` controls worker concurrency and defaults to 4. `TLA_CONFIG` can @@ -49,7 +63,9 @@ point at an alternate finite configuration. TLC proves the listed invariants and liveness property for the configured finite instance, not for arbitrary unbounded node and key sets. Larger models should be run periodically by increasing `Nodes`, `Origins`, `Keys`, `MaxSeq`, -`OplogBound`, and `MaxMessages`. +`OplogBound`, and `MaxMessages`. `check_matrix.sh` runs the protocol, snapshot +assembly, and peer-eviction models; set `TLA_EXTENDED=1` for the larger +anti-entropy configuration. The checked three-node default explores 1,835,826 states, finds 490,236 distinct states to a depth of 30, and completes in roughly 1 minute 40 seconds @@ -57,3 +73,11 @@ on the development machine used for the validation run. The snapshot-assembly model explores 15,681 states, finds 1,088 distinct states to a depth of 13, and completes in under a second on the same class of machine. + +The peer-eviction model explores 1,527,116 states, finds 238,120 distinct +states to a depth of 26, and completes in roughly 20 seconds on the development +machine used for validation. + +The extended two-key, three-sequence model explores 127,557,634 states, finds +32,238,304 distinct states to a depth of 34, and completes in roughly two hours +on the development machine used for validation. diff --git a/test/formal/check_matrix.sh b/test/formal/check_matrix.sh new file mode 100755 index 0000000..cd60e2e --- /dev/null +++ b/test/formal/check_matrix.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +: "${TLA_JAR:?TLA_JAR must point to tla2tools.jar}" + +run_check() { + local spec="$1" + local config="$2" + + TLA_SPEC="${script_dir}/${spec}.tla" \ + TLA_CONFIG="${script_dir}/${config}.cfg" \ + "${script_dir}/check.sh" +} + +run_check GroupAntiEntropy GroupAntiEntropy +run_check SnapshotAssembly SnapshotAssembly +run_check PeerEviction PeerEviction + +if [[ "${TLA_EXTENDED:-0}" == "1" ]]; then + run_check GroupAntiEntropy GroupAntiEntropyExtended +fi diff --git a/test/jepsen/.gitignore b/test/jepsen/.gitignore new file mode 100644 index 0000000..acbda9e --- /dev/null +++ b/test/jepsen/.gitignore @@ -0,0 +1,5 @@ +.cache/ +.lein-failures +.nrepl-port +store/ +target/ diff --git a/test/jepsen/Dockerfile.node b/test/jepsen/Dockerfile.node new file mode 100644 index 0000000..08a5f41 --- /dev/null +++ b/test/jepsen/Dockerfile.node @@ -0,0 +1,22 @@ +FROM elixir:1.19.5-otp-28-slim + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates iptables procps \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /group + +ENV MIX_ENV=prod + +COPY mix.exs mix.lock ./ +COPY lib ./lib +COPY test/jepsen/node.exs ./test/jepsen/node.exs + +RUN mix local.hex --force \ + && mix deps.get --only prod \ + && mix compile + +COPY test/jepsen/entrypoint.sh /usr/local/bin/group-jepsen-node +RUN chmod +x /usr/local/bin/group-jepsen-node + +ENTRYPOINT ["/usr/local/bin/group-jepsen-node"] diff --git a/test/jepsen/README.md b/test/jepsen/README.md new file mode 100644 index 0000000..d543df9 --- /dev/null +++ b/test/jepsen/README.md @@ -0,0 +1,149 @@ +# Group Jepsen model test + +This harness drives real Group instances in three independent BEAM VMs and +checks their quiescent state against a separate process-lifecycle oracle. It +tests failures outside the deterministic in-VM scheduler used by the +StreamData suite. + +Each node has eight independent client drivers, so operations on different +owners can overlap on the same node. Owners can hold multiple registry and PG +entries in the root, `red`, and `blue` cluster epochs. During the bounded-fault +prefix Jepsen: + +- registers, unregisters, joins, leaves, and kills real owner processes; +- closes and recreates named-cluster epochs while old traffic is stranded; +- creates a three-node registry conflict and requires every loser to die; +- partitions the full Erlang mesh, or only selected directed replica lanes; +- exercises isolation, all-way partition, and asymmetric one-way loss; +- resets transport sessions and kills/restarts complete BEAM nodes; and +- uses a 16-entry oplog and 1 KiB snapshot target so repair crosses pruning + and multi-chunk exact-snapshot paths. + +The replica lane is selectable without changing the workload or checker: + +- `distribution` delegates to Group's production Erlang-distribution adapter; +- `tcp` uses Group's production sideband TCP adapter while Erlang distribution + remains the control plane; and +- `chaos` is a local per-shard outbox which deterministically drops, + duplicates, delays, and reorders replica frames. + +After faults stop, every surviving node reconnects and the harness takes two +terminal snapshots. The independent checker requires: + +- exact, identical public registry and PG views on every survivor; +- every live owner claim to be visible, and no dead owner token to remain; +- deterministic resolution of registry conflicts with no unexpected owner + deaths; +- the expected peer set, including complete removal of a permanently retired + node; +- consistent registry, PG, cluster, claim, cursor, oplog, and remote-authority + indexes inside every shard; +- no staged partial snapshot and no retained data for a retired origin; +- coverage of delta batches, snapshot fallback, multi-chunk assembly, and + registry conflict termination; +- two identical quiescent observations per node; and +- every acknowledged Group operation below the configured latency ceiling. + +The oracle lives in owner processes outside Group's ETS and replica indexes. +Unexpected deaths and low-volume qualification evidence such as registry +conflict termination are also written to container-local logs which survive a +BEAM restart. A restart changes the boot component of new owner tokens, so a +stale generation, delayed delta, partial snapshot, or orphaned registry/PG row +cannot masquerade as a current owner. Every history explicitly restarts one +node after the deterministic conflict prelude, proving the checker does not +mistake restart-sensitive instrumentation for missing protocol coverage. + +## Requirements + +- Docker with Compose v2 +- Java 21 or newer +- `curl` + +The runner downloads Leiningen 2.12.0 into the ignored `.cache/` directory and +uses Jepsen 0.3.13. It installs nothing globally. + +## Run + +From the repository root: + +```bash +test/jepsen/run.sh +``` + +The default is a 60-second mixed-lifecycle run over three nodes and six client +workers. Normal Jepsen options and Group-specific options can be supplied: + +```bash +test/jepsen/run.sh test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 3n \ + --time-limit 300 \ + --test-count 10 \ + --key-count 32 \ + --owner-count 128 \ + --fault-interval 2 \ + --recovery-time 15 \ + --transport tcp \ + --scenario permanent \ + --max-operation-latency-ms 2000 +``` + +`--transport` accepts `distribution`, `tcp`, or `chaos`. `--scenario mixed` +restarts every transiently killed node; `--scenario permanent` finally retires +`n1` and proves that `n2` and `n3` converge after purging all of its public and +internal state. + +Run the sustained matrix over all transport and lifecycle combinations with: + +```bash +test/jepsen/campaign.sh +``` + +Its defaults are 20 five-minute runs for each of six combinations. The +`GROUP_JEPSEN_CAMPAIGN_*` environment variables in the script control duration, +count, concurrency, keys, owners, and recovery time. + +Run mutation qualification plus live positive- and negative-checker tests: + +```bash +test/jepsen/qualify.sh +``` + +This must reject production mutations which remove generation fencing, gap +detection, exact snapshot replacement, complete snapshot assembly, periodic +repair, or retirement purging. It then verifies that a healthy live history is +accepted and deliberately injected owner-death and internal-index corruption +are rejected. + +Results and histories are written below `test/jepsen/store/`. Containers are +removed after a run. Set `GROUP_JEPSEN_KEEP_CONTAINERS=1` to retain them and +inspect logs with: + +```bash +docker compose -f test/jepsen/docker-compose.yml logs +``` + +The pure checker qualification tests do not require Docker: + +```bash +test/jepsen/checker.sh +``` + +At the repository root, `mix test` runs this pure checker after the complete +ExUnit, StreamData, and deterministic-chaos suite. `mix test.soak` runs that +same PR gate followed by `campaign.sh`. + +## Scope + +This checker verifies Group's eventual lifecycle contract, not +linearizability. While communication is unavailable, each side may serve its +local view and accept new owners. The requirement begins after the explicitly +bounded fault prefix: surviving peers must then always resolve to the exact +same lifecycle view, while a peer which never returns must be completely +evicted after its lease expires. + +The formal models prove the protocol for finite state spaces; StreamData +generates and shrinks scheduler-controlled histories inside real Group nodes; +this harness tests OS sockets, independent VMs, VM death, and real transport +adapters. None of these layers alone is treated as a proof of the others. diff --git a/test/jepsen/campaign.sh b/test/jepsen/campaign.sh new file mode 100755 index 0000000..63ffe54 --- /dev/null +++ b/test/jepsen/campaign.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/../.." && pwd)" + +time_limit="${GROUP_JEPSEN_CAMPAIGN_TIME:-300}" +test_count="${GROUP_JEPSEN_CAMPAIGN_COUNT:-20}" +concurrency="${GROUP_JEPSEN_CAMPAIGN_CONCURRENCY:-4n}" +owner_count="${GROUP_JEPSEN_CAMPAIGN_OWNERS:-128}" +key_count="${GROUP_JEPSEN_CAMPAIGN_KEYS:-32}" +recovery_time="${GROUP_JEPSEN_CAMPAIGN_RECOVERY:-15}" +artifact_dir="${GROUP_JEPSEN_CAMPAIGN_ARTIFACT_DIR:-}" + +cd "${repo_dir}" + +if [[ "${GROUP_JEPSEN_SKIP_CHECKER:-0}" != "1" ]]; then + "${script_dir}/checker.sh" +fi + +export GROUP_JEPSEN_SKIP_CHECKER=1 +mkdir -p "${script_dir}/.cache" + +if [[ -z "${artifact_dir}" ]]; then + artifact_dir="$(mktemp -d "${script_dir}/.cache/campaign.XXXXXX")" +else + mkdir -p "${artifact_dir}" +fi + +echo "Jepsen campaign artifacts: ${artifact_dir}" + +for transport in distribution tcp chaos; do + for scenario in mixed permanent; do + log="${artifact_dir}/${transport}-${scenario}.log" + echo "Starting ${transport}/${scenario}: ${test_count} histories x ${time_limit}s" + + if "${script_dir}/run.sh" test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency "${concurrency}" \ + --time-limit "${time_limit}" \ + --test-count "${test_count}" \ + --key-count "${key_count}" \ + --owner-count "${owner_count}" \ + --fault-interval 2 \ + --recovery-time "${recovery_time}" \ + --transport "${transport}" \ + --scenario "${scenario}" >"${log}" 2>&1; then + valid_count="$(rg -c "Everything looks good" "${log}" || true)" + + if [[ "${valid_count}" != "${test_count}" ]]; then + echo "Failed ${transport}/${scenario}: expected ${test_count} valid histories, found ${valid_count:-0}" >&2 + tail -200 "${log}" >&2 + exit 1 + fi + + echo "Passed ${transport}/${scenario}: ${valid_count}/${test_count} valid histories" + else + status=$? + echo "Failed ${transport}/${scenario}; tail of ${log}:" >&2 + tail -200 "${log}" >&2 + exit "${status}" + fi + done +done + +echo "Jepsen campaign passed: ${artifact_dir}" diff --git a/test/jepsen/checker.sh b/test/jepsen/checker.sh new file mode 100755 index 0000000..336063f --- /dev/null +++ b/test/jepsen/checker.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +exec "${script_dir}/lein.sh" test diff --git a/test/jepsen/docker-compose.yml b/test/jepsen/docker-compose.yml new file mode 100644 index 0000000..e93b5bd --- /dev/null +++ b/test/jepsen/docker-compose.yml @@ -0,0 +1,49 @@ +services: + n1: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n1 + hostname: n1 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n1 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + ports: ["19081:9080"] + networks: [group] + + n2: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n2 + hostname: n2 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n2 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + ports: ["19082:9080"] + networks: [group] + + n3: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n3 + hostname: n3 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n3 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + ports: ["19083:9080"] + networks: [group] + +networks: + group: + name: group-jepsen-net diff --git a/test/jepsen/entrypoint.sh b/test/jepsen/entrypoint.sh new file mode 100755 index 0000000..8911254 --- /dev/null +++ b/test/jepsen/entrypoint.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GROUP_JEPSEN_NODE:?GROUP_JEPSEN_NODE is required}" + +exec elixir \ + --sname group \ + --cookie group_jepsen \ + --erl "-kernel net_ticktime 2" \ + -S mix run --no-compile --no-deps-check test/jepsen/node.exs diff --git a/test/jepsen/lein.sh b/test/jepsen/lein.sh new file mode 100755 index 0000000..5c70bef --- /dev/null +++ b/test/jepsen/lein.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cache_dir="${script_dir}/.cache" +lein="${cache_dir}/lein" + +mkdir -p "${cache_dir}" + +if [[ ! -x "${lein}" ]]; then + curl --fail --silent --show-error --location \ + https://raw.githubusercontent.com/technomancy/leiningen/2.12.0/bin/lein \ + --output "${lein}" + chmod +x "${lein}" +fi + +export LEIN_HOME="${cache_dir}/lein-home" +cd "${script_dir}" + +exec "${lein}" "$@" diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs new file mode 100644 index 0000000..76817fb --- /dev/null +++ b/test/jepsen/node.exs @@ -0,0 +1,1376 @@ +defmodule Group.Jepsen.Transport.Stats do + @moduledoc false + use GenServer + + @table :group_jepsen_transport_stats + @gate :group_jepsen_transport_gate + @persistent_event_log "/tmp/group-jepsen-persistent-events" + + def start_link(_opts), do: GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + + def increment(event, amount \\ 1) do + :ets.update_counter(@table, event, {2, amount}, {event, 0}) + :ok + rescue + ArgumentError -> :ok + end + + def increment_persistent(event, amount \\ 1) do + increment(event, amount) + File.write!(@persistent_event_log, "#{event}\t#{amount}\n", [:append]) + :ok + end + + def observe_max(event, value) when is_integer(value) and value >= 0 do + current = :ets.update_counter(@table, event, {2, 0}, {event, 0}) + if value > current, do: :ets.insert(@table, {event, value}) + :ok + rescue + ArgumentError -> :ok + end + + def snapshot do + persisted = persistent_events() + + @table + |> :ets.tab2list() + |> Map.new() + |> Map.merge(persisted, fn _event, current, durable -> max(current, durable) end) + end + + def block(target_node), do: :ets.insert(@gate, {target_node}) + def unblock(target_node), do: :ets.delete(@gate, target_node) + def heal, do: :ets.delete_all_objects(@gate) + def blocked?(target_node), do: :ets.member(@gate, target_node) + + @impl true + def init(:ok) do + _table = :ets.new(@table, [:named_table, :public, :set, write_concurrency: true]) + _gate = :ets.new(@gate, [:named_table, :public, :set, read_concurrency: true]) + {:ok, %{}} + end + + defp persistent_events do + case File.read(@persistent_event_log) do + {:ok, contents} -> + contents + |> String.split("\n", trim: true) + |> Enum.reduce(%{}, fn line, events -> + case String.split(line, "\t", parts: 2) do + [event, amount] -> + Map.update( + events, + String.to_existing_atom(event), + String.to_integer(amount), + &(&1 + String.to_integer(amount)) + ) + + _invalid -> + events + end + end) + + {:error, :enoent} -> + %{} + end + end +end + +defmodule Group.Jepsen.Transport.Common do + @moduledoc false + alias Group.Jepsen.Transport.Stats + + def try_send(delegate, group, target_node, shard, frame, opts) do + record(frame) + + if Stats.blocked?(target_node) do + Stats.increment(:logical_drop) + :ok + else + result = delegate.try_send(group, target_node, shard, frame, opts) + Stats.increment(transport_result(result)) + observe_outbox(group, shard) + result + end + end + + def record({:snapshot_chunk, _version, _stream, _seq, _index, chunk_count, _, _, _, _}) do + Stats.increment(:snapshot_chunk) + + if chunk_count > 1 do + Stats.increment(:multi_chunk_snapshot_chunk) + end + end + + def record({:delta_batch, _version, _runs}), do: Stats.increment(:delta_batch) + def record(_frame), do: Stats.increment(:other_frame) + + defp transport_result(:ok), do: :transport_ok + defp transport_result(:busy), do: :transport_busy + defp transport_result(:disconnected), do: :transport_disconnected + + defp observe_outbox(group, shard) do + case Process.whereis(Group.Replica.Transport.Outbox.name(group, shard)) do + pid when is_pid(pid) -> + case Process.info(pid, :message_queue_len) do + {:message_queue_len, length} -> Stats.observe_max(:outbox_mailbox_peak, length) + _ -> :ok + end + + nil -> + :ok + end + end +end + +defmodule Group.Jepsen.Transport.Distribution do + @moduledoc false + @behaviour Group.Replica.Transport + + alias Group.Jepsen.Transport.{Common, Stats} + alias Group.Replica.Transport.Distribution, as: Delegate + + @impl true + def id, do: Delegate.id() + + @impl true + def descriptor(group, opts), do: Delegate.descriptor(group, opts) + + @impl true + def child_spec(opts), do: {Stats, opts} + + @impl true + def try_send(group, target_node, shard, frame, opts) do + Common.try_send(Delegate, group, target_node, shard, frame, opts) + end +end + +defmodule Group.Jepsen.Transport.TCP do + @moduledoc false + @behaviour Group.Replica.Transport + + alias Group.Jepsen.Transport.Common + alias Group.Replica.Transport.TCP, as: Delegate + + @impl true + def id, do: Delegate.id() + + @impl true + def descriptor(group, opts), do: Delegate.descriptor(group, opts) + + @impl true + def child_spec(opts) do + %{ + id: {__MODULE__, Keyword.fetch!(opts, :name)}, + start: {Group.Jepsen.Transport.TCP.Supervisor, :start_link, [opts]}, + type: :supervisor + } + end + + @impl true + def try_send(group, target_node, shard, frame, opts) do + Common.try_send(Delegate, group, target_node, shard, frame, opts) + end + + @impl true + def peer_up(group, remote_node, descriptor, opts), + do: Delegate.peer_up(group, remote_node, descriptor, opts) + + @impl true + def peer_down(group, remote_node, opts), do: Delegate.peer_down(group, remote_node, opts) +end + +defmodule Group.Jepsen.Transport.TCP.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + children = [ + {Group.Jepsen.Transport.Stats, opts}, + Group.Replica.Transport.TCP.child_spec(opts) + ] + + Supervisor.init(children, strategy: :one_for_one) + end +end + +defmodule Group.Jepsen.Transport.Chaos do + @moduledoc false + @behaviour Group.Replica.Transport + + alias Group.Jepsen.Transport.{Common, Stats} + + @impl true + def id, do: :group_jepsen_unordered_v1 + + @impl true + def descriptor(_group, _opts), do: :group_jepsen_unordered_v1 + + @impl true + def child_spec(opts) do + %{ + id: {__MODULE__, Keyword.fetch!(opts, :name)}, + start: {Group.Jepsen.Transport.Chaos.Supervisor, :start_link, [opts]}, + type: :supervisor + } + end + + @impl true + def try_send(group, target_node, shard, frame, _opts) do + Common.record(frame) + + if Stats.blocked?(target_node) do + Stats.increment(:logical_drop) + :ok + else + case Process.whereis(worker_name(group, shard)) do + pid when is_pid(pid) -> + send(pid, {:send, target_node, frame}) + :ok + + nil -> + :disconnected + end + end + end + + def worker_name(group, shard), do: :"#{group}_jepsen_chaos_#{shard}" +end + +defmodule Group.Jepsen.Transport.Chaos.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + group = Keyword.fetch!(opts, :name) + num_shards = Keyword.fetch!(opts, :num_shards) + + workers = + for shard <- 0..(num_shards - 1) do + %{ + id: {Group.Jepsen.Transport.Chaos.Worker, shard}, + start: {Group.Jepsen.Transport.Chaos.Worker, :start_link, [group, shard]} + } + end + + Supervisor.init([{Group.Jepsen.Transport.Stats, opts} | workers], strategy: :one_for_one) + end +end + +defmodule Group.Jepsen.Transport.Chaos.Worker do + @moduledoc false + use GenServer + + alias Group.Jepsen.Transport.{Chaos, Stats} + + def start_link(group, shard) do + GenServer.start_link(__MODULE__, {group, shard}, name: Chaos.worker_name(group, shard)) + end + + @impl true + def init({group, shard}), do: {:ok, %{group: group, shard: shard, counter: 0}} + + @impl true + def handle_info({:send, target_node, frame}, state) do + counter = state.counter + 1 + next = %{state | counter: counter} + + if rem(counter, 5) == 0 do + Stats.increment(:chaos_drop) + else + delay = rem(counter * 17, 41) + Process.send_after(self(), {:deliver, target_node, frame}, delay) + + if rem(counter, 7) == 0 do + Stats.increment(:chaos_duplicate) + Process.send_after(self(), {:deliver, target_node, frame}, rem(delay + 19, 47)) + end + + if delay > 0, do: Stats.increment(:chaos_delay) + end + + {:noreply, next} + end + + def handle_info({:deliver, target_node, frame}, state) do + destination = {Group.Replica.shard_name(state.group, state.shard), target_node} + message = {:group_replica_frame, node(), frame} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> Stats.increment(:chaos_delivered) + false -> Stats.increment(:chaos_busy) + end + + {:noreply, state} + end +end + +defmodule Group.Jepsen.Transport.Control do + @moduledoc false + + alias Group.Jepsen.Transport.Stats + + def profile do + case System.get_env("GROUP_JEPSEN_TRANSPORT", "distribution") do + "distribution" -> :distribution + "tcp" -> :tcp + "chaos" -> :chaos + other -> raise "unknown GROUP_JEPSEN_TRANSPORT #{inspect(other)}" + end + end + + def transport(node_id) do + case profile() do + :distribution -> + Group.Jepsen.Transport.Distribution + + :chaos -> + Group.Jepsen.Transport.Chaos + + :tcp -> + {:ok, advertised_ip} = :inet.getaddr(String.to_charlist(node_id), :inet) + + {Group.Jepsen.Transport.TCP, + [ + ip: {0, 0, 0, 0}, + advertised_ip: advertised_ip, + port: 10_000, + max_queue: 32, + connect_timeout: 100, + send_timeout: 100, + reconnect_interval: 25, + outbox_batch_size: 16, + outbox_batch_bytes: 65_536, + outbox_flush_interval: 1, + outbox_deadline: 100 + ]} + end + end + + def block(target_node) do + Stats.block(target_node) + maybe_disconnect(target_node) + :ok + end + + def unblock(target_node) do + Stats.unblock(target_node) + maybe_reconnect(target_node) + :ok + end + + def heal(peer_nodes) do + Stats.heal() + Enum.each(peer_nodes, &maybe_reconnect/1) + :ok + end + + def reset(target_node) do + maybe_disconnect(target_node) + Process.sleep(10) + maybe_reconnect(target_node) + :ok + end + + defp maybe_disconnect(target_node) do + if profile() == :tcp do + Group.Replica.Transport.TCP.disconnect_peer(:jepsen_group, target_node) + end + catch + :exit, _ -> :ok + end + + defp maybe_reconnect(target_node) do + if profile() == :tcp do + Group.Replica.Transport.TCP.reconnect_peer(:jepsen_group, target_node) + end + catch + :exit, _ -> :ok + end +end + +defmodule Group.Jepsen.ConflictResolver do + @moduledoc false + + def resolve(_name, _key, {pid1, meta1, _time1}, {pid2, meta2, _time2}) do + if rank(meta1) >= rank(meta2), do: pid1, else: pid2 + end + + defp rank(%{revision: revision, token: token}), do: {revision, token} + defp rank(_meta), do: {-1, ""} +end + +defmodule Group.Jepsen.Owner do + @moduledoc false + use GenServer + + def start(token), do: GenServer.start(__MODULE__, token) + + @impl true + def init(token), do: {:ok, %{token: token, registrations: %{}, memberships: %{}}} + + @impl true + def handle_call({:mutate, :register, cluster, key, revision}, _from, state) do + meta = %{token: state.token, revision: revision} + + case safe_group_call(fn -> + Group.register(:jepsen_group, registry_key(key), meta, cluster_opts(cluster)) + end) do + :ok -> + entry = %{cluster: cluster, key: key, revision: revision} + state = put_in(state.registrations[{cluster, key}], entry) + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :unregister, cluster, key, _revision}, _from, state) do + owner_key = {cluster, key} + + if Map.has_key?(state.registrations, owner_key) do + case safe_group_call(fn -> + Group.unregister(:jepsen_group, registry_key(key), cluster_opts(cluster)) + end) do + :ok -> + state = %{state | registrations: Map.delete(state.registrations, owner_key)} + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + else + {:reply, {:error, :not_owned, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :join, cluster, key, revision}, _from, state) do + meta = %{token: state.token, revision: revision} + + case safe_group_call(fn -> + Group.join(:jepsen_group, pg_key(key), meta, cluster_opts(cluster)) + end) do + :ok -> + entry = %{cluster: cluster, key: key, revision: revision} + state = put_in(state.memberships[{cluster, key}], entry) + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :leave, cluster, key, _revision}, _from, state) do + owner_key = {cluster, key} + + if Map.has_key?(state.memberships, owner_key) do + case safe_group_call(fn -> + Group.leave(:jepsen_group, pg_key(key), cluster_opts(cluster)) + end) do + :ok -> + state = %{state | memberships: Map.delete(state.memberships, owner_key)} + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + else + {:reply, {:error, :not_owned, snapshot(state)}, state} + end + end + + def handle_call({:drop_cluster, cluster}, _from, state) do + registrations = drop_cluster(state.registrations, cluster) + memberships = drop_cluster(state.memberships, cluster) + state = %{state | registrations: registrations, memberships: memberships} + {:reply, :ok, state} + end + + def handle_call(:snapshot, _from, state), do: {:reply, snapshot(state), state} + + defp drop_cluster(entries, cluster) do + entries + |> Enum.reject(fn {{entry_cluster, _key}, _entry} -> entry_cluster == cluster end) + |> Map.new() + end + + defp snapshot(state) do + %{ + token: state.token, + registrations: state.registrations |> Map.values() |> sort_entries(), + memberships: state.memberships |> Map.values() |> sort_entries() + } + end + + defp sort_entries(entries), do: Enum.sort_by(entries, &{&1.cluster || "", &1.key}) + + defp safe_group_call(fun) do + fun.() + rescue + exception -> {:error, {:exception, Exception.message(exception)}} + catch + kind, reason -> {:error, {kind, reason}} + end + + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + defp registry_key(key), do: "jepsen/registry/#{key}" + defp pg_key(key), do: "jepsen/pg/#{key}" +end + +defmodule Group.Jepsen.Driver do + @moduledoc false + use GenServer + + alias Group.Jepsen.Transport.Stats + + @driver_count 8 + @unexpected_death_log "/tmp/group-jepsen-unexpected-deaths" + + def start_link(opts) do + index = Keyword.fetch!(opts, :index) + GenServer.start_link(__MODULE__, opts, name: name(index)) + end + + def child_specs(opts) do + for index <- 0..(@driver_count - 1) do + %{ + id: {__MODULE__, index}, + start: {__MODULE__, :start_link, [Keyword.put(opts, :index, index)]} + } + end + end + + def mutate(operation, logical_owner, cluster, key, revision) do + started_at = System.monotonic_time(:microsecond) + + response = + GenServer.call( + driver(logical_owner), + {:mutate, operation, logical_owner, cluster, key, revision}, + 10_000 + ) + + Map.put(response, :latency_us, System.monotonic_time(:microsecond) - started_at) + end + + def kill(logical_owner), do: GenServer.call(driver(logical_owner), {:kill, logical_owner}) + + def drop_cluster(cluster) do + Enum.each(names(), &GenServer.call(&1, {:drop_cluster, cluster}, 10_000)) + :ok + end + + def owner_snapshots do + names() + |> Enum.flat_map(&GenServer.call(&1, :owner_snapshots, 30_000)) + |> Enum.sort_by(& &1.token) + end + + def unexpected_deaths do + in_memory = Enum.flat_map(names(), &GenServer.call(&1, :unexpected_deaths, 30_000)) + + (in_memory ++ persisted_unexpected_deaths()) + |> Enum.uniq() + |> Enum.sort_by(& &1.token) + end + + @impl true + def init(opts) do + {:ok, + %{ + node_id: Keyword.fetch!(opts, :node_id), + boot_id: Keyword.fetch!(opts, :boot_id), + owners: %{}, + monitors: %{}, + incarnations: %{}, + unexpected_deaths: [] + }} + end + + @impl true + def handle_call({:mutate, operation, logical_owner, cluster, key, revision}, _from, state) do + {pid, state} = owner(state, logical_owner) + + try do + case GenServer.call(pid, {:mutate, operation, cluster, key, revision}, 8_000) do + {:ok, owner_state} -> + {:reply, %{status: :ok, owner: owner_state}, + put_owner_state(state, logical_owner, pid, owner_state)} + + {:error, reason, owner_state} -> + {:reply, %{status: :fail, error: inspect(reason), owner: owner_state}, + put_owner_state(state, logical_owner, pid, owner_state)} + end + catch + :exit, reason -> + {:reply, %{status: :unknown, error: inspect(reason)}, state} + end + end + + def handle_call({:kill, logical_owner}, _from, state) do + case Map.get(state.owners, logical_owner) do + nil -> + {:reply, %{status: :ok, killed: nil}, state} + + {pid, token, monitor_ref, _owner_state} -> + if Process.alive?(pid) do + Process.exit(pid, :kill) + Process.demonitor(monitor_ref, [:flush]) + + {:reply, %{status: :ok, killed: token}, + %{ + state + | owners: Map.delete(state.owners, logical_owner), + monitors: Map.delete(state.monitors, monitor_ref) + }} + else + {:reply, %{status: :unknown, error: "owner already dead"}, state} + end + end + end + + def handle_call({:drop_cluster, cluster}, _from, state) do + Enum.each(state.owners, fn {_logical_owner, {pid, _token, _monitor, _owner_state}} -> + if Process.alive?(pid), do: GenServer.call(pid, {:drop_cluster, cluster}, 10_000) + end) + + owners = + Map.new(state.owners, fn {logical_owner, {pid, token, monitor, _owner_state}} -> + owner_state = if Process.alive?(pid), do: GenServer.call(pid, :snapshot), else: nil + {logical_owner, {pid, token, monitor, owner_state}} + end) + + {:reply, :ok, %{state | owners: owners}} + end + + def handle_call(:owner_snapshots, _from, state) do + owners = + state.owners + |> Enum.flat_map(fn + {_logical_owner, {_pid, _token, _monitor_ref, nil}} -> [] + {_logical_owner, {_pid, _token, _monitor_ref, owner_state}} -> [owner_state] + end) + + {:reply, owners, state} + end + + def handle_call(:unexpected_deaths, _from, state) do + {:reply, state.unexpected_deaths, state} + end + + @impl true + def handle_info({:DOWN, monitor_ref, :process, _pid, reason}, state) do + case Map.pop(state.monitors, monitor_ref) do + {nil, monitors} -> + {:noreply, %{state | monitors: monitors}} + + {{logical_owner, token}, monitors} -> + owners = + case Map.get(state.owners, logical_owner) do + {_pid, ^token, ^monitor_ref, _owner_state} -> Map.delete(state.owners, logical_owner) + _newer_incarnation -> state.owners + end + + unexpected_deaths = + if match?({:group_registry_conflict, _key, _winner_meta}, reason) do + Stats.increment_persistent(:registry_conflict_death) + state.unexpected_deaths + else + death = %{token: token, reason: inspect(reason)} + :ok = persist_unexpected_death(death) + [death | state.unexpected_deaths] + end + + {:noreply, + %{state | owners: owners, monitors: monitors, unexpected_deaths: unexpected_deaths}} + end + end + + defp owner(state, logical_owner) do + case Map.get(state.owners, logical_owner) do + {pid, _token, _monitor_ref, _owner_state} when is_pid(pid) -> + if Process.alive?(pid), do: {pid, state}, else: start_owner(state, logical_owner) + + nil -> + start_owner(state, logical_owner) + end + end + + defp start_owner(state, logical_owner) do + incarnation = Map.get(state.incarnations, logical_owner, 0) + 1 + token = "#{state.node_id}/#{state.boot_id}/#{logical_owner}/#{incarnation}" + {:ok, pid} = Group.Jepsen.Owner.start(token) + monitor_ref = Process.monitor(pid) + owner_state = %{token: token, registrations: [], memberships: []} + + state = %{ + state + | owners: Map.put(state.owners, logical_owner, {pid, token, monitor_ref, owner_state}), + monitors: Map.put(state.monitors, monitor_ref, {logical_owner, token}), + incarnations: Map.put(state.incarnations, logical_owner, incarnation) + } + + {pid, state} + end + + defp put_owner_state(state, logical_owner, pid, owner_state) do + owners = + case Map.get(state.owners, logical_owner) do + {^pid, token, monitor_ref, _old_owner_state} -> + Map.put(state.owners, logical_owner, {pid, token, monitor_ref, owner_state}) + + _replaced_owner -> + state.owners + end + + %{state | owners: owners} + end + + defp names, do: Enum.map(0..(@driver_count - 1), &name/1) + defp driver(logical_owner), do: name(:erlang.phash2(logical_owner, @driver_count)) + defp name(index), do: :"group_jepsen_driver_#{index}" + + defp persist_unexpected_death(%{token: token, reason: reason}) do + File.write(@unexpected_death_log, token <> "\t" <> reason <> "\n", [:append]) + end + + defp persisted_unexpected_deaths do + case File.read(@unexpected_death_log) do + {:ok, contents} -> + contents + |> String.split("\n", trim: true) + |> Enum.flat_map(fn line -> + case String.split(line, "\t", parts: 2) do + [token, reason] -> [%{token: token, reason: reason}] + _invalid -> [] + end + end) + + {:error, :enoent} -> + [] + + {:error, reason} -> + [%{token: "ORACLE-READ-FAILURE", reason: inspect(reason)}] + end + end +end + +defmodule Group.Jepsen.Driver.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + + @impl true + def init(opts), + do: Supervisor.init(Group.Jepsen.Driver.child_specs(opts), strategy: :one_for_one) +end + +defmodule Group.Jepsen.Cluster do + @moduledoc false + use GenServer + + def start_link(clusters), do: GenServer.start_link(__MODULE__, clusters, name: __MODULE__) + def connect(cluster), do: GenServer.call(__MODULE__, {:connect, cluster}, 60_000) + def disconnect(cluster), do: GenServer.call(__MODULE__, {:disconnect, cluster}, 60_000) + def connect_all, do: GenServer.call(__MODULE__, :connect_all, 60_000) + + @impl true + def init(clusters) do + :ok = Group.connect(:jepsen_group, clusters) + {:ok, %{clusters: clusters}} + end + + @impl true + def handle_call({:connect, cluster}, _from, state) do + result = Group.connect(:jepsen_group, cluster) + {:reply, result(result), state} + end + + def handle_call({:disconnect, cluster}, _from, state) do + result = Group.disconnect(:jepsen_group, cluster) + if result == :ok, do: Group.Jepsen.Driver.drop_cluster(cluster) + {:reply, result(result), state} + end + + def handle_call(:connect_all, _from, state) do + result = Group.connect(:jepsen_group, state.clusters) + {:reply, result(result), state} + end + + defp result(:ok), do: %{status: :ok} + defp result({:error, reason}), do: %{status: :fail, error: inspect(reason)} +end + +defmodule Group.Jepsen.Invariant do + @moduledoc false + + alias Group.Replica.{Data, Protocol} + + def snapshot(retired_nodes) do + config = Group.get_config(:jepsen_group) + shards = 0..(config.num_shards - 1) + + errors = + check("dual indexes", &assert_dual_indexes/0) ++ + check("registry claims", &assert_registry_claims/0) ++ + check("oplog", &assert_oplogs/0) ++ + check("cursor authority", &assert_cursor_authority/0) ++ + check("retired origins", fn -> assert_retired_origins(retired_nodes) end) + + staging_count = + Enum.reduce(shards, 0, fn shard, total -> + state = :sys.get_state(Group.Replica.shard_name(:jepsen_group, shard)) + total + map_size(state.snapshot_transfers) + end) + + oplog_entries = + Enum.reduce(shards, 0, fn shard, total -> + total + :ets.info(Data.replica_oplog_order_table(:jepsen_group, shard), :size) + end) + + %{ + healthy: errors == [] and staging_count == 0, + errors: errors, + snapshot_staging_count: staging_count, + oplog_entries: oplog_entries, + oplog_max_entries_per_shard: config.replicated_oplog_max_entries, + shard_mailbox_max: + mailbox_max(Enum.map(shards, &Group.Replica.shard_name(:jepsen_group, &1))), + outbox_mailbox_max: + mailbox_max(Enum.map(shards, &Group.Replica.Transport.Outbox.name(:jepsen_group, &1))), + total_memory_bytes: :erlang.memory(:total) + } + rescue + exception -> + %{ + healthy: false, + errors: ["invariant snapshot failed: #{Exception.message(exception)}"], + snapshot_staging_count: -1 + } + end + + defp check(label, fun) do + fun.() + [] + rescue + exception -> ["#{label}: #{Exception.message(exception)}"] + catch + kind, reason -> ["#{label}: #{inspect({kind, reason})}"] + end + + defp assert_dual_indexes do + shards(fn shard -> + reg_key = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + reg_pid = + Data.reg_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{pid, cluster, key}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + pg_key = + Data.pg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key, pid}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + pg_pid = + Data.pg_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{pid, cluster, key}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + assert_equal!(reg_key, reg_pid, "registry dual indexes shard #{shard}") + assert_equal!(pg_key, pg_pid, "PG dual indexes shard #{shard}") + end) + + cluster_nodes = + Data.cluster_nodes_table(:jepsen_group) + |> :ets.tab2list() + |> MapSet.new(fn {cluster, origin} -> {cluster, origin} end) + + node_clusters = + Data.node_clusters_table(:jepsen_group) + |> :ets.tab2list() + |> MapSet.new(fn {origin, cluster} -> {cluster, origin} end) + + assert_equal!(cluster_nodes, node_clusters, "cluster dual indexes") + end + + defp assert_registry_claims do + shards(fn shard -> + by_key = + Data.reg_claim_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn + {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + by_pid = + Data.reg_claim_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn + {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + assert_equal!(by_key, by_pid, "registry claim indexes shard #{shard}") + + invalid_origin = + Enum.find(by_key, fn {_cluster, _key, pid, _meta, _time, origin, _gen, _epoch, _seq} -> + node(pid) != origin + end) + + if invalid_origin, do: raise("claim with invalid PID origin #{inspect(invalid_origin)}") + + authority = + MapSet.new(by_key, fn {cluster, key, pid, meta, time, origin, _gen, _epoch, _seq} -> + {cluster, key, pid, meta, time, origin} + end) + + visible = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + missing = MapSet.difference(visible, authority) + + if MapSet.size(missing) > 0, + do: raise("visible registry rows lack claims #{inspect(missing)}") + end) + end + + defp assert_oplogs do + max_entries = Group.get_config(:jepsen_group).replicated_oplog_max_entries + + shards(fn shard -> + oplog = + Data.replica_oplog_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream, seq}, append_id, _mutations} -> {append_id, stream, seq} end) + + order = + Data.replica_oplog_order_table(:jepsen_group, shard) |> :ets.tab2list() |> MapSet.new() + + assert_equal!(oplog, order, "oplog/order shard #{shard}") + + if MapSet.size(order) > max_entries do + raise "oplog bound exceeded shard #{shard}: #{MapSet.size(order)} > #{max_entries}" + end + + Data.replica_stream_meta_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream, head, floor, applied} -> + unless floor >= 1 and floor <= head + 1 and applied >= 0 and applied <= head do + raise "invalid stream bounds #{inspect({stream, head, floor, applied})}" + end + + retained = + oplog + |> Enum.filter(fn {_append, row_stream, _seq} -> row_stream == stream end) + |> Enum.map(&elem(&1, 2)) + |> Enum.sort() + + expected = if floor <= head, do: Enum.to_list(floor..head), else: [] + if retained != expected, do: raise("non-contiguous oplog #{inspect(stream)}") + end) + end) + end + + defp assert_cursor_authority do + shards(fn shard -> + Data.replica_cursor_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream, seq} -> + origin = Protocol.stream_origin(stream) + cluster = Protocol.stream_cluster(stream) + + valid? = + Protocol.stream_name(stream) == :jepsen_group and + Protocol.stream_shard(stream) == shard and + origin != node() and + Protocol.stream_generation(stream) == Data.remote_generation(:jepsen_group, origin) and + Protocol.stream_epoch(stream) == + Data.remote_cluster_epoch(:jepsen_group, origin, cluster) and seq >= 0 + + unless valid?, do: raise("cursor lacks current authority #{inspect({stream, seq})}") + end) + end) + end + + defp assert_retired_origins(retired_nodes) do + Enum.each(retired_nodes, fn origin -> + shards(fn shard -> + claims = + Data.reg_claim_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {{_cluster, _key, row_origin, _gen, _epoch}, _, _, _, _} -> + row_origin == origin + end) + + registry = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _pid, _meta, _time, row_origin} -> row_origin == origin end) + + pg = + Data.pg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _meta, _time, row_origin} -> row_origin == origin end) + + cursors = + Data.replica_cursor_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {stream, _seq} -> Protocol.stream_origin(stream) == origin end) + + view = Data.remote_view_generation(:jepsen_group, shard, origin) + + unless claims == [] and registry == [] and pg == [] and cursors == [] and is_nil(view) do + raise "retired origin retained on shard #{shard}: #{inspect(origin)}" + end + end) + + unless is_nil(Data.remote_generation(:jepsen_group, origin)) and + Data.clusters_for_node(:jepsen_group, origin) == [] do + raise "retired origin retained shared authority: #{inspect(origin)}" + end + end) + end + + defp shards(fun) do + num_shards = Group.get_config(:jepsen_group).num_shards + Enum.each(0..(num_shards - 1), fun) + end + + defp assert_equal!(left, right, label) do + if left != right do + raise "#{label}: left-only=#{inspect(MapSet.difference(left, right))} " <> + "right-only=#{inspect(MapSet.difference(right, left))}" + end + end + + defp mailbox_max(names) do + names + |> Enum.map(fn name -> + case Process.whereis(name) do + pid when is_pid(pid) -> + case Process.info(pid, :message_queue_len) do + {:message_queue_len, length} -> length + _ -> 0 + end + + nil -> + 0 + end + end) + |> Enum.max(fn -> 0 end) + end +end + +defmodule Group.Jepsen.Snapshot do + @moduledoc false + + def capture(node_id, boot_id, key_count, clusters, retired_nodes) do + owners = Group.Jepsen.Driver.owner_snapshots() + + registry = + Map.new([nil | clusters], fn cluster -> + values = + Map.new(0..(key_count - 1), fn key -> + value = + case Group.lookup(:jepsen_group, registry_key(key), cluster_opts(cluster)) do + nil -> nil + {_pid, %{token: token}} -> token + {_pid, other} -> "INVALID:#{inspect(other)}" + end + + {key, value} + end) + + {cluster_name(cluster), values} + end) + + pg = + Map.new([nil | clusters], fn cluster -> + values = + Map.new(0..(key_count - 1), fn key -> + tokens = + :jepsen_group + |> Group.members(pg_key(key), cluster_opts(cluster)) + |> Enum.map(fn + {_pid, %{token: token}} -> token + {_pid, other} -> "INVALID:#{inspect(other)}" + end) + |> Enum.sort() + + {key, tokens} + end) + + {cluster_name(cluster), values} + end) + + %{ + status: :ok, + snapshot: %{ + node: node_id, + boot: boot_id, + peers: Group.nodes(:jepsen_group) |> Enum.map(&Atom.to_string/1) |> Enum.sort(), + owners: owners, + unexpected_deaths: Group.Jepsen.Driver.unexpected_deaths(), + transport_events: Group.Jepsen.Transport.Stats.snapshot(), + transport_profile: Group.Jepsen.Transport.Control.profile(), + internal: Group.Jepsen.Invariant.snapshot(retired_nodes), + registry: registry, + pg: pg + } + } + end + + defp cluster_name(nil), do: "root" + defp cluster_name(cluster), do: cluster + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + defp registry_key(key), do: "jepsen/registry/#{key}" + defp pg_key(key), do: "jepsen/pg/#{key}" +end + +defmodule Group.Jepsen.EDN do + @moduledoc false + + def encode(nil), do: "nil" + def encode(true), do: "true" + def encode(false), do: "false" + def encode(value) when is_integer(value), do: Integer.to_string(value) + def encode(value) when is_binary(value), do: inspect(value) + + def encode(value) when is_atom(value) do + ":" <> (value |> Atom.to_string() |> String.replace("_", "-")) + end + + def encode(value) when is_list(value) do + "[" <> Enum.map_join(value, " ", &encode/1) <> "]" + end + + def encode(%MapSet{} = value) do + "#" <> "{" <> (value |> Enum.sort() |> Enum.map_join(" ", &encode/1)) <> "}" + end + + def encode(value) when is_map(value) do + encoded = + value + |> Enum.map(fn {key, inner} -> {encode(key), encode(inner)} end) + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.map_join(" ", fn {key, inner} -> key <> " " <> inner end) + + "{" <> encoded <> "}" + end +end + +defmodule Group.Jepsen.Wire do + @moduledoc false + + def serve(port, context) do + {:ok, listener} = + :gen_tcp.listen(port, [:binary, packet: 4, active: false, reuseaddr: true]) + + accept(listener, context) + end + + defp accept(listener, context) do + {:ok, socket} = :gen_tcp.accept(listener) + spawn(fn -> connection(socket, context) end) + accept(listener, context) + end + + defp connection(socket, context) do + case :gen_tcp.recv(socket, 0) do + {:ok, payload} -> + response = payload |> command(context) |> Group.Jepsen.EDN.encode() + :ok = :gen_tcp.send(socket, response) + connection(socket, context) + + {:error, _reason} -> + :gen_tcp.close(socket) + end + end + + defp command(payload, context) do + case String.split(payload, "\t") do + ["ping"] -> + %{status: :ok} + + ["ready", expected] -> + expected = String.to_integer(expected) + + if length(Group.nodes(:jepsen_group)) == expected - 1 do + %{status: :ok} + else + %{status: :retry, peers: length(Group.nodes(:jepsen_group))} + end + + ["mutate", operation, logical_owner, cluster, key, revision] -> + Group.Jepsen.Driver.mutate( + String.to_existing_atom(operation), + logical_owner, + parse_cluster(cluster), + String.to_integer(key), + String.to_integer(revision) + ) + + ["kill", logical_owner] -> + Group.Jepsen.Driver.kill(logical_owner) + + ["cluster", "connect", cluster] -> + Group.Jepsen.Cluster.connect(cluster) + + ["cluster", "disconnect", cluster] -> + Group.Jepsen.Cluster.disconnect(cluster) + + ["cluster", "connect-all"] -> + Group.Jepsen.Cluster.connect_all() + + ["transport", "block", target] -> + :ok = Group.Jepsen.Transport.Control.block(String.to_atom(target)) + %{status: :ok} + + ["transport", "unblock", target] -> + :ok = Group.Jepsen.Transport.Control.unblock(String.to_atom(target)) + %{status: :ok} + + ["transport", "heal"] -> + :ok = Group.Jepsen.Transport.Control.heal(context.peers) + %{status: :ok} + + ["transport", "reset", target] -> + :ok = Group.Jepsen.Transport.Control.reset(String.to_atom(target)) + %{status: :ok} + + ["snapshot", key_count, clusters, retired] -> + Group.Jepsen.Snapshot.capture( + context.node_id, + context.boot_id, + String.to_integer(key_count), + parse_list(clusters), + retired |> parse_list() |> Enum.map(&String.to_atom/1) + ) + + ["corrupt", mode] -> + corrupt(mode) + + other -> + %{status: :fail, error: "unknown command #{inspect(other)}"} + end + rescue + exception -> %{status: :unknown, error: Exception.message(exception)} + catch + kind, reason -> %{status: :unknown, error: inspect({kind, reason})} + end + + defp corrupt("unexpected-death") do + File.write!( + "/tmp/group-jepsen-unexpected-deaths", + "oracle-self-test\t:injected\n", + [:append] + ) + + %{status: :ok} + end + + defp corrupt("internal-index") do + table = Group.Replica.Data.reg_by_pid_table(:jepsen_group, 0) + :ets.insert(table, {{self(), nil, "jepsen/registry/corrupt"}, %{}, 0, node()}) + %{status: :ok} + end + + defp corrupt(other), do: %{status: :fail, error: "unknown corruption #{inspect(other)}"} + defp parse_cluster("root"), do: nil + defp parse_cluster(cluster), do: cluster + defp parse_list(""), do: [] + defp parse_list(value), do: String.split(value, ",", trim: true) +end + +defmodule Group.Jepsen.Main do + @moduledoc false + @clusters ["red", "blue"] + + def run(argv) do + {opts, _rest, []} = + OptionParser.parse(argv, + strict: [node: :string, port: :integer, peers: :string] + ) + + node_id = Keyword.get(opts, :node) || System.fetch_env!("GROUP_JEPSEN_NODE") + + port = + Keyword.get(opts, :port) || + System.get_env("GROUP_JEPSEN_PORT", "9080") |> String.to_integer() + + peers = + (Keyword.get(opts, :peers) || + System.get_env("GROUP_JEPSEN_PEERS", "group@n1,group@n2,group@n3")) + |> String.split(",", trim: true) + |> Enum.map(&String.to_atom/1) + + {:ok, _apps} = Application.ensure_all_started(:group) + + {:ok, group} = + Group.start_link( + name: :jepsen_group, + shards: 4, + log: false, + resolve_registry_conflict: {Group.Jepsen.ConflictResolver, :resolve, []}, + replica_transport: Group.Jepsen.Transport.Control.transport(node_id), + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 16, + replicated_snapshot_chunk_target_bytes: 1_024, + replicated_anti_entropy_interval: 50, + replicated_peer_lease_timeout: 750 + ) + + Process.unlink(group) + boot_id = Base.encode16(:crypto.strong_rand_bytes(8), case: :lower) + + {:ok, _drivers} = + Group.Jepsen.Driver.Supervisor.start_link(node_id: node_id, boot_id: boot_id) + + {:ok, _cluster} = Group.Jepsen.Cluster.start_link(@clusters) + spawn_link(fn -> reconnect_loop(peers) end) + + Group.Jepsen.Wire.serve(port, %{ + node_id: node_id, + boot_id: boot_id, + peers: Enum.reject(peers, &(&1 == node())) + }) + end + + defp reconnect_loop(peers) do + Enum.each(peers, fn peer -> + if peer != node(), do: Node.connect(peer) + end) + + Process.sleep(100) + reconnect_loop(peers) + end +end + +Group.Jepsen.Main.run(System.argv()) diff --git a/test/jepsen/project.clj b/test/jepsen/project.clj new file mode 100644 index 0000000..05284a5 --- /dev/null +++ b/test/jepsen/project.clj @@ -0,0 +1,8 @@ +(defproject group-jepsen "0.1.0-SNAPSHOT" + :description "Jepsen lifecycle and convergence tests for Group" + :url "https://github.com/phoenixframework/group" + :license {:name "MIT"} + :dependencies [[org.clojure/clojure "1.12.4"] + [jepsen "0.3.13"]] + :main group.jepsen.core + :jvm-opts ["-Xmx4g" "-Djava.awt.headless=true" "-server"]) diff --git a/test/jepsen/qualify.sh b/test/jepsen/qualify.sh new file mode 100755 index 0000000..5de63c1 --- /dev/null +++ b/test/jepsen/qualify.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/../.." && pwd)" +artifact_dir="$(mktemp -d "${script_dir}/.cache/qualification.XXXXXX")" + +cd "${repo_dir}" + +mix run test/mutation/run.exs \ + accept_old_generation \ + advance_cursor_across_gap \ + registry_snapshot_is_additive \ + commit_incomplete_snapshot \ + disable_periodic_heads \ + skip_generation_purge + +run_jepsen() { + local expectation="$1" + local corruption="$2" + local log="${artifact_dir}/${expectation}-${corruption}.log" + local status=0 + + set +e + "${script_dir}/run.sh" test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 2n \ + --time-limit 6 \ + --fault-interval 1 \ + --recovery-time 5 \ + --transport distribution \ + --scenario mixed \ + --corruption "${corruption}" >"${log}" 2>&1 + status=$? + set -e + + if [[ "${expectation}" == "pass" ]] && [[ "${status}" -ne 0 ]]; then + echo "healthy Jepsen baseline failed; see ${log}" >&2 + return 1 + fi + + if [[ "${expectation}" == "fail" ]] && [[ "${status}" -eq 0 ]]; then + echo "Jepsen checker accepted corruption ${corruption}; see ${log}" >&2 + return 1 + fi + + echo "${expectation}: ${corruption} (${log})" +} + +run_jepsen pass none +run_jepsen fail unexpected-death +run_jepsen fail internal-index + +echo "mutation and live checker qualification passed" diff --git a/test/jepsen/run.sh b/test/jepsen/run.sh new file mode 100755 index 0000000..9ee7f33 --- /dev/null +++ b/test/jepsen/run.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +compose_file="${script_dir}/docker-compose.yml" + +if [[ "${GROUP_JEPSEN_SKIP_CHECKER:-0}" != "1" ]]; then + "${script_dir}/checker.sh" +fi + +cleanup() { + if [[ "${GROUP_JEPSEN_KEEP_CONTAINERS:-0}" != "1" ]]; then + docker compose --file "${compose_file}" down --volumes >/dev/null + fi +} + +trap cleanup EXIT + +transport="${GROUP_JEPSEN_TRANSPORT:-distribution}" +args=("$@") + +for ((index = 0; index < ${#args[@]}; index++)); do + if [[ "${args[index]}" == "--transport" ]] && ((index + 1 < ${#args[@]})); then + transport="${args[index + 1]}" + fi +done + +export GROUP_JEPSEN_TRANSPORT="${transport}" + +docker compose --file "${compose_file}" up --detach --build --force-recreate + +if [[ "$#" -eq 0 ]]; then + set -- test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 2n \ + --time-limit 60 \ + --transport "${transport}" +fi + +"${script_dir}/lein.sh" run -- "$@" diff --git a/test/jepsen/src/group/jepsen/client.clj b/test/jepsen/src/group/jepsen/client.clj new file mode 100644 index 0000000..e6de357 --- /dev/null +++ b/test/jepsen/src/group/jepsen/client.clj @@ -0,0 +1,132 @@ +(ns group.jepsen.client + (:require [clojure.edn :as edn] + [clojure.string :as str] + [group.jepsen.docker :as docker] + [jepsen.client :as client]) + (:import (java.io DataInputStream DataOutputStream) + (java.net InetSocketAddress Socket) + (java.nio.charset StandardCharsets))) + +(defn request! + ([node fields] (request! node fields 3000)) + ([node fields timeout-ms] + (with-open [socket (Socket.)] + (let [^String host "127.0.0.1"] + (.connect socket (InetSocketAddress. host (int (docker/port node))) 1000)) + (.setSoTimeout socket timeout-ms) + (let [payload (.getBytes (str/join "\t" fields) StandardCharsets/UTF_8) + out (DataOutputStream. (.getOutputStream socket)) + in (DataInputStream. (.getInputStream socket))] + (.writeInt out (alength payload)) + (.write out payload) + (.flush out) + (let [length (.readInt in) + response (byte-array length)] + (.readFully in response) + (edn/read-string (String. response StandardCharsets/UTF_8))))))) + +(defn wait-ready! + ([node expected] (wait-ready! node expected 30000)) + ([node expected timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (let [response (try + (request! node ["ready" (str expected)] 1000) + (catch Exception _ nil))] + (cond + (= :ok (:status response)) true + (< (System/currentTimeMillis) deadline) + (do (Thread/sleep 100) (recur)) + :else + (throw (ex-info "Group node did not become ready" + {:node node, :last-response response})))))))) + +(defn wait-listening! + ([node] (wait-listening! node 15000)) + ([node timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (if (try + (= :ok (:status (request! node ["ping"] 1000))) + (catch Exception _ false)) + true + (if (< (System/currentTimeMillis) deadline) + (do (Thread/sleep 100) (recur)) + (throw (ex-info "Group node did not start listening" {:node node})))))))) + +(defn cluster-field [cluster] + (if (nil? cluster) "root" cluster)) + +(defn command [op] + (let [logical-owner (str (or (get-in op [:value :owner]) (:process op))) + cluster (cluster-field (get-in op [:value :cluster]))] + (case (:f op) + :register + ["mutate" "register" logical-owner cluster + (str (get-in op [:value :key])) + (str (get-in op [:value :revision]))] + + :unregister + ["mutate" "unregister" logical-owner cluster + (str (get-in op [:value :key])) "0"] + + :join + ["mutate" "join" logical-owner cluster + (str (get-in op [:value :key])) + (str (get-in op [:value :revision]))] + + :leave + ["mutate" "leave" logical-owner cluster + (str (get-in op [:value :key])) "0"] + + :kill + ["kill" logical-owner] + + :cluster-connect + ["cluster" "connect" (get-in op [:value :cluster])] + + :cluster-disconnect + ["cluster" "disconnect" (get-in op [:value :cluster])] + + :connect-all + ["cluster" "connect-all"] + + :corrupt + ["corrupt" (name (get-in op [:value :mode]))] + + :snapshot + ["snapshot" + (str (get-in op [:value :key-count])) + (str/join "," (get-in op [:value :clusters])) + (->> (get-in op [:value :retired-nodes]) + (map #(str "group@" (name %))) + (str/join ","))]))) + +(defrecord GroupClient [node] + client/Client + (open! [this _test node] (assoc this :node (name node))) + (setup! [this _test] this) + + (invoke! [_this _test op] + (let [target (name (or (get-in op [:value :target]) node))] + (try + (let [response (request! target (command op) 10000) + value (if (= :snapshot (:f op)) + (:snapshot response) + {:request (:value op), :response response, :node target})] + (case (:status response) + :ok (assoc op :type :ok, :value value) + :fail (assoc op :type :fail, :value value, :error (:error response)) + (assoc op :type :info, :value value, :error (:error response)))) + (catch Exception exception + (assoc op :type :info + :error {:class (str (class exception)) + :message (.getMessage exception)}))))) + + (teardown! [this _test] this) + (close! [_this _test]) + + client/Reusable + (reusable? [_this _test] true)) + +(defn client [], (GroupClient. nil)) diff --git a/test/jepsen/src/group/jepsen/core.clj b/test/jepsen/src/group/jepsen/core.clj new file mode 100644 index 0000000..2db2c8c --- /dev/null +++ b/test/jepsen/src/group/jepsen/core.clj @@ -0,0 +1,187 @@ +(ns group.jepsen.core + (:gen-class) + (:require [group.jepsen.client :as group-client] + [group.jepsen.db :as group-db] + [group.jepsen.model :as model] + [group.jepsen.nemesis :as group-nemesis] + [jepsen.cli :as cli] + [jepsen.generator :as gen] + [jepsen.os :as os] + [jepsen.tests :as tests])) + +(def clusters ["red" "blue"]) + +(defn mutation [revision key-count owner-count] + (let [key (rand-int key-count) + owner (rand-int owner-count) + cluster (rand-nth [nil nil nil "red" "blue"]) + rev #(swap! revision inc)] + (case (long (rand-int 16)) + 0 {:f :kill, :value {:owner owner}} + 1 {:f :unregister, :value {:owner owner, :cluster cluster, :key key}} + 2 {:f :leave, :value {:owner owner, :cluster cluster, :key key}} + 3 {:f :cluster-disconnect, :value {:cluster (rand-nth clusters)}} + 4 {:f :cluster-connect, :value {:cluster (rand-nth clusters)}} + 5 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 6 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 7 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 8 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + {:f :register, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}}))) + +(defn fault-cycle [fault-interval] + (cycle [(gen/sleep fault-interval) + {:type :info, :f :replica-partition-start} + (gen/sleep fault-interval) + {:type :info, :f :replica-reset} + (gen/sleep fault-interval) + {:type :info, :f :kill-node} + (gen/sleep fault-interval) + {:type :info, :f :restart-node} + (gen/sleep fault-interval) + {:type :info, :f :replica-partition-stop} + (gen/sleep fault-interval) + {:type :info, :f :partition-start} + (gen/sleep fault-interval) + {:type :info, :f :partition-stop}])) + +(defn targeted [target f value] + {:f f, :value (assoc value :target target)}) + +(defn prelude [] + [(gen/log "Opening a three-way replica partition for deterministic conflict and epoch fencing") + (gen/nemesis {:type :info, :f :replica-partition-start, :value {:shape :all}}) + (gen/sleep 0.25) + (gen/clients + (gen/once + (targeted :n1 :register + {:owner "epoch-old", :cluster "red", :key 0, :revision 1000}))) + (gen/clients + (gen/once (targeted :n1 :cluster-disconnect {:cluster "red"}))) + (gen/clients + (gen/once (targeted :n1 :cluster-connect {:cluster "red"}))) + (gen/clients + (gen/once + (targeted :n1 :register + {:owner "epoch-new", :cluster "red", :key 0, :revision 1001}))) + (gen/log "Creating three independent claims for one root registry key") + (gen/clients + [(targeted :n1 :register {:owner "triple-n1", :cluster nil, :key 0, :revision 2001}) + (targeted :n2 :register {:owner "triple-n2", :cluster nil, :key 0, :revision 2002}) + (targeted :n3 :register {:owner "triple-n3", :cluster nil, :key 0, :revision 2003})]) + (gen/sleep 0.25) + (gen/nemesis {:type :info, :f :replica-partition-stop}) + (gen/sleep 1) + (gen/log "Restarting n2 after conflict resolution to prove durable qualification evidence") + (gen/nemesis {:type :info, :f :kill-node, :value {:node :n2}}) + (gen/nemesis {:type :info, :f :restart-node}) + (gen/sleep 1)]) + +(defn terminal-snapshot [opts] + {:f :snapshot + :value {:key-count (:key-count opts) + :clusters clusters + :retired-nodes (:retired-nodes opts)}}) + +(defn snapshot-round [opts] + (let [read (terminal-snapshot opts) + permanent? (= "permanent" (:scenario opts))] + (gen/clients + (gen/each-thread + (if permanent? + (gen/once read) + (gen/until-ok (repeat read))))))) + +(defn terminal-phases [opts] + (let [permanent? (= "permanent" (:scenario opts)) + corruption (keyword (:corruption opts))] + (cond-> + [(gen/log "Healing every fault and restarting transiently killed nodes") + (gen/nemesis {:type :info, :f :restart-node}) + (gen/nemesis {:type :info, :f :partition-stop}) + (gen/nemesis {:type :info, :f :replica-partition-stop}) + (gen/clients + (gen/each-thread + (gen/until-ok (repeat {:f :connect-all, :value {}})))) + (gen/sleep (:recovery-time opts))] + permanent? + (conj (gen/log "Retiring n1 permanently and waiting for complete eviction") + (gen/nemesis {:type :info, :f :retire-node, :value {:node :n1}}) + (gen/sleep (:recovery-time opts))) + + (not= :none corruption) + (conj (gen/log "Injecting a checker-qualification corruption") + (gen/clients + (gen/once + (targeted (first (:terminal-nodes opts)) :corrupt {:mode corruption})))) + + true + (conj (gen/log "Collecting first terminal model snapshot") + (snapshot-round opts) + (gen/sleep 1) + (gen/log "Collecting stable terminal model snapshot") + (snapshot-round opts))))) + +(defn workload [opts] + (let [revision (atom 3000) + active (->> (repeatedly #(mutation revision (:key-count opts) (:owner-count opts))) + (gen/stagger 0.005) + (gen/nemesis (fault-cycle (:fault-interval opts))) + (gen/time-limit (:time-limit opts)))] + (apply gen/phases (concat (prelude) [active] (terminal-phases opts))))) + +(defn group-test [opts] + (let [db (group-db/db) + permanent? (= "permanent" (:scenario opts)) + terminal-nodes (if permanent? (vec (rest (:nodes opts))) (:nodes opts)) + retired-nodes (if permanent? [(first (:nodes opts))] []) + opts (assoc opts + :clusters clusters + :terminal-nodes terminal-nodes + :retired-nodes retired-nodes)] + (merge tests/noop-test + opts + {:name (str "group lifecycle convergence (" (:transport opts) "/" + (:scenario opts) ")") + :os os/noop + :db db + :client (group-client/client) + :nemesis (group-nemesis/nemesis db) + :pure-generators true + :generator (workload opts) + :checker (model/checker)}))) + +(def cli-options + [[nil "--key-count NUMBER" "Number of keys in each cluster and data type" + :default 8 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--fault-interval SECONDS" "Seconds between fault transitions" + :default 2 + :parse-fn #(Double/parseDouble %) + :validate [pos? "Must be positive"]] + [nil "--owner-count NUMBER" "Logical owner slots per node" + :default 32 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--recovery-time SECONDS" "Fault-free convergence time before checking" + :default 8 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--transport PROFILE" "Replica transport: distribution, tcp, or chaos" + :default "distribution" + :validate [#{"distribution" "tcp" "chaos"} "Unsupported transport"]] + [nil "--scenario SCENARIO" "Lifecycle scenario: mixed or permanent" + :default "mixed" + :validate [#{"mixed" "permanent"} "Unsupported scenario"]] + [nil "--corruption MODE" "Checker qualification: none, unexpected-death, internal-index" + :default "none" + :validate [#{"none" "unexpected-death" "internal-index"} "Unsupported corruption"]] + [nil "--max-operation-latency-ms MILLIS" "Maximum acknowledged Group call latency" + :default 2000 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]]]) + +(defn -main [& args] + (cli/run! + (cli/single-test-cmd {:test-fn group-test, :opt-spec cli-options}) + args)) diff --git a/test/jepsen/src/group/jepsen/db.clj b/test/jepsen/src/group/jepsen/db.clj new file mode 100644 index 0000000..702b41c --- /dev/null +++ b/test/jepsen/src/group/jepsen/db.clj @@ -0,0 +1,28 @@ +(ns group.jepsen.db + (:require [group.jepsen.client :as group-client] + [group.jepsen.docker :as docker] + [jepsen.db :as db])) + +(defrecord DockerDB [] + db/DB + (setup! [_this test node] + (docker/heal! (:nodes test)) + (docker/restart! node) + (docker/reset-oracle! node) + (group-client/wait-ready! node (count (:nodes test)))) + + (teardown! [_this test _node] + (docker/heal! (:nodes test))) + + db/Kill + (kill! [_this _test node] + (docker/stop! node)) + + (start! [_this _test node] + (docker/start! node) + (group-client/wait-listening! node)) + + db/LogFiles + (log-files [_this _test _node] [])) + +(defn db [], (DockerDB.)) diff --git a/test/jepsen/src/group/jepsen/docker.clj b/test/jepsen/src/group/jepsen/docker.clj new file mode 100644 index 0000000..6fc987f --- /dev/null +++ b/test/jepsen/src/group/jepsen/docker.clj @@ -0,0 +1,121 @@ +(ns group.jepsen.docker + (:require [clojure.java.shell :as shell] + [clojure.string :as str])) + +(def containers + {"n1" "group-jepsen-n1" + "n2" "group-jepsen-n2" + "n3" "group-jepsen-n3"}) + +(def ports + {"n1" 19081 + "n2" 19082 + "n3" 19083}) + +(def full-chain "GROUP_JEPSEN_FULL") +(def replica-chain "GROUP_JEPSEN_REPLICA") +(def replica-port 10000) + +(defn container [node] + (or (get containers (name node)) + (throw (ex-info "unknown Jepsen node" {:node node})))) + +(defn port [node] + (or (get ports (name node)) + (throw (ex-info "unknown Jepsen node" {:node node})))) + +(defn shell! + [& args] + (let [{:keys [exit out err]} (apply shell/sh args)] + (when-not (zero? exit) + (throw (ex-info "command failed" + {:command args, :exit exit, :out out, :err err}))) + (str/trim out))) + +(defn docker! + [& args] + (apply shell! "docker" args)) + +(defn running? [node] + (= "true" + (try + (docker! "inspect" "--format" "{{.State.Running}}" (container node)) + (catch Exception _ "false")))) + +(defn start! [node] + (docker! "start" (container node))) + +(defn stop! [node] + (when (running? node) + (docker! "stop" "--time" "0" (container node)))) + +(defn restart! [node] + (docker! "restart" "--time" "0" (container node))) + +(defn ip [node] + (docker! "inspect" + "--format" + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" + (container node))) + +(defn exec-sh! + [node script] + (docker! "exec" (container node) "sh" "-c" script)) + +(defn reset-oracle! [node] + (exec-sh! node + "rm -f /tmp/group-jepsen-unexpected-deaths /tmp/group-jepsen-persistent-events")) + +(defn ensure-firewall-chain! [node chain] + (exec-sh! + node + (str "iptables -N " chain " 2>/dev/null || true; " + "iptables -C INPUT -j " chain " 2>/dev/null || " + "iptables -I INPUT 1 -j " chain "; " + "iptables -C OUTPUT -j " chain " 2>/dev/null || " + "iptables -I OUTPUT 1 -j " chain))) + +(defn flush-chain! [node chain] + (when (running? node) + (ensure-firewall-chain! node chain) + (exec-sh! node (str "iptables -F " chain)))) + +(defn heal-full! [nodes] + (doseq [node nodes] + (flush-chain! node full-chain))) + +(defn heal-replica! [nodes] + (doseq [node nodes] + (flush-chain! node replica-chain))) + +(defn heal! [nodes] + (heal-full! nodes) + (heal-replica! nodes)) + +(defn isolate! + "Cuts one node off from every other DB node while preserving client traffic." + [nodes isolated] + (heal-full! nodes) + (let [ips (into {} (map (juxt identity ip) nodes))] + (doseq [node nodes + peer nodes + :when (and (not= node peer) + (or (= node isolated) (= peer isolated)))] + (exec-sh! + node + (str "iptables -A " full-chain + " -d " (get ips peer) " -j DROP; " + "iptables -A " full-chain + " -s " (get ips peer) " -j DROP"))))) + +(defn partition-replica! + "Drops only sideband TCP packets for the supplied directed node pairs." + [nodes edges] + (heal-replica! nodes) + (let [ips (into {} (map (juxt identity ip) nodes))] + (doseq [[source target] edges] + (exec-sh! + source + (str "iptables -A " replica-chain + " -p tcp -d " (get ips target) + " --dport " replica-port " -j DROP"))))) diff --git a/test/jepsen/src/group/jepsen/model.clj b/test/jepsen/src/group/jepsen/model.clj new file mode 100644 index 0000000..b0e782a --- /dev/null +++ b/test/jepsen/src/group/jepsen/model.clj @@ -0,0 +1,219 @@ +(ns group.jepsen.model + (:require [clojure.set :as set] + [jepsen.checker :as checker] + [jepsen.history :as history])) + +(defn successful-snapshots [history] + (->> history + (remove history/invoke?) + (filter #(and (= :snapshot (:f %)) (= :ok (:type %)))) + (sort-by :index))) + +(defn snapshots-by-node [history] + (reduce (fn [snapshots op] + (update snapshots (get-in op [:value :node]) (fnil conj []) (:value op))) + {} + (successful-snapshots history))) + +(defn latest-snapshots [history] + (->> (successful-snapshots history) + (reduce (fn [snapshots op] + (assoc snapshots (get-in op [:value :node]) (:value op))) + {}))) + +(defn spaces [test] + (cons "root" (:clusters test))) + +(defn empty-view [test empty-value] + (into {} + (for [cluster (spaces test)] + [cluster (zipmap (range (:key-count test)) (repeat empty-value))]))) + +(defn owner-entries [owner field] + (or (get owner field) [])) + +(defn expected-state [test snapshots] + (let [owners (->> snapshots vals (mapcat :owners) (map (juxt :token identity)) (into {})) + registry-candidates + (reduce (fn [by-key [_ owner]] + (reduce (fn [entries {:keys [cluster key]}] + (update entries [(or cluster "root") key] + (fnil conj #{}) (:token owner))) + by-key + (owner-entries owner :registrations))) + {} + owners) + conflicts (into {} (filter (comp #(< 1 %) count val) registry-candidates)) + registry + (reduce (fn [view [[cluster key] tokens]] + (assoc-in view [cluster key] (first tokens))) + (empty-view test nil) + registry-candidates) + pg + (reduce (fn [view [_ owner]] + (reduce (fn [entries {:keys [cluster key]}] + (update-in entries [(or cluster "root") key] + (fnil conj #{}) (:token owner))) + view + (owner-entries owner :memberships))) + (empty-view test #{}) + owners)] + {:owners owners, :registry registry, :pg pg, :conflicts conflicts})) + +(defn normalize-view [test snapshot] + {:registry + (into {} + (for [cluster (spaces test)] + [cluster + (into {} + (for [key (range (:key-count test))] + [key (get-in snapshot [:registry cluster key])]))])) + :pg + (into {} + (for [cluster (spaces test)] + [cluster + (into {} + (for [key (range (:key-count test))] + [key (set (get-in snapshot [:pg cluster key]))]))]))}) + +(defn stable-internal [snapshot] + (select-keys (:internal snapshot) + [:healthy :errors :snapshot-staging-count :oplog-entries])) + +(defn snapshot-fingerprint [test snapshot] + {:owners (set (:owners snapshot)) + :peers (set (:peers snapshot)) + :unexpected-deaths (set (:unexpected-deaths snapshot)) + :view (normalize-view test snapshot) + :internal (stable-internal snapshot)}) + +(defn operation-latencies [history] + (->> history + (remove history/invoke?) + (keep #(get-in % [:value :response :latency-us])))) + +(defn analyze [test history] + (let [observations (snapshots-by-node history) + snapshots (latest-snapshots history) + required-nodes (set (map name (or (:terminal-nodes test) (:nodes test)))) + relevant-observations (select-keys observations required-nodes) + relevant-snapshots (select-keys snapshots required-nodes) + missing-nodes (set/difference required-nodes (set (keys relevant-snapshots))) + minimum-observations (get test :terminal-snapshots-per-node 2) + insufficient-observations + (into {} + (keep (fn [node] + (let [observation-count (count (get relevant-observations node))] + (when (< observation-count minimum-observations) + [node observation-count])))) + required-nodes) + unstable-observations + (into {} + (keep (fn [[node node-observations]] + (let [fingerprints + (set (map #(snapshot-fingerprint test %) + node-observations))] + (when (< 1 (count fingerprints)) + [node fingerprints])))) + relevant-observations) + expected (expected-state test relevant-snapshots) + expected-view (select-keys expected [:registry :pg]) + views (into {} (map (fn [[node snapshot]] + [node (normalize-view test snapshot)])) + relevant-snapshots) + mismatches (into {} (remove (comp #(= expected-view %) val) views)) + peer-mismatches + (into {} + (keep (fn [[node snapshot]] + (let [expected-peers (->> required-nodes + (remove #(= node %)) + (map #(str "group@" %)) + set) + actual-peers (set (:peers snapshot))] + (when (not= expected-peers actual-peers) + [node {:expected expected-peers, :actual actual-peers}])))) + relevant-snapshots) + transport-events + (reduce #(merge-with + %1 %2) + {} + (map #(or (:transport-events %) {}) (vals relevant-snapshots))) + required-transport-events + (get test :required-transport-events + #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk + :registry-conflict-death}) + missing-transport-events + (set (remove #(pos? (get transport-events % 0)) required-transport-events)) + expected-profile (keyword (:transport test)) + transport-profile-mismatches + (into {} + (keep (fn [[node snapshot]] + (when (not= expected-profile (:transport-profile snapshot)) + [node (:transport-profile snapshot)]))) + relevant-snapshots) + internal-errors + (into {} + (keep (fn [[node snapshot]] + (let [internal (:internal snapshot)] + (when (or (not= true (:healthy internal)) + (seq (:errors internal)) + (not= 0 (:snapshot-staging-count internal))) + [node internal])))) + relevant-snapshots) + unexpected-deaths (->> relevant-snapshots vals (mapcat :unexpected-deaths) set) + live-tokens (set (keys (:owners expected))) + actual-tokens (->> views + vals + (mapcat (fn [{:keys [registry pg]}] + (concat + (->> registry vals (mapcat vals) (remove nil?)) + (->> pg vals (mapcat vals) (mapcat identity))))) + set) + expected-tokens + (set/union + (->> (:registry expected) vals (mapcat vals) (remove nil?) set) + (->> (:pg expected) vals (mapcat vals) (mapcat identity) set)) + orphaned (set/difference actual-tokens live-tokens) + missing-live (set/difference expected-tokens actual-tokens) + latencies (operation-latencies history) + max-latency-us (if (seq latencies) (apply max latencies) 0) + latency-limit-us (* 1000 (get test :max-operation-latency-ms 2000)) + latency-violation? (> max-latency-us latency-limit-us) + valid? (and (empty? missing-nodes) + (empty? insufficient-observations) + (empty? unstable-observations) + (empty? peer-mismatches) + (empty? missing-transport-events) + (empty? transport-profile-mismatches) + (empty? internal-errors) + (empty? (:conflicts expected)) + (empty? mismatches) + (empty? unexpected-deaths) + (empty? orphaned) + (empty? missing-live) + (not latency-violation?))] + {:valid? valid? + :snapshots (set (keys relevant-snapshots)) + :missing-nodes missing-nodes + :insufficient-terminal-observations insufficient-observations + :unstable-terminal-observations unstable-observations + :peer-mismatches peer-mismatches + :transport-events transport-events + :missing-transport-events missing-transport-events + :transport-profile-mismatches transport-profile-mismatches + :internal-invariant-errors internal-errors + :max-group-operation-latency-ms (/ max-latency-us 1000.0) + :group-operation-latency-limit-ms (/ latency-limit-us 1000.0) + :live-owner-count (count live-tokens) + :live-registry-conflicts (:conflicts expected) + :mismatched-views mismatches + :unexpected-owner-deaths unexpected-deaths + :orphaned-owner-tokens orphaned + :missing-live-owner-tokens missing-live + :expected expected-view})) + +(defrecord LifecycleChecker [] + checker/Checker + (check [_this test history _opts] + (analyze test history))) + +(defn checker [], (LifecycleChecker.)) diff --git a/test/jepsen/src/group/jepsen/nemesis.clj b/test/jepsen/src/group/jepsen/nemesis.clj new file mode 100644 index 0000000..e11c40f --- /dev/null +++ b/test/jepsen/src/group/jepsen/nemesis.clj @@ -0,0 +1,166 @@ +(ns group.jepsen.nemesis + (:require [group.jepsen.client :as group-client] + [group.jepsen.docker :as docker] + [jepsen.db :as db] + [jepsen.nemesis :as nemesis])) + +(defn ordered-pairs [nodes] + (for [source nodes, target nodes :when (not= source target)] [source target])) + +(defn partition-shape [nodes requested] + (let [nodes (vec nodes) + shape (or requested (rand-nth [:isolate :all :asymmetric]))] + (case shape + :all {:shape :all, :edges (vec (ordered-pairs nodes))} + :asymmetric + (let [source (rand-nth nodes) + target (rand-nth (vec (remove #(= source %) nodes)))] + {:shape :asymmetric, :edges [[source target]]}) + :isolate + (let [isolated (rand-nth nodes)] + {:shape :isolate + :isolated isolated + :edges (vec (filter (fn [[source target]] + (or (= source isolated) (= target isolated))) + (ordered-pairs nodes)))})))) + +(defn logical-block! [edges] + (doseq [[source target] edges] + (when (docker/running? source) + (group-client/request! source ["transport" "block" (str "group@" (name target))])))) + +(defn logical-heal! [nodes] + (doseq [node nodes] + (when (docker/running? node) + (try + (group-client/request! node ["transport" "heal"]) + (catch Exception _ nil))))) + +(defrecord PartitionNemesis [isolated] + nemesis/Nemesis + (setup! [this test] + (docker/heal-full! (:nodes test)) + this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @isolated + (assoc op :type :info, :value {:already-isolated @isolated}) + (let [node (rand-nth (vec (:nodes test)))] + (docker/isolate! (:nodes test) node) + (reset! isolated node) + (assoc op :type :info, :value {:isolated node}))) + + :stop + (do + (docker/heal-full! (:nodes test)) + (let [node @isolated] + (reset! isolated nil) + (assoc op :type :info, :value {:healed node}))))) + + (teardown! [_this test] + (docker/heal-full! (:nodes test)))) + +(defrecord ReplicaNemesis [active] + nemesis/Nemesis + (setup! [this test] + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)) + this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @active + (assoc op :type :info, :value {:already-active @active}) + (let [requested (get-in op [:value :shape]) + fault (partition-shape (:nodes test) requested)] + (if (= "tcp" (:transport test)) + (docker/partition-replica! (:nodes test) (:edges fault)) + (logical-block! (:edges fault))) + (reset! active fault) + (assoc op :type :info, :value fault))) + + :stop + (do + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)) + (let [fault @active] + (reset! active nil) + (assoc op :type :info, :value {:healed fault}))) + + :reset + (let [nodes (vec (filter docker/running? (:nodes test))) + source (when (seq nodes) (rand-nth nodes)) + targets (when source (vec (remove #(= source %) nodes))) + target (when (seq targets) (rand-nth targets))] + (when (and source target) + (group-client/request! + source + ["transport" "reset" (str "group@" (name target))])) + (assoc op :type :info, :value {:source source, :target target})))) + + (teardown! [_this test] + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)))) + +(defrecord ProcessNemesis [db killed] + nemesis/Nemesis + (setup! [this _test] this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @killed + (assoc op :type :info, :value {:already-killed @killed}) + (let [node (or (get-in op [:value :node]) + (rand-nth (vec (:nodes test))))] + (db/kill! db test node) + (reset! killed node) + (assoc op :type :info, :value {:killed node}))) + + :stop + (if-let [node @killed] + (do + (db/start! db test node) + (reset! killed nil) + (assoc op :type :info, :value {:restarted node})) + (assoc op :type :info, :value {:restarted nil})))) + + (teardown! [_this test] + (when-let [node @killed] + (db/start! db test node) + (reset! killed nil)))) + +(defrecord RetirementNemesis [db retired] + nemesis/Nemesis + (setup! [this _test] this) + + (invoke! [_this test op] + (let [node (or (get-in op [:value :node]) (first (:nodes test)))] + (when (and (nil? @retired) (docker/running? node)) + (db/kill! db test node) + (reset! retired node)) + (assoc op :type :info, :value {:retired @retired}))) + + (teardown! [_this test] + (when-let [node @retired] + (db/start! db test node) + (reset! retired nil)))) + +(defn nemesis [db] + (nemesis/compose + {{:partition-start :start, :partition-stop :stop} + (PartitionNemesis. (atom nil)) + + {:replica-partition-start :start, + :replica-partition-stop :stop, + :replica-reset :reset} + (ReplicaNemesis. (atom nil)) + + {:kill-node :start, :restart-node :stop} + (ProcessNemesis. db (atom nil)) + + {:retire-node :retire} + (RetirementNemesis. db (atom nil))})) diff --git a/test/jepsen/test/group/jepsen/model_test.clj b/test/jepsen/test/group/jepsen/model_test.clj new file mode 100644 index 0000000..241fb15 --- /dev/null +++ b/test/jepsen/test/group/jepsen/model_test.clj @@ -0,0 +1,208 @@ +(ns group.jepsen.model-test + (:require [clojure.test :refer :all] + [group.jepsen.model :as model])) + +(def test-map + {:nodes ["n1" "n2" "n3"] + :terminal-nodes ["n1" "n2" "n3"] + :key-count 2 + :clusters ["red"] + :transport "distribution" + :terminal-snapshots-per-node 1 + :required-transport-events #{}}) + +(defn peers-for [node nodes] + (->> nodes + (remove #(= node %)) + (map #(str "group@" %)) + sort + vec)) + +(defn public-view [root red] + {"root" root, "red" red}) + +(defn healthy-internal [] + {:healthy true + :errors [] + :snapshot-staging-count 0 + :oplog-entries 2}) + +(defn snapshot-op + ([index node owners registry pg] + (snapshot-op index node (:terminal-nodes test-map) owners registry pg)) + ([index node nodes owners registry pg] + {:index index + :process index + :type :ok + :f :snapshot + :value {:node node + :peers (peers-for node nodes) + :owners owners + :unexpected-deaths [] + :transport-events {} + :transport-profile :distribution + :internal (healthy-internal) + :registry registry + :pg pg}})) + +(defn owner [token registrations memberships] + {:token token, :registrations registrations, :memberships memberships}) + +(defn registration [cluster key revision] + {:cluster cluster, :key key, :revision revision}) + +(defn membership [cluster key revision] + {:cluster cluster, :key key, :revision revision}) + +(defn empty-registry [] + (public-view {0 nil, 1 nil} {0 nil, 1 nil})) + +(defn empty-pg [] + (public-view {0 [], 1 []} {0 [], 1 []})) + +(defn with-unexpected-death [op token] + (assoc-in op [:value :unexpected-deaths] [{:token token, :reason ":boom"}])) + +(deftest accepts-an-exact-converged-multi-cluster-view + (let [owners [(owner "a" [(registration nil 0 1) (registration "red" 1 2)] []) + (owner "b" [] [(membership nil 1 2) (membership "red" 0 3)])] + registry (public-view {0 "a", 1 nil} {0 nil, 1 "a"}) + pg (public-view {0 [], 1 ["b"]} {0 ["b"], 1 []}) + history [(snapshot-op 1 "n1" owners registry pg) + (snapshot-op 2 "n2" [] registry pg) + (snapshot-op 3 "n3" [] registry pg)]] + (is (:valid? (model/analyze test-map history))))) + +(deftest does-not-require-an-owner-without-group-intent + (let [idle-owner (owner "idle" [] []) + history [(snapshot-op 1 "n1" [idle-owner] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))]] + (is (:valid? (model/analyze test-map history))))) + +(deftest rejects-an-incomplete-terminal-observation + (let [history [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= #{"n3"} (:missing-nodes result))))) + +(deftest accepts-a-permanently-retired-node-and-requires-its-absence + (let [survivors ["n2" "n3"] + permanent-test (assoc test-map :terminal-nodes survivors) + history [(snapshot-op 1 "n2" survivors [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n3" survivors [] (empty-registry) (empty-pg))]] + (is (:valid? (model/analyze permanent-test history))))) + +(deftest rejects-zombies-missing-live-owners-and-divergence + (let [live (owner "live" [(registration nil 0 1)] []) + stale-registry (assoc-in (empty-registry) ["root" 0] "dead") + result (model/analyze + test-map + [(snapshot-op 1 "n1" [live] stale-registry (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= #{"dead"} (:orphaned-owner-tokens result))) + (is (= #{"live"} (:missing-live-owner-tokens result))) + (is (seq (:mismatched-views result))))) + +(deftest rejects-a-live-unresolved-registry-conflict + (let [owners [(owner "a" [(registration nil 0 1)] []) + (owner "b" [(registration nil 0 2)] [])] + registry (assoc-in (empty-registry) ["root" 0] "b") + history [(snapshot-op 1 "n1" owners registry (empty-pg)) + (snapshot-op 2 "n2" [] registry (empty-pg)) + (snapshot-op 3 "n3" [] registry (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= {["root" 0] #{"a" "b"}} (:live-registry-conflicts result))))) + +(deftest rejects-an-unexpected-owner-death-even-after-cleanup + (let [result (model/analyze + test-map + [(with-unexpected-death + (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + "lost-owner") + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= #{{:token "lost-owner", :reason ":boom"}} + (:unexpected-owner-deaths result))))) + +(deftest rejects-terminal-state-which-keeps-changing + (let [stale-registry (assoc-in (empty-registry) ["root" 0] "stale") + history [(snapshot-op 1 "n1" [] stale-registry (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg)) + (snapshot-op 4 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 5 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 6 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze + (assoc test-map :terminal-snapshots-per-node 2) + history)] + (is (false? (:valid? result))) + (is (contains? (:unstable-terminal-observations result) "n1")))) + +(deftest rejects-a-node-without-all-control-plane-peers + (let [history [(assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :peers] + ["group@n2"]) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= #{"group@n2" "group@n3"} + (get-in result [:peer-mismatches "n1" :expected]))))) + +(deftest rejects-a-run-which-did-not-exercise-required-repair-paths + (let [history [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze + (assoc test-map + :required-transport-events + #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk}) + history)] + (is (false? (:valid? result))) + (is (= #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk} + (:missing-transport-events result))))) + +(deftest rejects-internal-corruption-or-leftover-snapshot-staging + (let [bad (-> (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (assoc-in [:value :internal :healthy] false) + (assoc-in [:value :internal :snapshot-staging-count] 1) + (assoc-in [:value :internal :errors] ["broken index"])) + result (model/analyze + test-map + [bad + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= 1 (get-in result [:internal-invariant-errors "n1" + :snapshot-staging-count]))))) + +(deftest rejects-the-wrong-transport-profile + (let [wrong (assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :transport-profile] + :tcp) + result (model/analyze + test-map + [wrong + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= {"n1" :tcp} (:transport-profile-mismatches result))))) + +(deftest rejects-a-blocking-group-operation + (let [base [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + slow {:index 4 + :process 0 + :type :ok + :f :register + :value {:response {:latency-us 2500000}}} + result (model/analyze test-map (conj base slow))] + (is (false? (:valid? result))) + (is (= 2500.0 (:max-group-operation-latency-ms result))))) diff --git a/test/replica_adversarial_test.exs b/test/replica_adversarial_test.exs index 6b1437a..9bcfa4c 100644 --- a/test/replica_adversarial_test.exs +++ b/test/replica_adversarial_test.exs @@ -13,14 +13,14 @@ defmodule Group.ReplicaAdversarialTest do @seed seed @tag chaos_seed: seed - test "seeded mixed-operation transport chaos converges without zombies (seed #{@seed})" do + test "seeded three-node transport chaos converges without zombies (seed #{@seed})" do seed = @seed :rand.seed(:exsss, {seed, seed * 3 + 1, seed * 7 + 2}) - peers = TestCluster.start_peers(2) + peers = TestCluster.start_peers(3) on_exit(fn -> TestCluster.stop_peers(peers) end) - [{_, node_a}, {_, node_b}] = peers + nodes = Enum.map(peers, &elem(&1, 1)) name = :"replica_chaos_#{seed}_#{System.unique_integer([:positive])}" opts = [ @@ -40,30 +40,29 @@ defmodule Group.ReplicaAdversarialTest do TestCluster.assert_eventually( fn -> - Enum.all?([node_a, node_b], fn node -> - length(TestCluster.rpc!(node, Group, :nodes, [name])) == 1 and + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == length(nodes) - 1 and Enum.all?(@clusters, fn cluster -> - length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 2 + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == length(nodes) end) end) end, timeout: 10_000 ) - :ok = - TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ - name, - {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]} - ]) + chaos_modes = [ + {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]}, + {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]}, + {:chaos, [drop_every: 4, duplicate_every: 9, max_delay: 70]} + ] - :ok = - TestCluster.rpc!(node_b, Group.TestReplicaTransport, :set_mode, [ - name, - {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]} - ]) + Enum.zip(nodes, chaos_modes) + |> Enum.each(fn {node, mode} -> + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, mode]) + end) initial = %{ - active: %{node_a => MapSet.new(@clusters), node_b => MapSet.new(@clusters)}, + active: Map.new(nodes, &{&1, MapSet.new(@clusters)}), counter: 0, pg_keys: MapSet.new(), pids: [], @@ -73,37 +72,36 @@ defmodule Group.ReplicaAdversarialTest do state = Enum.reduce(1..72, initial, fn step, state -> - apply_random_operation(state, step, seed, name, node_a, node_b) + apply_random_operation(state, step, seed, name, nodes) end) - for node <- [node_a, node_b] do + for node <- nodes do :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) :ok = TestCluster.rpc!(node, Group, :connect, [name, @clusters]) end - assert_converges(name, node_a, node_b, state) + assert_converges(name, nodes, state) # Let delayed frames from the chaos phase arrive, then prove they are # duplicates/stale rather than a source of resurrection. Process.sleep(150) - TestCluster.flush_shards(node_a, name) - TestCluster.flush_shards(node_b, name) - assert_converges(name, node_a, node_b, state) + Enum.each(nodes, &TestCluster.flush_shards(&1, name)) + assert_converges(name, nodes, state) - for node <- [node_a, node_b] do + for node <- nodes do assert :ok = TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) end TestCluster.assert_eventually( - fn -> retained_owners_alive?(name, [node_a, node_b]) end, + fn -> retained_owners_alive?(name, nodes) end, timeout: 15_000 ) end end - defp apply_random_operation(state, step, seed, name, node_a, node_b) do - nodes = [node_a, node_b] + defp apply_random_operation(state, step, seed, name, nodes) do + [node_a, node_b | _rest] = nodes case :rand.uniform(12) do choice when choice in 1..3 -> @@ -244,17 +242,19 @@ defmodule Group.ReplicaAdversarialTest do {"chaos/#{seed}/#{kind}/#{step}/#{counter}", %{state | counter: counter}} end - defp assert_converges(name, node_a, node_b, state) do + defp assert_converges(name, nodes, state) do TestCluster.assert_eventually( fn -> - length(TestCluster.rpc!(node_a, Group, :nodes, [name])) == 1 and - length(TestCluster.rpc!(node_b, Group, :nodes, [name])) == 1 and + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == length(nodes) - 1 + end) and Enum.all?(@clusters, fn cluster -> - length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and - length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == length(nodes) + end) end) and - registry_equal?(name, node_a, node_b, state.reg_keys) and - memberships_equal?(name, node_a, node_b, state.pg_keys) + registry_equal?(name, nodes, state.reg_keys) and + memberships_equal?(name, nodes, state.pg_keys) end, timeout: 20_000, interval: 75 @@ -263,19 +263,21 @@ defmodule Group.ReplicaAdversarialTest do error -> flunk( "chaos convergence failed: #{Exception.message(error)}\n" <> - "differences=#{inspect(convergence_differences(name, node_a, node_b, state), limit: :infinity)}\n" <> + "differences=#{inspect(convergence_differences(name, nodes, state), limit: :infinity)}\n" <> "recent operations=#{inspect(Enum.take(state.trace, 20), limit: :infinity)}" ) end - defp convergence_differences(name, node_a, node_b, state) do + defp convergence_differences(name, nodes, state) do registry = state.reg_keys |> Enum.flat_map(fn {cluster, key} -> args = [name, key, cluster_opts(cluster)] - value_a = TestCluster.rpc!(node_a, Group, :lookup, args) - value_b = TestCluster.rpc!(node_b, Group, :lookup, args) - if value_a == value_b, do: [], else: [{:registry, cluster, key, value_a, value_b}] + values = Map.new(nodes, &{&1, TestCluster.rpc!(&1, Group, :lookup, args)}) + + if values |> Map.values() |> Enum.uniq() |> length() == 1, + do: [], + else: [{:registry, cluster, key, values}] end) |> Enum.take(10) @@ -283,17 +285,17 @@ defmodule Group.ReplicaAdversarialTest do state.pg_keys |> Enum.flat_map(fn {cluster, key} -> args = [name, key, cluster_opts(cluster)] - value_a = TestCluster.rpc!(node_a, Group, :members, args) |> Enum.sort() - value_b = TestCluster.rpc!(node_b, Group, :members, args) |> Enum.sort() - if value_a == value_b, do: [], else: [{:pg, cluster, key, value_a, value_b}] + values = Map.new(nodes, &{&1, TestCluster.rpc!(&1, Group, :members, args) |> Enum.sort()}) + + if values |> Map.values() |> Enum.uniq() |> length() == 1, + do: [], + else: [{:pg, cluster, key, values}] end) |> Enum.take(10) - nodes = + topology = for cluster <- [nil | @clusters] do - value_a = group_nodes(node_a, name, cluster) - value_b = group_nodes(node_b, name, cluster) - {cluster, value_a, value_b} + {cluster, Map.new(nodes, &{&1, group_nodes(&1, name, cluster)})} end cluster_trace = @@ -305,12 +307,12 @@ defmodule Group.ReplicaAdversarialTest do end) protocol = - for node <- [node_a, node_b] do + for node <- nodes do {node, TestCluster.rpc!(node, Group.TestCluster, :replica_protocol_state, [name])} end [ - nodes: nodes, + nodes: topology, registry: registry, pg: pg, cluster_trace: cluster_trace, @@ -323,21 +325,23 @@ defmodule Group.ReplicaAdversarialTest do defp group_nodes(node, name, cluster), do: TestCluster.rpc!(node, Group, :nodes, [name, cluster]) - defp registry_equal?(name, node_a, node_b, keys) do + defp registry_equal?(name, nodes, keys) do Enum.all?(keys, fn {cluster, key} -> args = [name, key, cluster_opts(cluster)] - TestCluster.rpc!(node_a, Group, :lookup, args) == - TestCluster.rpc!(node_b, Group, :lookup, args) + nodes |> Enum.map(&TestCluster.rpc!(&1, Group, :lookup, args)) |> Enum.uniq() |> length() == + 1 end) end - defp memberships_equal?(name, node_a, node_b, keys) do + defp memberships_equal?(name, nodes, keys) do Enum.all?(keys, fn {cluster, key} -> args = [name, key, cluster_opts(cluster)] - members_a = TestCluster.rpc!(node_a, Group, :members, args) |> Enum.sort() - members_b = TestCluster.rpc!(node_b, Group, :members, args) |> Enum.sort() - members_a == members_b + + nodes + |> Enum.map(&(TestCluster.rpc!(&1, Group, :members, args) |> Enum.sort())) + |> Enum.uniq() + |> length() == 1 end) end From 1e85161f302089ad4d4178a7c137e91daba0262d Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Tue, 11 Aug 2026 12:52:35 +0000 Subject: [PATCH 6/7] Refine replica transport boundary --- CHANGELOG.md | 6 +- CLAUDE.md | 9 ++- README.md | 33 +++++----- lib/group.ex | 2 +- lib/group/replica.ex | 55 ++++++++-------- lib/group/replica/transport.ex | 66 +++++++++++--------- lib/group/replica/transport/outbox.ex | 64 +++++++++---------- lib/group/replica/transport/tcp.ex | 26 ++++---- test/README.md | 17 ++--- test/distributed_test.exs | 26 ++++---- test/jepsen/README.md | 2 +- test/jepsen/node.exs | 32 +++++----- test/replica_snapshot_distributed_test.exs | 4 +- test/replica_transport_outbox_test.exs | 38 +++++------ test/support/controlled_replica_transport.ex | 10 +-- test/support/replica_model_scheduler.ex | 18 +++--- test/support/test_replica_transport.ex | 66 ++++++++++---------- 17 files changed, 245 insertions(+), 229 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f41d3d..45bec76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ oracle across distribution, sideband TCP, and lossy/reordering transports. `mix test` is the every-PR ExUnit/property/checker gate and `mix test.soak` runs the six-profile nightly/release campaign. +- **Breaking**: the replica transport boundary now names logical direction + rather than implementation mechanics: adapters implement `outgoing/5`, + sideband adapters use `Group.Replica.Transport.Outbox.push/5`, and receiving + adapters call `incoming/4` or `incoming_batch/4`. - **Breaking**: replica protocol v2 splits exact snapshots into transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage chunks in shard-owned private ETS and advance the stream cursor only after an @@ -44,7 +48,7 @@ the losing process's `{:group_registry_conflict, key, winner_meta}` exit reason. - **Breaking**: `Group.dispatch/4` remote sends and process-DOWN replication are now non-suspending and never auto-connect. Busy dispatch drops still force a disconnect and - bounded reconnect retry; replica frames are dropped and repaired by anti-entropy without + bounded reconnect retry; replica messages are dropped and repaired by anti-entropy without disturbing the dist-Erlang control connection. Previously dispatch could block the caller and initiate new connections. - Configured function-form `extract_meta` callbacks are now applied on reads and lifecycle diff --git a/CLAUDE.md b/CLAUDE.md index 9c326c2..15ba7ad 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -134,7 +134,7 @@ dependencies on additive full-state merge. ## Anti-Entropy -Replica data frames are: +Replica data messages are: - `heads`: stream, retained floor, and head; - `delta_batch`: one or more contiguous stream runs; @@ -163,7 +163,7 @@ visible state. Staging expires after one peer-lease interval without progress. All cross-node Group control sends use `:erlang.send_nosuspend(..., [:noconnect])`. The default distribution replica adapter sends directly the same way and adds no local hop. `:busy` and -`:disconnected` mean “drop this frame”; periodic anti-entropy repairs it. +`:disconnected` mean “drop this message”; periodic anti-entropy repairs it. A sideband adapter may use one local `Group.Replica.Transport.Outbox` per shard. Outboxes batch by peer, impose deadlines, and run bounded socket work @@ -174,9 +174,8 @@ identity and carries authority. TCP is not encrypted. Transport ordering is not required for correctness. Per-shard ordered delivery is a fast path; stream sequences reject duplicate/out-of-order data, and -generation/epoch/lane fences handle control/data reordering. Ingress must derive -`source_node` from the authenticated connection, never from payload data, and -must reassemble any transport segmentation before `deliver_batch/4`. +generation/epoch/lane fences handle control/data reordering. A transport must +reassemble any transport segmentation before `incoming_batch/4`. ## Registry Projection and Process Ownership diff --git a/README.md b/README.md index 8a00c32..51ac159 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ All operations are **eventually consistent**: milliseconds. Defaults to 5. - **`busy_dist_retry_attempts`** — reconnect attempts after a non-suspending remote dispatch reports a busy dist link. Defaults to 300. Replica transport - frames are simply dropped and repaired instead of forcing a disconnect. + messages are simply dropped and repaired instead of forcing a disconnect. - **`busy_dist_retry_interval`** — milliseconds between dispatch busy-link reconnect attempts. Defaults to 1,000. - **`replicated_pg_receiver_local_request_quota`** — legacy-named quota for @@ -298,7 +298,7 @@ All operations are **eventually consistent**: - **`replica_transport`** — a module implementing `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, - or `:disconnected`. Dropped and busy frames are repaired by anti-entropy. + or `:disconnected`. Dropped and busy messages are repaired by anti-entropy. `Group.Replica.Transport.TCP` is an included sideband adapter with local per-shard batching and bounded per-peer writer queues; its socket owners are separate processes, so socket backpressure cannot block a Group shard. @@ -307,7 +307,7 @@ All operations are **eventually consistent**: peer acknowledgements; a peer behind the retained floor receives an exact snapshot. - **`replicated_snapshot_chunk_target_bytes`** — target maximum encoded size - of each exact-snapshot frame. Defaults to 1 MiB and applies above every + of each exact-snapshot message. Defaults to 1 MiB and applies above every transport, including dist Erlang. A single row larger than the target is sent alone. Receivers stage chunks in shard-owned private ETS and replace visible state only after the complete exact slice is present. @@ -435,7 +435,7 @@ fenced, stream-head exchange on the replica transport catches the peer up. Every local mutation is first appended to a stream identified by `{group, origin_node, origin_generation, shard, cluster, cluster_epoch}` and a strictly increasing sequence number. It is then applied to the materialized -ETS view and batched into one delta frame per target. Process-death registry +ETS view and batched into one delta message per target. Process-death registry and PG removals can share one record and retain their one-event-batch behavior. Receivers advance a cursor only across a contiguous sequence prefix. A gap @@ -448,7 +448,7 @@ There are no leaders, quorum acknowledgements, per-entry replicated tombstones, or known-membership retention barriers. Oplog memory is bounded locally and independently of slow peers. Deletes are normal ordered records while retained, and exact snapshots close gaps after pruning. Exact snapshots are split into -transport-neutral byte-bounded frames; loss, duplication, or reordering leaves +transport-neutral byte-bounded messages; loss, duplication, or reordering leaves the old visible slice and cursor untouched until all chunks arrive. Incomplete staging expires after a peer-lease interval without progress and is destroyed automatically with its owning shard. Named-cluster close uses only a temporary @@ -467,8 +467,8 @@ writes, each stream numbers them, and receivers reject gaps and duplicates. Per-shard ordered delivery is still a useful fast path. Cross-stream order is not a correctness dependency; cluster epochs reject data racing a disconnect or reconnect, and generation fencing rejects data from a restarted origin. -An alternative sideband adapter authenticates the peer as a dist-Erlang node -and calls `Group.Replica.Transport.deliver/4` locally. +An alternative sideband adapter passes incoming messages to +`Group.Replica.Transport.incoming/4` locally. For example, replica data can use the included sideband TCP adapter while authority and membership remain on dist Erlang: @@ -495,22 +495,21 @@ control/data ordering relationship; the generation/epoch lane barrier and stream sequence checks supply correctness. The default distribution adapter still sends directly to the remote shard and -does not pay for a local outbox. Sideband adapters can delegate `try_send/5` to -`Group.Replica.Transport.Outbox.try_send/5` and supervise one outbox per shard -with `Group.Replica.Transport.Outbox.child_spec/1`. An outbox groups frames by +does not pay for a local outbox. Sideband adapters can delegate `outgoing/5` to +`Group.Replica.Transport.Outbox.push/5` and supervise one outbox per shard +with `Group.Replica.Transport.Outbox.child_spec/1`. An outbox groups messages by target and invokes the adapter's `send_batch/4` callback. Calls that expire or return `:busy`/`:disconnected` are dropped without a local retry; the next anti-entropy exchange repairs them. A message-oriented backend fits this callback shape by obtaining a connection once from `init_outbox/3`, then sending each `send_batch/4` result to a -registered ingress name on the target node. Queue pressure maps to `:busy` and -a missing session maps to `:disconnected`. Ingress must attach the authenticated -connection's source node; an adapter must never trust a source node supplied -inside the payload. Exact snapshots are already bounded by Group. A transport -with a smaller maximum frame may additionally segment an encoded batch, but it -must completely reassemble that batch before calling -`Group.Replica.Transport.deliver_batch/4`. +registered incoming name on the target node. Queue pressure maps to `:busy` and +a missing session maps to `:disconnected`. The adapter passes the trusted peer's +source node alongside each message. Exact snapshots are already bounded by +Group. A transport with a smaller maximum frame may additionally segment an +encoded batch, but it must completely reassemble that batch before calling +`Group.Replica.Transport.incoming_batch/4`. ### Named Cluster TTL Leases diff --git a/lib/group.ex b/lib/group.ex index 0e949e8..17a3df6 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -234,7 +234,7 @@ defmodule Group do data or cluster controls are busy (default: `8`) - `:replica_transport` — replica data transport module or `{module, opts}` tuple. Defaults to `Group.Replica.Transport.Distribution`. The transport must be - nonblocking and may return `:busy`; anti-entropy repairs dropped frames. + nonblocking and may return `:busy`; anti-entropy repairs dropped messages. Sideband transports can use `Group.Replica.Transport.Outbox` for lossy, batched, per-shard isolation without adding a hop to the default transport. - `:replicated_oplog_max_entries` — maximum retained replica records per shard diff --git a/lib/group/replica.ex b/lib/group/replica.ex index d0f888d..d70b462 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -64,11 +64,11 @@ defmodule Group.Replica do the requested prefix has already been pruned. Receivers stage chunks in a private ETS table and expose nothing until every chunk is present. - Every stream field is validated against the authenticated source node and + Every stream field is validated against the source node and current generation/epoch. An old generation, a closed epoch, a wrong shard, or a transitive claim for another node's pid is rejected. Control/data - reordering is safe: early frames are ignored and repeated heads repair them; - late frames fail their generation or epoch fence. Snapshot chunks may be + reordering is safe: early messages are ignored and repeated heads repair them; + late messages fail their generation or epoch fence. Snapshot chunks may be lost, duplicated, reordered, or mixed across retransmissions at the same stream head; exact row counts and set insertion prevent partial commits. @@ -88,10 +88,10 @@ defmodule Group.Replica do All cross-node control messages use :erlang.send_nosuspend/3 with :noconnect. The default replica adapter does the same. Transport callbacks return :ok, - :busy, or :disconnected; failure drops the frame and anti-entropy repairs it. + :busy, or :disconnected; failure drops the message and anti-entropy repairs it. Replica shards never remotely monitor or exit member processes. - The transport need not order frames for correctness. The local shard + The transport need not order messages for correctness. The local shard serializes writes, sequence numbers establish per-stream order, and receivers reject duplicates and gaps. TCP shard-to-shard ordering remains the efficient fast path. No semantic operation spans clusters, so cross-stream ordering is @@ -764,20 +764,21 @@ defmodule Group.Replica do end end - def handle_info({:group_replica_frame, remote_pid, frame}, state) when is_pid(remote_pid) do + def handle_info({:group_replica_frame, remote_pid, message}, state) when is_pid(remote_pid) do remote_node = node(remote_pid) - state = handle_replica_frame(state, remote_node, frame) + state = handle_replica_message(state, remote_node, message) {:noreply, take_priority_turn(state)} end - def handle_info({:group_replica_frame, remote_node, frame}, state) when is_atom(remote_node) do - state = handle_replica_frame(state, remote_node, frame) + def handle_info({:group_replica_frame, remote_node, message}, state) + when is_atom(remote_node) do + state = handle_replica_message(state, remote_node, message) {:noreply, take_priority_turn(state)} end - def handle_info({:group_replica_batch, remote_node, frames}, state) - when is_atom(remote_node) and is_list(frames) do - state = Enum.reduce(frames, state, &handle_replica_frame(&2, remote_node, &1)) + def handle_info({:group_replica_batch, remote_node, messages}, state) + when is_atom(remote_node) and is_list(messages) do + state = Enum.reduce(messages, state, &handle_replica_message(&2, remote_node, &1)) {:noreply, take_priority_turn(state)} end @@ -2712,7 +2713,7 @@ defmodule Group.Replica do {stream_id, first_seq, records, head} end) - try_send_replica_frame(state, target_node, {:delta_batch, Protocol.version(), runs}) + outgoing_replica_message(state, target_node, {:delta_batch, Protocol.version(), runs}) end defp group_broadcast_ops_by_target(ops, state, cluster_fun) do @@ -2812,12 +2813,12 @@ defmodule Group.Replica do end end - defp try_send_replica_frame(state, target_node, frame) do - case state.replica_transport.try_send( + defp outgoing_replica_message(state, target_node, message) do + case state.replica_transport.outgoing( state.name, target_node, state.shard_index, - frame, + message, state.replica_transport_opts ) do :ok -> :ok @@ -3230,7 +3231,7 @@ defmodule Group.Replica do if heads == [] do state else - try_send_replica_frame(state, target_node, {:heads, Protocol.version(), heads}) + outgoing_replica_message(state, target_node, {:heads, Protocol.version(), heads}) end end @@ -3287,7 +3288,7 @@ defmodule Group.Replica do (is_nil(cluster) or cluster_member?(state.name, cluster)) end - defp handle_replica_frame(state, source_node, {:heads, version, heads}) + defp handle_replica_message(state, source_node, {:heads, version, heads}) when version == @protocol_version do needs = Enum.flat_map(heads, fn {stream_id, _floor, head} -> @@ -3302,11 +3303,11 @@ defmodule Group.Replica do needs |> Enum.chunk_every(state.replicated_sender_buffer_size) |> Enum.reduce(state, fn chunk, acc -> - try_send_replica_frame(acc, source_node, {:needs, Protocol.version(), chunk}) + outgoing_replica_message(acc, source_node, {:needs, Protocol.version(), chunk}) end) end - defp handle_replica_frame(state, source_node, {:delta_batch, version, runs}) + defp handle_replica_message(state, source_node, {:delta_batch, version, runs}) when version == @protocol_version do state = flush_pending_replicated_sender_barrier(state) @@ -3315,7 +3316,7 @@ defmodule Group.Replica do end) end - defp handle_replica_frame(state, source_node, {:need, version, stream_id, next_seq}) + defp handle_replica_message(state, source_node, {:need, version, stream_id, next_seq}) when version == @protocol_version do if Protocol.stream_origin(stream_id) == node() and Protocol.stream_shard(stream_id) == state.shard_index and @@ -3326,12 +3327,12 @@ defmodule Group.Replica do end end - defp handle_replica_frame(state, source_node, {:needs, version, needs}) + defp handle_replica_message(state, source_node, {:needs, version, needs}) when version == @protocol_version do send_replica_repairs(state, source_node, needs) end - defp handle_replica_frame( + defp handle_replica_message( state, source_node, {:snapshot_chunk, version, stream_id, snapshot_seq, chunk_index, chunk_count, @@ -3380,7 +3381,7 @@ defmodule Group.Replica do end end - defp handle_replica_frame(state, _source_node, _frame), do: state + defp handle_replica_message(state, _source_node, _message), do: state defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do valid_remote_stream?(state, source_node, stream_id) and @@ -3857,7 +3858,7 @@ defmodule Group.Replica do end defp request_replica_need(state, target_node, stream_id, next_seq) do - try_send_replica_frame( + outgoing_replica_message( state, target_node, {:needs, Protocol.version(), [{stream_id, next_seq}]} @@ -3884,7 +3885,7 @@ defmodule Group.Replica do state runs -> - try_send_replica_frame( + outgoing_replica_message( state, target_node, {:delta_batch, Protocol.version(), Enum.reverse(runs)} @@ -3944,7 +3945,7 @@ defmodule Group.Replica do snapshot.chunks |> Enum.with_index(1) |> Enum.reduce(state, fn {{reg_chunk, pg_chunk}, chunk_index}, acc -> - try_send_replica_frame( + outgoing_replica_message( acc, target_node, {:snapshot_chunk, Protocol.version(), stream_id, head, chunk_index, chunk_count, diff --git a/lib/group/replica/transport.ex b/lib/group/replica/transport.ex index b7f46c1..46bb532 100644 --- a/lib/group/replica/transport.ex +++ b/lib/group/replica/transport.ex @@ -3,38 +3,45 @@ defmodule Group.Replica.Transport do Transport contract for Group replica data. Implementations must return promptly and must never wait for socket or remote - mailbox backpressure. This applies to `try_send/5` and the optional lifecycle + mailbox backpressure. This applies to `outgoing/5` and the optional lifecycle callbacks. Returning `:busy` or `:disconnected` is safe: replica anti-entropy will retransmit the missing state. Erlang distribution remains Group's control plane and supplies the stable node identity used here. A sideband adapter can use its `descriptor/2` in the - control hello to exchange endpoints, authenticate the connection as that - node, and pass inbound frames to `deliver/4`. + control hello to exchange endpoints and pass incoming messages to + `incoming/4` or `incoming_batch/4`. Adapters do not need to preserve ordering. Group serializes writes per shard and sequences each origin/generation/shard/cluster/epoch stream; receivers discard duplicates and request gaps. Per-shard ordered delivery avoids repair traffic and is therefore the preferred fast path. - A sideband implementation can delegate `try_send/5` to - `Group.Replica.Transport.Outbox.try_send/5`. That adds one local send only for + A sideband implementation can delegate `outgoing/5` to + `Group.Replica.Transport.Outbox.push/5`. That adds one local send only for the configured sideband transport; the default distribution adapter retains its direct remote `:erlang.send_nosuspend/3` path. """ - @type frame :: term() - @type send_result :: :ok | :busy | :disconnected + @type message :: term() + @type outgoing_result :: :ok | :busy | :disconnected @callback id() :: term() @callback descriptor(group :: atom(), opts :: keyword()) :: term() - @callback try_send( + @doc """ + Called when Group has an outgoing replica message for another node. + + This callback must return promptly and must never wait for transport or + remote backpressure. `:ok` means the transport took responsibility for the + message, not that the remote shard received it. + """ + @callback outgoing( group :: atom(), target_node :: node(), shard :: non_neg_integer(), - frame(), + message(), opts :: keyword() - ) :: send_result() + ) :: outgoing_result() @callback child_spec(keyword()) :: Supervisor.child_spec() | :ignore @callback peer_up(group :: atom(), node(), descriptor :: term(), opts :: keyword()) :: :ok @@ -43,31 +50,34 @@ defmodule Group.Replica.Transport do @optional_callbacks child_spec: 1, peer_up: 4, peer_down: 3 @doc """ - Delivers a frame received by a transport adapter to the local replica shard. + Passes an incoming replica message to the corresponding local shard. - `source_node` must come from the adapter's authenticated peer identity, never - from untrusted frame contents. Delivery is a local mailbox operation; stream - generation, epoch, group, shard, and origin are validated by the replica. + This is a local mailbox operation. Stream generation, epoch, group, shard, + and origin are validated by the replica. """ - def deliver(group, source_node, shard, frame) + def incoming(group, source_node, shard, message) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 do - send(Group.Replica.shard_name(group, shard), {:group_replica_frame, source_node, frame}) + send( + Group.Replica.shard_name(group, shard), + {:group_replica_frame, source_node, message} + ) + :ok end @doc """ - Delivers a complete batch received from one authenticated peer. + Passes a complete incoming batch to the corresponding local shard. - A finite-frame transport may segment the encoded batch on the wire, but it - must authenticate the peer and reassemble every segment before calling this - function. Group never observes or applies a partial batch. + A finite-message transport may segment the encoded batch on the wire, but it + must reassemble every segment before calling this function. Group never + observes or applies a partial batch. """ - def deliver_batch(group, source_node, shard, frames) + def incoming_batch(group, source_node, shard, messages) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 and - is_list(frames) do + is_list(messages) do send( Group.Replica.shard_name(group, shard), - {:group_replica_batch, source_node, frames} + {:group_replica_batch, source_node, messages} ) :ok @@ -84,7 +94,7 @@ defmodule Group.Replica.Transport do def validate!({module, _opts} = transport) do Code.ensure_loaded!(module) - for {function, arity} <- [id: 0, descriptor: 2, try_send: 5] do + for {function, arity} <- [id: 0, descriptor: 2, outgoing: 5] do unless function_exported?(module, function, arity) do raise ArgumentError, "replica transport #{inspect(module)} must implement #{function}/#{arity}" @@ -99,10 +109,10 @@ defmodule Group.Replica.Transport.Distribution do @moduledoc """ Default nonblocking replica transport over Erlang distribution. - Frames are sent directly to the matching remote shard with + Messages are sent directly to the matching remote shard with `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a busy distribution socket and never initiates a connection. A busy or absent - link returns `:busy`; Group drops that frame and repairs it through periodic + link returns `:busy`; Group drops that message and repairs it through periodic anti-entropy. """ @behaviour Group.Replica.Transport @@ -116,9 +126,9 @@ defmodule Group.Replica.Transport.Distribution do def descriptor(_group, _opts), do: :erlang_distribution @impl true - def try_send(group, target_node, shard, frame, _opts) do + def outgoing(group, target_node, shard, replica_message, _opts) do destination = {Replica.shard_name(group, shard), target_node} - message = {:group_replica_frame, node(), frame} + message = {:group_replica_frame, node(), replica_message} case :erlang.send_nosuspend(destination, message, [:noconnect]) do true -> :ok diff --git a/lib/group/replica/transport/outbox.ex b/lib/group/replica/transport/outbox.ex index aee2ba6..43c23b3 100644 --- a/lib/group/replica/transport/outbox.ex +++ b/lib/group/replica/transport/outbox.ex @@ -4,12 +4,12 @@ defmodule Group.Replica.Transport.Outbox do This module is an implementation helper, not a replacement for `Group.Replica.Transport`. Distribution can continue sending directly with - `:erlang.send_nosuspend/3`. A sideband adapter delegates `try_send/5` to - `try_send/5`, which performs only a local `send/2` to the matching shard + `:erlang.send_nosuspend/3`. A sideband adapter delegates `outgoing/5` to + `push/5`, which performs only a local `send/2` to the matching shard outbox. - Each outbox batches frames by target node outside the Group shard. Expired - frames and batches rejected by the backend are deliberately dropped; + Each outbox batches messages by target node outside the Group shard. Expired + messages and batches rejected by the backend are deliberately dropped; anti-entropy repairs them. Backends may perform bounded blocking work in `send_batch/4` because they run in the outbox rather than a Group process. @@ -19,35 +19,35 @@ defmodule Group.Replica.Transport.Outbox do def init_outbox(group, shard, opts), do: {:ok, backend_state} - def send_batch(target_node, frames, deadline, backend_state) do + def send_batch(target_node, messages, deadline, backend_state) do # Return promptly once `deadline` has passed. It is safe to drop. {:ok, backend_state} end - The backend is responsible for authenticated ingress and must pass only - complete logical frames to `Group.Replica.Transport.deliver_batch/4`. + The backend must pass only complete logical messages to + `Group.Replica.Transport.incoming_batch/4`. ## Options - * `:outbox_batch_size` - maximum logical frames collected per flush, + * `:outbox_batch_size` - maximum logical messages collected per flush, default `64` * `:outbox_batch_bytes` - approximate external-term bytes collected per flush, default `1_048_576` * `:outbox_flush_interval` - maximum batching delay in milliseconds, default `1` - * `:outbox_deadline` - maximum useful residence time for an outbound frame + * `:outbox_deadline` - maximum useful residence time for an outgoing message in milliseconds, default `100` The deadline bounds stale work, not mailbox memory. A backend must also put a finite bound on every socket enqueue or write it performs. Exact snapshot - frames are independently bounded by `:replicated_snapshot_chunk_target_bytes`. - Other logical frames or a whole batch may still exceed `:outbox_batch_bytes`; + messages are independently bounded by `:replicated_snapshot_chunk_target_bytes`. + Other logical messages or a whole batch may still exceed `:outbox_batch_bytes`; a transport with a smaller finite frame size must segment and completely reassemble those batches before local delivery. """ - @type frame :: Group.Replica.Transport.frame() - @type send_result :: Group.Replica.Transport.send_result() + @type message :: Group.Replica.Transport.message() + @type outgoing_result :: Group.Replica.Transport.outgoing_result() @type backend_state :: term() @callback init_outbox(group :: atom(), shard :: non_neg_integer(), opts :: keyword()) :: @@ -55,10 +55,10 @@ defmodule Group.Replica.Transport.Outbox do @callback send_batch( target_node :: node(), - frames :: [frame()], + messages :: [message()], deadline :: integer(), backend_state() - ) :: {send_result(), backend_state()} + ) :: {outgoing_result(), backend_state()} @default_deadline 100 @@ -81,20 +81,20 @@ defmodule Group.Replica.Transport.Outbox do end @doc """ - Enqueues a frame into its local shard outbox. + Pushes a replica message into its shard's local outbox. - This performs no backend or socket operation. `:ok` only means the message - was sent to the current local outbox PID; the outbox may later drop it on - expiry or backpressure. A concurrently terminating outbox can also lose an - accepted message, which anti-entropy repairs. + This operation is local and nonblocking and performs no backend or socket + work. `:ok` only means the outbox was available; the message may later expire + or be dropped under transport pressure. A concurrently terminating outbox + can also lose an accepted message, which anti-entropy repairs. """ - def try_send(group, target_node, shard, frame, opts) + def push(group, target_node, shard, message, opts) when is_atom(group) and is_atom(target_node) and is_integer(shard) and shard >= 0 and is_list(opts) do case Process.whereis(name(group, shard)) do pid when is_pid(pid) -> deadline = monotonic_ms() + deadline(opts) - send(pid, {:group_replica_outbox_send, target_node, deadline, frame}) + send(pid, {:group_replica_outbox_push, target_node, deadline, message}) :ok nil -> @@ -192,12 +192,12 @@ defmodule Group.Replica.Transport.Outbox.Worker do end @impl true - def handle_info({:group_replica_outbox_send, target_node, deadline, frame}, state) + def handle_info({:group_replica_outbox_push, target_node, deadline, message}, state) when is_atom(target_node) and is_integer(deadline) do if deadline <= Outbox.monotonic_ms() do {:noreply, state} else - bytes = :erlang.external_size({target_node, frame}) + bytes = :erlang.external_size({target_node, message}) state = if state.pending_count > 0 and @@ -208,7 +208,7 @@ defmodule Group.Replica.Transport.Outbox.Worker do state end - state = enqueue(state, target_node, deadline, frame, bytes) + state = put_pending(state, target_node, deadline, message, bytes) if state.pending_count >= state.batch_size or state.pending_bytes >= state.batch_bytes do {:noreply, flush(state)} @@ -225,8 +225,8 @@ defmodule Group.Replica.Transport.Outbox.Worker do def handle_info({:group_replica_outbox_flush, _stale_ref}, state), do: {:noreply, state} def handle_info(_message, state), do: {:noreply, state} - defp enqueue(state, target_node, deadline, frame, bytes) do - entry = {target_node, deadline, frame} + defp put_pending(state, target_node, deadline, message, bytes) do + entry = {target_node, deadline, message} %{ state @@ -254,22 +254,22 @@ defmodule Group.Replica.Transport.Outbox.Worker do batches = state.pending |> Enum.reverse() - |> Enum.reject(fn {_target_node, deadline, _frame} -> deadline <= now end) - |> Enum.group_by(fn {target_node, _deadline, _frame} -> target_node end) + |> Enum.reject(fn {_target_node, deadline, _message} -> deadline <= now end) + |> Enum.group_by(fn {target_node, _deadline, _message} -> target_node end) backend_state = Enum.reduce(batches, state.backend_state, fn {target_node, entries}, backend_state -> - frames = Enum.map(entries, fn {_target_node, _deadline, frame} -> frame end) + messages = Enum.map(entries, fn {_target_node, _deadline, message} -> message end) deadline = entries - |> Enum.map(fn {_target_node, deadline, _frame} -> deadline end) + |> Enum.map(fn {_target_node, deadline, _message} -> deadline end) |> Enum.min() if deadline <= Outbox.monotonic_ms() do backend_state else - case state.backend.send_batch(target_node, frames, deadline, backend_state) do + case state.backend.send_batch(target_node, messages, deadline, backend_state) do {result, next_backend_state} when result in [:ok, :busy, :disconnected] -> next_backend_state diff --git a/lib/group/replica/transport/tcp.ex b/lib/group/replica/transport/tcp.ex index 0cedaa8..d074099 100644 --- a/lib/group/replica/transport/tcp.ex +++ b/lib/group/replica/transport/tcp.ex @@ -3,17 +3,17 @@ defmodule Group.Replica.Transport.TCP do Sideband TCP transport for replica data. Erlang distribution still carries Group discovery and authority controls. - Replica frames use independent TCP connections, so there is no ordering + Replica messages use independent TCP connections, so there is no ordering relationship between a control message and its data lane. - `try_send/5` only sends to a local per-shard outbox. The outbox batches - frames and forwards each target batch to a bounded per-peer writer queue. + `outgoing/5` only pushes to a local per-shard outbox. The outbox batches + messages and forwards each target batch to a bounded per-peer writer queue. The writer may block up to `:send_timeout` without blocking a Group shard. Expired, busy, and disconnected batches are dropped and repaired by anti-entropy. The endpoint capability in the dist-Erlang hello prevents an unrelated - socket client from injecting frames. This transport is intended for trusted + socket client from injecting messages. This transport is intended for trusted cluster networks; it does not encrypt traffic. Put it behind a private network or a TLS/WebSocket tunnel when confidentiality is required. @@ -64,20 +64,20 @@ defmodule Group.Replica.Transport.TCP do end @impl true - def try_send(group, target_node, shard, frame, opts), - do: Outbox.try_send(group, target_node, shard, frame, opts) + def outgoing(group, target_node, shard, message, opts), + do: Outbox.push(group, target_node, shard, message, opts) @impl Group.Replica.Transport.Outbox def init_outbox(group, shard, _opts), do: {:ok, %{group: group, shard: shard}} @impl Group.Replica.Transport.Outbox - def send_batch(target_node, frames, deadline, %{group: group, shard: shard} = state) do + def send_batch(target_node, messages, deadline, %{group: group, shard: shard} = state) do result = try do case :ets.lookup(route_table(group), target_node) do [{^target_node, writer, queued, max_queue}] -> if :atomics.add_get(queued, 1, 1) <= max_queue do - send(writer, {:replica_batch, deadline, shard, frames}) + send(writer, {:replica_batch, deadline, shard, messages}) :ok else :atomics.sub(queued, 1, 1) @@ -379,12 +379,12 @@ defmodule Group.Replica.Transport.TCP do defp writer_loop(socket, manager, remote_node, queued) do receive do - {:replica_batch, deadline, shard, frames} -> + {:replica_batch, deadline, shard, messages} -> result = if deadline <= Outbox.monotonic_ms() do :expired else - :gen_tcp.send(socket, :erlang.term_to_binary({:batch, shard, frames})) + :gen_tcp.send(socket, :erlang.term_to_binary({:batch, shard, messages})) end :atomics.sub(queued, 1, 1) @@ -438,9 +438,9 @@ defmodule Group.Replica.Transport.TCP do case :gen_tcp.recv(socket, 0) do {:ok, payload} -> case decode_authenticated_frame(payload) do - {:ok, {:batch, shard, frames}} - when is_integer(shard) and shard >= 0 and is_list(frames) -> - :ok = Group.Replica.Transport.deliver_batch(group, source_node, shard, frames) + {:ok, {:batch, shard, messages}} + when is_integer(shard) and shard >= 0 and is_list(messages) -> + :ok = Group.Replica.Transport.incoming_batch(group, source_node, shard, messages) reader_loop(socket, group, source_node) _ -> diff --git a/test/README.md b/test/README.md index 224f250..ffb2986 100644 --- a/test/README.md +++ b/test/README.md @@ -33,7 +33,7 @@ release qualification rather than individual edits. ## Model-based and formal checks `replica_model_property_test.exs` runs real Group instances on three peer VMs. -The controlled transport queues each replica frame so generated commands can +The controlled transport queues each replica message so generated commands can deliver, duplicate, drop, reorder, or strand it. After the bounded-fault prefix, the test enables fair delivery and compares every tracked registry and PG key against an independent application-level lifecycle oracle. It also @@ -251,11 +251,12 @@ timestamp. ### Replica transport fault injection `Group.TestReplicaTransport` implements the production transport behaviour but -can return `:busy`, drop selected frame types, duplicate or delay frames, and -capture frames for explicit stale-generation/epoch replay. Its `{:chaos, opts}` -mode is deterministic for a given frame, which makes failures reproducible. +can return `:busy`, drop selected message types, duplicate or delay messages, +and capture messages for explicit stale-generation/epoch replay. Its +`{:chaos, opts}` mode is deterministic for a given message, which makes failures +reproducible. -`Group.ControlledReplicaTransport` is the model-test transport. It queues frames +`Group.ControlledReplicaTransport` is the model-test transport. It queues messages at the test process without scheduling timers; `Group.ReplicaModelScheduler` then owns the exact delivery schedule. These roles are separate so the existing timing-oriented regressions retain their original mechanics while property @@ -263,7 +264,7 @@ failures can be replayed and shrunk exactly. The distributed anti-entropy tests cover dropped creates and deletes, cursor gaps, globally pruned multi-stream oplogs, exact snapshot fallback, malformed -authority, stale frame replay, lease expiry on a live VM, and multi-shard +authority, stale message replay, lease expiry on a live VM, and multi-shard generation recovery. They also restart a suspended data lane after deliberately losing its cluster-close fence and require the lane to sweep the stale registry and PG slices from shared authority. Authority topology tests suspend every @@ -272,7 +273,7 @@ the full epoch snapshot, nonzero shards receive constant-size lane hellos, and incremental opens stay on their matching shard. Separate tests suspend a backlogged authority shard while other replica lanes continue converging and deliver data before authority to prove rejection does not advance the cursor -and the same frame applies after authority repair. Concurrent snapshot tests +and the same message applies after authority repair. Concurrent snapshot tests require every advertised revision to contain exactly that many unique named epochs, and heartbeat tests prove observed revisions cannot advance the exact authority marker. Crash-window tests interrupt journal, dual-index, receive @@ -283,7 +284,7 @@ requires snapshot recovery without changing the third node's independent registry or PG state. `replica_transport_outbox_test.exs` proves that a blocked sideband backend -cannot delay the Group-facing local send, frames expire behind that backend, +cannot delay the Group-facing local push, messages expire behind that backend, busy batches are not retried locally, and batching preserves per-target order. The real three-node TCP recovery test runs through the same outbox path. diff --git a/test/distributed_test.exs b/test/distributed_test.exs index 3a7c2d7..ebe8278 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -4606,7 +4606,7 @@ defmodule Group.DistributedTest do ]) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 1, @@ -4635,7 +4635,7 @@ defmodule Group.DistributedTest do end) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 1, @@ -4722,7 +4722,7 @@ defmodule Group.DistributedTest do end @tag timeout: 60_000 - test "duplicate and reordered replica frames are idempotent and emit one lifecycle event" do + test "duplicate and reordered replica messages are idempotent and emit one lifecycle event" do peers = TestCluster.start_peers(2) on_exit(fn -> TestCluster.stop_peers(peers) end) @@ -4827,7 +4827,7 @@ defmodule Group.DistributedTest do Enum.each(generation_frames, fn {_target, shard, frame} -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -4882,7 +4882,7 @@ defmodule Group.DistributedTest do Enum.each(epoch_frames, fn {_target, shard, frame} -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -5096,7 +5096,7 @@ defmodule Group.DistributedTest do ]} :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 0, @@ -5113,7 +5113,7 @@ defmodule Group.DistributedTest do ]) == 0 :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_b, 0, @@ -5368,7 +5368,7 @@ defmodule Group.DistributedTest do mutation = {:register, nil, key, pid, meta, System.monotonic_time(), node_a} :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 0, @@ -5432,7 +5432,7 @@ defmodule Group.DistributedTest do Map.fetch!(frames_by_first_seq, 2) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -5454,7 +5454,7 @@ defmodule Group.DistributedTest do for frame <- [first_frame, second_frame, third_frame] do :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -5492,7 +5492,7 @@ defmodule Group.DistributedTest do for frame <- [first_frame, second_frame, third_frame] do :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -5600,7 +5600,7 @@ defmodule Group.DistributedTest do # time this frame runs, but shard 1 must still reject it until its own # old-generation purge has completed. :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 1, @@ -5633,7 +5633,7 @@ defmodule Group.DistributedTest do ]) == 0 :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 1, diff --git a/test/jepsen/README.md b/test/jepsen/README.md index d543df9..aeba2ce 100644 --- a/test/jepsen/README.md +++ b/test/jepsen/README.md @@ -25,7 +25,7 @@ The replica lane is selectable without changing the workload or checker: - `tcp` uses Group's production sideband TCP adapter while Erlang distribution remains the control plane; and - `chaos` is a local per-shard outbox which deterministically drops, - duplicates, delays, and reorders replica frames. + duplicates, delays, and reorders replica messages. After faults stop, every surviving node reconnects and the harness takes two terminal snapshots. The independent checker requires: diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index 76817fb..c72774b 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -80,14 +80,14 @@ defmodule Group.Jepsen.Transport.Common do @moduledoc false alias Group.Jepsen.Transport.Stats - def try_send(delegate, group, target_node, shard, frame, opts) do - record(frame) + def outgoing(delegate, group, target_node, shard, message, opts) do + record(message) if Stats.blocked?(target_node) do Stats.increment(:logical_drop) :ok else - result = delegate.try_send(group, target_node, shard, frame, opts) + result = delegate.outgoing(group, target_node, shard, message, opts) Stats.increment(transport_result(result)) observe_outbox(group, shard) result @@ -103,7 +103,7 @@ defmodule Group.Jepsen.Transport.Common do end def record({:delta_batch, _version, _runs}), do: Stats.increment(:delta_batch) - def record(_frame), do: Stats.increment(:other_frame) + def record(_message), do: Stats.increment(:other_message) defp transport_result(:ok), do: :transport_ok defp transport_result(:busy), do: :transport_busy @@ -140,8 +140,8 @@ defmodule Group.Jepsen.Transport.Distribution do def child_spec(opts), do: {Stats, opts} @impl true - def try_send(group, target_node, shard, frame, opts) do - Common.try_send(Delegate, group, target_node, shard, frame, opts) + def outgoing(group, target_node, shard, message, opts) do + Common.outgoing(Delegate, group, target_node, shard, message, opts) end end @@ -168,8 +168,8 @@ defmodule Group.Jepsen.Transport.TCP do end @impl true - def try_send(group, target_node, shard, frame, opts) do - Common.try_send(Delegate, group, target_node, shard, frame, opts) + def outgoing(group, target_node, shard, message, opts) do + Common.outgoing(Delegate, group, target_node, shard, message, opts) end @impl true @@ -219,8 +219,8 @@ defmodule Group.Jepsen.Transport.Chaos do end @impl true - def try_send(group, target_node, shard, frame, _opts) do - Common.record(frame) + def outgoing(group, target_node, shard, message, _opts) do + Common.record(message) if Stats.blocked?(target_node) do Stats.increment(:logical_drop) @@ -228,7 +228,7 @@ defmodule Group.Jepsen.Transport.Chaos do else case Process.whereis(worker_name(group, shard)) do pid when is_pid(pid) -> - send(pid, {:send, target_node, frame}) + send(pid, {:outgoing, target_node, message}) :ok nil -> @@ -277,7 +277,7 @@ defmodule Group.Jepsen.Transport.Chaos.Worker do def init({group, shard}), do: {:ok, %{group: group, shard: shard, counter: 0}} @impl true - def handle_info({:send, target_node, frame}, state) do + def handle_info({:outgoing, target_node, message}, state) do counter = state.counter + 1 next = %{state | counter: counter} @@ -285,11 +285,11 @@ defmodule Group.Jepsen.Transport.Chaos.Worker do Stats.increment(:chaos_drop) else delay = rem(counter * 17, 41) - Process.send_after(self(), {:deliver, target_node, frame}, delay) + Process.send_after(self(), {:forward, target_node, message}, delay) if rem(counter, 7) == 0 do Stats.increment(:chaos_duplicate) - Process.send_after(self(), {:deliver, target_node, frame}, rem(delay + 19, 47)) + Process.send_after(self(), {:forward, target_node, message}, rem(delay + 19, 47)) end if delay > 0, do: Stats.increment(:chaos_delay) @@ -298,9 +298,9 @@ defmodule Group.Jepsen.Transport.Chaos.Worker do {:noreply, next} end - def handle_info({:deliver, target_node, frame}, state) do + def handle_info({:forward, target_node, replica_message}, state) do destination = {Group.Replica.shard_name(state.group, state.shard), target_node} - message = {:group_replica_frame, node(), frame} + message = {:group_replica_frame, node(), replica_message} case :erlang.send_nosuspend(destination, message, [:noconnect]) do true -> Stats.increment(:chaos_delivered) diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index a517770..89804b8 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -556,7 +556,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do ]) :ok = - TestCluster.rpc!(node_a, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_a, Group.Replica.Transport, :incoming, [ name, node_b, 0, @@ -581,7 +581,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do defp deliver_frames(node_b, node_a, name, frames) do Enum.each(frames, fn frame -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 0, diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs index 329ed96..63c3a4e 100644 --- a/test/replica_transport_outbox_test.exs +++ b/test/replica_transport_outbox_test.exs @@ -19,10 +19,10 @@ defmodule Group.ReplicaTransportOutboxTest do end @impl true - def send_batch(target_node, frames, deadline, state) do + def send_batch(target_node, messages, deadline, state) do send( state.controller, - {:outbox_batch, state.group, state.shard, target_node, frames, deadline} + {:outbox_batch, state.group, state.shard, target_node, messages, deadline} ) if state.sleep > 0, do: Process.sleep(state.sleep) @@ -30,7 +30,7 @@ defmodule Group.ReplicaTransportOutboxTest do end end - test "batches frames per target while preserving per-target order" do + test "batches messages per target while preserving per-target order" do group = unique_group(:batch) target_a = :"outbox-a@test" target_b = :"outbox-b@test" @@ -40,20 +40,20 @@ defmodule Group.ReplicaTransportOutboxTest do outbox_flush_interval: 1_000 ) - assert :ok = Outbox.try_send(group, target_a, 0, {:frame, 1}, outbox_deadline: 1_000) - assert :ok = Outbox.try_send(group, target_b, 0, {:frame, 2}, outbox_deadline: 1_000) - assert :ok = Outbox.try_send(group, target_a, 0, {:frame, 3}, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target_a, 0, {:message, 1}, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target_b, 0, {:message, 2}, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target_a, 0, {:message, 3}, outbox_deadline: 1_000) batches = for _ <- 1..2, into: %{} do - assert_receive {:outbox_batch, ^group, 0, target, frames, deadline}, 1_000 + assert_receive {:outbox_batch, ^group, 0, target, messages, deadline}, 1_000 assert deadline > Outbox.monotonic_ms() - {target, frames} + {target, messages} end assert batches == %{ - target_a => [{:frame, 1}, {:frame, 3}], - target_b => [{:frame, 2}] + target_a => [{:message, 1}, {:message, 3}], + target_b => [{:message, 2}] } end @@ -66,21 +66,21 @@ defmodule Group.ReplicaTransportOutboxTest do backend_sleep: 200 ) - assert :ok = Outbox.try_send(group, target, 0, :first, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target, 0, :first, outbox_deadline: 1_000) assert_receive {:outbox_batch, ^group, 0, ^target, [:first], _deadline}, 1_000 caller = self() spawn(fn -> - result = Outbox.try_send(group, target, 0, :expires_behind_backend, outbox_deadline: 10) - send(caller, {:try_send_returned, result}) + result = Outbox.push(group, target, 0, :expires_behind_backend, outbox_deadline: 10) + send(caller, {:push_returned, result}) end) - assert_receive {:try_send_returned, :ok}, 100 + assert_receive {:push_returned, :ok}, 100 refute_receive {:outbox_batch, ^group, 0, ^target, [:expires_behind_backend], _deadline}, 300 end - test "expired frames and backend backpressure are dropped without local retries" do + test "expired messages and backend backpressure are dropped without local retries" do expired_group = unique_group(:expired) target = :"outbox-expired@test" @@ -89,7 +89,7 @@ defmodule Group.ReplicaTransportOutboxTest do ) assert :ok = - Outbox.try_send(expired_group, target, 0, :expired, outbox_deadline: 5) + Outbox.push(expired_group, target, 0, :expired, outbox_deadline: 5) refute_receive {:outbox_batch, ^expired_group, 0, ^target, [:expired], _deadline}, 100 @@ -100,12 +100,12 @@ defmodule Group.ReplicaTransportOutboxTest do backend_result: :busy ) - assert :ok = Outbox.try_send(busy_group, target, 0, :busy, outbox_deadline: 1_000) + assert :ok = Outbox.push(busy_group, target, 0, :busy, outbox_deadline: 1_000) assert_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 1_000 refute_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 100 end - test "complete inbound batches use one authenticated local delivery" do + test "complete incoming batches use one local mailbox operation" do group = unique_group(:deliver) source_node = :"outbox-source@test" parent = self() @@ -124,7 +124,7 @@ defmodule Group.ReplicaTransportOutboxTest do assert_receive :receiver_ready assert :ok = - Group.Replica.Transport.deliver_batch( + Group.Replica.Transport.incoming_batch( group, source_node, 0, diff --git a/test/support/controlled_replica_transport.ex b/test/support/controlled_replica_transport.ex index 70d7d48..9658457 100644 --- a/test/support/controlled_replica_transport.ex +++ b/test/support/controlled_replica_transport.ex @@ -19,15 +19,15 @@ defmodule Group.ControlledReplicaTransport do end @impl true - def try_send(group, target_node, shard, frame, opts) do + def outgoing(group, target_node, shard, message, opts) do case :persistent_term.get({__MODULE__, group, :mode}, :capture) do :capture -> controller = Keyword.fetch!(opts, :controller) - send(controller, {__MODULE__, :frame, group, node(), target_node, shard, frame}) + send(controller, {__MODULE__, :message, group, node(), target_node, shard, message}) :ok :pass -> - deliver(group, target_node, shard, frame) + forward(group, target_node, shard, message) :busy -> :busy @@ -37,9 +37,9 @@ defmodule Group.ControlledReplicaTransport do end end - defp deliver(group, target_node, shard, frame) do + defp forward(group, target_node, shard, replica_message) do destination = {Group.Replica.shard_name(group, shard), target_node} - message = {:group_replica_frame, node(), frame} + message = {:group_replica_frame, node(), replica_message} case :erlang.send_nosuspend(destination, message, [:noconnect]) do true -> :ok diff --git a/test/support/replica_model_scheduler.ex b/test/support/replica_model_scheduler.ex index b698324..12955db 100644 --- a/test/support/replica_model_scheduler.ex +++ b/test/support/replica_model_scheduler.ex @@ -5,10 +5,10 @@ defmodule Group.ReplicaModelScheduler do defmodule Envelope do @moduledoc false - defstruct [:id, :source, :target, :shard, :frame] + defstruct [:id, :source, :target, :shard, :message] end - defstruct [:name, :nodes, :model, :group_opts, owners: %{}, queue: [], next_frame_id: 1] + defstruct [:name, :nodes, :model, :group_opts, owners: %{}, queue: [], next_message_id: 1] def new(name, nodes, group_opts \\ []) do %__MODULE__{ @@ -247,18 +247,18 @@ defmodule Group.ReplicaModelScheduler do def drain(%__MODULE__{} = state, wait_ms \\ 0) do receive do - {ControlledReplicaTransport, :frame, group, source, target, shard, frame} + {ControlledReplicaTransport, :message, group, source, target, shard, message} when group == state.name -> envelope = %Envelope{ - id: state.next_frame_id, + id: state.next_message_id, source: source, target: target, shard: shard, - frame: frame + message: message } drain( - %{state | queue: state.queue ++ [envelope], next_frame_id: state.next_frame_id + 1}, + %{state | queue: state.queue ++ [envelope], next_message_id: state.next_message_id + 1}, wait_ms ) after @@ -391,8 +391,8 @@ defmodule Group.ReplicaModelScheduler do TestCluster.rpc!( envelope.target, Group.Replica.Transport, - :deliver, - [state.name, envelope.source, envelope.shard, envelope.frame] + :incoming, + [state.name, envelope.source, envelope.shard, envelope.message] ) TestCluster.flush_shards(envelope.target, state.name) @@ -505,7 +505,7 @@ defmodule Group.ReplicaModelScheduler do defp drain_transport_messages(name) do receive do - {ControlledReplicaTransport, :frame, ^name, _source, _target, _shard, _frame} -> + {ControlledReplicaTransport, :message, ^name, _source, _target, _shard, _message} -> drain_transport_messages(name) after 0 -> :ok diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex index 5eef3d0..4efc7b2 100644 --- a/test/support/test_replica_transport.ex +++ b/test/support/test_replica_transport.ex @@ -36,7 +36,7 @@ defmodule Group.TestReplicaTransport do end @impl true - def try_send(group, target_node, shard, frame, _opts) do + def outgoing(group, target_node, shard, message, _opts) do case :persistent_term.get({__MODULE__, group}, :pass) do :drop -> :ok @@ -45,42 +45,44 @@ defmodule Group.TestReplicaTransport do :busy :duplicate -> - deliver(group, target_node, shard, frame) - deliver(group, target_node, shard, frame) + forward(group, target_node, shard, message) + forward(group, target_node, shard, message) {:drop_types, types} -> - if frame_type(frame) in types, do: :ok, else: deliver(group, target_node, shard, frame) + if message_type(message) in types, + do: :ok, + else: forward(group, target_node, shard, message) {:duplicate_types, types} -> - if frame_type(frame) in types do - deliver(group, target_node, shard, frame) - deliver(group, target_node, shard, frame) + if message_type(message) in types do + forward(group, target_node, shard, message) + forward(group, target_node, shard, message) else - deliver(group, target_node, shard, frame) + forward(group, target_node, shard, message) end {:delay_types, delays, default_delay} -> - delay = Map.get(delays, frame_type(frame), default_delay) - delayed_deliver(group, target_node, shard, frame, delay) + delay = Map.get(delays, message_type(message), default_delay) + delayed_forward(group, target_node, shard, message, delay) {:capture_drop, types} -> - if frame_type(frame) in types, do: capture(group, target_node, shard, frame) + if message_type(message) in types, do: capture(group, target_node, shard, message) :ok {:capture_pass, types} -> - if frame_type(frame) in types, do: capture(group, target_node, shard, frame) - deliver(group, target_node, shard, frame) + if message_type(message) in types, do: capture(group, target_node, shard, message) + forward(group, target_node, shard, message) {:chaos, opts} -> - chaos_deliver(group, target_node, shard, frame, opts) + chaos_forward(group, target_node, shard, message, opts) :pass -> - deliver(group, target_node, shard, frame) + forward(group, target_node, shard, message) end end - defp chaos_deliver(group, target_node, shard, frame, opts) do - hash = :erlang.phash2({target_node, shard, frame}, 1_000_003) + defp chaos_forward(group, target_node, shard, message, opts) do + hash = :erlang.phash2({target_node, shard, message}, 1_000_003) drop_every = Keyword.get(opts, :drop_every, 0) duplicate_every = Keyword.get(opts, :duplicate_every, 0) max_delay = Keyword.get(opts, :max_delay, 0) @@ -91,43 +93,43 @@ defmodule Group.TestReplicaTransport do duplicate_every > 0 and rem(hash, duplicate_every) == 0 -> delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 - delayed_deliver(group, target_node, shard, frame, delay) - delayed_deliver(group, target_node, shard, frame, max(max_delay - delay, 0)) + delayed_forward(group, target_node, shard, message, delay) + delayed_forward(group, target_node, shard, message, max(max_delay - delay, 0)) true -> delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 - delayed_deliver(group, target_node, shard, frame, delay) + delayed_forward(group, target_node, shard, message, delay) end end - defp delayed_deliver(group, target_node, shard, frame, delay) when delay <= 0, - do: deliver(group, target_node, shard, frame) + defp delayed_forward(group, target_node, shard, message, delay) when delay <= 0, + do: forward(group, target_node, shard, message) - defp delayed_deliver(group, target_node, shard, frame, delay) do + defp delayed_forward(group, target_node, shard, message, delay) do source_node = node() spawn(fn -> receive do after - delay -> deliver(group, target_node, shard, frame, source_node) + delay -> forward(group, target_node, shard, message, source_node) end end) :ok end - defp capture(group, target_node, shard, frame) do + defp capture(group, target_node, shard, message) do key = {__MODULE__, group, :captured} captured = :persistent_term.get(key, []) - :persistent_term.put(key, [{target_node, shard, frame} | captured]) + :persistent_term.put(key, [{target_node, shard, message} | captured]) end - defp deliver(group, target_node, shard, frame), - do: deliver(group, target_node, shard, frame, node()) + defp forward(group, target_node, shard, message), + do: forward(group, target_node, shard, message, node()) - defp deliver(group, target_node, shard, frame, source_node) do + defp forward(group, target_node, shard, replica_message, source_node) do destination = {Group.Replica.shard_name(group, shard), target_node} - message = {:group_replica_frame, source_node, frame} + message = {:group_replica_frame, source_node, replica_message} case :erlang.send_nosuspend(destination, message, [:noconnect]) do true -> :ok @@ -135,6 +137,6 @@ defmodule Group.TestReplicaTransport do end end - defp frame_type(frame) when is_tuple(frame), do: elem(frame, 0) - defp frame_type(_frame), do: :unknown + defp message_type(message) when is_tuple(message), do: elem(message, 0) + defp message_type(_message), do: :unknown end From 1f81c498151633ce1a3694a6f4be62fa48ad6807 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Wed, 12 Aug 2026 18:21:09 +0000 Subject: [PATCH 7/7] Move replica transports to public namespace --- CHANGELOG.md | 16 ++- CLAUDE.md | 26 ++-- README.md | 56 ++++----- lib/group.ex | 4 +- lib/group/replica.ex | 116 +++++++++--------- lib/group/replica/data.ex | 72 +++++------ lib/group/replica/snapshot.ex | 2 +- .../replica/{protocol.ex => wire_protocol.ex} | 2 +- lib/group/supervisor.ex | 6 +- lib/group/{replica => }/transport.ex | 48 ++------ lib/group/transport/dist_erl.ex | 32 +++++ lib/group/{replica => }/transport/outbox.ex | 24 ++-- test/README.md | 5 +- test/distributed_test.exs | 63 +++++----- test/group_test.exs | 8 +- test/jepsen/Dockerfile.node | 1 + test/jepsen/README.md | 2 +- test/jepsen/node.exs | 39 +++--- test/mutation/run.exs | 8 +- test/replica_snapshot_distributed_test.exs | 15 ++- test/replica_snapshot_test.exs | 4 +- test/replica_transport_outbox_test.exs | 4 +- test/support/controlled_replica_transport.ex | 2 +- test/support/replica_model_scheduler.ex | 2 +- test/support/test_cluster.ex | 14 +-- test/support/test_replica_transport.ex | 2 +- .../support/test_tcp_transport.ex | 58 +++------ 27 files changed, 304 insertions(+), 327 deletions(-) rename lib/group/replica/{protocol.ex => wire_protocol.ex} (96%) rename lib/group/{replica => }/transport.ex (73%) create mode 100644 lib/group/transport/dist_erl.ex rename lib/group/{replica => }/transport/outbox.ex (92%) rename lib/group/replica/transport/tcp.ex => test/support/test_tcp_transport.ex (84%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45bec76..47fdd90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,21 @@ - Add layered anti-entropy qualification: three-node StreamData lifecycle models, seeded adversarial transport histories, TLA+ models for convergence, chunk assembly, and permanent peer eviction, plus a Docker-backed Jepsen - oracle across distribution, sideband TCP, and lossy/reordering transports. + oracle across distribution, a test-only sideband TCP lane, and + lossy/reordering transports. `mix test` is the every-PR ExUnit/property/checker gate and `mix test.soak` runs the six-profile nightly/release campaign. -- **Breaking**: the replica transport boundary now names logical direction +- **Breaking**: move the replica transport API from + `Group.Replica.Transport.*` to `Group.Transport.*`; the default adapter is + now `Group.Transport.DistErl`. The boundary also names logical direction rather than implementation mechanics: adapters implement `outgoing/5`, - sideband adapters use `Group.Replica.Transport.Outbox.push/5`, and receiving - adapters call `incoming/4` or `incoming_batch/4`. + sideband adapters use `Group.Transport.Outbox.push/5`, and receiving adapters + call `Group.Transport.incoming/4` or `incoming_batch/4`. No compatibility + aliases are provided. +- Rename the internal replica wire helper from `Group.Replica.Protocol` to + `Group.Replica.WireProtocol` to avoid overloading Elixir protocol terminology. + The standalone TCP adapter is retained only as hidden test infrastructure; + Group ships the transport contract, dist-Erlang adapter, and outbox helper. - **Breaking**: replica protocol v2 splits exact snapshots into transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage chunks in shard-owned private ETS and advance the stream cursor only after an diff --git a/CLAUDE.md b/CLAUDE.md index 15ba7ad..fe5f774 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,12 +18,13 @@ lib/ group/peer_reconnect.ex — bounded retry for busy dispatch links group/replica.ex — sharded writes, control, AE, projection group/replica/data.ex — ETS owner, journal, authority, indexes - group/replica/protocol.ex — stream identity and mutation helpers + group/replica/wire_protocol.ex — wire version, stream identity, mutations group/replica/snapshot.ex — byte-targeted snapshot chunks/staging - group/replica/transport.ex — replica transport contract + dist adapter - group/replica/transport/outbox.ex — optional lossy sideband outboxes - group/replica/transport/tcp.ex — included sideband TCP adapter + group/transport.ex — replica transport contract + group/transport/dist_erl.ex — default dist-Erlang adapter + group/transport/outbox.ex — optional lossy sideband outboxes test/ + support/test_tcp_transport.ex — test-only independent-socket adapter replica_model_property_test.exs — shrinkable real-node lifecycle model replica_adversarial_test.exs — seeded three-node transport chaos replica_snapshot_* — chunk/assembly failure coverage @@ -165,12 +166,11 @@ All cross-node Group control sends use adapter sends directly the same way and adds no local hop. `:busy` and `:disconnected` mean “drop this message”; periodic anti-entropy repairs it. -A sideband adapter may use one local `Group.Replica.Transport.Outbox` per -shard. Outboxes batch by peer, impose deadlines, and run bounded socket work -outside Group shards. Queue overflow, expiry, or socket backpressure drops the -batch. The included TCP adapter adds bounded per-peer writer queues and -capability-authenticated ingress while distribution still authenticates node -identity and carries authority. TCP is not encrypted. +A sideband adapter may use one local `Group.Transport.Outbox` per shard. +Outboxes batch by peer, impose deadlines, and run bounded socket work outside +Group shards. Queue overflow, expiry, or socket backpressure drops the batch. +The test suite's hidden TCP adapter exercises a genuinely independent socket +lane; it is validation infrastructure, not a supported production transport. Transport ordering is not required for correctness. Per-shard ordered delivery is a fast path; stream sequences reject duplicate/out-of-order data, and @@ -238,8 +238,10 @@ FIFO local-request turn to prevent replica pressure from starving callers. 2. Exact snapshots replace one origin slice; they are never additive merges. 3. Authority requires generation, exact epoch revision, and installed lane readiness. Observed heartbeats/controls are not exact authority. -4. A stale generation, epoch, lane, shard, transitive pid, or unauthenticated - source is rejected before applying replica data. +4. A stale generation, epoch, lane, shard, transitive pid, or stream whose + origin differs from the transport-reported source is rejected before + applying replica data. Group trusts the adapter's source identity; + authenticating a sideband peer belongs to that transport. 5. Registry claims are retained per origin until that origin deletes them or is retired; the visible winner is reconstructible from remaining claims. 6. Only an owner node monitors, retires, or exits its member processes. diff --git a/README.md b/README.md index 51ac159..43c6cb4 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ All operations are **eventually consistent**: busy_dist_retry_attempts: 300, busy_dist_retry_interval: 1_000, replicated_pg_receiver_local_request_quota: 8, - replica_transport: Group.Replica.Transport.Distribution, + replica_transport: Group.Transport.DistErl, replicated_oplog_max_entries: 65_536, replicated_snapshot_chunk_target_bytes: 1_048_576, replicated_anti_entropy_interval: 1_000, @@ -296,12 +296,12 @@ All operations are **eventually consistent**: queued local shard requests drained per fairness turn while replica data or cluster controls are busy. Defaults to 8. - **`replica_transport`** — a module implementing - `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses - `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, - or `:disconnected`. Dropped and busy messages are repaired by anti-entropy. - `Group.Replica.Transport.TCP` is an included sideband adapter with local - per-shard batching and bounded per-peer writer queues; its socket owners are - separate processes, so socket backpressure cannot block a Group shard. + `Group.Transport`, or `{module, opts}`. The default + `Group.Transport.DistErl` adapter uses `:erlang.send_nosuspend/3`; adapters + must return promptly with `:ok`, `:busy`, or `:disconnected`. Dropped and busy + messages are repaired by anti-entropy. Sideband implementations can use + `Group.Transport.Outbox` to move bounded batching and socket work outside the + Group shard. - **`replicated_oplog_max_entries`** — maximum retained replica records per shard across all local streams. Defaults to 65,536. Pruning never waits for peer acknowledgements; a peer behind the retained floor receives an exact @@ -468,36 +468,20 @@ Per-shard ordered delivery is still a useful fast path. Cross-stream order is not a correctness dependency; cluster epochs reject data racing a disconnect or reconnect, and generation fencing rejects data from a restarted origin. An alternative sideband adapter passes incoming messages to -`Group.Replica.Transport.incoming/4` locally. - -For example, replica data can use the included sideband TCP adapter while +`Group.Transport.incoming/4` locally. Configure a custom adapter while authority and membership remain on dist Erlang: ```elixir replica_transport: - {Group.Replica.Transport.TCP, - [ - ip: {0, 0, 0, 0}, - advertised_ip: {10, 0, 1, 12}, - port: 44_321, - max_queue: 1_024, - outbox_batch_size: 64, - outbox_batch_bytes: 1_048_576, - outbox_flush_interval: 1, - outbox_deadline: 100 - ]} + {MyApp.GroupTransport, + [outbox_batch_size: 64, outbox_batch_bytes: 1_048_576, + outbox_flush_interval: 1, outbox_deadline: 100]} ``` -Each node advertises its own reachable address. TCP frames are capability -authenticated by the dist-Erlang hello but are not encrypted, so use a trusted -network or place the connection behind TLS. The adapter deliberately has no -control/data ordering relationship; the generation/epoch lane barrier and -stream sequence checks supply correctness. - -The default distribution adapter still sends directly to the remote shard and +The default `Group.Transport.DistErl` adapter sends directly to the remote shard and does not pay for a local outbox. Sideband adapters can delegate `outgoing/5` to -`Group.Replica.Transport.Outbox.push/5` and supervise one outbox per shard -with `Group.Replica.Transport.Outbox.child_spec/1`. An outbox groups messages by +`Group.Transport.Outbox.push/5` and supervise one outbox per shard +with `Group.Transport.Outbox.child_spec/1`. An outbox groups messages by target and invokes the adapter's `send_batch/4` callback. Calls that expire or return `:busy`/`:disconnected` are dropped without a local retry; the next anti-entropy exchange repairs them. @@ -505,11 +489,13 @@ anti-entropy exchange repairs them. A message-oriented backend fits this callback shape by obtaining a connection once from `init_outbox/3`, then sending each `send_batch/4` result to a registered incoming name on the target node. Queue pressure maps to `:busy` and -a missing session maps to `:disconnected`. The adapter passes the trusted peer's -source node alongside each message. Exact snapshots are already bounded by -Group. A transport with a smaller maximum frame may additionally segment an -encoded batch, but it must completely reassemble that batch before calling -`Group.Replica.Transport.incoming_batch/4`. +a missing session maps to `:disconnected`. The adapter passes its trusted peer +identity as the source node; Group verifies that stream origins and member pids +match that identity but does not authenticate the sideband connection itself. +Exact snapshots are already bounded by Group. A transport with a smaller +maximum frame may additionally segment an encoded batch, but it must completely +reassemble that batch before calling +`Group.Transport.incoming_batch/4`. ### Named Cluster TTL Leases diff --git a/lib/group.ex b/lib/group.ex index 17a3df6..284b8da 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -233,9 +233,9 @@ defmodule Group do local shard requests drained in each fairness turn, including while replica data or cluster controls are busy (default: `8`) - `:replica_transport` — replica data transport module or `{module, opts}` tuple. - Defaults to `Group.Replica.Transport.Distribution`. The transport must be + Defaults to `Group.Transport.DistErl`. The transport must be nonblocking and may return `:busy`; anti-entropy repairs dropped messages. - Sideband transports can use `Group.Replica.Transport.Outbox` for lossy, + Sideband transports can use `Group.Transport.Outbox` for lossy, batched, per-shard isolation without adding a hop to the default transport. - `:replicated_oplog_max_entries` — maximum retained replica records per shard before old prefixes are pruned and lagging peers require a snapshot diff --git a/lib/group/replica.ex b/lib/group/replica.ex index d70b462..39ea943 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -10,7 +10,7 @@ defmodule Group.Replica do @anti_entropy_timer :group_replica_anti_entropy @local_request_tag :group_local_request @local_reply_tag :group_local_reply - @protocol_version Group.Replica.Protocol.version() + @protocol_version Group.Replica.WireProtocol.version() _archdoc = ~S""" Sharded control process for local writes, replica transport, anti-entropy, @@ -55,7 +55,7 @@ defmodule Group.Replica do creating remote process monitors. A generation/epoch-revision mismatch requests a fresh authoritative hello. - Replica state uses the configured Group.Replica.Transport: + Replica state uses the configured Group.Transport: - heads advertises {stream, retained_floor, head}. - delta_batch carries one or more contiguous stream runs. @@ -126,7 +126,7 @@ defmodule Group.Replica do require Logger - alias Group.Replica.{Data, Protocol, Snapshot} + alias Group.Replica.{Data, Snapshot, WireProtocol} defstruct [ :name, @@ -387,7 +387,7 @@ defmodule Group.Replica do end) cond do - version != Protocol.version() or transport_id != state.replica_transport.id() -> + version != WireProtocol.version() or transport_id != state.replica_transport.id() -> Logger.error( "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" ) @@ -448,7 +448,7 @@ defmodule Group.Replica do state = flush_pending_replicated_message_barrier(state) remote_node = node(remote_pid) - if version == Protocol.version() and transport_id == state.replica_transport.id() do + if version == WireProtocol.version() and transport_id == state.replica_transport.id() do if function_exported?(state.replica_transport, :peer_up, 4) do :ok = state.replica_transport.peer_up( @@ -737,14 +737,14 @@ defmodule Group.Replica do state = cond do - version == Protocol.version() and + version == WireProtocol.version() and replica_authority_current?(state, remote_node, generation, epoch_revision) and replica_view_current?(state, remote_node) -> state |> put_remote_shard(remote_node, remote_pid) |> touch_replica_peer(remote_node) - version == Protocol.version() and + version == WireProtocol.version() and replica_authority_current?(state, remote_node, generation, epoch_revision) -> state @@ -1781,7 +1781,7 @@ defmodule Group.Replica do end defp append_local_replica_record(state, op) do - cluster = Protocol.op_cluster(op) + cluster = WireProtocol.op_cluster(op) case Data.local_stream_id(state.name, state.shard_index, cluster) do nil -> @@ -2713,7 +2713,7 @@ defmodule Group.Replica do {stream_id, first_seq, records, head} end) - outgoing_replica_message(state, target_node, {:delta_batch, Protocol.version(), runs}) + outgoing_replica_message(state, target_node, {:delta_batch, WireProtocol.version(), runs}) end defp group_broadcast_ops_by_target(ops, state, cluster_fun) do @@ -2772,7 +2772,7 @@ defmodule Group.Replica do do: registry_op_cluster(op) defp sequenced_op_cluster({:sequenced, stream_id, _seq, _mutations}), - do: Protocol.stream_cluster(stream_id) + do: WireProtocol.stream_cluster(stream_id) defp replicated_op_for_active_cluster?(name, op, cluster_fun) when is_function(cluster_fun, 1) do @@ -2922,14 +2922,14 @@ defmodule Group.Replica do send_remote_control_message( state, target_node, - {:replica_hello, self(), Protocol.version(), generation, epoch_revision, cluster_epochs, - state.replica_transport.id(), descriptor} + {:replica_hello, self(), WireProtocol.version(), generation, epoch_revision, + cluster_epochs, state.replica_transport.id(), descriptor} ) else send_remote_shard_message( state, target_node, - {:replica_lane_hello, self(), Protocol.version(), Data.generation(state.name), + {:replica_lane_hello, self(), WireProtocol.version(), Data.generation(state.name), Data.local_cluster_epoch_revision(state.name), state.replica_transport.id(), descriptor} ) end @@ -3104,7 +3104,7 @@ defmodule Group.Replica do send_remote_shard_message( acc, target_node, - {:replica_heartbeat, self(), Protocol.version(), Data.generation(acc.name), + {:replica_heartbeat, self(), WireProtocol.version(), Data.generation(acc.name), Data.local_cluster_epoch_revision(acc.name)} ) @@ -3231,7 +3231,7 @@ defmodule Group.Replica do if heads == [] do state else - outgoing_replica_message(state, target_node, {:heads, Protocol.version(), heads}) + outgoing_replica_message(state, target_node, {:heads, WireProtocol.version(), heads}) end end @@ -3263,27 +3263,27 @@ defmodule Group.Replica do end defp replica_stream_target?(state, stream_id, target_node) do - Protocol.stream_name(stream_id) == state.name and - Protocol.stream_origin(stream_id) == node() and - Protocol.stream_shard(stream_id) == state.shard_index and - Protocol.stream_generation(stream_id) == Data.generation(state.name) and - Protocol.stream_epoch(stream_id) == - Data.local_cluster_epoch(state.name, Protocol.stream_cluster(stream_id)) and - case Protocol.stream_cluster(stream_id) do + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_generation(stream_id) == Data.generation(state.name) and + WireProtocol.stream_epoch(stream_id) == + Data.local_cluster_epoch(state.name, WireProtocol.stream_cluster(stream_id)) and + case WireProtocol.stream_cluster(stream_id) do nil -> Map.has_key?(state.peer_last_seen, target_node) cluster -> target_node in Data.cluster_nodes(state.name, cluster) end end defp valid_remote_stream?(state, source_node, stream_id) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) - Protocol.stream_name(stream_id) == state.name and - Protocol.stream_origin(stream_id) == source_node and - Protocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == source_node and + WireProtocol.stream_shard(stream_id) == state.shard_index and replica_view_current?(state, source_node) and - Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and - Protocol.stream_epoch(stream_id) == + WireProtocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and + WireProtocol.stream_epoch(stream_id) == Data.remote_cluster_epoch(state.name, source_node, cluster) and (is_nil(cluster) or cluster_member?(state.name, cluster)) end @@ -3303,7 +3303,7 @@ defmodule Group.Replica do needs |> Enum.chunk_every(state.replicated_sender_buffer_size) |> Enum.reduce(state, fn chunk, acc -> - outgoing_replica_message(acc, source_node, {:needs, Protocol.version(), chunk}) + outgoing_replica_message(acc, source_node, {:needs, WireProtocol.version(), chunk}) end) end @@ -3318,8 +3318,8 @@ defmodule Group.Replica do defp handle_replica_message(state, source_node, {:need, version, stream_id, next_seq}) when version == @protocol_version do - if Protocol.stream_origin(stream_id) == node() and - Protocol.stream_shard(stream_id) == state.shard_index and + if WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == state.shard_index and replica_stream_target?(state, stream_id, source_node) do send_replica_repair(state, source_node, stream_id, next_seq) else @@ -3407,7 +3407,7 @@ defmodule Group.Replica do end defp valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) Enum.all?(reg_data, fn {key, pid, _meta, _time} when is_pid(pid) -> @@ -3534,7 +3534,7 @@ defmodule Group.Replica do state = if valid_snapshot_stream?(state, source_node, stream_id, transfer.snapshot_seq) do state = flush_pending_replicated_barrier(state) - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) affected_registry_keys = Data.replace_registry_claims_for_stream_from_staging( @@ -3587,7 +3587,7 @@ defmodule Group.Replica do pg_data ) do state = flush_pending_replicated_barrier(state) - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) affected_registry_keys = Data.replace_registry_claims_for_stream( @@ -3672,8 +3672,8 @@ defmodule Group.Replica do do: {Enum.reverse(acc), next_seq} defp valid_replica_mutations?(stream_id, mutations) do - origin = Protocol.stream_origin(stream_id) - cluster = Protocol.stream_cluster(stream_id) + origin = WireProtocol.stream_origin(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) mutations != [] and Enum.all?(mutations, &valid_replica_mutation?(&1, cluster, origin)) end @@ -3846,7 +3846,7 @@ defmodule Group.Replica do end) |> Enum.uniq() - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) Enum.reduce(keys, {state, []}, fn key, {acc, events} -> reconcile_registry_projection(acc, cluster, key, :reconcile, events) @@ -3861,15 +3861,15 @@ defmodule Group.Replica do outgoing_replica_message( state, target_node, - {:needs, Protocol.version(), [{stream_id, next_seq}]} + {:needs, WireProtocol.version(), [{stream_id, next_seq}]} ) end defp send_replica_repairs(state, target_node, needs) do {state, runs} = Enum.reduce(needs, {state, []}, fn {stream_id, next_seq}, {acc, runs} -> - if Protocol.stream_origin(stream_id) == node() and - Protocol.stream_shard(stream_id) == acc.shard_index and + if WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == acc.shard_index and replica_stream_target?(acc, stream_id, target_node) do case replica_repair(acc, target_node, stream_id, next_seq) do {:run, run} -> {acc, [run | runs]} @@ -3888,7 +3888,7 @@ defmodule Group.Replica do outgoing_replica_message( state, target_node, - {:delta_batch, Protocol.version(), Enum.reverse(runs)} + {:delta_batch, WireProtocol.version(), Enum.reverse(runs)} ) end end @@ -3925,7 +3925,7 @@ defmodule Group.Replica do end defp send_replica_snapshot(state, target_node, stream_id, head) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) pg_data = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, node()) @@ -3948,7 +3948,7 @@ defmodule Group.Replica do outgoing_replica_message( acc, target_node, - {:snapshot_chunk, Protocol.version(), stream_id, head, chunk_index, chunk_count, + {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, chunk_count, snapshot.registry_count, snapshot.pg_count, reg_chunk, pg_chunk} ) end) @@ -4114,7 +4114,7 @@ defmodule Group.Replica do stream_ids = Enum.map(cluster_epochs, fn {cluster, epoch} -> - Protocol.stream_id( + WireProtocol.stream_id( state.name, remote_node, generation, @@ -4171,7 +4171,7 @@ defmodule Group.Replica do superseded = Enum.flat_map(current_epochs, fn {cluster, current_epoch} -> current_stream = - Protocol.stream_id( + WireProtocol.stream_id( state.name, remote_node, generation, @@ -4207,7 +4207,7 @@ defmodule Group.Replica do # rather than rebuilding the node-wide epoch map in every lane. current_epochs = streams - |> Enum.map(&Protocol.stream_cluster/1) + |> Enum.map(&WireProtocol.stream_cluster/1) |> Enum.uniq() |> Map.new(fn cluster -> {cluster, Data.remote_cluster_epoch(state.name, remote_node, cluster)} @@ -4215,9 +4215,9 @@ defmodule Group.Replica do superseded = Enum.reject(streams, fn stream_id -> - Protocol.stream_generation(stream_id) == generation and - Map.get(current_epochs, Protocol.stream_cluster(stream_id)) == - Protocol.stream_epoch(stream_id) + WireProtocol.stream_generation(stream_id) == generation and + Map.get(current_epochs, WireProtocol.stream_cluster(stream_id)) == + WireProtocol.stream_epoch(stream_id) end) purge_superseded_remote_streams(state, remote_node, current_epochs, superseded) @@ -4234,7 +4234,7 @@ defmodule Group.Replica do generation = Data.remote_generation(state.name, remote_node) superseded - |> Enum.group_by(&Protocol.stream_cluster/1) + |> Enum.group_by(&WireProtocol.stream_cluster/1) |> Enum.reduce(state, fn {cluster, cluster_streams}, acc -> affected_keys = Data.purge_registry_claims_for_streams( @@ -4265,7 +4265,7 @@ defmodule Group.Replica do current_epoch -> current_stream = - Protocol.stream_id( + WireProtocol.stream_id( state.name, remote_node, generation, @@ -4343,7 +4343,7 @@ defmodule Group.Replica do records |> Enum.reduce(%{}, fn {:sequenced, _stream_id, _seq, [op | _]} = record, acc -> - cluster = Protocol.op_cluster(op) + cluster = WireProtocol.op_cluster(op) Enum.reduce(process_down_targets(state, cluster), acc, fn target_node, inner -> Map.update(inner, target_node, [record], &[record | &1]) @@ -4520,13 +4520,13 @@ defmodule Group.Replica do end defp current_local_stream?(state, stream_id) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) - Protocol.stream_name(stream_id) == state.name and - Protocol.stream_origin(stream_id) == node() and - Protocol.stream_generation(stream_id) == Data.generation(state.name) and - Protocol.stream_shard(stream_id) == state.shard_index and - Protocol.stream_epoch(stream_id) == Data.local_cluster_epoch(state.name, cluster) + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_generation(stream_id) == Data.generation(state.name) and + WireProtocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_epoch(stream_id) == Data.local_cluster_epoch(state.name, cluster) end defp apply_registry_claim_mutations(state, stream_id, seq, mutations) do diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 43e4d47..3f26250 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -2,7 +2,7 @@ defmodule Group.Replica.Data do @moduledoc false use GenServer - alias Group.Replica.Protocol + alias Group.Replica.WireProtocol _archdoc = """ GenServer that owns ETS tables for all shards. @@ -375,7 +375,7 @@ defmodule Group.Replica.Data do nil epoch -> - Group.Replica.Protocol.stream_id( + Group.Replica.WireProtocol.stream_id( name, node(), generation(name), @@ -472,8 +472,8 @@ defmodule Group.Replica.Data do drop_local_stream( name, shard, - Protocol.stream_cluster(stream_id), - Protocol.stream_epoch(stream_id) + WireProtocol.stream_cluster(stream_id), + WireProtocol.stream_epoch(stream_id) ) end end) @@ -518,13 +518,13 @@ defmodule Group.Replica.Data do end defp current_local_stream?(name, shard, stream_id) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) - Protocol.stream_name(stream_id) == name and - Protocol.stream_origin(stream_id) == node() and - Protocol.stream_generation(stream_id) == generation(name) and - Protocol.stream_shard(stream_id) == shard and - Protocol.stream_epoch(stream_id) == local_cluster_epoch(name, cluster) + WireProtocol.stream_name(stream_id) == name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_generation(stream_id) == generation(name) and + WireProtocol.stream_shard(stream_id) == shard and + WireProtocol.stream_epoch(stream_id) == local_cluster_epoch(name, cluster) end defp await_closed_local_clusters(name, clusters, timeout, started_at) do @@ -605,7 +605,7 @@ defmodule Group.Replica.Data do {{cluster, _key, _pid}, _meta, _time, _entry_node} -> cluster end), Enum.map(:ets.tab2list(replica_cursor_table(name, shard)), fn {stream_id, _seq} -> - Protocol.stream_cluster(stream_id) + WireProtocol.stream_cluster(stream_id) end) ]) |> Enum.reject(&is_nil/1) @@ -748,7 +748,7 @@ defmodule Group.Replica.Data do def drop_local_stream(name, shard, cluster, epoch) do stream_id = - Group.Replica.Protocol.stream_id(name, node(), generation(name), shard, cluster, epoch) + Group.Replica.WireProtocol.stream_id(name, node(), generation(name), shard, cluster, epoch) append_rows = :ets.select(replica_oplog_table(name, shard), [ @@ -867,10 +867,10 @@ defmodule Group.Replica.Data do # ===================================================================== def put_registry_claim(name, shard, stream_id, seq, key, pid, meta, time) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) claim_key = {cluster, key, origin_node, generation, epoch} by_key = reg_claim_by_key_table(name, shard) @@ -904,10 +904,10 @@ defmodule Group.Replica.Data do end def delete_registry_claim(name, shard, stream_id, seq, key, pid) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) claim_key = {cluster, key, origin_node, generation, epoch} case :ets.lookup(reg_claim_by_key_table(name, shard), claim_key) do @@ -934,10 +934,10 @@ defmodule Group.Replica.Data do end def registry_claims_for_stream(name, shard, stream_id) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) :ets.select(reg_claim_by_key_table(name, shard), [ {{{cluster, :"$1", origin_node, generation, epoch}, :"$2", :"$3", :"$4", :_}, [], @@ -946,10 +946,10 @@ defmodule Group.Replica.Data do end def replace_registry_claims_for_stream(name, shard, stream_id, snapshot_seq, claims) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) existing = registry_claims_for_stream(name, shard, stream_id) Enum.each(existing, fn {key, pid, _meta, _time} -> @@ -979,10 +979,10 @@ defmodule Group.Replica.Data do staging_table, chunk_count ) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) existing = registry_claims_for_stream(name, shard, stream_id) keys = @@ -1042,10 +1042,10 @@ defmodule Group.Replica.Data do streams = MapSet.new(stream_ids, fn stream_id -> { - Group.Replica.Protocol.stream_cluster(stream_id), - Group.Replica.Protocol.stream_origin(stream_id), - Group.Replica.Protocol.stream_generation(stream_id), - Group.Replica.Protocol.stream_epoch(stream_id) + Group.Replica.WireProtocol.stream_cluster(stream_id), + Group.Replica.WireProtocol.stream_origin(stream_id), + Group.Replica.WireProtocol.stream_generation(stream_id), + Group.Replica.WireProtocol.stream_epoch(stream_id) } end) diff --git a/lib/group/replica/snapshot.ex b/lib/group/replica/snapshot.ex index 13dc29a..8287407 100644 --- a/lib/group/replica/snapshot.ex +++ b/lib/group/replica/snapshot.ex @@ -47,7 +47,7 @@ defmodule Group.Replica.Snapshot do # empty list. Reserve that once for each domain. The large chunk integers # ensure every practical index/count uses no more space than this envelope. :erlang.external_size( - {:snapshot_chunk, Group.Replica.Protocol.version(), stream_id, snapshot_seq, + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, snapshot_seq, @max_compact_chunk_count, @max_compact_chunk_count, registry_count, pg_count, [], []} ) + 10 end diff --git a/lib/group/replica/protocol.ex b/lib/group/replica/wire_protocol.ex similarity index 96% rename from lib/group/replica/protocol.ex rename to lib/group/replica/wire_protocol.ex index a79d983..b8e1908 100644 --- a/lib/group/replica/protocol.ex +++ b/lib/group/replica/wire_protocol.ex @@ -1,4 +1,4 @@ -defmodule Group.Replica.Protocol do +defmodule Group.Replica.WireProtocol do @moduledoc false @version 2 diff --git a/lib/group/supervisor.ex b/lib/group/supervisor.ex index df2e97d..3c66484 100644 --- a/lib/group/supervisor.ex +++ b/lib/group/supervisor.ex @@ -44,9 +44,9 @@ defmodule Group.Supervisor do replica_transport = opts - |> Keyword.get(:replica_transport, Group.Replica.Transport.Distribution) - |> Group.Replica.Transport.normalize() - |> Group.Replica.Transport.validate!() + |> Keyword.get(:replica_transport, Group.Transport.DistErl) + |> Group.Transport.normalize() + |> Group.Transport.validate!() replicated_oplog_max_entries = positive_integer_opt(opts, :replicated_oplog_max_entries, 65_536) diff --git a/lib/group/replica/transport.ex b/lib/group/transport.ex similarity index 73% rename from lib/group/replica/transport.ex rename to lib/group/transport.ex index 46bb532..30eba2d 100644 --- a/lib/group/replica/transport.ex +++ b/lib/group/transport.ex @@ -1,4 +1,4 @@ -defmodule Group.Replica.Transport do +defmodule Group.Transport do @moduledoc """ Transport contract for Group replica data. @@ -10,7 +10,9 @@ defmodule Group.Replica.Transport do Erlang distribution remains Group's control plane and supplies the stable node identity used here. A sideband adapter can use its `descriptor/2` in the control hello to exchange endpoints and pass incoming messages to - `incoming/4` or `incoming_batch/4`. + `incoming/4` or `incoming_batch/4`. Group trusts the `source_node` supplied + by the adapter and validates stream origins and member pids against it; peer + authentication, when needed, belongs to the transport. Adapters do not need to preserve ordering. Group serializes writes per shard and sequences each origin/generation/shard/cluster/epoch stream; receivers @@ -18,7 +20,7 @@ defmodule Group.Replica.Transport do traffic and is therefore the preferred fast path. A sideband implementation can delegate `outgoing/5` to - `Group.Replica.Transport.Outbox.push/5`. That adds one local send only for + `Group.Transport.Outbox.push/5`. That adds one local send only for the configured sideband transport; the default distribution adapter retains its direct remote `:erlang.send_nosuspend/3` path. """ @@ -52,8 +54,9 @@ defmodule Group.Replica.Transport do @doc """ Passes an incoming replica message to the corresponding local shard. - This is a local mailbox operation. Stream generation, epoch, group, shard, - and origin are validated by the replica. + This is a local mailbox operation. `source_node` is the trusted peer identity + established by the adapter. Stream generation, epoch, group, shard, origin, + and member-pid ownership are validated by the replica. """ def incoming(group, source_node, shard, message) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 do @@ -70,7 +73,8 @@ defmodule Group.Replica.Transport do A finite-message transport may segment the encoded batch on the wire, but it must reassemble every segment before calling this function. Group never - observes or applies a partial batch. + observes or applies a partial batch. `source_node` has the same trusted-peer + meaning as in `incoming/4`. """ def incoming_batch(group, source_node, shard, messages) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 and @@ -104,35 +108,3 @@ defmodule Group.Replica.Transport do transport end end - -defmodule Group.Replica.Transport.Distribution do - @moduledoc """ - Default nonblocking replica transport over Erlang distribution. - - Messages are sent directly to the matching remote shard with - `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a - busy distribution socket and never initiates a connection. A busy or absent - link returns `:busy`; Group drops that message and repairs it through periodic - anti-entropy. - """ - @behaviour Group.Replica.Transport - - alias Group.Replica - - @impl true - def id, do: :erlang_distribution - - @impl true - def descriptor(_group, _opts), do: :erlang_distribution - - @impl true - def outgoing(group, target_node, shard, replica_message, _opts) do - destination = {Replica.shard_name(group, shard), target_node} - message = {:group_replica_frame, node(), replica_message} - - case :erlang.send_nosuspend(destination, message, [:noconnect]) do - true -> :ok - false -> :busy - end - end -end diff --git a/lib/group/transport/dist_erl.ex b/lib/group/transport/dist_erl.ex new file mode 100644 index 0000000..9812fb5 --- /dev/null +++ b/lib/group/transport/dist_erl.ex @@ -0,0 +1,32 @@ +defmodule Group.Transport.DistErl do + @moduledoc """ + Default nonblocking replica transport over Erlang distribution. + + Messages are sent directly to the matching remote shard with + `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a + busy distribution socket and never initiates a connection. A busy or absent + link returns `:busy`; Group drops that message and repairs it through periodic + anti-entropy. + """ + + @behaviour Group.Transport + + alias Group.Replica + + @impl true + def id, do: :erlang_distribution + + @impl true + def descriptor(_group, _opts), do: :erlang_distribution + + @impl true + def outgoing(group, target_node, shard, replica_message, _opts) do + destination = {Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, node(), replica_message} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end +end diff --git a/lib/group/replica/transport/outbox.ex b/lib/group/transport/outbox.ex similarity index 92% rename from lib/group/replica/transport/outbox.ex rename to lib/group/transport/outbox.ex index 43c23b3..4b523f1 100644 --- a/lib/group/replica/transport/outbox.ex +++ b/lib/group/transport/outbox.ex @@ -1,9 +1,9 @@ -defmodule Group.Replica.Transport.Outbox do +defmodule Group.Transport.Outbox do @moduledoc """ Lossy per-shard outboxes for sideband replica transports. This module is an implementation helper, not a replacement for - `Group.Replica.Transport`. Distribution can continue sending directly with + `Group.Transport`. Distribution can continue sending directly with `:erlang.send_nosuspend/3`. A sideband adapter delegates `outgoing/5` to `push/5`, which performs only a local `send/2` to the matching shard outbox. @@ -15,7 +15,7 @@ defmodule Group.Replica.Transport.Outbox do A backend using this helper implements: - @behaviour Group.Replica.Transport.Outbox + @behaviour Group.Transport.Outbox def init_outbox(group, shard, opts), do: {:ok, backend_state} @@ -25,7 +25,7 @@ defmodule Group.Replica.Transport.Outbox do end The backend must pass only complete logical messages to - `Group.Replica.Transport.incoming_batch/4`. + `Group.Transport.incoming_batch/4`. ## Options @@ -46,8 +46,8 @@ defmodule Group.Replica.Transport.Outbox do reassemble those batches before local delivery. """ - @type message :: Group.Replica.Transport.message() - @type outgoing_result :: Group.Replica.Transport.outgoing_result() + @type message :: Group.Transport.message() + @type outgoing_result :: Group.Transport.outgoing_result() @type backend_state :: term() @callback init_outbox(group :: atom(), shard :: non_neg_integer(), opts :: keyword()) :: @@ -73,7 +73,7 @@ defmodule Group.Replica.Transport.Outbox do %{ id: {__MODULE__, group}, - start: {Group.Replica.Transport.Outbox.Supervisor, :start_link, [opts]}, + start: {Group.Transport.Outbox.Supervisor, :start_link, [opts]}, type: :supervisor, restart: :permanent, shutdown: :infinity @@ -119,7 +119,7 @@ defmodule Group.Replica.Transport.Outbox do end end -defmodule Group.Replica.Transport.Outbox.Supervisor do +defmodule Group.Transport.Outbox.Supervisor do @moduledoc false use Supervisor @@ -143,8 +143,8 @@ defmodule Group.Replica.Transport.Outbox.Supervisor do children = for shard <- 0..(num_shards - 1) do %{ - id: {Group.Replica.Transport.Outbox.Worker, group, shard}, - start: {Group.Replica.Transport.Outbox.Worker, :start_link, [opts, shard]}, + id: {Group.Transport.Outbox.Worker, group, shard}, + start: {Group.Transport.Outbox.Worker, :start_link, [opts, shard]}, restart: :permanent, shutdown: 5_000 } @@ -154,11 +154,11 @@ defmodule Group.Replica.Transport.Outbox.Supervisor do end end -defmodule Group.Replica.Transport.Outbox.Worker do +defmodule Group.Transport.Outbox.Worker do @moduledoc false use GenServer - alias Group.Replica.Transport.Outbox + alias Group.Transport.Outbox @default_batch_size 64 @default_batch_bytes 1_048_576 diff --git a/test/README.md b/test/README.md index ffb2986..198a961 100644 --- a/test/README.md +++ b/test/README.md @@ -61,7 +61,8 @@ The Docker-backed Jepsen harness lives in [`jepsen/`](jepsen/). It drives three independent BEAM containers through concurrent, multi-entry owner lifecycles, named-cluster epoch churn, directed/full partitions, transport session resets, and VM restarts. The same workload runs over distribution, -real sideband TCP, and a lossy/duplicating/reordering transport. After healing, +a test-only real sideband TCP lane, and a lossy/duplicating/reordering +transport. After healing, its independent oracle checks exact public views and the internal registry, PG, claim, cluster, cursor, oplog, snapshot-staging, and retired-origin invariants. Its permanent-retirement scenario proves eviction even when a peer @@ -278,7 +279,7 @@ require every advertised revision to contain exactly that many unique named epochs, and heartbeat tests prove observed revisions cannot advance the exact authority marker. Crash-window tests interrupt journal, dual-index, receive cursor, and named-cluster close updates, then require startup repair to remove -every invisible row and temporary close barrier. A three-node sideband TCP test +every invisible row and temporary close barrier. A three-node test-only TCP test disconnects one origin's real socket, prunes its oplog, reconnects it, and requires snapshot recovery without changing the third node's independent registry or PG state. diff --git a/test/distributed_test.exs b/test/distributed_test.exs index ebe8278..ed8cf1b 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -2452,8 +2452,8 @@ defmodule Group.DistributedTest do |> Enum.map(fn %{shard: shard, cursors: cursors} -> relevant = Enum.filter(cursors, fn {stream_id, _seq} -> - Group.Replica.Protocol.stream_origin(stream_id) == node_a and - Group.Replica.Protocol.stream_cluster(stream_id) == dropped_cluster + Group.Replica.WireProtocol.stream_origin(stream_id) == node_a and + Group.Replica.WireProtocol.stream_cluster(stream_id) == dropped_cluster end) {shard, relevant} @@ -4464,7 +4464,7 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_b, :erlang, :send, [ shard_name(name, 2), - {:replica_heartbeat, a_lane, Group.Replica.Protocol.version(), + {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]), latest_revision} ]) @@ -4606,7 +4606,7 @@ defmodule Group.DistributedTest do ]) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 1, @@ -4626,7 +4626,7 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_b, :erlang, :send, [ shard_name(name, 1), - {:replica_heartbeat, a_lane, Group.Replica.Protocol.version(), generation, revision} + {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), generation, revision} ]) TestCluster.assert_eventually(fn -> @@ -4635,7 +4635,7 @@ defmodule Group.DistributedTest do end) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 1, @@ -4827,7 +4827,7 @@ defmodule Group.DistributedTest do Enum.each(generation_frames, fn {_target, shard, frame} -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -4882,7 +4882,7 @@ defmodule Group.DistributedTest do Enum.each(epoch_frames, fn {_target, shard, frame} -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -4999,7 +4999,7 @@ defmodule Group.DistributedTest do node_a |> TestCluster.rpc!(Group.Replica.Data, :replica_stream_heads, [name, 0]) |> Enum.find(fn {stream_id, _floor, _head} -> - Group.Replica.Protocol.stream_cluster(stream_id) == "cold" + Group.Replica.WireProtocol.stream_cluster(stream_id) == "cold" end) assert floor > 2 @@ -5083,7 +5083,7 @@ defmodule Group.DistributedTest do invalid_key = "anti-entropy/authority/forged" invalid_frame = - {:delta_batch, Group.Replica.Protocol.version(), + {:delta_batch, Group.Replica.WireProtocol.version(), [ {stream_id, 1, [ @@ -5096,7 +5096,7 @@ defmodule Group.DistributedTest do ]} :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 0, @@ -5113,7 +5113,7 @@ defmodule Group.DistributedTest do ]) == 0 :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_b, 0, @@ -5356,7 +5356,7 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) stream_id = - Group.Replica.Protocol.stream_id( + Group.Replica.WireProtocol.stream_id( name, node_a, make_ref(), @@ -5368,11 +5368,12 @@ defmodule Group.DistributedTest do mutation = {:register, nil, key, pid, meta, System.monotonic_time(), node_a} :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 0, - {:delta_batch, Group.Replica.Protocol.version(), [{stream_id, 1, [{1, [mutation]}], 1}]} + {:delta_batch, Group.Replica.WireProtocol.version(), + [{stream_id, 1, [{1, [mutation]}], 1}]} ]) TestCluster.flush_shards(node_b, name) @@ -5432,7 +5433,7 @@ defmodule Group.DistributedTest do Map.fetch!(frames_by_first_seq, 2) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -5454,7 +5455,7 @@ defmodule Group.DistributedTest do for frame <- [first_frame, second_frame, third_frame] do :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -5492,7 +5493,7 @@ defmodule Group.DistributedTest do for frame <- [first_frame, second_frame, third_frame] do :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -5577,7 +5578,7 @@ defmodule Group.DistributedTest do |> then(&"anti-entropy/authority-fanout/new/#{&1}") stream_id = - Group.Replica.Protocol.stream_id( + Group.Replica.WireProtocol.stream_id( name, node_a, new_generation, @@ -5590,7 +5591,7 @@ defmodule Group.DistributedTest do {:register, nil, new_key, pid, %{generation: :new}, System.system_time(), node_a} new_frame = - {:delta_batch, Group.Replica.Protocol.version(), + {:delta_batch, Group.Replica.WireProtocol.version(), [{stream_id, 1, [{1, [new_mutation]}], 1}]} :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_lane]) @@ -5600,7 +5601,7 @@ defmodule Group.DistributedTest do # time this frame runs, but shard 1 must still reject it until its own # old-generation purge has completed. :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 1, @@ -5609,7 +5610,7 @@ defmodule Group.DistributedTest do send( b_control, - {:replica_hello, a_control, Group.Replica.Protocol.version(), new_generation, 0, + {:replica_hello, a_control, Group.Replica.WireProtocol.version(), new_generation, 0, [{nil, new_generation}], Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} ) @@ -5633,7 +5634,7 @@ defmodule Group.DistributedTest do ]) == 0 :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 1, @@ -5660,7 +5661,7 @@ defmodule Group.DistributedTest do name: name, shards: 2, replica_transport: - {Group.Replica.Transport.TCP, + {Group.TestTCPTransport, [ max_queue: 16, connect_timeout: 250, @@ -5683,7 +5684,7 @@ defmodule Group.DistributedTest do Enum.all?(nodes -- [source], fn target -> TestCluster.rpc!( source, - Group.Replica.Transport.TCP, + Group.TestTCPTransport, :connected?, [name, target] ) @@ -5728,15 +5729,15 @@ defmodule Group.DistributedTest do end) :ok = - TestCluster.rpc!(node_a, Group.Replica.Transport.TCP, :disconnect_peer, [name, node_b]) + TestCluster.rpc!(node_a, Group.TestTCPTransport, :disconnect_peer, [name, node_b]) old_reader = - TestCluster.rpc!(node_b, Group.Replica.Transport.TCP, :status, [name]) + TestCluster.rpc!(node_b, Group.TestTCPTransport, :status, [name]) |> get_in([:inbound, node_a]) refute TestCluster.rpc!( node_a, - Group.Replica.Transport.TCP, + Group.TestTCPTransport, :connected?, [name, node_b] ) @@ -5765,11 +5766,11 @@ defmodule Group.DistributedTest do ) :ok = - TestCluster.rpc!(node_a, Group.Replica.Transport.TCP, :reconnect_peer, [name, node_b]) + TestCluster.rpc!(node_a, Group.TestTCPTransport, :reconnect_peer, [name, node_b]) TestCluster.assert_eventually(fn -> new_reader = - TestCluster.rpc!(node_b, Group.Replica.Transport.TCP, :status, [name]) + TestCluster.rpc!(node_b, Group.TestTCPTransport, :status, [name]) |> get_in([:inbound, node_a]) is_pid(new_reader) and new_reader != old_reader @@ -5779,7 +5780,7 @@ defmodule Group.DistributedTest do fn -> TestCluster.rpc!( node_a, - Group.Replica.Transport.TCP, + Group.TestTCPTransport, :connected?, [name, node_b] ) and diff --git a/test/group_test.exs b/test/group_test.exs index f6356a5..25a5f68 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2985,7 +2985,7 @@ defmodule GroupTest do Enum.to_list(1..operations_per_shard) assert Enum.all?(order_rows, fn {_append_id, stream_id, _seq} -> - Group.Replica.Protocol.stream_shard(stream_id) == shard + Group.Replica.WireProtocol.stream_shard(stream_id) == shard end) oplog_rows = @@ -3097,8 +3097,8 @@ defmodule GroupTest do :ok = Group.join(name, pg_key, %{kind: :pg}) stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) :ets.delete(Group.Replica.Data.reg_by_pid_table(name, 0), {self(), nil, reg_key}) :ets.delete(Group.Replica.Data.pg_by_pid_table(name, 0), {self(), nil, pg_key}) @@ -3155,7 +3155,7 @@ defmodule GroupTest do :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) stream_id = Group.Replica.Data.local_stream_id(name, 0, cluster) - old_epoch = Group.Replica.Protocol.stream_epoch(stream_id) + old_epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) # Group.disconnect/3 closes authority and routing before its request # reaches every shard. Model a shard kill in that exact window. diff --git a/test/jepsen/Dockerfile.node b/test/jepsen/Dockerfile.node index 08a5f41..5f1107e 100644 --- a/test/jepsen/Dockerfile.node +++ b/test/jepsen/Dockerfile.node @@ -11,6 +11,7 @@ ENV MIX_ENV=prod COPY mix.exs mix.lock ./ COPY lib ./lib COPY test/jepsen/node.exs ./test/jepsen/node.exs +COPY test/support/test_tcp_transport.ex ./test/support/test_tcp_transport.ex RUN mix local.hex --force \ && mix deps.get --only prod \ diff --git a/test/jepsen/README.md b/test/jepsen/README.md index aeba2ce..42b13cc 100644 --- a/test/jepsen/README.md +++ b/test/jepsen/README.md @@ -22,7 +22,7 @@ prefix Jepsen: The replica lane is selectable without changing the workload or checker: - `distribution` delegates to Group's production Erlang-distribution adapter; -- `tcp` uses Group's production sideband TCP adapter while Erlang distribution +- `tcp` uses Group's hidden test-only TCP adapter while Erlang distribution remains the control plane; and - `chaos` is a local per-shard outbox which deterministically drops, duplicates, delays, and reorders replica messages. diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index c72774b..c2ef0eb 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -1,3 +1,5 @@ +Code.require_file("../support/test_tcp_transport.ex", __DIR__) + defmodule Group.Jepsen.Transport.Stats do @moduledoc false use GenServer @@ -110,7 +112,7 @@ defmodule Group.Jepsen.Transport.Common do defp transport_result(:disconnected), do: :transport_disconnected defp observe_outbox(group, shard) do - case Process.whereis(Group.Replica.Transport.Outbox.name(group, shard)) do + case Process.whereis(Group.Transport.Outbox.name(group, shard)) do pid when is_pid(pid) -> case Process.info(pid, :message_queue_len) do {:message_queue_len, length} -> Stats.observe_max(:outbox_mailbox_peak, length) @@ -125,10 +127,10 @@ end defmodule Group.Jepsen.Transport.Distribution do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport alias Group.Jepsen.Transport.{Common, Stats} - alias Group.Replica.Transport.Distribution, as: Delegate + alias Group.Transport.DistErl, as: Delegate @impl true def id, do: Delegate.id() @@ -147,10 +149,10 @@ end defmodule Group.Jepsen.Transport.TCP do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport alias Group.Jepsen.Transport.Common - alias Group.Replica.Transport.TCP, as: Delegate + alias Group.TestTCPTransport, as: Delegate @impl true def id, do: Delegate.id() @@ -190,7 +192,7 @@ defmodule Group.Jepsen.Transport.TCP.Supervisor do def init(opts) do children = [ {Group.Jepsen.Transport.Stats, opts}, - Group.Replica.Transport.TCP.child_spec(opts) + Group.TestTCPTransport.child_spec(opts) ] Supervisor.init(children, strategy: :one_for_one) @@ -199,7 +201,7 @@ end defmodule Group.Jepsen.Transport.Chaos do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport alias Group.Jepsen.Transport.{Common, Stats} @@ -380,7 +382,7 @@ defmodule Group.Jepsen.Transport.Control do defp maybe_disconnect(target_node) do if profile() == :tcp do - Group.Replica.Transport.TCP.disconnect_peer(:jepsen_group, target_node) + Group.TestTCPTransport.disconnect_peer(:jepsen_group, target_node) end catch :exit, _ -> :ok @@ -388,7 +390,7 @@ defmodule Group.Jepsen.Transport.Control do defp maybe_reconnect(target_node) do if profile() == :tcp do - Group.Replica.Transport.TCP.reconnect_peer(:jepsen_group, target_node) + Group.TestTCPTransport.reconnect_peer(:jepsen_group, target_node) end catch :exit, _ -> :ok @@ -813,7 +815,7 @@ end defmodule Group.Jepsen.Invariant do @moduledoc false - alias Group.Replica.{Data, Protocol} + alias Group.Replica.{Data, WireProtocol} def snapshot(retired_nodes) do config = Group.get_config(:jepsen_group) @@ -846,7 +848,7 @@ defmodule Group.Jepsen.Invariant do shard_mailbox_max: mailbox_max(Enum.map(shards, &Group.Replica.shard_name(:jepsen_group, &1))), outbox_mailbox_max: - mailbox_max(Enum.map(shards, &Group.Replica.Transport.Outbox.name(:jepsen_group, &1))), + mailbox_max(Enum.map(shards, &Group.Transport.Outbox.name(:jepsen_group, &1))), total_memory_bytes: :erlang.memory(:total) } rescue @@ -1002,15 +1004,16 @@ defmodule Group.Jepsen.Invariant do Data.replica_cursor_table(:jepsen_group, shard) |> :ets.tab2list() |> Enum.each(fn {stream, seq} -> - origin = Protocol.stream_origin(stream) - cluster = Protocol.stream_cluster(stream) + origin = WireProtocol.stream_origin(stream) + cluster = WireProtocol.stream_cluster(stream) valid? = - Protocol.stream_name(stream) == :jepsen_group and - Protocol.stream_shard(stream) == shard and + WireProtocol.stream_name(stream) == :jepsen_group and + WireProtocol.stream_shard(stream) == shard and origin != node() and - Protocol.stream_generation(stream) == Data.remote_generation(:jepsen_group, origin) and - Protocol.stream_epoch(stream) == + WireProtocol.stream_generation(stream) == + Data.remote_generation(:jepsen_group, origin) and + WireProtocol.stream_epoch(stream) == Data.remote_cluster_epoch(:jepsen_group, origin, cluster) and seq >= 0 unless valid?, do: raise("cursor lacks current authority #{inspect({stream, seq})}") @@ -1041,7 +1044,7 @@ defmodule Group.Jepsen.Invariant do cursors = Data.replica_cursor_table(:jepsen_group, shard) |> :ets.tab2list() - |> Enum.filter(fn {stream, _seq} -> Protocol.stream_origin(stream) == origin end) + |> Enum.filter(fn {stream, _seq} -> WireProtocol.stream_origin(stream) == origin end) view = Data.remote_view_generation(:jepsen_group, shard, origin) diff --git a/test/mutation/run.exs b/test/mutation/run.exs index 6efefbe..19c9f51 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -16,7 +16,7 @@ defmodule Group.MutationCampaign do name: "accept_old_generation", file: "lib/group/replica.ex", correct_source: - "Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and", + "WireProtocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and", faulty_source: "true and", test: ["test/distributed_test.exs:5328"] }, @@ -24,7 +24,7 @@ defmodule Group.MutationCampaign do name: "accept_old_epoch", file: "lib/group/replica.ex", correct_source: """ - Protocol.stream_epoch(stream_id) == + WireProtocol.stream_epoch(stream_id) == Data.remote_cluster_epoch(state.name, source_node, cluster) and """, faulty_source: """ @@ -369,11 +369,11 @@ defmodule Group.MutationCampaign do name: "accept_shared_authority_before_lane_install", file: "lib/group/replica.ex", correct_source: """ - Protocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_shard(stream_id) == state.shard_index and replica_view_current?(state, source_node) and """, faulty_source: """ - Protocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_shard(stream_id) == state.shard_index and true and """, test: ["test/distributed_test.exs:5531"] diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index 89804b8..9987963 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -234,7 +234,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do TestCluster.flush_shards(node_a, name) old_stream = local_stream(node_a, name, cluster) - old_epoch = Group.Replica.Protocol.stream_epoch(old_stream) + old_epoch = Group.Replica.WireProtocol.stream_epoch(old_stream) frames = capture_snapshot(node_a, node_b, name, old_stream, 1) assert length(frames) > 1 {partial, [last]} = Enum.split(frames, -1) @@ -307,7 +307,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do new_stream = local_stream(node_a, name, nil) refute new_stream == old_stream - new_generation = Group.Replica.Protocol.stream_generation(new_stream) + new_generation = Group.Replica.WireProtocol.stream_generation(new_stream) TestCluster.assert_eventually(fn -> TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == @@ -471,9 +471,8 @@ defmodule Group.ReplicaSnapshotDistributedTest do send( target_control, - {:replica_hello, source_control, Group.Replica.Protocol.version(), generation, revision, - epochs, Group.Replica.Transport.Distribution.id(), - Group.Replica.Transport.Distribution.descriptor(name, [])} + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), generation, revision, + epochs, Group.Transport.DistErl.id(), Group.Transport.DistErl.descriptor(name, [])} ) _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) @@ -556,11 +555,11 @@ defmodule Group.ReplicaSnapshotDistributedTest do ]) :ok = - TestCluster.rpc!(node_a, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ name, node_b, 0, - {:needs, Group.Replica.Protocol.version(), [{stream_id, next_seq}]} + {:needs, Group.Replica.WireProtocol.version(), [{stream_id, next_seq}]} ]) TestCluster.flush_shards(node_a, name) @@ -581,7 +580,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do defp deliver_frames(node_b, node_a, name, frames) do Enum.each(frames, fn frame -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 0, diff --git a/test/replica_snapshot_test.exs b/test/replica_snapshot_test.exs index d87700f..2728870 100644 --- a/test/replica_snapshot_test.exs +++ b/test/replica_snapshot_test.exs @@ -42,8 +42,8 @@ defmodule Group.ReplicaSnapshotTest do Enum.with_index(snapshot.chunks, 1) |> Enum.each(fn {{registry, pg}, index} -> frame = - {:snapshot_chunk, Group.Replica.Protocol.version(), stream_id, 123, index, chunk_count, - snapshot.registry_count, snapshot.pg_count, registry, pg} + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, 123, index, + chunk_count, snapshot.registry_count, snapshot.pg_count, registry, pg} assert :erlang.external_size(frame) <= target end) diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs index 63c3a4e..c2739ea 100644 --- a/test/replica_transport_outbox_test.exs +++ b/test/replica_transport_outbox_test.exs @@ -1,7 +1,7 @@ defmodule Group.ReplicaTransportOutboxTest do use ExUnit.Case, async: true - alias Group.Replica.Transport.Outbox + alias Group.Transport.Outbox defmodule Backend do @behaviour Outbox @@ -124,7 +124,7 @@ defmodule Group.ReplicaTransportOutboxTest do assert_receive :receiver_ready assert :ok = - Group.Replica.Transport.incoming_batch( + Group.Transport.incoming_batch( group, source_node, 0, diff --git a/test/support/controlled_replica_transport.ex b/test/support/controlled_replica_transport.ex index 9658457..aecef9b 100644 --- a/test/support/controlled_replica_transport.ex +++ b/test/support/controlled_replica_transport.ex @@ -1,6 +1,6 @@ defmodule Group.ControlledReplicaTransport do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport @impl true def id, do: :group_controlled_replica_transport diff --git a/test/support/replica_model_scheduler.ex b/test/support/replica_model_scheduler.ex index 12955db..fc766ff 100644 --- a/test/support/replica_model_scheduler.ex +++ b/test/support/replica_model_scheduler.ex @@ -390,7 +390,7 @@ defmodule Group.ReplicaModelScheduler do :ok = TestCluster.rpc!( envelope.target, - Group.Replica.Transport, + Group.Transport, :incoming, [state.name, envelope.source, envelope.shard, envelope.message] ) diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index 80cd37e..ab11a61 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -728,7 +728,7 @@ defmodule Group.TestCluster do Group.Replica.Data.replica_cursor_table(name, shard) |> :ets.tab2list() |> Enum.filter(fn {stream_id, _seq} -> - Group.Replica.Protocol.stream_origin(stream_id) == origin + Group.Replica.WireProtocol.stream_origin(stream_id) == origin end) retained_view = Group.Replica.Data.remote_view_generation(name, shard, origin) @@ -908,14 +908,14 @@ defmodule Group.TestCluster do Group.Replica.Data.replica_cursor_table(name, shard) |> :ets.tab2list() |> Enum.each(fn {stream_id, seq} -> - origin = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + origin = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) valid? = - Group.Replica.Protocol.stream_name(stream_id) == name and - Group.Replica.Protocol.stream_shard(stream_id) == shard and + Group.Replica.WireProtocol.stream_name(stream_id) == name and + Group.Replica.WireProtocol.stream_shard(stream_id) == shard and origin != node() and generation == Group.Replica.Data.remote_generation(name, origin) and epoch == Group.Replica.Data.remote_cluster_epoch(name, origin, cluster) and diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex index 4efc7b2..f5ff194 100644 --- a/test/support/test_replica_transport.ex +++ b/test/support/test_replica_transport.ex @@ -1,6 +1,6 @@ defmodule Group.TestReplicaTransport do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport @impl true def id, do: :group_test_transport diff --git a/lib/group/replica/transport/tcp.ex b/test/support/test_tcp_transport.ex similarity index 84% rename from lib/group/replica/transport/tcp.ex rename to test/support/test_tcp_transport.ex index d074099..39dbce2 100644 --- a/lib/group/replica/transport/tcp.ex +++ b/test/support/test_tcp_transport.ex @@ -1,44 +1,15 @@ -defmodule Group.Replica.Transport.TCP do - @moduledoc """ - Sideband TCP transport for replica data. - - Erlang distribution still carries Group discovery and authority controls. - Replica messages use independent TCP connections, so there is no ordering - relationship between a control message and its data lane. - - `outgoing/5` only pushes to a local per-shard outbox. The outbox batches - messages and forwards each target batch to a bounded per-peer writer queue. - The writer may block up to `:send_timeout` without blocking a Group shard. - Expired, busy, and disconnected batches are dropped and repaired by - anti-entropy. - - The endpoint capability in the dist-Erlang hello prevents an unrelated - socket client from injecting messages. This transport is intended for trusted - cluster networks; it does not encrypt traffic. Put it behind a private - network or a TLS/WebSocket tunnel when confidentiality is required. - - ## Options - - * `:ip` - listen address, default `{127, 0, 0, 1}` - * `:advertised_ip` - address placed in the hello, defaults to `:ip` - * `:port` - listen port, default `0` (ephemeral) - * `:max_queue` - maximum queued batches per peer, default `1_024` - * `:connect_timeout` - outbound connect timeout in milliseconds, default `1_000` - * `:send_timeout` - writer socket send timeout in milliseconds, default `1_000` - * `:reconnect_interval` - retry delay in milliseconds, default `50` - - See `Group.Replica.Transport.Outbox` for batching and deadline options. - """ +defmodule Group.TestTCPTransport do + @moduledoc false use GenServer - @behaviour Group.Replica.Transport - @behaviour Group.Replica.Transport.Outbox + @behaviour Group.Transport + @behaviour Group.Transport.Outbox - alias Group.Replica.Transport.Outbox + alias Group.Transport.Outbox @impl true - def id, do: :group_sideband_tcp_v2 + def id, do: :group_test_sideband_tcp_v2 @impl true def child_spec(opts) do @@ -46,7 +17,7 @@ defmodule Group.Replica.Transport.TCP do %{ id: {__MODULE__, name}, - start: {Group.Replica.Transport.TCP.Supervisor, :start_link, [opts]}, + start: {Group.TestTCPTransport.Supervisor, :start_link, [opts]}, type: :supervisor, restart: :permanent, shutdown: :infinity @@ -67,10 +38,10 @@ defmodule Group.Replica.Transport.TCP do def outgoing(group, target_node, shard, message, opts), do: Outbox.push(group, target_node, shard, message, opts) - @impl Group.Replica.Transport.Outbox + @impl Group.Transport.Outbox def init_outbox(group, shard, _opts), do: {:ok, %{group: group, shard: shard}} - @impl Group.Replica.Transport.Outbox + @impl Group.Transport.Outbox def send_batch(target_node, messages, deadline, %{group: group, shard: shard} = state) do result = try do @@ -144,7 +115,7 @@ defmodule Group.Replica.Transport.TCP do {:ok, {_listen_ip, listen_port}} = :inet.sockname(listener) capability = :erlang.term_to_binary({node(), make_ref(), System.unique_integer()}) - descriptor = {:group_sideband_tcp_v2, advertised_ip, listen_port, capability} + descriptor = {:group_test_sideband_tcp_v2, advertised_ip, listen_port, capability} :persistent_term.put({__MODULE__, group, :descriptor}, descriptor) :ets.new(route_table(group), [ @@ -343,7 +314,7 @@ defmodule Group.Replica.Transport.TCP do manager, group, remote_node, - {:group_sideband_tcp_v2, host, port, capability}, + {:group_test_sideband_tcp_v2, host, port, capability}, connect_timeout, send_timeout ) do @@ -440,7 +411,7 @@ defmodule Group.Replica.Transport.TCP do case decode_authenticated_frame(payload) do {:ok, {:batch, shard, messages}} when is_integer(shard) and shard >= 0 and is_list(messages) -> - :ok = Group.Replica.Transport.incoming_batch(group, source_node, shard, messages) + :ok = Group.Transport.incoming_batch(group, source_node, shard, messages) reader_loop(socket, group, source_node) _ -> @@ -471,11 +442,12 @@ defmodule Group.Replica.Transport.TCP do defp route_table(group), do: :"#{group}_replica_tcp_routes" end -defmodule Group.Replica.Transport.TCP.Supervisor do +defmodule Group.TestTCPTransport.Supervisor do @moduledoc false use Supervisor - alias Group.Replica.Transport.{Outbox, TCP} + alias Group.TestTCPTransport, as: TCP + alias Group.Transport.Outbox def start_link(opts), do: Supervisor.start_link(__MODULE__, opts)