From ad7d8e88e7cec0949bf691f6af5df49e2b14c73e Mon Sep 17 00:00:00 2001 From: pgnickb Date: Thu, 17 Sep 2026 15:15:50 +0200 Subject: [PATCH 1/4] feat(coredump): capture PostgreSQL-only core dumps on the AMI - LimitCORE=infinity, scoped to postgresql.service only (not machine-wide) so an unrelated crash still produces no core - CoredumpFilter excludes shared_buffers (an anonymous shared mapping) from every core, since it can be many GB - this is what actually bounds core size - systemd-coredump gets its own conservative storage limits (compression, size caps), and is installed explicitly (not assumed to already be present) - confirm postgres_prestart.sh never resets ulimit -c (regression guard) - testinfra: LimitCORE, coredump_filter value, storage limits, coredump dir permissions, and a real end-to-end crash test (crash produces a core, an unrelated process's crash doesn't) --- .../postgresql_config/coredump-storage.conf | 7 + ansible/files/postgresql_config/coredump.conf | 2 + .../postgresql_config/postgresql.service | 4 + ansible/tasks/setup-postgres.yml | 37 ++++ testinfra/test_ami_nix.py | 180 ++++++++++++++++++ 5 files changed, 230 insertions(+) create mode 100644 ansible/files/postgresql_config/coredump-storage.conf create mode 100644 ansible/files/postgresql_config/coredump.conf diff --git a/ansible/files/postgresql_config/coredump-storage.conf b/ansible/files/postgresql_config/coredump-storage.conf new file mode 100644 index 0000000000..fcd9c211e8 --- /dev/null +++ b/ansible/files/postgresql_config/coredump-storage.conf @@ -0,0 +1,7 @@ +[Coredump] +Storage=external +Compress=yes +ProcessSizeMax=8G +ExternalSizeMax=8G +MaxUse=10G +KeepFree=5G diff --git a/ansible/files/postgresql_config/coredump.conf b/ansible/files/postgresql_config/coredump.conf new file mode 100644 index 0000000000..7c45cd49e5 --- /dev/null +++ b/ansible/files/postgresql_config/coredump.conf @@ -0,0 +1,2 @@ +[Service] +LimitCORE=infinity diff --git a/ansible/files/postgresql_config/postgresql.service b/ansible/files/postgresql_config/postgresql.service index 68c37140bd..c7ed7cad5d 100644 --- a/ansible/files/postgresql_config/postgresql.service +++ b/ansible/files/postgresql_config/postgresql.service @@ -11,6 +11,10 @@ Type=notify User=postgres ExecStart=/usr/lib/postgresql/bin/postgres -D /etc/postgresql ExecStartPre=+/usr/local/bin/postgres_prestart.sh +# Excludes shared_buffers (an anonymous *shared* mapping) from core dumps - +# without this, every core would include the whole (potentially many-GB) +# buffer pool. Inherited by every process postgres forks. +CoredumpFilter=private-anonymous elf-headers private-huge ExecReload=/bin/kill -HUP $MAINPID KillMode=mixed KillSignal=SIGINT diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 1bcc0420e4..12bc85a640 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -292,6 +292,43 @@ state: 'directory' become: true +# Required for this (plus CoredumpFilter= in postgresql.service and the +# storage limits below) to do anything at all: without systemd-coredump, +# core_pattern stays at the kernel's plain "core" default and none of this +# gets exercised (confirmed on a local test VM). Unconditional - every +# Postgres flavor's capture depends on it, not just OrioleDB. +- name: ensure systemd-coredump is installed + ansible.builtin.apt: + name: systemd-coredump + become: true + +- name: copy PostgreSQL coredump systemd drop-in + ansible.builtin.copy: + dest: '/etc/systemd/system/postgresql.service.d/coredump.conf' + group: 'root' + mode: '0644' + owner: 'root' + src: 'files/postgresql_config/coredump.conf' + become: true + +- name: Create a systemd-coredump conf.d dir for PostgreSQL storage limits + ansible.builtin.file: + group: 'root' + mode: '0755' + owner: 'root' + path: '/etc/systemd/coredump.conf.d' + state: 'directory' + become: true + +- name: copy systemd-coredump storage limits + ansible.builtin.copy: + dest: '/etc/systemd/coredump.conf.d/postgres.conf' + group: 'root' + mode: '0644' + owner: 'root' + src: 'files/postgresql_config/coredump-storage.conf' + become: true + - name: Ensure PostgreSQL starts after tuned become: true community.general.ini_file: diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index cfd77554d6..869ab3a020 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -1407,3 +1407,183 @@ def test_apparmor_denies_access_to_sensitive_paths(host): f"to have succeeded.\nstdout: {result['stdout']}\nstderr: {result['stderr']}" ) print(f"Confirmed: access to {test_file} denied by AppArmor") + + +def test_postgresql_service_allows_unlimited_core_dumps(host): + """Verify the postgresql.service coredump drop-in sets LimitCORE=infinity. + + Coredumps are intentionally scoped to the postgresql unit only (via a + systemd.service.d drop-in), not enabled machine-wide, so other services + must keep the default core limit. + """ + result = run_ssh_command(host["ssh"], "systemctl show postgresql -p LimitCORE") + assert result["succeeded"], f"systemctl show failed: {result['stderr']}" + assert "LimitCORE=infinity" in result["stdout"], ( + f"Expected postgresql.service to have LimitCORE=infinity, got:\n{result['stdout']}" + ) + + +def test_coredump_storage_limits_configured(host): + """Verify /etc/systemd/coredump.conf.d/postgres.conf sets conservative, + bounded storage limits for the systemd-coredump storage that backs + Postgres core capture.""" + result = run_ssh_command( + host["ssh"], "cat /etc/systemd/coredump.conf.d/postgres.conf" + ) + assert result["succeeded"], ( + f"Could not read coredump storage config: {result['stderr']}" + ) + for expected in [ + "Storage=external", + "Compress=yes", + "ProcessSizeMax=", + "ExternalSizeMax=", + "MaxUse=", + "KeepFree=", + ]: + assert expected in result["stdout"], ( + f"Expected '{expected}' in /etc/systemd/coredump.conf.d/postgres.conf, " + f"got:\n{result['stdout']}" + ) + + +def test_postgres_coredump_filter_excludes_shared_buffers(host): + """Verify the running postmaster's /proc/[pid]/coredump_filter is 0x31 + (49): private mappings + ELF headers, but not anonymous-shared mappings. + + shared_buffers is mmap(MAP_SHARED|MAP_ANONYMOUS) (shared_memory_type + defaults to 'mmap' and is not overridden), which the kernel classifies + as an anonymous shared mapping - excluding it from every core is what + actually keeps core size bounded, since shared_buffers can be many GB. + postgresql.service sets this via the native systemd + 'CoredumpFilter=private-anonymous elf-headers private-huge' directive + (systemd >= 246), which coredump_filter (inherited across fork(2) and + preserved across execve(2)) then propagates to everything postgres forks. + """ + pid = run_ssh_command( + host["ssh"], "systemctl show postgresql -p MainPID --value" + )["stdout"].strip() + assert pid.isdigit() and pid != "0", f"Could not resolve postgresql.service MainPID: {pid}" + + result = run_ssh_command(host["ssh"], f"cat /proc/{pid}/coredump_filter") + assert result["succeeded"], f"Could not read coredump_filter for pid {pid}: {result['stderr']}" + assert result["stdout"].strip() == "31", ( + f"Expected /proc/{pid}/coredump_filter to be '31' (0x31 = 49 decimal), " + f"got '{result['stdout'].strip()}'" + ) + + +def test_coredump_storage_directory_root_only(host): + """Verify /var/lib/systemd/coredump is root-only, since it can hold + core files containing customer data.""" + result = run_ssh_command( + host["ssh"], "stat -c '%a %U:%G' /var/lib/systemd/coredump" + ) + assert result["succeeded"], f"stat failed: {result['stderr']}" + mode, owner = result["stdout"].strip().split() + assert mode in ("700", "750"), ( + f"Expected /var/lib/systemd/coredump to be root-only, got mode {mode}" + ) + assert owner.startswith("root:"), ( + f"Expected /var/lib/systemd/coredump to be owned by root, got {owner}" + ) + + +def test_postgres_prestart_does_not_reset_core_limit(host): + """Regression guard: postgres_prestart.sh must never touch 'ulimit -c', + or it would silently defeat the postgresql.service LimitCORE=infinity + coredump drop-in.""" + result = run_ssh_command( + host["ssh"], "cat /usr/local/bin/postgres_prestart.sh" + ) + assert result["succeeded"], f"Could not read prestart script: {result['stderr']}" + assert "ulimit -c" not in result["stdout"], ( + "postgres_prestart.sh must not set 'ulimit -c' - doing so would silently " + "defeat the postgresql.service coredump drop-in" + ) + + +def _crash_a_backend_and_wait_for_recovery(host): + """Grab a real Postgres backend pid, SIGSEGV it, and wait for PostgreSQL's + normal crash-recovery to bring the instance back on its own + (Restart=always / auto-reinit, same as production). Returns the crashed + backend's pid. Used by tests that need a real, fresh core to appear.""" + backend_pid = run_ssh_command( + host["ssh"], + "sudo -u postgres psql -U supabase_admin -h localhost -d postgres " + "-tAc 'select pg_backend_pid()'", + )["stdout"].strip() + assert backend_pid.isdigit(), ( + f"Could not resolve a Postgres backend pid: {backend_pid}" + ) + run_ssh_command(host["ssh"], f"sudo kill -SEGV {backend_pid}") + + recovered = False + for _ in range(30): + sleep(2) + probe = run_ssh_command( + host["ssh"], + "sudo -u postgres psql -U supabase_admin -h localhost -d postgres " + "-tAc 'select 1'", + ) + if probe["succeeded"] and probe["stdout"].strip() == "1": + recovered = True + break + assert recovered, ( + "PostgreSQL did not come back up within 60s after the induced backend crash" + ) + return backend_pid + + +def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(host): + """End-to-end capture check: a segfaulted Postgres backend must produce a + coredumpctl-visible core, while an unrelated process crashing the same way + must not. + + This intentionally crashes a live backend (mirroring a real SIGSEGV, e.g. + the background-writer crash in incident ORI-261) and relies on + PostgreSQL's normal crash-recovery to bring the instance back on its own + (Restart=always / auto-reinit), the same as in production - it does not + reinstall data or otherwise reset the shared test instance. + """ + before = run_ssh_command( + host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true" + ) + before_lines = set(before["stdout"].splitlines()) + + # Unrelated process: launch a disposable process outside the + # LimitCORE=infinity-scoped postgresql.service and SIGSEGV it - it must + # not produce a core, since the default core limit is unchanged for it. + run_ssh_command( + host["ssh"], + "setsid bash -c 'sleep 60 & echo $! > /tmp/unrelated_pid; wait' >/dev/null 2>&1 &", + ) + sleep(1) + unrelated_pid = run_ssh_command(host["ssh"], "cat /tmp/unrelated_pid")[ + "stdout" + ].strip() + assert unrelated_pid.isdigit(), ( + f"Could not resolve the disposable unrelated process pid: {unrelated_pid}" + ) + run_ssh_command(host["ssh"], f"kill -SEGV {unrelated_pid}") + sleep(2) + + # Postgres backend: grab a real backend pid and crash it the same way. + backend_pid = _crash_a_backend_and_wait_for_recovery(host) + + after = run_ssh_command( + host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true" + ) + new_lines = [ + line for line in after["stdout"].splitlines() if line not in before_lines + ] + assert any("postgres" in line for line in new_lines), ( + f"Expected a new postgres core dump after SIGSEGV to backend {backend_pid}, " + f"but coredumpctl list shows:\n{after['stdout']}" + ) + assert not any(unrelated_pid in line for line in new_lines), ( + f"Unrelated process {unrelated_pid} should not have produced a core dump:\n" + f"{after['stdout']}" + ) + + From 9381966ef7d030f612091249db688316caf6a970 Mon Sep 17 00:00:00 2001 From: pgnickb Date: Thu, 17 Sep 2026 15:16:06 +0200 Subject: [PATCH 2/4] test(coredump): verify shipped debug symbols actually resolve - postgres/orioledb debug+source packages already ship on every AMI (postgres-env bundle) - nothing new to package here, verification only - check build-ID match: shipped postgres binary and orioledb.so vs the installed debug package - check GDB can read real source content through the source package (not just that a filename is known - info sources lists names regardless of whether the file is actually reachable on disk) --- testinfra/test_ami_nix.py | 107 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index 869ab3a020..d629fc49e2 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -1587,3 +1587,110 @@ def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(hos ) +def _build_id_of(host, path): + """Return the ELF build-id (hex string) of the binary/library at path, or None.""" + import re + + result = run_ssh_command(host["ssh"], f"readelf -n {path} 2>/dev/null") + if not result["succeeded"]: + return None + match = re.search(r"Build ID:\s*([0-9a-f]+)", result["stdout"]) + return match.group(1) if match else None + + +def _debug_file_exists_for_build_id(host, build_id): + """Check whether the postgres nix-profile's debug output has a + .build-id/xx/yyyy...debug file matching the given build-id.""" + prefix, rest = build_id[:2], build_id[2:] + debug_path = ( + f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug" + ) + result = run_ssh_command(host["ssh"], f"test -f {debug_path} && echo present") + return result["succeeded"] and "present" in result["stdout"] + + +def test_postgres_binary_build_id_matches_shipped_debug_symbols(host): + """Verify the installed 'postgres' binary's build-id has a matching + .build-id/xx/yyyy.debug file in the postgres-env debug output, so a + future coredump-processing GDB session can actually resolve symbols.""" + build_id = _build_id_of(host, "/usr/lib/postgresql/bin/postgres") + assert build_id, "Could not read a build-id from /usr/lib/postgresql/bin/postgres" + assert _debug_file_exists_for_build_id(host, build_id), ( + f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/" + f".build-id/ matching postgres build-id {build_id} - the shipped " + f"_debug package may be out of sync with the shipped binary" + ) + + +def test_orioledb_library_build_id_matches_shipped_debug_symbols(host): + """Verify orioledb.so's build-id has a matching debug file, same as for + the postgres binary - orioledb.so is built by the same derivation + (isOrioleDB flavor) so its debug info ships in the same _debug output.""" + orioledb_so = "/usr/lib/postgresql/lib/orioledb.so" + exists = run_ssh_command(host["ssh"], f"test -f {orioledb_so} && echo present") + if "present" not in exists["stdout"]: + pytest.skip("orioledb.so not present on this AMI (not an OrioleDB build)") + + build_id = _build_id_of(host, orioledb_so) + assert build_id, f"Could not read a build-id from {orioledb_so}" + assert _debug_file_exists_for_build_id(host, build_id), ( + f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/" + f".build-id/ matching orioledb.so build-id {build_id}" + ) + + +def test_gdb_resolves_postgres_source_via_shipped_src_package(host): + """Verify GDB can read actual source *content* for the installed postgres + binary via the shipped _src package - not just that a filename is known + from debug info (which 'info sources' would show regardless of whether + the file is reachable on disk). + + The _src package mirrors the exact build-time source tree under the + nix-profile root (see nix/postgresql/src.nix), but the debug info records + the original nix build sandbox directory (DW_AT_comp_dir, e.g. + /build/postgres-) as each file's location. GDB needs a + 'substitute-path' from that recorded build directory to the profile root + to find the files - this test discovers that build directory dynamically + (rather than hardcoding a guess) and confirms 'list main' then prints + real source lines instead of falling back to the 'in ' placeholder + GDB uses when a source file can't be found. + """ + import re + + build_id = _build_id_of(host, "/usr/lib/postgresql/bin/postgres") + assert build_id, "Could not read a build-id from /usr/lib/postgresql/bin/postgres" + prefix, rest = build_id[:2], build_id[2:] + debug_file = f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug" + + comp_dir = run_ssh_command( + host["ssh"], + f"readelf --debug-dump=info {debug_file} 2>/dev/null " + "| grep -m1 DW_AT_comp_dir | grep -oE '/[^ ]+$'", + )["stdout"].strip() + assert comp_dir, f"Could not determine DW_AT_comp_dir from {debug_file}" + + result = run_ssh_command( + host["ssh"], + "sudo -u postgres gdb --batch -quiet " + "-ex 'set debug-file-directory /var/lib/postgresql/.nix-profile/lib/debug' " + f"-ex 'set substitute-path {comp_dir} /var/lib/postgresql/.nix-profile' " + "-ex 'file /usr/lib/postgresql/bin/postgres' " + "-ex 'list main' " + "2>&1", + ) + assert result["succeeded"], f"gdb invocation failed: {result['stderr']}" + assert "No debugging symbols found" not in result["stdout"], ( + f"GDB could not find debug symbols for postgres:\n{result['stdout']}" + ) + assert not re.search(r"^\d+\tin /", result["stdout"], re.MULTILINE), ( + f"GDB fell back to the 'in ' placeholder, meaning it could not " + f"actually read the source file even with substitute-path set from " + f"{comp_dir} to /var/lib/postgresql/.nix-profile:\n{result['stdout']}" + ) + numbered_lines = re.findall(r"^\d+\t.+$", result["stdout"], re.MULTILINE) + assert len(numbered_lines) >= 3, ( + f"Expected 'list main' to print several lines of real source code " + f"content via the shipped _src package, got:\n{result['stdout']}" + ) + + From 5984bb453f23a70c72db5ece1ce475bfacba1974 Mon Sep 17 00:00:00 2001 From: pgnickb Date: Thu, 17 Sep 2026 15:16:37 +0200 Subject: [PATCH 3/4] feat(coredump): process captured cores into diagnostic bundles - orioledb-coredump.path (triggers on new cores) + .timer (10-min fallback sweep) both run a flock-guarded oneshot service - gdb and binutils are installed explicitly for this (scoped to OrioleDB images via is_psql_oriole, since only this processor needs them) - processor script: keeps only postgres-owned cores, extracts a bundle via GDB (cmds.gdb - matches OrioleDB's own CI debugging script: full backtrace, argv, shared libraries, registers, lock state), deletes the raw core, quarantines metadata-only after repeated failures, and enforces its own age/size retention independent of systemd-coredump's own limits - handles /usr/lib/postgresql/bin/postgres being a Nix wrapper script (not the real ELF) throughout: the executable filter, readelf, and gdb all use the real per-crash path reported by coredumpctl instead of a fixed guess - matches/deletes cores by PID, not raw file path (coredumpctl has no "rm" verb on this systemd version, and path-based matching was unreliable) - reads the active postgresql log path from current_logfiles at runtime, since this AMI logs via csvlog with no fixed filename - rolled out to OrioleDB images only for now (is_psql_oriole) - testinfra: end-to-end check that a real crash produces a diagnostic bundle and the raw core gets deleted --- ansible/files/coredump/cmds.gdb | 20 ++ ansible/files/coredump/orioledb-coredump.path | 9 + .../files/coredump/orioledb-coredump.service | 14 + .../files/coredump/orioledb-coredump.timer | 10 + .../coredump/process-orioledb-coredumps.sh | 281 ++++++++++++++++++ ansible/playbook.yml | 4 + ansible/tasks/setup-coredump-processing.yml | 75 +++++ testinfra/test_ami_nix.py | 54 ++++ 8 files changed, 467 insertions(+) create mode 100644 ansible/files/coredump/cmds.gdb create mode 100644 ansible/files/coredump/orioledb-coredump.path create mode 100644 ansible/files/coredump/orioledb-coredump.service create mode 100644 ansible/files/coredump/orioledb-coredump.timer create mode 100644 ansible/files/coredump/process-orioledb-coredumps.sh create mode 100644 ansible/tasks/setup-coredump-processing.yml diff --git a/ansible/files/coredump/cmds.gdb b/ansible/files/coredump/cmds.gdb new file mode 100644 index 0000000000..93b4599453 --- /dev/null +++ b/ansible/files/coredump/cmds.gdb @@ -0,0 +1,20 @@ +# Matches orioledb/ci/cmds.gdb (OrioleDB's own CI crash-debugging script). +# debug-file-directory/file/core-file are set by the caller before this +# file is sourced (see process-orioledb-coredumps.sh). +# +# GDB aborts the rest of a sourced script on the first command error, so +# order matters here: the reliable, high-value output runs first; the +# OrioleDB-internal lock-state dumps run last, since they can fail if +# orioledb.so's own debug symbols aren't available (a known, separate issue) +thread apply all bt full +up 99999 +set $i=0 +set $end=argc +while ($i < $end) +p argv[$i++] +end +info sharedlibrary +info registers +eval "p *((LWLockHandle (*) [%u]) held_lwlocks)", num_held_lwlocks +eval "p *((MyLockedPage (*) [%u]) myLockedPages)", numberOfMyLockedPages +quit diff --git a/ansible/files/coredump/orioledb-coredump.path b/ansible/files/coredump/orioledb-coredump.path new file mode 100644 index 0000000000..c9a5ce83d1 --- /dev/null +++ b/ansible/files/coredump/orioledb-coredump.path @@ -0,0 +1,9 @@ +[Unit] +Description=Watch for PostgreSQL/OrioleDB core dumps + +[Path] +PathChanged=/var/lib/systemd/coredump +Unit=orioledb-coredump.service + +[Install] +WantedBy=multi-user.target diff --git a/ansible/files/coredump/orioledb-coredump.service b/ansible/files/coredump/orioledb-coredump.service new file mode 100644 index 0000000000..49e574438d --- /dev/null +++ b/ansible/files/coredump/orioledb-coredump.service @@ -0,0 +1,14 @@ +[Unit] +Description=Process PostgreSQL/OrioleDB core dumps into a redacted diagnostic bundle +After=systemd-coredump@.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/process-orioledb-coredumps.sh +User=root +Group=root +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/var/lib/systemd/coredump /var/lib/orioledb-coredumps /run diff --git a/ansible/files/coredump/orioledb-coredump.timer b/ansible/files/coredump/orioledb-coredump.timer new file mode 100644 index 0000000000..15d332ba66 --- /dev/null +++ b/ansible/files/coredump/orioledb-coredump.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Periodic fallback sweep for PostgreSQL/OrioleDB core dumps + +[Timer] +OnBootSec=10min +OnUnitActiveSec=10min +Unit=orioledb-coredump.service + +[Install] +WantedBy=timers.target diff --git a/ansible/files/coredump/process-orioledb-coredumps.sh b/ansible/files/coredump/process-orioledb-coredumps.sh new file mode 100644 index 0000000000..896e96a380 --- /dev/null +++ b/ansible/files/coredump/process-orioledb-coredumps.sh @@ -0,0 +1,281 @@ +#!/bin/bash +# Process PostgreSQL/OrioleDB core dumps captured by systemd-coredump into a +# text summary. +# +# Key ideas: +# - Only cores of the postgres binary are processed; any other core left +# untouched +# - No environment variables are ever collected. The GDB extraction (see +# cmds.gdb, matching OrioleDB's own CI debugging script) does run +# `thread apply all bt full`, which prints local variable values and can +# surface fragments of in-memory data (buffer/tuple pointers etc.) - this +# is a deliberate, reviewed trade-off in favor of debuggability, not an +# oversight. +# - Cores are deleted after a successful run or quarantined (metadata only) +# after MAX_ATTEMPTS failures. + +set -euo pipefail + +STATE_DIR=/var/lib/orioledb-coredumps/state +OUTPUT_DIR=/var/lib/orioledb-coredumps/diagnostics +QUARANTINE_DIR=/var/lib/orioledb-coredumps/quarantine +LOCK_FILE=/run/orioledb-coredump.lock + +MAX_ATTEMPTS=3 +EXTRACTION_TIMEOUT=120 +MAX_AGE_DAYS=7 +MAX_TOTAL_BYTES=$((5 * 1024 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse + +GDB_DEBUG_DIR=/var/lib/postgresql/.nix-profile/lib/debug +GDB_CMDS_FILE=/usr/local/sbin/orioledb-coredump-cmds.gdb +PGDATA_CURRENT_LOGFILES=/var/lib/postgresql/data/current_logfiles + +log() { + echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" +} + +# /usr/lib/postgresql/bin/postgres is a Nix wrapper *script* (sets +# NIX_PGLIBDIR, then execs the real ELF at .../bin/.postgres-wrapped) - not +# an executable itself. A crashing backend's real, kernel-recorded +# Executable: is that wrapped path (basename ".postgres-wrapped"), never the +# wrapper. Normalize both forms to "postgres" for comparison, and always use +# the per-crash Executable: path (not a hardcoded one) for readelf/gdb. +normalize_exe_basename() { + local base + base=$(basename "$1") + base="${base#.}" + base="${base%-wrapped}" + printf '%s' "$base" +} + +# orioledb.so has no fixed, predictable path either - there is no +# /usr/lib/postgresql/lib mirror of it. It lives in the same nix store +# derivation as the resolved postgres executable, just under lib/ instead +# of bin/, e.g. .../postgresql-and-plugins-17_20/{bin/.postgres-wrapped, +# lib/orioledb.so} - so derive it from $exe rather than guessing a path. +orioledb_lib_path() { + local exe_dir + exe_dir=$(dirname "$(dirname "$1")") + printf '%s/lib/orioledb.so' "$exe_dir" +} + +# coredumpctl on this systemd version (255.4) has no "rm"/"delete" verb - the +# only reliable way to remove a core is to delete its on-disk Storage: path +# directly. Returns success only if the path is actually gone afterward. +delete_core() { + local path="$1" + [ -z "$path" ] && return 1 + rm -f -- "$path" + [ ! -e "$path" ] +} + +# The active postgresql log file has an unpredictable name and can be +# csvlog, stderr-text, or both depending on config (this AMI defaults to +# csvlog-only, e.g. /var/log/postgresql/postgresql.csv - the stderr-format +# postgresql.log stops receiving anything the moment the logging collector +# switches over at startup). PGDATA/current_logfiles is postgres's own, +# always-current record of the real path(s); prefer csvlog, fall back to +# stderr. Either format still starts each line with a literal timestamp, so +# the grep -F substring match below works unchanged either way. +current_postgres_log() { + [ -f "$PGDATA_CURRENT_LOGFILES" ] || return 0 + awk '$1 == "csvlog" {p = $2} $1 == "stderr" && !p {p = $2} END {print p}' "$PGDATA_CURRENT_LOGFILES" +} + +enforce_retention() { + find "$OUTPUT_DIR" -maxdepth 1 -type f -mtime "+${MAX_AGE_DAYS}" -delete 2>/dev/null || true + + while true; do + total=$(du -sb "$OUTPUT_DIR" 2>/dev/null | cut -f1) + [ -z "$total" ] && break + [ "$total" -le "$MAX_TOTAL_BYTES" ] && break + oldest=$(find "$OUTPUT_DIR" -maxdepth 1 -type f -printf '%T@ %p\n' 2>/dev/null | sort -n | head -1 | cut -d' ' -f2-) + [ -z "$oldest" ] && break + log "retention: removing oldest bundle $oldest to stay under ${MAX_TOTAL_BYTES} bytes" + rm -f "$oldest" + done +} + +# Handles one candidate crash, given its PID: decide whether it's ours to +# process, extract a diagnostic bundle via GDB, then delete the raw core. +# (1) look up the crash via coredumpctl +# (2) decide keep/ignore/quarantine based on prior attempts +# (3) export the core and run GDB against it +# (4) write the bundle and delete the raw core. Returns 1 only for failures +# worth retrying next run (main() logs those); every other outcome is +# ignored, quarantined, or successfully processed - returns 0. +process_one() { + local pid="$1" + + # coredumpctl matches reliably by PID; matching by the raw storage path + # (which encodes an escaped comm, e.g. "core.\x2epostgres-wrapp....zst" + # for the wrapped binary below) was found to fail in practice. + local meta + if ! meta=$(coredumpctl info "$pid" --no-pager 2>/dev/null); then + log "pid ${pid}: coredumpctl info failed" + return 1 + fi + + local exe boot_id storage_path + exe=$(awk -F': ' '/^ *Executable:/ {print $2; exit}' <<<"$meta") + boot_id=$(awk -F': ' '/^ *Boot ID:/ {print $2; exit}' <<<"$meta") + # "Storage: /path/to/core (present)" -> "/path/to/core" + storage_path=$(sed -n 's/^ *Storage: \(.*\) (.*)$/\1/p' <<<"$meta" | head -1) + # PID alone isn't a safe dedup key long-term (PIDs get reused across + # boots), so pair it with boot ID - matches "state keyed by boot ID plus + # dump identifier" from the original design. + local key="${boot_id}-${pid}" + local state_file="${STATE_DIR}/${key}" + + # .done means "final decision made, never look at this dump again" + # (processed successfully, ignored as non-postgres, or quarantined). + # .attempts only counts *failed* tries, to cap retries before quarantine. + [ -f "${state_file}.done" ] && return 0 + + local attempts=0 + [ -f "${state_file}.attempts" ] && attempts=$(cat "${state_file}.attempts") + if [ "$attempts" -ge "$MAX_ATTEMPTS" ]; then + log "pid ${pid}: exceeded ${MAX_ATTEMPTS} attempts, discarding core (metadata kept in ${QUARANTINE_DIR})" + echo "$meta" >"${QUARANTINE_DIR}/${key}.info" + if delete_core "$storage_path"; then + log "pid ${pid}: raw core deleted" + else + log "pid ${pid}: WARNING - could not delete raw core at '${storage_path}'" + fi + touch "${state_file}.done" + return 0 + fi + + # /usr/lib/postgresql/bin/postgres is a wrapper script that execs the + # real ELF at .../bin/.postgres-wrapped - the kernel (and therefore + # coredumpctl's Executable:) always records the latter. Normalize both + # forms before comparing, so we don't silently ignore every real crash. + if [[ "$(normalize_exe_basename "$exe")" != "postgres" ]]; then + log "pid ${pid}: executable '${exe}' is not postgres, ignoring" + touch "${state_file}.done" + return 0 + fi + + # From here on we're committed to actually processing this dump, so + # count it as an attempt before doing any of the risky (slow, can fail) + # work below - a crash/timeout past this point still gets retried, up + # to MAX_ATTEMPTS + echo $((attempts + 1)) >"${state_file}.attempts" + + # coredumpctl stores the core compressed; pull a private, working copy + # out into a root-only scratch dir before handing it to GDB. The trap + # guarantees that scratch dir is removed when this function returns, no + # matter which of the several `return`s below fires. + local tmpdir + tmpdir=$(mktemp -d /tmp/coredump-XXXXXX) + chmod 700 "$tmpdir" + # suppress shellcheck warning about quoting $tmpdir in the trap command - + # it's correct to quote it, and the trap is evaluated at runtime, + # not parse time. + # shellcheck disable=SC2064 + trap "rm -rf '$tmpdir'" RETURN + + if ! timeout "$EXTRACTION_TIMEOUT" coredumpctl dump "$pid" --output "${tmpdir}/core" >/dev/null 2>&1; then + log "pid ${pid}: export failed or timed out" + return 1 + fi + + # Already have this in $meta from the coredumpctl info call above - + # just pulling out the two fields the bundle header needs. + local signal timestamp + signal=$(awk -F': ' '/^ *Signal:/ {print $2; exit}' <<<"$meta") + timestamp=$(awk -F': ' '/^ *Timestamp:/ {print $2; exit}' <<<"$meta") + + # Everything from here to the closing "}" is the diagnostic bundle + # itself, one section at a time, redirected straight to $bundle - + # there's no in-memory buffering of it, so a slow/hanging step just + # shows up as a truncated file rather than blocking the whole write. + local bundle="${OUTPUT_DIR}/${key}.txt" + { + echo "== OrioleDB/PostgreSQL coredump diagnostic bundle ==" + echo "generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')" + echo "pid: ${pid}" + echo "boot_id: ${boot_id}" + echo "signal: ${signal}" + echo "crash_timestamp: ${timestamp}" + echo "executable: ${exe}" + echo + + echo "== build ids ==" + echo "postgres (${exe}):" + readelf -n "$exe" 2>/dev/null | grep 'Build ID' || echo " (could not read build id)" + orioledb_lib=$(orioledb_lib_path "$exe") + if [ -f "$orioledb_lib" ]; then + echo "orioledb.so (${orioledb_lib}):" + readelf -n "$orioledb_lib" 2>/dev/null | grep 'Build ID' || echo " (could not read build id)" + fi + echo + + echo "== gdb backtrace (full), lwlocks, locked pages, argv, shared libraries, registers ==" + timeout "$EXTRACTION_TIMEOUT" gdb --batch -quiet \ + -ex "set debug-file-directory ${GDB_DEBUG_DIR}" \ + -ex "file ${exe}" \ + -ex "core-file ${tmpdir}/core" \ + -x "$GDB_CMDS_FILE" \ + 2>&1 || echo "(gdb extraction failed or timed out)" + echo + + echo "== postgresql.log excerpt around crash ==" + # $timestamp looks like "Thu 2026-09-17 11:25:49 UTC (1s ago)" - pull + # out just the "YYYY-MM-DD HH:MM:SS" portion to match against + # postgres's own log line prefix (a fixed offset previously grabbed + # the leading weekday name instead and never matched anything). + log_ts=$(grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}' <<<"$timestamp" | head -1) + postgres_log=$(current_postgres_log) + if [ -n "$log_ts" ] && [ -n "$postgres_log" ] && [ -f "$postgres_log" ]; then + grep -F "$log_ts" -A 5 -B 20 "$postgres_log" 2>/dev/null | tail -200 || + echo "(no log lines found matching ${log_ts} in ${postgres_log})" + else + echo "(no crash timestamp or log file available)" + fi + } >"$bundle" + chmod 600 "$bundle" + + # Bundle is written either way at this point, even if the raw-core + # delete below fails - we don't want a delete failure to make us + # reprocess (and re-append to) an already-complete bundle next run. + touch "${state_file}.done" + if delete_core "$storage_path"; then + log "pid ${pid}: wrote ${bundle}, deleted raw core" + else + log "pid ${pid}: wrote ${bundle}, but WARNING - could not delete raw core at '${storage_path}'" + fi +} + +main() { + mkdir -p "$STATE_DIR" "$OUTPUT_DIR" "$QUARANTINE_DIR" + chmod 700 "$STATE_DIR" "$OUTPUT_DIR" "$QUARANTINE_DIR" + + enforce_retention + + # Enumerate via coredumpctl (per INSTRUCTIONS.md's original design), + # restricted to entries whose raw core is still on disk ("present") - + # already-removed/historical journal entries are skipped without + # needing a coredumpctl info round-trip. Field positions match the + # observed `coredumpctl list` table layout (systemd 255): + # TIME(4 tokens) PID UID GID SIG COREFILE EXE SIZE + local rc=0 + local pid + while read -r pid; do + [ -z "$pid" ] && continue + process_one "$pid" || { + log "pid ${pid}: processing failed, will retry on next run" + rc=1 + } + done < <(coredumpctl --no-legend --no-pager list 2>/dev/null | awk '$9 == "present" {print $5}') + + return "$rc" +} + +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + log "another processor run is already in progress, exiting" + exit 0 +fi + +main diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 8aea9e4cba..46f883cf16 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -39,6 +39,10 @@ - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml + - name: Install PostgreSQL/OrioleDB coredump processing + when: stage2 and is_psql_oriole + import_tasks: tasks/setup-coredump-processing.yml + - name: Install PgBouncer import_tasks: tasks/setup-pgbouncer.yml tags: diff --git a/ansible/tasks/setup-coredump-processing.yml b/ansible/tasks/setup-coredump-processing.yml new file mode 100644 index 0000000000..6696b44d74 --- /dev/null +++ b/ansible/tasks/setup-coredump-processing.yml @@ -0,0 +1,75 @@ +--- +# gdb and binutils (readelf) are only used by the processor script below, +# which is only installed for OrioleDB images due to is_psql_oriole +- name: ensure gdb and binutils are installed + ansible.builtin.apt: + name: + - gdb + - binutils + become: true + +- name: Create orioledb-coredumps state/output directories + ansible.builtin.file: + group: 'root' + mode: '0700' + owner: 'root' + path: "{{ coredump_processing_item }}" + state: 'directory' + become: true + loop: + - '/var/lib/orioledb-coredumps' + - '/var/lib/orioledb-coredumps/state' + - '/var/lib/orioledb-coredumps/diagnostics' + - '/var/lib/orioledb-coredumps/quarantine' + loop_control: + loop_var: 'coredump_processing_item' + +- name: copy PostgreSQL/OrioleDB coredump processor script + ansible.builtin.copy: + dest: '/usr/local/sbin/process-orioledb-coredumps.sh' + group: 'root' + mode: '0700' + owner: 'root' + src: 'coredump/process-orioledb-coredumps.sh' + become: true + +- name: copy GDB extraction commands (matches orioledb/ci/cmds.gdb) + ansible.builtin.copy: + dest: '/usr/local/sbin/orioledb-coredump-cmds.gdb' + group: 'root' + mode: '0600' + owner: 'root' + src: 'coredump/cmds.gdb' + become: true + +- name: copy PostgreSQL/OrioleDB coredump systemd units + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ coredump_unit_item }}" + group: 'root' + mode: '0644' + owner: 'root' + src: "coredump/{{ coredump_unit_item }}" + become: true + loop: + - 'orioledb-coredump.path' + - 'orioledb-coredump.timer' + - 'orioledb-coredump.service' + loop_control: + loop_var: 'coredump_unit_item' + +- name: reload systemd for coredump processing units + ansible.builtin.systemd_service: + daemon_reload: true + become: true + +- name: enable coredump processing path and timer units + ansible.builtin.systemd_service: + name: "{{ coredump_enable_item }}" + enabled: true + state: 'started' + become: true + loop: + - 'orioledb-coredump.path' + - 'orioledb-coredump.timer' + loop_control: + loop_var: 'coredump_enable_item' diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index d629fc49e2..ba1b27e4b5 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -1694,3 +1694,57 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): ) +def test_coredump_processor_produces_diagnostic_bundle_and_deletes_raw_core(host): + """End-to-end processor check: after a real Postgres backend crash, the + orioledb-coredump.path-triggered processor must turn the raw core into a + small diagnostic bundle and delete the raw core - not leave it sitting in + /var/lib/systemd/coredump indefinitely. + + Only runs on images where the coredump processor is installed (OrioleDB + builds, per the gated rollout); skipped otherwise. + """ + unit_check = run_ssh_command( + host["ssh"], "systemctl list-unit-files orioledb-coredump.path --no-legend" + ) + if "orioledb-coredump.path" not in unit_check["stdout"]: + pytest.skip("coredump processor not installed on this AMI (not an OrioleDB build)") + + before_bundles = set( + run_ssh_command( + host["ssh"], "sudo find /var/lib/orioledb-coredumps/diagnostics -maxdepth 1 -type f" + )["stdout"].splitlines() + ) + + _crash_a_backend_and_wait_for_recovery(host) + + new_bundle = None + for _ in range(30): + sleep(2) + current = set( + run_ssh_command( + host["ssh"], + "sudo find /var/lib/orioledb-coredumps/diagnostics -maxdepth 1 -type f", + )["stdout"].splitlines() + ) + new = current - before_bundles + if new: + new_bundle = sorted(new)[0] + break + assert new_bundle, ( + "Expected a new diagnostic bundle under /var/lib/orioledb-coredumps/diagnostics " + "after the induced backend crash, but none appeared within 60s" + ) + + bundle_contents = run_ssh_command(host["ssh"], f"sudo cat {new_bundle}")["stdout"] + assert "gdb backtrace" in bundle_contents, ( + f"Expected the diagnostic bundle to contain a gdb backtrace section, got:\n" + f"{bundle_contents[:500]}" + ) + + remaining_cores = run_ssh_command( + host["ssh"], "sudo find /var/lib/systemd/coredump -maxdepth 1 -type f" + )["stdout"].strip() + assert remaining_cores == "", ( + f"Expected the raw core to be deleted after successful processing, but " + f"/var/lib/systemd/coredump still has:\n{remaining_cores}" + ) From 3b62d097d2ae9df826bdc4befaebdf2bf4d0b727 Mon Sep 17 00:00:00 2001 From: pgnickb Date: Thu, 17 Sep 2026 17:12:13 +0200 Subject: [PATCH 4/4] Attempt to fix the gid collision --- ansible/tasks/setup-postgres.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 12bc85a640..7c7bcd2f92 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -292,6 +292,26 @@ state: 'directory' become: true +# Pin the systemd-coredump group/user before installing the package: its +# postinst creates them with a dynamically-assigned system gid/uid (whatever +# is next free at that point), which is not reproducible across AMI variants +# that install a different set of packages before this point - it has +# collided with the vector group's own pinned gid (989) on some builds. +- name: add systemd-coredump system group + ansible.builtin.group: + name: systemd-coredump + gid: 986 + system: yes + +- name: add systemd-coredump system user + ansible.builtin.user: + name: systemd-coredump + uid: 986 + group: systemd-coredump + system: yes + create_home: false + shell: /usr/sbin/nologin + # Required for this (plus CoredumpFilter= in postgresql.service and the # storage limits below) to do anything at all: without systemd-coredump, # core_pattern stays at the kernel's plain "core" default and none of this