From c9716ed029bf8f5f59b97959655109285d0bdc99 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Tue, 18 Aug 2026 07:49:11 -0700
Subject: [PATCH 01/18] fix(install): let the explorer install on a stack that
 has no coins yet

The explorer reports healthy only once it holds a DB pool, and its pools
come from the coin stacks. Installing every service puts the explorer in
the shared bucket, which runs BEFORE any coin exists, so the readiness
check demanded a reply the explorer could not give: it answered 503
degraded for ten seconds and the whole stack install failed with
"Couldn't install the explorer module". A first install of the platform
on a clean host was unsatisfiable by construction.

It never surfaced on a dev box or a CI venue because both already carry
coin DBs from an earlier install, and the hosted e2e workflow had never
been driven past its clone step, so nothing had booted a genuinely empty
host since the check was written.

ExplorerConnector gains probe(), splitting the single boolean into the two
facts that differ here: a 503 means the server ANSWERED and holds no pool,
while a refused or timed-out connection means nothing answered. The
install now accepts answering-with-no-pools only while no coin is
installed, and keeps the full health check the moment one is, so a
genuinely broken explorer on a populated host still fails. ping() is
unchanged for its other callers.

Adds 10 tests (6 connector, 4 install). Full CI green (1518, 73, 58).
---
 src/ExplorerConnector.js            | 31 +++++++++++---
 src/services/ExplorerService.js     | 23 +++++++++-
 test/unit/ExplorerConnector.test.js | 60 ++++++++++++++++++++++++++
 test/unit/ExplorerService.test.js   | 66 +++++++++++++++++++++++++++++
 4 files changed, 171 insertions(+), 9 deletions(-)

diff --git a/src/ExplorerConnector.js b/src/ExplorerConnector.js
index 4db7dc9..422f719 100644
--- a/src/ExplorerConnector.js
+++ b/src/ExplorerConnector.js
@@ -24,13 +24,22 @@ class ExplorerConnector {
         this.port = port
     }
 
-    async ping(){
+    // Two independent facts about the explorer, which ping() collapses into one.
+    //
+    // `answering` means the HTTP server accepted the request and the JSON-RPC
+    // handler replied AT ALL, a 503 included. `healthy` means that reply was a
+    // success result, which the explorer only gives once it holds at least one
+    // DB pool. The distinction exists because "up but holding no pools" is the
+    // CORRECT state of an explorer on a host with no coin stack yet, and the
+    // install path has to be able to tell it apart from a container that never
+    // started. See installExplorerModule.
+    async probe(){
         const data = {
             jsonrpc: '2.0',
             method: 'ping',
             id: 1
         }
-        
+
         var response = null
         try {
             // Bounded timeout (matches HubConnector): without it, an explorer that
@@ -38,16 +47,24 @@ class ExplorerConnector {
             // flow forever, since installExplorerModule awaits this ping.
             response = await axios.post(this.url, data, { timeout: 5000 })
         } catch (err) {
+            // An HTTP status means the server answered and axios rejected on the
+            // CODE; no status means nothing answered (refused, reset, timed out).
+            if (err.response && err.response.status) {
+                return { answering: true, healthy: false }
+            }
             console.error('ExplorerConnector: failed to check explorer connectivity:', err.message);
-            return false
+            return { answering: false, healthy: false }
         }
 
-        if (response.data && response.data.result) {
-            return true;
-        } else {
-            return false
+        return {
+            answering: true,
+            healthy: !!(response.data && response.data.result)
         }
     }
+
+    async ping(){
+        return (await this.probe()).healthy
+    }
 }
 
 module.exports = ExplorerConnector
\ No newline at end of file
diff --git a/src/services/ExplorerService.js b/src/services/ExplorerService.js
index 762608c..e47f11f 100644
--- a/src/services/ExplorerService.js
+++ b/src/services/ExplorerService.js
@@ -130,10 +130,25 @@ async function installExplorerModule(force = false, branch = null) {
     await getStatus(null, null, false)
     console.log("Waiting for the xchain-explorer to respond")
 
+    // A healthy explorer is one holding at least one DB pool, and its pools come
+    // from the COIN stacks. So on a host with no coin installed yet there is no
+    // reply that can satisfy a health check, and demanding one made the first
+    // install of a stack unsatisfiable by construction: `install <ref> all` puts
+    // the explorer in the shared ("",  "") bucket, which runs BEFORE any coin
+    // exists, so the explorer answered 503 degraded for ten seconds and the whole
+    // install failed. Measured on a clean hosted runner 2026-08-18; it never
+    // surfaced on a dev box or a CI venue because both already carry coin DBs
+    // from an earlier install.
+    //
+    // So the bar is "answering" when there is no coin to serve, and stays the
+    // full health check the moment there is one: with a coin installed, an
+    // explorer holding no pools is a real fault and must still fail the install.
+    const coinsPresent = Object.keys(await getInstalledCoinsAndNetworks()).length > 0
+
     let tries = 10
     while (tries > 0) {
-        const ping = await explorerConnector.ping()
-        if (ping) {
+        const { answering, healthy } = await explorerConnector.probe()
+        if (healthy || (answering && !coinsPresent)) {
             try {
                 await updateExplorer()
             } catch {
@@ -141,6 +156,10 @@ async function installExplorerModule(force = false, branch = null) {
                 await sleep(1000)
                 continue
             }
+            if (!healthy) {
+                console.log("xchain-explorer is up with no coin data to serve yet;" +
+                    " it starts serving as each coin stack is installed.")
+            }
             return true
         } else {
             await sleep(1000)
diff --git a/test/unit/ExplorerConnector.test.js b/test/unit/ExplorerConnector.test.js
index 50bbeea..4363310 100644
--- a/test/unit/ExplorerConnector.test.js
+++ b/test/unit/ExplorerConnector.test.js
@@ -85,3 +85,63 @@ describe('ExplorerConnector', function () {
         })
     })
 })
+
+// probe() splits the single boolean ping() reports into the two facts the install
+// path needs. The explorer answers 503 whenever it holds no DB pool, which is the
+// CORRECT state on a host whose coin stacks are not installed yet, and a 503 is an
+// answer while a refused connection is not. Collapsing both to false made the
+// first install of a stack unsatisfiable: the explorer is installed before any
+// coin, so it could never report healthy and the install failed after ten seconds.
+describe('ExplorerConnector.probe()', function () {
+
+    const rpcOk = { data: { jsonrpc: '2.0', result: { status: 'success', db: true }, id: 1 } }
+
+    function connectorWith(axiosStub) {
+        const ExplorerConnector = loadConnector(axiosStub)
+        return new ExplorerConnector('localhost', 18080)
+    }
+
+    it('a success result is answering AND healthy', async function () {
+        const axiosStub = makeAxiosStub()
+        axiosStub.post.resolves(rpcOk)
+        expect(await connectorWith(axiosStub).probe()).to.deep.equal({ answering: true, healthy: true })
+    })
+
+    it('a 503 is ANSWERING but not healthy: the server replied, it just holds no pool', async function () {
+        const axiosStub = makeAxiosStub()
+        const err = new Error('Request failed with status code 503')
+        err.response = { status: 503, data: { result: undefined } }
+        axiosStub.post.rejects(err)
+        expect(await connectorWith(axiosStub).probe()).to.deep.equal({ answering: true, healthy: false })
+    })
+
+    it('a refused connection is neither: nothing answered', async function () {
+        const axiosStub = makeAxiosStub()
+        axiosStub.post.rejects(new Error('connect ECONNREFUSED 127.0.0.1:18080'))
+        expect(await connectorWith(axiosStub).probe()).to.deep.equal({ answering: false, healthy: false })
+    })
+
+    it('a timeout is neither, so a hung explorer cannot pass as installed', async function () {
+        const axiosStub = makeAxiosStub()
+        axiosStub.post.rejects(new Error('timeout of 5000ms exceeded'))
+        expect(await connectorWith(axiosStub).probe()).to.deep.equal({ answering: false, healthy: false })
+    })
+
+    it('a 200 carrying no result is answering but not healthy', async function () {
+        const axiosStub = makeAxiosStub()
+        axiosStub.post.resolves({ data: { jsonrpc: '2.0', error: { message: 'nope' }, id: 1 } })
+        expect(await connectorWith(axiosStub).probe()).to.deep.equal({ answering: true, healthy: false })
+    })
+
+    it('ping() keeps its old contract, reporting probe()s health half only', async function () {
+        const okStub = makeAxiosStub()
+        okStub.post.resolves(rpcOk)
+        expect(await connectorWith(okStub).ping()).to.be.true
+
+        const degraded = makeAxiosStub()
+        const err = new Error('Request failed with status code 503')
+        err.response = { status: 503 }
+        degraded.post.rejects(err)
+        expect(await connectorWith(degraded).ping()).to.be.false
+    })
+})
diff --git a/test/unit/ExplorerService.test.js b/test/unit/ExplorerService.test.js
index d0cbe2b..4726a2c 100644
--- a/test/unit/ExplorerService.test.js
+++ b/test/unit/ExplorerService.test.js
@@ -39,6 +39,7 @@ function makeExplorerServiceStubs(overrides = {}) {
         cloneGit:         overrides.cloneGit         || sinon.stub().resolves(true),
         buildAndUp:       overrides.buildAndUp       || sinon.stub().resolves('c'.repeat(64)),
         explorerPing:     overrides.explorerPing     || sinon.stub().resolves(false),
+        explorerProbe:    overrides.explorerProbe    || null,
         // Default is the no-active-release answer the real service gives: the
         // caller's ref passes through unpinned.
         resolveComponentRef: overrides.resolveComponentRef
@@ -50,6 +51,15 @@ function loadExplorerService(stubs) {
     // Build a mock ExplorerConnector class so we can control ping()
     const MockExplorerConnector = sinon.stub()
     MockExplorerConnector.prototype.ping = stubs.explorerPing
+    // probe() is what the install path reads. Default it to the real class's own
+    // relationship between the two (a healthy explorer answers; an unhealthy one
+    // is assumed silent) so every pre-existing case keeps its meaning, and let a
+    // test override it to express the third state: answering but degraded.
+    MockExplorerConnector.prototype.probe = stubs.explorerProbe
+        || (async function () {
+            const healthy = await stubs.explorerPing()
+            return { answering: healthy, healthy }
+        })
 
     return proxyquire('../../src/services/ExplorerService', {
         '../config/constants': {
@@ -314,6 +324,58 @@ describe('ExplorerService: installExplorerModule() honours the install ref', fun
     })
 })
 
+// The explorer only reports healthy once it holds a DB pool, and its pools come
+// from the coin stacks. `install <ref> all` installs it in the shared bucket,
+// BEFORE any coin exists, so requiring health there made a first install
+// unsatisfiable: measured on a clean hosted runner, the explorer answered 503
+// degraded for ten seconds and the whole stack install failed. It never showed on
+// a dev box or CI venue, both of which already carry coin DBs.
+describe('ExplorerService: installExplorerModule() on a stack with no coins yet', function () {
+
+    it('accepts an answering-but-degraded explorer when no coin is installed', async function () {
+        const stubs = makeExplorerServiceStubs({
+            explorerProbe: sinon.stub().resolves({ answering: true, healthy: false }),
+            getInstalledCoinsAndNetworks: sinon.stub().resolves({})
+        })
+        const es = loadExplorerService(stubs)
+        expect(await es.installExplorerModule(false)).to.be.true
+        expect(stubs.buildAndUp.called).to.be.true
+    })
+
+    it('still REFUSES an answering-but-degraded explorer once a coin is installed', async function () {
+        // With a coin present the explorer should hold a pool for it, so degraded
+        // is a real fault and must not be waved through by the carve-out above.
+        const stubs = makeExplorerServiceStubs({
+            explorerProbe: sinon.stub().resolves({ answering: true, healthy: false }),
+            getInstalledCoinsAndNetworks: sinon.stub().resolves({ bitcoin: ['regtest'] })
+        })
+        const es = loadExplorerService(stubs)
+        let threw = null
+        try { await es.installExplorerModule(false) } catch (err) { threw = err }
+        expect(threw).to.match(/Couldn't install the explorer module/)
+    })
+
+    it('refuses a container that never answers at all, coins or not', async function () {
+        const stubs = makeExplorerServiceStubs({
+            explorerProbe: sinon.stub().resolves({ answering: false, healthy: false }),
+            getInstalledCoinsAndNetworks: sinon.stub().resolves({})
+        })
+        const es = loadExplorerService(stubs)
+        let threw = null
+        try { await es.installExplorerModule(false) } catch (err) { threw = err }
+        expect(threw).to.match(/Couldn't install the explorer module/)
+    })
+
+    it('a healthy explorer is accepted whether or not a coin is installed', async function () {
+        const stubs = makeExplorerServiceStubs({
+            explorerProbe: sinon.stub().resolves({ answering: true, healthy: true }),
+            getInstalledCoinsAndNetworks: sinon.stub().resolves({ bitcoin: ['regtest'] })
+        })
+        const es = loadExplorerService(stubs)
+        expect(await es.installExplorerModule(false)).to.be.true
+    })
+})
+
 describe('ExplorerService: installExplorerModule() force=true', function () {
 
     it('kills and removes existing container when force=true and container exists', async function () {
@@ -419,6 +481,10 @@ describe('ExplorerService: installExplorerModule() updateExplorer error in loop'
 
         const MockExplorerConnector = sinon.stub()
         MockExplorerConnector.prototype.ping = stubs.explorerPing
+        MockExplorerConnector.prototype.probe = async () => {
+            const healthy = await stubs.explorerPing()
+            return { answering: healthy, healthy }
+        }
 
         const es = proxyquire('../../src/services/ExplorerService', {
             '../config/constants': { EXPLORER_MODULE_NAME: 'xchain-explorer' },

From 76fb1b0761aebc92795dcef1e5d03d297827c093 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Tue, 18 Aug 2026 08:06:13 -0700
Subject: [PATCH 02/18] fix(install): return only once the explorer is actually
 serving the new coins

The explorer learns which coins exist by POLLING the hub, once a minute by
default. It is installed in the shared bucket, ahead of the coin stacks, so
after a fresh install it keeps answering 503 until a poll lands. Measured on
a clean host: the coin stack finished at 14:55:47 and a suite starting at
14:57:04 still got a degraded explorer.

The defect is that install returned success there. A caller that installs a
stack and then reads it inherits the race with nothing to warn it, which is
what broke the e2e gate: its suites start the instant install returns, and
its preflight ping got the 503.

installModules now converges the explorer before returning, and only when
the run actually installed a coin stack. It waits on an explorer that is
TALKING, since a degraded reply is a service mid-convergence; silence means
there is no explorer on this host to converge, and whether it came up at all
is installExplorerModule's question, already asked. Failing to converge
warns rather than fails: the stack is installed either way, and saying so is
better than pretending it is ready.

Adds 7 tests (4 on the wait, 3 on the install wiring). Full CI green
(1525, 73, 58).
---
 src/operations/moduleOperations.js | 16 ++++++++++
 src/services/ExplorerService.js    | 50 +++++++++++++++++++++++++++++-
 test/unit/ExplorerService.test.js  | 43 +++++++++++++++++++++++++
 test/unit/moduleOperations.test.js | 43 +++++++++++++++++++++++++
 4 files changed, 151 insertions(+), 1 deletion(-)

diff --git a/src/operations/moduleOperations.js b/src/operations/moduleOperations.js
index 5612f99..d38889c 100644
--- a/src/operations/moduleOperations.js
+++ b/src/operations/moduleOperations.js
@@ -101,6 +101,22 @@ async function installModules(servicesList, ref = null) {
                 .map(s => `${s.module} (${s.coin} ${s.network})`).join(', ')
                 + ' - already installed. Use `update` to rebuild.')
         }
+
+        // The explorer is installed in the shared bucket, which runs BEFORE the
+        // coin stacks, and it learns its coins by polling the hub. So a run that
+        // installed a coin leaves it serving 503 for up to a poll interval after
+        // this loop ends. Returning there hands every caller a stack that reports
+        // installed and answers nothing; the first one to be bitten was the e2e
+        // gate, whose suite starts the moment install returns.
+        const installedACoin = outcome.installed.some(i => i.coin && i.network)
+        if (installedACoin) {
+            const { waitForExplorerReady } = require('../services/ExplorerService')
+            if (!await waitForExplorerReady()) {
+                console.warn('install: the xchain-explorer is still not serving coin data.' +
+                    ' The stack is installed; the explorer either cannot reach the hub or the hub' +
+                    ' has no config for these coins yet. Check it before running anything that reads it.')
+            }
+        }
         return outcome
     })
 }
diff --git a/src/services/ExplorerService.js b/src/services/ExplorerService.js
index e47f11f..e48a64e 100644
--- a/src/services/ExplorerService.js
+++ b/src/services/ExplorerService.js
@@ -170,7 +170,55 @@ async function installExplorerModule(force = false, branch = null) {
     throw "Couldn't install the explorer module"
 }
 
+// Block until the explorer is actually serving, or the budget runs out.
+//
+// The explorer learns its coins by POLLING the hub (UPDATE_CONFIG_INTERVAL,
+// one minute by default), so an explorer that booted before the coin stacks
+// existed keeps answering 503 for up to a poll after they arrive. That gap is
+// invisible and it is long: measured on a clean host, the coin stack finished
+// at 14:55:47 and a suite that started at 14:57:04 still got a degraded
+// explorer, because the tick that would have fixed it had not landed.
+//
+// An install that returns while a service it just installed serves nothing is
+// reporting the wrong thing, and everything downstream inherits the race. This
+// converges it instead of leaving each caller to discover it.
+//
+// Returns true when healthy, false when the budget expired: a slow explorer is
+// not a reason to fail an install that otherwise succeeded, and the caller says
+// so out loud rather than pretending the stack is ready.
+// It waits only on an explorer that is TALKING. A degraded reply is a service
+// mid-convergence and worth the budget; silence is a host with no explorer to
+// converge (a coin-only install, a stack that never installed one), and grinding
+// the full budget against it would add minutes to every such run for nothing.
+// Whether the explorer came up at all is installExplorerModule's question, asked
+// and answered before this is ever reached.
+async function waitForExplorerReady(timeoutMs = 150000, silenceGraceMs = 6000) {
+    const defaultConfig = await getDefaultConfig(EXPLORER_MODULE_NAME, null, null)
+    const explorerConnector = new ExplorerConnector(defaultConfig["EXPLORER_HOST"], defaultConfig["EXPLORER_PORT"])
+
+    const started = Date.now()
+    const deadline = started + timeoutMs
+    let announced = false
+    let everAnswered = false
+
+    while (Date.now() < deadline) {
+        const { answering, healthy } = await explorerConnector.probe()
+        if (healthy) return true
+        if (answering) everAnswered = true
+        if (!everAnswered && Date.now() - started >= silenceGraceMs) return true
+
+        if (answering && !announced) {
+            console.log("Waiting for the xchain-explorer to pick up the installed coins" +
+                " (it polls the hub for config, so this takes up to a poll interval)...")
+            announced = true
+        }
+        await sleep(2000)
+    }
+    return false
+}
+
 module.exports = {
     updateExplorer,
-    installExplorerModule
+    installExplorerModule,
+    waitForExplorerReady
 }
diff --git a/test/unit/ExplorerService.test.js b/test/unit/ExplorerService.test.js
index 4726a2c..f8fdb64 100644
--- a/test/unit/ExplorerService.test.js
+++ b/test/unit/ExplorerService.test.js
@@ -601,3 +601,46 @@ describe('ExplorerService: installExplorerModule() full happy path', function ()
         expect(stubs.buildAndUp.calledWith(EXPLORER_MODULE_NAME, null, null)).to.be.true
     })
 })
+
+// The wait exists because the explorer polls the hub for its coins, so a fresh
+// install returns while it is still answering 503. It must converge a service
+// that is talking, and must NOT burn its budget on a host where no explorer is
+// listening at all (a coin-only install), which is also what keeps it out of the
+// way of suites that run against a fully mocked stack.
+describe('ExplorerService: waitForExplorerReady()', function () {
+
+    it('returns true as soon as the explorer reports healthy', async function () {
+        const stubs = makeExplorerServiceStubs({
+            explorerProbe: sinon.stub().resolves({ answering: true, healthy: true })
+        })
+        const es = loadExplorerService(stubs)
+        expect(await es.waitForExplorerReady(10000)).to.be.true
+    })
+
+    it('keeps waiting through degraded replies, then succeeds when it converges', async function () {
+        const probe = sinon.stub()
+        probe.onCall(0).resolves({ answering: true, healthy: false })
+        probe.onCall(1).resolves({ answering: true, healthy: false })
+        probe.resolves({ answering: true, healthy: true })
+        const stubs = makeExplorerServiceStubs({ explorerProbe: probe })
+        const es = loadExplorerService(stubs)
+        expect(await es.waitForExplorerReady(10000)).to.be.true
+        expect(probe.callCount).to.be.greaterThan(2)
+    })
+
+    it('gives up early, reporting no problem, when nothing is listening at all', async function () {
+        // Silence is "no explorer on this host", not "an explorer converging".
+        // Grinding the full budget here would add minutes to every coin-only install.
+        const probe = sinon.stub().resolves({ answering: false, healthy: false })
+        const stubs = makeExplorerServiceStubs({ explorerProbe: probe, sleep: sinon.stub().resolves() })
+        const es = loadExplorerService(stubs)
+        expect(await es.waitForExplorerReady(150000, 0)).to.be.true
+    })
+
+    it('reports false when a talking explorer never converges inside the budget', async function () {
+        const probe = sinon.stub().resolves({ answering: true, healthy: false })
+        const stubs = makeExplorerServiceStubs({ explorerProbe: probe, sleep: sinon.stub().resolves() })
+        const es = loadExplorerService(stubs)
+        expect(await es.waitForExplorerReady(10)).to.be.false
+    })
+})
diff --git a/test/unit/moduleOperations.test.js b/test/unit/moduleOperations.test.js
index 7e18b38..596704b 100644
--- a/test/unit/moduleOperations.test.js
+++ b/test/unit/moduleOperations.test.js
@@ -20,6 +20,10 @@ const proxyquire = require('proxyquire').noCallThru()
 
 function makeStubs() {
     return {
+        // Converges by default: the tests that care about the wait assert on it
+        // directly, and every other install case would otherwise sit through a
+        // real poll loop.
+        waitForExplorerReady: sinon.stub().resolves(true),
         db: {
             getModuleContainer: sinon.stub().resolves('container-id-123'),
             removeModuleContainer: sinon.stub().resolves(true),
@@ -100,6 +104,9 @@ function loadOperations(stubs) {
             installModule: stubs.installModule,
             uninstallModule: stubs.uninstallModule
         },
+        '../services/ExplorerService': {
+            waitForExplorerReady: stubs.waitForExplorerReady
+        },
         '../services/SkewGuardService': {
             assertHubNotBehind: stubs.assertHubNotBehind
         },
@@ -141,6 +148,42 @@ describe('moduleOperations', function () {
             expect(stubs.createDockerNetwork.calledOnce).to.be.true
         })
 
+        // The explorer is installed in the shared bucket, ahead of the coin stacks,
+        // and learns its coins by polling the hub. Returning the moment the loop ends
+        // therefore hands the caller a stack that says installed and serves 503 for up
+        // to a poll interval; the e2e gate, which starts testing the instant install
+        // returns, was the first thing it broke.
+        it('waits for the explorer to serve once a coin was installed', async function () {
+            const stubs = makeStubs()
+            const ops = loadOperations(stubs)
+            await ops.installModules({ bitcoin: { regtest: ['xchain-indexer'] } })
+            expect(stubs.waitForExplorerReady.calledOnce).to.be.true
+        })
+
+        it('does not wait when the run installed no coin stack', async function () {
+            // A shared-only install (the ""/"" bucket) has nothing for the explorer
+            // to serve, so waiting would just burn the whole budget on every run.
+            const stubs = makeStubs()
+            const ops = loadOperations(stubs)
+            await ops.installModules({ '': { '': ['xchain-explorer'] } })
+            expect(stubs.waitForExplorerReady.called).to.be.false
+        })
+
+        it('warns but still succeeds when the explorer never converges', async function () {
+            const stubs = makeStubs()
+            stubs.waitForExplorerReady = sinon.stub().resolves(false)
+            const ops = loadOperations(stubs)
+            const warn = sinon.stub(console, 'warn')
+            let result
+            try {
+                result = await ops.installModules({ bitcoin: { regtest: ['xchain-indexer'] } })
+            } finally {
+                warn.restore()
+            }
+            expect(result.installed.length).to.equal(1)
+            expect(warn.args.some(a => /not serving coin data/.test(String(a[0])))).to.be.true
+        })
+
         it('builds database before installing modules', async function () {
             const stubs = makeStubs()
             const ops = loadOperations(stubs)

From cc6e94d6a553e7bba17786ae68ee333897d48041 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Tue, 18 Aug 2026 08:25:57 -0700
Subject: [PATCH 03/18] fix(install): tell the shared services about coins the
 same run just created

updateHub and updateExplorer push coin config to the hub and JOIN the hub
and explorer containers to each coin's docker network. They run in preCheck,
which fires BEFORE the action, so an install that creates brand-new coin
stacks finished without either shared service ever hearing about them.

The explorer is the visible casualty: installed in the shared bucket ahead
of the coin stacks, it ends up on no network from which the hub is
reachable, so it cannot read config, cannot populate a DB pool, and answers
503 forever. Measured on a clean host, it stayed degraded through a full
150-second readiness wait, which is what ruled out the poll-interval race
this was first taken for.

The new step runs from the install ACTION rather than inside installModules,
because it reconciles against live docker while installModules is also
driven directly by suites whose container registry is fixture data: run
there, the reconcile purged those rows and left the regression suite
asserting on an empty registry.

Nothing here fails the command. The modules are installed either way, and
the next command's preCheck runs the same two calls.

Adds 5 tests. Full CI green (1527, 73, 58).
---
 src/cli.js                         |   8 ++-
 src/operations/moduleOperations.js |  42 ++++++++---
 test/unit/moduleOperations.test.js | 111 +++++++++++++++++++----------
 3 files changed, 114 insertions(+), 47 deletions(-)

diff --git a/src/cli.js b/src/cli.js
index 766fc3d..dd5b25f 100644
--- a/src/cli.js
+++ b/src/cli.js
@@ -23,6 +23,7 @@ const { filterCommandParameters, resolveArgs } = require('./services/ConfigServi
 const { redactSecrets } = require('./utils/helpers')
 const {
     installModules,
+    syncSharedServicesAfterInstall,
     updateModules,
     recreateModules,
     uninstallModules,
@@ -207,7 +208,12 @@ async function parseCommand() {
             // make the documented default install a branch install forever.
             const resolved = resolveArgs([branch, service, chain, network], { expectBranch: true, defaultBranch: null })
             const serviceList = filterCommandParameters(null, resolved.service, resolved.chain, resolved.network)
-            await installModules(serviceList, resolved.branch)
+            const installed = await installModules(serviceList, resolved.branch)
+            // A coin installed by THIS run is unknown to the hub and explorer until
+            // something tells them, and the thing that does runs in preCheck, ahead
+            // of this action. Without it the command returns a stack whose explorer
+            // serves 503 to everything.
+            await syncSharedServicesAfterInstall(installed)
             return process.exit(0)
         })
 
diff --git a/src/operations/moduleOperations.js b/src/operations/moduleOperations.js
index d38889c..87c69c8 100644
--- a/src/operations/moduleOperations.js
+++ b/src/operations/moduleOperations.js
@@ -108,19 +108,42 @@ async function installModules(servicesList, ref = null) {
         // this loop ends. Returning there hands every caller a stack that reports
         // installed and answers nothing; the first one to be bitten was the e2e
         // gate, whose suite starts the moment install returns.
-        const installedACoin = outcome.installed.some(i => i.coin && i.network)
-        if (installedACoin) {
-            const { waitForExplorerReady } = require('../services/ExplorerService')
-            if (!await waitForExplorerReady()) {
-                console.warn('install: the xchain-explorer is still not serving coin data.' +
-                    ' The stack is installed; the explorer either cannot reach the hub or the hub' +
-                    ' has no config for these coins yet. Check it before running anything that reads it.')
-            }
-        }
         return outcome
     })
 }
 
+// Make the coins this run installed usable before the command returns.
+//
+// updateHub and updateExplorer push coin config to the hub and JOIN the hub and
+// explorer containers to each coin's docker network. They run in preCheck, which
+// fires BEFORE the action, so an install that creates brand-new coin stacks ends
+// without either shared service having heard about them: the explorer sits on no
+// network from which the hub is reachable, never populates a DB pool, and answers
+// 503 until some later command's preCheck happens to fix it. Measured on a clean
+// host, it stayed degraded through a full 150-second readiness wait.
+//
+// This is a COMMAND-level step, not part of the install primitive: it reconciles
+// against live docker, and installModules is also driven directly by suites whose
+// container registry is fixture data that such a reconcile would purge.
+//
+// Nothing here fails the command. The modules are installed either way, and the
+// next command's preCheck runs the same two calls.
+async function syncSharedServicesAfterInstall(outcome) {
+    if (!outcome || !outcome.installed.some(i => i.coin && i.network)) return
+
+    const { updateHub } = require('../services/HubService')
+    const { updateExplorer, waitForExplorerReady } = require('../services/ExplorerService')
+
+    try { await updateHub() }      catch (err) { console.warn('install: could not push config to the hub: ' + err) }
+    try { await updateExplorer() } catch (err) { console.warn('install: could not attach the explorer to the new coin networks: ' + err) }
+
+    if (!await waitForExplorerReady()) {
+        console.warn('install: the xchain-explorer is still not serving coin data.' +
+            ' The stack is installed; the explorer either cannot reach the hub or the hub' +
+            ' has no config for these coins yet. Check it before running anything that reads it.')
+    }
+}
+
 async function updateModules(servicesList, ref = null) {
     const { isReleaseRef } = require('../services/ReleaseManifestService')
 
@@ -888,6 +911,7 @@ async function resetModules(service, coin, network, force = false) {
 
 module.exports = {
     installModules,
+    syncSharedServicesAfterInstall,
     updateModules,
     recreateModules,
     uninstallModules,
diff --git a/test/unit/moduleOperations.test.js b/test/unit/moduleOperations.test.js
index 596704b..6d6d5ff 100644
--- a/test/unit/moduleOperations.test.js
+++ b/test/unit/moduleOperations.test.js
@@ -24,6 +24,8 @@ function makeStubs() {
         // directly, and every other install case would otherwise sit through a
         // real poll loop.
         waitForExplorerReady: sinon.stub().resolves(true),
+        updateExplorer: sinon.stub().resolves(true),
+        updateHub: sinon.stub().resolves(true),
         db: {
             getModuleContainer: sinon.stub().resolves('container-id-123'),
             removeModuleContainer: sinon.stub().resolves(true),
@@ -105,7 +107,11 @@ function loadOperations(stubs) {
             uninstallModule: stubs.uninstallModule
         },
         '../services/ExplorerService': {
-            waitForExplorerReady: stubs.waitForExplorerReady
+            waitForExplorerReady: stubs.waitForExplorerReady,
+            updateExplorer: stubs.updateExplorer
+        },
+        '../services/HubService': {
+            updateHub: stubs.updateHub
         },
         '../services/SkewGuardService': {
             assertHubNotBehind: stubs.assertHubNotBehind
@@ -148,42 +154,6 @@ describe('moduleOperations', function () {
             expect(stubs.createDockerNetwork.calledOnce).to.be.true
         })
 
-        // The explorer is installed in the shared bucket, ahead of the coin stacks,
-        // and learns its coins by polling the hub. Returning the moment the loop ends
-        // therefore hands the caller a stack that says installed and serves 503 for up
-        // to a poll interval; the e2e gate, which starts testing the instant install
-        // returns, was the first thing it broke.
-        it('waits for the explorer to serve once a coin was installed', async function () {
-            const stubs = makeStubs()
-            const ops = loadOperations(stubs)
-            await ops.installModules({ bitcoin: { regtest: ['xchain-indexer'] } })
-            expect(stubs.waitForExplorerReady.calledOnce).to.be.true
-        })
-
-        it('does not wait when the run installed no coin stack', async function () {
-            // A shared-only install (the ""/"" bucket) has nothing for the explorer
-            // to serve, so waiting would just burn the whole budget on every run.
-            const stubs = makeStubs()
-            const ops = loadOperations(stubs)
-            await ops.installModules({ '': { '': ['xchain-explorer'] } })
-            expect(stubs.waitForExplorerReady.called).to.be.false
-        })
-
-        it('warns but still succeeds when the explorer never converges', async function () {
-            const stubs = makeStubs()
-            stubs.waitForExplorerReady = sinon.stub().resolves(false)
-            const ops = loadOperations(stubs)
-            const warn = sinon.stub(console, 'warn')
-            let result
-            try {
-                result = await ops.installModules({ bitcoin: { regtest: ['xchain-indexer'] } })
-            } finally {
-                warn.restore()
-            }
-            expect(result.installed.length).to.equal(1)
-            expect(warn.args.some(a => /not serving coin data/.test(String(a[0])))).to.be.true
-        })
-
         it('builds database before installing modules', async function () {
             const stubs = makeStubs()
             const ops = loadOperations(stubs)
@@ -1297,3 +1267,70 @@ describe('moduleOperations', function () {
         })
     })
 })
+
+// A coin installed by THIS run is unknown to the hub and explorer until something
+// tells them, and the thing that does (preCheck) fires BEFORE the action. Left
+// unsynced, the explorer sits on no network from which the hub is reachable,
+// populates no DB pool, and answers 503 to everything: measured on a clean host it
+// stayed degraded through a full 150-second readiness wait, which is what ruled out
+// the poll-interval race this was first mistaken for.
+//
+// It lives beside installModules rather than inside it because it reconciles
+// against LIVE docker, and installModules is driven directly by suites whose
+// container registry is fixture data that such a reconcile purges.
+describe('moduleOperations: syncSharedServicesAfterInstall()', function () {
+
+    const coinInstalled = { installed: [{ module: 'xchain-indexer', coin: 'bitcoin', network: 'regtest' }], skipped: [] }
+    const sharedOnly    = { installed: [{ module: 'xchain-explorer', coin: '', network: '' }], skipped: [] }
+
+    it('pushes hub config, attaches the explorer, then waits for it to serve', async function () {
+        const stubs = makeStubs()
+        const ops = loadOperations(stubs)
+        await ops.syncSharedServicesAfterInstall(coinInstalled)
+        expect(stubs.updateHub.calledOnce).to.be.true
+        expect(stubs.updateExplorer.calledOnce).to.be.true
+        expect(stubs.updateExplorer.calledBefore(stubs.waitForExplorerReady)).to.be.true
+    })
+
+    it('does nothing when the run installed no coin stack', async function () {
+        // A shared-only install has no new network to join and nothing to serve.
+        const stubs = makeStubs()
+        const ops = loadOperations(stubs)
+        await ops.syncSharedServicesAfterInstall(sharedOnly)
+        expect(stubs.updateHub.called).to.be.false
+        expect(stubs.waitForExplorerReady.called).to.be.false
+    })
+
+    it('tolerates a missing outcome rather than throwing at the end of an install', async function () {
+        const stubs = makeStubs()
+        const ops = loadOperations(stubs)
+        await ops.syncSharedServicesAfterInstall(undefined)
+        expect(stubs.updateHub.called).to.be.false
+    })
+
+    it('still waits when the hub push fails, and never fails the command', async function () {
+        const stubs = makeStubs()
+        stubs.updateHub = sinon.stub().rejects(new Error('hub unreachable'))
+        const ops = loadOperations(stubs)
+        const warn = sinon.stub(console, 'warn')
+        try {
+            await ops.syncSharedServicesAfterInstall(coinInstalled)
+        } finally {
+            warn.restore()
+        }
+        expect(stubs.waitForExplorerReady.calledOnce).to.be.true
+    })
+
+    it('warns, without throwing, when the explorer never converges', async function () {
+        const stubs = makeStubs()
+        stubs.waitForExplorerReady = sinon.stub().resolves(false)
+        const ops = loadOperations(stubs)
+        const warn = sinon.stub(console, 'warn')
+        try {
+            await ops.syncSharedServicesAfterInstall(coinInstalled)
+        } finally {
+            warn.restore()
+        }
+        expect(warn.args.some(a => /not serving coin data/.test(String(a[0])))).to.be.true
+    })
+})

From 02e3373d48c6b779679e1c17e4031072f5aa40ef Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Tue, 18 Aug 2026 09:15:13 -0700
Subject: [PATCH 04/18] ci(e2e): capture the stack's own logs and topology when
 the job fails

A failure here carried off the runner only the e2e suite's log, which records
that a service answered wrong and never why. The service's own log and the
docker network topology are what hold the answer, and without them a remote
diagnosis is guesswork: the explorer refusing to serve on a clean runner cost
three wrong hypotheses, one fifteen-minute run at a time.

On failure only, dumps docker ps, the xchain networks and their members, each
container's network attachments, and the last 400 log lines per container into
the artifact that already uploads. Every command tolerates its own failure so
evidence-gathering can never mask the real error, and container inspection
takes network attachments rather than Config.Env, which carries credentials.
---
 .github/workflows/nightly-e2e.yml | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
index 360ffd2..5a265f6 100644
--- a/.github/workflows/nightly-e2e.yml
+++ b/.github/workflows/nightly-e2e.yml
@@ -196,6 +196,33 @@ jobs:
         # Live-stack per-service latency budgets (regression ceilings).
         run: node src/index.js e2etest "$COIN" --script test:perf:budget --ref "$STACK_REF"
 
+      # What the stack itself said, which is the one thing a failure here has never
+      # carried off the runner. The suite log records that a service answered
+      # wrong; only the service's own log and the network topology say why, and
+      # without them a remote diagnosis is guesswork - three wrong hypotheses in a
+      # row were paid for one 15-minute run at a time.
+      - name: Capture stack diagnostics
+        if: failure()
+        run: |
+          OUT="$XCHAIN_NODE_DATA_DIR/e2e-logs/diagnostics"
+          mkdir -p "$OUT"
+          # Never let a diagnostic failure mask the real one: this step is
+          # evidence-gathering, and the job is already failing.
+          docker ps -a                  > "$OUT/docker-ps.txt"        2>&1 || true
+          docker network ls             > "$OUT/docker-networks.txt"  2>&1 || true
+          for net in $(docker network ls --filter name=xchain --format '{{.Name}}'); do
+            echo "=== $net ==="        >> "$OUT/network-inspect.txt"
+            docker network inspect "$net" >> "$OUT/network-inspect.txt" 2>&1 || true
+          done
+          for c in $(docker ps -a --format '{{.Names}}'); do
+            echo "=== $c ==="          >> "$OUT/container-inspect.txt"
+            # Config.Env carries credentials; take only what identifies the
+            # container and how it is attached.
+            docker inspect "$c" --format '{{json .NetworkSettings.Networks}}' >> "$OUT/container-inspect.txt" 2>&1 || true
+            docker logs --tail 400 "$c" > "$OUT/log-$c.txt"           2>&1 || true
+          done
+          echo "captured $(ls -1 "$OUT" | wc -l) diagnostic files"
+
       - name: Upload e2e logs
         if: always()
         uses: actions/upload-artifact@v4

From 96010ee2568aac75f2cb6e04de7e152d7cc64a12 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Tue, 18 Aug 2026 11:36:54 -0700
Subject: [PATCH 05/18] fix(e2e): stage xchain-sync so the consensus hash
 drift-lock actually runs

The e2e image stages its siblings from LIBRARY_BUNDLES and the suites reach
them at ../../../xchain-<name>. xchain-sync was required by tests but never
staged, and the consequence is worse than a failure: consensusHashConformance
catches the missing require and SKIPS, so it reported green while never
running.

That suite is the only place sync's BlockHasher meets the indexer's committed
ledger, actions and contract hashes over real stack data. The unit goldens in
each repo lock their own serialization; only this one recomputes every indexed
block through the full pipeline and compares the two implementations. Drift
between them is exactly what it exists to redden, and on a consensus release
it had been silently absent.

Two tests pin the bundle list, since a sibling dropped from it costs a skipped
guard rather than a red one.
---
 src/config/constants.js            |  8 +++++++-
 test/unit/moduleOperations.test.js | 21 +++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/config/constants.js b/src/config/constants.js
index 7d803d0..42078b7 100644
--- a/src/config/constants.js
+++ b/src/config/constants.js
@@ -177,7 +177,13 @@ const LIBRARY_BUNDLES = {
     // attestationHelper and the integration/parity/regression suites can
     // resolve its consensus-critical primitives instead of dying with
     // MODULE_NOT_FOUND at load.
-    "xchain-e2e-test": ["xchain-hub", "xchain-sdk", "xchain-contracts", "xchain-indexer"]
+    // xchain-sync rides along for ONE suite that cannot be replaced by a unit
+    // golden: consensusHashConformance recomputes every indexed block's hashes with
+    // sync's BlockHasher and compares them to the indexer's committed values, which
+    // is the only place the two implementations meet over real stack data. Absent,
+    // it does not fail - it SKIPS, so the drift-lock reported green while never
+    // running (measured on the first hosted e2e run to reach the suites).
+    "xchain-e2e-test": ["xchain-hub", "xchain-sdk", "xchain-contracts", "xchain-indexer", "xchain-sync"]
 }
 
 // Single source of truth for the per-service Docker run-args and the
diff --git a/test/unit/moduleOperations.test.js b/test/unit/moduleOperations.test.js
index 6d6d5ff..359ad12 100644
--- a/test/unit/moduleOperations.test.js
+++ b/test/unit/moduleOperations.test.js
@@ -1334,3 +1334,24 @@ describe('moduleOperations: syncSharedServicesAfterInstall()', function () {
         expect(warn.args.some(a => /not serving coin data/.test(String(a[0])))).to.be.true
     })
 })
+
+// The e2e image stages its siblings from LIBRARY_BUNDLES, and the suites reach
+// them at ../../../xchain-<name>. A sibling that is required but not staged does
+// not redden: the suite that needs it SKIPS, which is indistinguishable from
+// green in the tally. consensusHashConformance is the one that matters most,
+// being the only place sync's BlockHasher meets the indexer's committed hashes
+// over real stack data, and it skipped silently until sync was added here.
+describe('constants: the e2e image stages every sibling its suites require', function () {
+
+    const { LIBRARY_BUNDLES } = require('../../src/config/constants')
+
+    it('bundles sync, so the consensus hash drift-lock can run instead of skipping', function () {
+        expect(LIBRARY_BUNDLES['xchain-e2e-test']).to.include('xchain-sync')
+    })
+
+    it('keeps the siblings the other suites resolve directly', function () {
+        for (const lib of ['xchain-hub', 'xchain-sdk', 'xchain-contracts', 'xchain-indexer']) {
+            expect(LIBRARY_BUNDLES['xchain-e2e-test']).to.include(lib)
+        }
+    })
+})

From 45d2eaf2b6b21551ec2ed312295f34c40abc8145 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Tue, 18 Aug 2026 12:10:36 -0700
Subject: [PATCH 06/18] fix(install): fail when the explorer never starts
 serving coin data

install already waited for the explorer and warned when it never converged,
then exited 0 anyway, so the stack reported success while serving 503 to
every read. Callers hit the failure at their first query instead of here,
which puts the error a long way from its cause. The command now exits
non-zero; XCHAIN_NODE_ALLOW_DEGRADED_EXPLORER=1 keeps the old behaviour for
install-then-repair flows.
---
 src/cli.js                         |  6 +++--
 src/operations/moduleOperations.js | 28 +++++++++++++++-----
 test/unit/moduleOperations.test.js | 42 ++++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/src/cli.js b/src/cli.js
index dd5b25f..2ec61ab 100644
--- a/src/cli.js
+++ b/src/cli.js
@@ -213,8 +213,10 @@ async function parseCommand() {
             // something tells them, and the thing that does runs in preCheck, ahead
             // of this action. Without it the command returns a stack whose explorer
             // serves 503 to everything.
-            await syncSharedServicesAfterInstall(installed)
-            return process.exit(0)
+            // Exit non-zero when the explorer never came up serving coins, so the
+            // caller stops here rather than at its first read of a 503 stack.
+            const usable = await syncSharedServicesAfterInstall(installed)
+            return process.exit(usable ? 0 : 1)
         })
 
     program
diff --git a/src/operations/moduleOperations.js b/src/operations/moduleOperations.js
index 87c69c8..1917dfd 100644
--- a/src/operations/moduleOperations.js
+++ b/src/operations/moduleOperations.js
@@ -126,10 +126,11 @@ async function installModules(servicesList, ref = null) {
 // against live docker, and installModules is also driven directly by suites whose
 // container registry is fixture data that such a reconcile would purge.
 //
-// Nothing here fails the command. The modules are installed either way, and the
-// next command's preCheck runs the same two calls.
+// Returns whether the stack is usable. The modules are installed either way, but
+// reporting success for a stack whose explorer serves 503 makes every later
+// failure land on the caller's first read instead of here.
 async function syncSharedServicesAfterInstall(outcome) {
-    if (!outcome || !outcome.installed.some(i => i.coin && i.network)) return
+    if (!outcome || !outcome.installed.some(i => i.coin && i.network)) return true
 
     const { updateHub } = require('../services/HubService')
     const { updateExplorer, waitForExplorerReady } = require('../services/ExplorerService')
@@ -137,11 +138,24 @@ async function syncSharedServicesAfterInstall(outcome) {
     try { await updateHub() }      catch (err) { console.warn('install: could not push config to the hub: ' + err) }
     try { await updateExplorer() } catch (err) { console.warn('install: could not attach the explorer to the new coin networks: ' + err) }
 
-    if (!await waitForExplorerReady()) {
-        console.warn('install: the xchain-explorer is still not serving coin data.' +
-            ' The stack is installed; the explorer either cannot reach the hub or the hub' +
-            ' has no config for these coins yet. Check it before running anything that reads it.')
+    if (await waitForExplorerReady()) return true
+
+    console.warn('install: the xchain-explorer is still not serving coin data.' +
+        ' The stack is installed; the explorer either cannot reach the hub or the hub' +
+        ' has no config for these coins yet. Check it before running anything that reads it.')
+
+    // Escape hatch for the install-then-fix flows: the modules ARE installed, so a
+    // caller that intends to repair the explorer by hand can still treat this as success.
+    if (allowDegradedExplorer()) {
+        console.warn('install: continuing anyway (XCHAIN_NODE_ALLOW_DEGRADED_EXPLORER is set).')
+        return true
     }
+    return false
+}
+
+// Opt-out for callers that knowingly accept a stack whose explorer serves no coins.
+function allowDegradedExplorer() {
+    return ['1', 'true', 'yes'].includes(String(process.env.XCHAIN_NODE_ALLOW_DEGRADED_EXPLORER || '').toLowerCase())
 }
 
 async function updateModules(servicesList, ref = null) {
diff --git a/test/unit/moduleOperations.test.js b/test/unit/moduleOperations.test.js
index 359ad12..ef1d842 100644
--- a/test/unit/moduleOperations.test.js
+++ b/test/unit/moduleOperations.test.js
@@ -1333,6 +1333,48 @@ describe('moduleOperations: syncSharedServicesAfterInstall()', function () {
         }
         expect(warn.args.some(a => /not serving coin data/.test(String(a[0])))).to.be.true
     })
+
+    it('reports the stack usable when the explorer converges', async function () {
+        const stubs = makeStubs()
+        const ops = loadOperations(stubs)
+        expect(await ops.syncSharedServicesAfterInstall(coinInstalled)).to.be.true
+    })
+
+    it('reports the stack UNUSABLE when the explorer never converges', async function () {
+        // The caller exits non-zero on this, so a gate stops at the boot step
+        // instead of at its first read of a 503 explorer.
+        const stubs = makeStubs()
+        stubs.waitForExplorerReady = sinon.stub().resolves(false)
+        const ops = loadOperations(stubs)
+        const warn = sinon.stub(console, 'warn')
+        try {
+            expect(await ops.syncSharedServicesAfterInstall(coinInstalled)).to.be.false
+        } finally {
+            warn.restore()
+        }
+    })
+
+    it('honours XCHAIN_NODE_ALLOW_DEGRADED_EXPLORER for install-then-fix flows', async function () {
+        const stubs = makeStubs()
+        stubs.waitForExplorerReady = sinon.stub().resolves(false)
+        const ops = loadOperations(stubs)
+        const warn = sinon.stub(console, 'warn')
+        const prior = process.env.XCHAIN_NODE_ALLOW_DEGRADED_EXPLORER
+        process.env.XCHAIN_NODE_ALLOW_DEGRADED_EXPLORER = '1'
+        try {
+            expect(await ops.syncSharedServicesAfterInstall(coinInstalled)).to.be.true
+        } finally {
+            warn.restore()
+            if (prior === undefined) delete process.env.XCHAIN_NODE_ALLOW_DEGRADED_EXPLORER
+            else process.env.XCHAIN_NODE_ALLOW_DEGRADED_EXPLORER = prior
+        }
+    })
+
+    it('reports usable when the run installed no coin stack', async function () {
+        const stubs = makeStubs()
+        const ops = loadOperations(stubs)
+        expect(await ops.syncSharedServicesAfterInstall(sharedOnly)).to.be.true
+    })
 })
 
 // The e2e image stages its siblings from LIBRARY_BUNDLES, and the suites reach

From 7aae55555f2c6f49f706c00e1e34f3fe773c50da Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Tue, 18 Aug 2026 12:10:42 -0700
Subject: [PATCH 07/18] ci(e2e): capture what the hub serves the explorer on a
 failed run

The explorer answering 503 with no DB pools is the failure this job keeps
hitting, and a container log alone cannot say whether the hub ever served it
coin config. Records the explorer ping, its hub-pointing env, and the key
structure of the hub config response. That response is in the hub sensitive
-read tier and carries DB credentials, so only key names are written and the
raw body is kept out of the uploaded directory.
---
 .github/workflows/nightly-e2e.yml | 43 +++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
index 5a265f6..5ef6344 100644
--- a/.github/workflows/nightly-e2e.yml
+++ b/.github/workflows/nightly-e2e.yml
@@ -221,6 +221,49 @@ jobs:
             docker inspect "$c" --format '{{json .NetworkSettings.Networks}}' >> "$OUT/container-inspect.txt" 2>&1 || true
             docker logs --tail 400 "$c" > "$OUT/log-$c.txt"           2>&1 || true
           done
+
+          # The explorer answering 503 with zero DB pools is the failure this job
+          # keeps hitting, and the container log alone cannot say whether the hub
+          # ever served it coin config. Capture both sides of that exchange.
+          curl -s -m 10 -X POST -H 'Content-Type: application/json' \
+            -d '{"jsonrpc":"2.0","method":"ping","params":{},"id":1}' \
+            http://127.0.0.1:18080/ > "$OUT/explorer-ping.json" 2>&1 || true
+
+          # getallconfigs is in the hub's sensitive-read tier: its response carries DB
+          # credentials. Record the KEY STRUCTURE only, never a value, and keep the raw
+          # body outside the uploaded directory so it cannot reach the artifact.
+          RAW="$RUNNER_TEMP/hub-config-raw.json"
+          curl -s -m 10 -X POST -H 'Content-Type: application/json' \
+            -d '{"jsonrpc":"2.0","method":"getallconfigs","params":{"since_updated_at":0},"id":1}' \
+            http://127.0.0.1:10000/ > "$RAW" 2>&1 || true
+          RAW="$RAW" node -e '
+            const fs = require("fs");
+            let parsed;
+            try { parsed = JSON.parse(fs.readFileSync(process.env.RAW, "utf8")); }
+            catch (e) { console.log("hub response unparseable or empty"); process.exit(0); }
+            const result = (parsed && parsed.result) || {};
+            const tree   = result.configs || result;
+            const shape  = {};
+            for (const coin of Object.keys(tree || {})) {
+              shape[coin] = {};
+              for (const net of Object.keys(tree[coin] || {})) {
+                shape[coin][net] = {};
+                for (const mod of Object.keys(tree[coin][net] || {}))
+                  shape[coin][net][mod] = Object.keys(tree[coin][net][mod] || {});
+              }
+            }
+            console.log("seq=" + result.seq + " watermark=" + result.watermark);
+            console.log("coins served: " + (Object.keys(shape).join(", ") || "(none)"));
+            console.log(JSON.stringify(shape, null, 2));
+          ' > "$OUT/hub-config-shape.txt" 2>&1 || true
+          rm -f "$RAW"
+
+          # Allowlisted so no credential-bearing var can be swept in.
+          docker inspect xchain-node-xchain-explorer \
+            --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null \
+            | grep -E '^(HUB_API_HOST|HUB_PORT|HUB_VALIDATORS|NO_HUB|UPDATE_CONFIG_INTERVAL)=' \
+            > "$OUT/explorer-hub-env.txt" 2>&1 || true
+
           echo "captured $(ls -1 "$OUT" | wc -l) diagnostic files"
 
       - name: Upload e2e logs

From 067eb2438795addb0631eb8b6a24feb90342a561 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Wed, 19 Aug 2026 07:15:01 -0700
Subject: [PATCH 08/18] ci(e2e): keep enough container log to diagnose an early
 suite

400 lines covers a boot failure, which is what the limit was chosen for, but
a two-hour suite scrolls far past it. A defect in an early suite has already
been discarded by the time the job fails, so the one log that would explain
it arrives empty.

Hit while diagnosing an attestation execution that confirms on-chain and
never gets an execution row: the indexer's own account of that action was
gone, and the file's own note already said to raise this when early-run
evidence is needed.
---
 .github/workflows/nightly-e2e.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
index 5ef6344..a8e7a04 100644
--- a/.github/workflows/nightly-e2e.yml
+++ b/.github/workflows/nightly-e2e.yml
@@ -219,7 +219,10 @@ jobs:
             # Config.Env carries credentials; take only what identifies the
             # container and how it is attached.
             docker inspect "$c" --format '{{json .NetworkSettings.Networks}}' >> "$OUT/container-inspect.txt" 2>&1 || true
-            docker logs --tail 400 "$c" > "$OUT/log-$c.txt"           2>&1 || true
+            # 400 lines covers a boot failure but not a 2-hour suite: a defect in an
+            # EARLY suite has already scrolled away by the time the job fails, which is
+            # what made the attestation execution-row failure undiagnosable off-runner.
+            docker logs --tail 20000 "$c" > "$OUT/log-$c.txt"         2>&1 || true
           done
 
           # The explorer answering 503 with zero DB pools is the failure this job

From 4885cf451f088050d1ce621ec08490957a2d5719 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Wed, 19 Aug 2026 07:20:26 -0700
Subject: [PATCH 09/18] ci(e2e): capture whole container logs, on every run

The diagnostics step kept the last 400 lines of each container and ran only
on failure. Both discard evidence that cannot be collected again, because the
runner is destroyed with the job.

Any line cap is a bet on where the next defect lands, and this one lost: an
attestation execution that confirms on-chain and never gets an execution row
was undiagnosable off-runner, because the indexer's account of it had
scrolled away hours before the job failed. The whole suite log for a
two-hour run is under a megabyte, so keeping everything is affordable.

Capturing on success as well gives a healthy baseline to read a later
failure against, which is the comparison that was missing here. Artifacts are
per-run, so no run's evidence overwrites another's.
---
 .github/workflows/nightly-e2e.yml | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
index a8e7a04..2acf284 100644
--- a/.github/workflows/nightly-e2e.yml
+++ b/.github/workflows/nightly-e2e.yml
@@ -202,7 +202,10 @@ jobs:
       # without them a remote diagnosis is guesswork - three wrong hypotheses in a
       # row were paid for one 15-minute run at a time.
       - name: Capture stack diagnostics
-        if: failure()
+        # Runs on every outcome, not just failure. A passing run is the baseline a
+        # later failure is read against, and there is no second chance to collect it:
+        # the runner is destroyed with the job.
+        if: always()
         run: |
           OUT="$XCHAIN_NODE_DATA_DIR/e2e-logs/diagnostics"
           mkdir -p "$OUT"
@@ -219,10 +222,11 @@ jobs:
             # Config.Env carries credentials; take only what identifies the
             # container and how it is attached.
             docker inspect "$c" --format '{{json .NetworkSettings.Networks}}' >> "$OUT/container-inspect.txt" 2>&1 || true
-            # 400 lines covers a boot failure but not a 2-hour suite: a defect in an
-            # EARLY suite has already scrolled away by the time the job fails, which is
-            # what made the attestation execution-row failure undiagnosable off-runner.
-            docker logs --tail 20000 "$c" > "$OUT/log-$c.txt"         2>&1 || true
+            # NO --tail. Any cap is a bet on where the next defect lands, and the
+            # previous 400 lost an early-suite failure in a 2-hour run. The whole suite
+            # log for such a run is under a megabyte, so completeness is affordable and
+            # the artifact is per-run, so nothing here overwrites a prior run's evidence.
+            docker logs "$c"              > "$OUT/log-$c.txt"         2>&1 || true
           done
 
           # The explorer answering 503 with zero DB pools is the failure this job

From 6bdcfab97a367b074f1f114d2a7a4dcb518733b8 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Wed, 19 Aug 2026 07:31:04 -0700
Subject: [PATCH 10/18] ci(e2e): allow a single action suite per run

Fixing one defect cost a full two-hour pass, nearly all of it re-running
suites that already passed, and the cycle repeated for every fix. The runner
already accepts a suite name; this exposes it, so a fix costs a boot plus the
one suite it touches.

The security and performance suites stay skipped while a subset runs. They
grade the whole stack, and a partial action pass has not earned them.

Empty runs everything, which is what a release gate must do. A subset proves
a fix; only a full pass proves the train.
---
 .github/workflows/nightly-e2e.yml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
index 2acf284..5d4c258 100644
--- a/.github/workflows/nightly-e2e.yml
+++ b/.github/workflows/nightly-e2e.yml
@@ -46,6 +46,16 @@ on:
         type: choice
         options: [bitcoin, litecoin, dogecoin]
         default: bitcoin
+      suite:
+        # ONE suite instead of the whole action set, so fixing a defect costs a boot
+        # plus that suite rather than a two-hour full pass. Takes the file-name stem
+        # e2etest already accepts (e.g. "attestation", "order"). Empty runs everything,
+        # which is what a release gate must do; a subset proves a fix, never the train.
+        # The security and performance suites are skipped while a subset is running,
+        # because they grade the whole stack and a partial action pass does not earn them.
+        description: 'Single action suite to run (file stem, e.g. attestation). Empty = full suite.'
+        type: string
+        default: ''
       ref:
         # The release ceremony's freeze step needs a driven e2e run against the
         # EXACT content a release is being cut from, not against a branch tip
@@ -87,6 +97,7 @@ jobs:
       # stack it installs. Splitting those two was how "we tested the release"
       # could mean "we tested master's installer against the release".
       STACK_REF: ${{ github.event.inputs.ref || 'develop' }}
+      SUITE: ${{ github.event.inputs.suite || '' }}
       # Headless DB: point xchain-node at an external MariaDB instead of its
       # bundled DB container, so the first install does NOT stop on the
       # interactive root-password prompt (see DatabaseService.getExternalDbConfig
@@ -185,14 +196,16 @@ jobs:
       - name: Run the e2e action suite
         # Now exits non-zero on failure (cli.js e2etest propagates the suite's
         # exit code), so this step natively gates the job.
-        run: node src/index.js e2etest "$COIN" --ref "$STACK_REF"
+        run: node src/index.js e2etest "$COIN" $SUITE --ref "$STACK_REF"
 
       - name: Run the e2e Security suite (test/security)
+        if: env.SUITE == ''
         # Stack-dependent on-chain security suite (VM sandbox-escape / gas-bomb /
         # deploy-reject). Driven via the e2etest --script option.
         run: node src/index.js e2etest "$COIN" --script test:security --ref "$STACK_REF"
 
       - name: Run the e2e Performance suite (test/perf)
+        if: env.SUITE == ''
         # Live-stack per-service latency budgets (regression ceilings).
         run: node src/index.js e2etest "$COIN" --script test:perf:budget --ref "$STACK_REF"
 

From a45626ae864a48121d56a8fa0f30d6b62c77fc06 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Wed, 19 Aug 2026 13:24:31 -0700
Subject: [PATCH 11/18] feat(config): give the e2e container the validator
 pubkey it was told to guess

The validator-onboarding suite STAKEs the hub's signing pubkey and asserts
the indexer then admits it to each capability set, so it has to know which
key the hub actually runs as. It read VALIDATOR_PUBKEY from the env and
skipped when unset, and the only way to set it was for an operator to
hand-copy the hex out of `validator status` into the coin config - so it
skipped everywhere nobody had, CI included, while reporting as a pending
test rather than as missing coverage.

Derive it from the same settings file the hub's own env is built from, so
the two can never name different keys. Public half only: the seed stays in
signing.key and goes to the hub alone. A standalone node has no validator,
so the var is absent and the suite still skips - correctly, because there
is no identity to onboard.
---
 src/services/ConfigService.js   | 20 ++++++++++++++++
 test/unit/ConfigService.test.js | 42 +++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/src/services/ConfigService.js b/src/services/ConfigService.js
index d5cc200..79c4952 100644
--- a/src/services/ConfigService.js
+++ b/src/services/ConfigService.js
@@ -435,6 +435,26 @@ async function getDefaultConfig(module, coin, network) {
         if (module === XChainService.XCHAIN_E2E_TEST) {
             defaultValues["COIN"] = coin
             defaultValues["XCHAIN_CONTRACTS_DIR"] = "/XChainE2ETest/xchain-contracts"
+
+            // The validator-onboarding suite STAKEs the hub's own signing pubkey and
+            // asserts the indexer then admits it to each capability set, so it needs
+            // to know which key the hub actually runs as. It read VALIDATOR_PUBKEY
+            // from the env and skipped when unset, which meant the only way to run it
+            // was for an operator to hand-copy the hex out of `validator status` into
+            // the coin config - so it skipped everywhere nobody had, including CI.
+            // Derive it from the same settings file the hub's own env comes from
+            // (getValidatorEnv above), so the two can never name different keys.
+            //
+            // PUBLIC half only. The seed stays in signing.key / SIGNING_PRIVKEY_HEX
+            // and goes to the hub alone; the test needs the pubkey and nothing else.
+            //
+            // A standalone node has no validator, so this is absent and the suite
+            // still skips - correctly, because there is no identity to onboard.
+            const { getValidatorSettings } = require('./ValidatorService')
+            const validatorSettings = getValidatorSettings()
+            if (validatorSettings && validatorSettings.pubkey) {
+                defaultValues["VALIDATOR_PUBKEY"] = validatorSettings.pubkey
+            }
         }
 
         // Genesis-ledger bootstrap env (xchain-indexer only). The indexer binds its
diff --git a/test/unit/ConfigService.test.js b/test/unit/ConfigService.test.js
index 14b4bcc..e813666 100644
--- a/test/unit/ConfigService.test.js
+++ b/test/unit/ConfigService.test.js
@@ -336,6 +336,48 @@ describe('ConfigService', function () {
                 expect(config['NODE_PASSWORD']).to.not.equal('rpc')
             })
 
+            // The e2e-test container learns which key the hub runs as from the same
+            // settings ValidatorService hands the hub, rather than from a hex string an
+            // operator remembered to paste into the coin config.
+            function makeServiceWithValidator(validatorSettings) {
+                const fsStub = {
+                    createReadStream: sinon.stub().callsFake(() => streamFromString('')),
+                    existsSync: sinon.stub().returns(true),
+                    readFileSync: sinon.stub().returns(''),
+                    appendFileSync: sinon.stub(),
+                    writeFileSync: sinon.stub(),
+                    rmSync: sinon.stub(),
+                    mkdirSync: sinon.stub()
+                }
+                return proxyquire('../../src/services/ConfigService', {
+                    'fs': fsStub,
+                    './ValidatorService': {
+                        getValidatorSettings: () => validatorSettings,
+                        getValidatorEnv: () => ({})
+                    }
+                })
+            }
+
+            it('passes the validator pubkey to the e2e-test container when one is configured', async function () {
+                const pubkey = 'ab'.repeat(32)
+                const cs = makeServiceWithValidator({ enabled: true, pubkey })
+                const config = await cs.getDefaultConfig(XChainService.XCHAIN_E2E_TEST, 'bitcoin', 'regtest')
+                expect(config['VALIDATOR_PUBKEY']).to.equal(pubkey)
+            })
+
+            it('never hands the e2e-test container the signing seed, only the public half', async function () {
+                const cs = makeServiceWithValidator({ enabled: true, pubkey: 'ab'.repeat(32), seedHex: 'cd'.repeat(32) })
+                const config = await cs.getDefaultConfig(XChainService.XCHAIN_E2E_TEST, 'bitcoin', 'regtest')
+                expect(config).to.not.have.property('SIGNING_PRIVKEY_HEX')
+                expect(JSON.stringify(config)).to.not.include('cd'.repeat(32))
+            })
+
+            it('omits VALIDATOR_PUBKEY on a standalone node, so the onboarding suite skips rather than staking a key nothing runs as', async function () {
+                const cs = makeServiceWithValidator(null)
+                const config = await cs.getDefaultConfig(XChainService.XCHAIN_E2E_TEST, 'bitcoin', 'regtest')
+                expect(config).to.not.have.property('VALIDATOR_PUBKEY')
+            })
+
             // A memory-backed fs so generate -> persist -> read-back is observable across
             // calls (the default makeServiceWithConfig stub no-ops writes). Keyed by the
             // exact paths ConfigService resolves: config/<coin>-<network>, its .local

From 9f174e3b28b5d8dd906af1638800b055a831af5f Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Wed, 19 Aug 2026 13:24:41 -0700
Subject: [PATCH 12/18] ci(e2e): opt-in validator mode, so the onboarding suite
 has a hub to onboard

The hosted stack's hub has never been a validator: p2pConfig is built only
when P2P_VALIDATOR_ADDR is set, that comes from the validator settings, and
initValidator is reachable from exactly one place - the explicit CLI
subcommand - which the install path the workflow runs never calls. So the
onboarding suite skipped on every hosted run for want of an identity.

Opt-in, default false, rather than always-on. Validator mode is not a
superset of standalone: peerManager exists, so startOracle() proceeds and
the hub begins finalizing its own price rounds, and several action suites
branch on precisely that (the fee fixtures size an output from a pair they
seed, and skip when the venue publishes its own XCHAIN/USD). Turning it on
for the whole matrix would quietly change what the gate measures, and a
single-suite probe of the onboarding test could not have shown it.

The step runs before install because ConfigService renders both the hub env
and the e2e container's VALIDATOR_PUBKEY at install time. It exports the two
vars a validator hub fails loud on and the standalone path never needs:
HUB_NETWORK, and ORACLE_MIN_SUBMISSIONS=1 because a lone validator can never
reach the default two-hub diversity floor, so no round would finalize and
every indexer's price-sync barrier would stall.
---
 .github/workflows/nightly-e2e.yml | 50 +++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
index 5d4c258..a1d80c5 100644
--- a/.github/workflows/nightly-e2e.yml
+++ b/.github/workflows/nightly-e2e.yml
@@ -56,6 +56,24 @@ on:
         description: 'Single action suite to run (file stem, e.g. attestation). Empty = full suite.'
         type: string
         default: ''
+      validator:
+        # Boot the stack's hub as a full VALIDATOR rather than the standalone
+        # config oracle it runs as by default.
+        #
+        # Opt-in, and default false on purpose. Validator mode is not a superset
+        # of standalone: peerManager exists, so startOracle() proceeds and the hub
+        # begins finalizing its own price rounds. Several action suites branch on
+        # exactly that (the fee fixtures compute an output size FROM a pair they
+        # seed, and skip when the venue publishes its own XCHAIN/USD), so flipping
+        # this for the whole matrix would silently change what the gate measures.
+        #
+        # Turn it on for the validator-onboarding suite, which STAKEs the hub's
+        # signing pubkey and asserts the indexer admits it - the one suite that
+        # cannot run against a hub with no identity, and which skips itself when
+        # VALIDATOR_PUBKEY is absent.
+        description: 'Run the hub as a validator (needed by the onboardValidator suite; changes the price regime)'
+        type: boolean
+        default: false
       ref:
         # The release ceremony's freeze step needs a driven e2e run against the
         # EXACT content a release is being cut from, not against a branch tip
@@ -181,6 +199,38 @@ jobs:
             git config --global url."https://github.com/".insteadOf "git@github.com:"
           fi
 
+      - name: Initialize the validator identity (opt-in; must precede install)
+        if: github.event.inputs.validator == 'true'
+        # MUST run before install: ConfigService reads the settings this writes
+        # when it generates the hub's container env (getValidatorEnv) and the
+        # e2e container's VALIDATOR_PUBKEY. Initializing afterwards would leave
+        # both already rendered from a standalone config.
+        #
+        # The hub fails loud in validator mode on two vars the standalone path
+        # never needs, and both are host-env passthroughs rather than anything
+        # `validator init` writes, so they are exported here for every later step:
+        #   HUB_NETWORK           consensus gating (STAKE_WEIGHTED_QUORUM height
+        #                         is per network); hub exits 1 when blank/invalid
+        #   ORACLE_MIN_SUBMISSIONS  a lone validator can never reach the default
+        #                         2-hub diversity floor, so no round would ever
+        #                         finalize and every indexer's price-sync barrier
+        #                         would stall
+        # ORACLE_EPOCH_START is also required, but `validator init` persists it
+        # into validator.json and getValidatorEnv injects it, so it is passed as
+        # a flag rather than exported. Its value only has to be shared across a
+        # federation; this venue is a single hub, so the run's own clock is fine.
+        run: |
+          node src/index.js validator init \
+            --oracle-epoch-start "$(date -u +%s)000" \
+            --capabilities price,cross_chain,oracle_publish,attestation
+          {
+            echo "HUB_NETWORK=regtest"
+            echo "ORACLE_MIN_SUBMISSIONS=1"
+          } >> "$GITHUB_ENV"
+          # Prints the pubkey and never the seed (validator status reads the
+          # settings file; the 0600 signing.key is not echoed).
+          node src/index.js validator status
+
       - name: Boot the regtest stack (clones every service at ${{ github.event.inputs.ref || 'develop' }})
         # First heavy step - repo clones + docker image builds + coin daemon.
         # Watch disk and image-build time here (the 120-min job timeout covers it).

From 58173e813f5d06778ab49e3d2a9565171987b6d2 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Wed, 19 Aug 2026 13:32:35 -0700
Subject: [PATCH 13/18] ci(e2e): raise the job budget so the suites behind the
 action pass can run

A BTC full action suite alone runs ~1h50m, and the security and performance
suites are sequenced after it, so at 120 minutes they shared whatever the
action suite left - in practice nothing. That is why they had never executed
once: the binding constraint was the job budget, not the tests, and no amount
of fixing the action suite could have reached them.

360 is the platform ceiling for a hosted job, so this imposes no limit the
runner would not anyway. Deliberate: those two suites have no measured
duration to size headroom against, and guessing low would re-create the same
invisible truncation one tier further along. The tradeoff is that a wedged run
takes the full six hours to report; the answer to that is a per-step timeout
on the suite steps, not a lower job budget that also caps a long green pass.
---
 .github/workflows/nightly-e2e.yml | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
index a1d80c5..523d0fd 100644
--- a/.github/workflows/nightly-e2e.yml
+++ b/.github/workflows/nightly-e2e.yml
@@ -107,7 +107,24 @@ permissions:
 jobs:
   e2e:
     runs-on: ubuntu-latest
-    timeout-minutes: 120
+    # A BTC full action suite alone runs ~1h50m of wall clock, and the security
+    # and performance suites are sequenced AFTER it, so at 120 the two of them
+    # shared whatever minutes the action suite happened to leave - usually none.
+    # They had never once executed, and no amount of fixing the action suite
+    # could have changed that: the budget, not the tests, was the binding
+    # constraint.
+    #
+    # 360 is the platform ceiling for a hosted job, i.e. this sets no limit the
+    # runner would not impose anyway. That is deliberate: the two suites behind
+    # the action pass have never run, so there is no measured duration to size
+    # headroom against, and guessing low would re-create the same invisible
+    # truncation one tier further along. The cost is that a WEDGED run (a hung
+    # container, a stalled indexer poll) now takes the full 6 hours to report
+    # instead of failing earlier - so if that starts happening, add a
+    # `timeout-minutes` to the individual suite steps rather than clawing this
+    # number back down; a per-step bound catches a hang without also capping a
+    # legitimately long pass.
+    timeout-minutes: 360
     env:
       COIN: ${{ github.event.inputs.coin || 'bitcoin' }}
       # Drives BOTH the xchain-node checkout below and the `install` boot

From e70e22a9170720ab7fc506833e78facf7190187f Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Wed, 19 Aug 2026 19:07:54 -0700
Subject: [PATCH 14/18] feat(explorer): self-synced checkpoint mirror config,
 and the grants it needs

The explorer's checkpoint, proof and cross-chain routes read hub-mirrored
tables from a local schema that xchain-sync deliberately never replicates.
Deployments with no externally-maintained hub schema colocated with the
explorer can now opt in (EXPLORER_CHECKPOINT_SELF_SYNC) to a checkpoint
descriptor the explorer's own mirror writer self-provisions, using the
indexer's DB identity so the two paths cannot drift. The indexer DB grant
covers the mirror schema on non-mainnet networks, and the Read Contract
flag passes through verbatim so the string-exact readers see it.
---
 src/services/ConfigService.js   | 28 ++++++++++++++++++
 src/services/DatabaseService.js | 30 +++++++++++++++++++
 src/services/HubService.js      | 51 ++++++++++++++++++++++++++++++++-
 3 files changed, 108 insertions(+), 1 deletion(-)

diff --git a/src/services/ConfigService.js b/src/services/ConfigService.js
index 79c4952..a887d94 100644
--- a/src/services/ConfigService.js
+++ b/src/services/ConfigService.js
@@ -608,6 +608,34 @@ async function getDefaultConfig(module, coin, network) {
             defaultValues.CORS_ORIGIN = process.env.CORS_ORIGIN
         }
 
+        if (module === EXPLORER_MODULE_NAME) {
+            // Self-synced hub-mirror checkpoint schema (row 39, #4138 decoupling):
+            // HubMirrorSyncManager needs the hub's own REST base URL to pull
+            // state_checkpoints / capability_snapshots / cross_chain_matches, which
+            // is a DIFFERENT thing from HUB_API_HOST/HUB_PORT above (those feed the
+            // explorer's ordinary getallconfigs config poll, not the mirror writer).
+            // Opt-in via host env EXPLORER_CHECKPOINT_SELF_SYNC, read directly by
+            // HubService.buildHubModuleConfig's checkpoint injection (see there for
+            // why this stays a second knob instead of piggybacking
+            // ALLOW_NO_COLOCATED_HUB_DB: that flag only downgrades the fatal
+            // startup assertion to a warning and says nothing about whether a
+            // local mirror should be provisioned). Emitted only when opted in, so
+            // a deployment that never uses self-sync carries no unused hub URL.
+            if (process.env.EXPLORER_CHECKPOINT_SELF_SYNC !== undefined && process.env.EXPLORER_CHECKPOINT_SELF_SYNC !== "") {
+                defaultValues.HUB_API_URL = process.env.HUB_API_URL ||
+                    ("http://" + getDockerContainerImageName(HUB_MODULE_NAME, "", "") + ":" + defaultValues.HUB_PORT)
+            }
+
+            // Read Contract simulation (contract.html #contract-read-card) is
+            // default-off; the readers test for the exact STRING 'true', not any
+            // truthy value, so pass the host env through verbatim rather than
+            // coercing it. Sourced from host env so it persists across `update`/
+            // `recreate`, mirroring the other explorer passthroughs here.
+            if (process.env.EXPLORER_VM_QUERY_ENABLED !== undefined && process.env.EXPLORER_VM_QUERY_ENABLED !== "") {
+                defaultValues.EXPLORER_VM_QUERY_ENABLED = process.env.EXPLORER_VM_QUERY_ENABLED
+            }
+        }
+
         // The explorer resolves each coin's utxo-tracker and decoder from
         // UTXO_TRACKER_URL_<CODE> (e.g. UTXO_TRACKER_URL_RBTC) and
         // DECODER_API_URL_<COIN>_<NETWORK> (e.g. DECODER_API_URL_BTC_REGTEST).
diff --git a/src/services/DatabaseService.js b/src/services/DatabaseService.js
index cc21ffc..54fb409 100644
--- a/src/services/DatabaseService.js
+++ b/src/services/DatabaseService.js
@@ -573,6 +573,26 @@ async function addUserPasswordToDatabase(module, coin, network, databaseName, us
                 console.log(redactSecrets("DrillB parity-database permissions granted to " + mariadbUser + "!"))
             }
 
+            // Same shape again, for row 39's self-synced checkpoint mirror (#4138
+            // decoupling): HubService.buildCheckpointConfig names the schema
+            // `<INDEXER_DB_NAME>_HubMirror`, and the explorer's own
+            // HubMirrorSyncManager/HubMirrorPool.ensureDatabase() runs `CREATE
+            // DATABASE IF NOT EXISTS` on it under THIS SAME indexer account
+            // (db.js's _checkpointSource only honours a checkpoint entry whose
+            // host/port/user/pass exactly match the indexer DB, so the mirror
+            // writer has no separate credential to hold a separate grant).
+            // Escaped underscores, so the pattern matches nothing but a
+            // `_HubMirror` schema; gated to non-mainnet like DrillB, since
+            // self-sync is currently an opt-in for deployments with no
+            // externally-maintained hub schema colocated with the explorer.
+            if (module === XChainService.XCHAIN_INDEXER && network && network !== "mainnet") {
+                await executeDockerMariaDbCommand(mariadbContainerId, mariadbRootPassword,
+                    "GRANT ALL PRIVILEGES ON `XChain\\_%\\_HubMirror`.* TO " + mariadbUser
+                )
+                await executeDockerMariaDbCommand(mariadbContainerId, mariadbRootPassword, "FLUSH PRIVILEGES")
+                console.log(redactSecrets("Checkpoint hub-mirror database permissions granted to " + mariadbUser + "!"))
+            }
+
             return true
         } catch (err) {
             console.log(err)
@@ -641,6 +661,16 @@ async function addUserPasswordToDatabase(module, coin, network, databaseName, us
                 await executeNativeMariaDbCommand(externalCfg, "FLUSH PRIVILEGES")
                 console.log(redactSecrets("DrillB parity-database permissions granted to " + mariadbUser + "!"))
             }
+
+            // See the docker branch above: the same row-39 checkpoint hub-mirror
+            // grant, since the native-DB venues can self-sync too.
+            if (module === XChainService.XCHAIN_INDEXER && network && network !== "mainnet") {
+                await executeNativeMariaDbCommand(externalCfg,
+                    "GRANT ALL PRIVILEGES ON `XChain\\_%\\_HubMirror`.* TO " + mariadbUser
+                )
+                await executeNativeMariaDbCommand(externalCfg, "FLUSH PRIVILEGES")
+                console.log(redactSecrets("Checkpoint hub-mirror database permissions granted to " + mariadbUser + "!"))
+            }
             return true
         } catch (err) {
             console.log(err)
diff --git a/src/services/HubService.js b/src/services/HubService.js
index 9b241ff..53949de 100644
--- a/src/services/HubService.js
+++ b/src/services/HubService.js
@@ -17,7 +17,7 @@
 
 const {
     HUB_MODULE_NAME, EXPLORER_MODULE_NAME, SYNC_MODULE_NAME,
-    EXTERNAL_DB, SERVICE_REGISTRY
+    EXTERNAL_DB, SERVICE_REGISTRY, XChainService
 } = require('../config/constants')
 
 // Build the hub/explorer per-module config descriptor from the table-driven
@@ -45,6 +45,39 @@ function buildHubModuleConfig(nextModule, defaultConfigCoinNetwork, ctx) {
     }
     return config
 }
+
+// Row 39 (#4138 decoupling): the explorer's checkpoint/proof/cross-chain routes
+// read state_checkpoints / capability_snapshots / cross_chain_matches from a
+// LOCAL schema (config database.checkpoint), because xchain-sync deliberately
+// never replicates those hub-mirrored tables. A deployment with no externally-
+// maintained hub schema colocated with the explorer needs one the explorer's
+// own HubMirrorSyncManager self-provisions and keeps live over the hub's
+// /hub-db feed instead (self_sync: true). Wired in below behind the
+// EXPLORER_CHECKPOINT_SELF_SYNC opt-in (paired with the HUB_API_URL
+// passthrough in ConfigService, which the mirror writer needs to reach the
+// hub); a deployment that already points database.checkpoint at a real hub
+// schema by hand, or wants the routes to just 500 (ALLOW_NO_COLOCATED_HUB_DB),
+// leaves this env unset and is unaffected.
+//
+// db.js's _checkpointSource only honours an entry whose host/port/user/pass
+// EXACTLY match the indexer DB (db.js:481), so this reads the SAME
+// defaultConfigCoinNetwork fields buildHubModuleConfig('xchain-indexer', ...)
+// reads above, rather than re-deriving them, to guarantee byte-identical
+// values instead of two independent paths that could drift apart.
+function buildCheckpointConfig(defaultConfigCoinNetwork) {
+    return {
+        db_host:   defaultConfigCoinNetwork.INDEXER_DB_HOST,
+        db_port:   defaultConfigCoinNetwork.INDEXER_DB_PORT,
+        user:      defaultConfigCoinNetwork.INDEXER_DB_USER,
+        pass:      defaultConfigCoinNetwork.INDEXER_DB_PASS,
+        // A dedicated schema beside the indexer DB, never the indexer schema
+        // itself: HubMirrorPool.ensureDatabase() runs CREATE DATABASE IF NOT
+        // EXISTS on this name under the same indexer DB user, which must
+        // therefore be able to create it (or it must already exist, pre-granted).
+        name:      defaultConfigCoinNetwork.INDEXER_DB_NAME + '_HubMirror',
+        self_sync: true
+    }
+}
 const { db, getLastStatus, isStatusUpdated, isVerbose } = require('../state')
 const { sleep, redactSecrets }                 = require('../utils/helpers')
 const { getDefaultConfig, getDockerContainerImageName, getDockerNetwork } = require('./ConfigService')
@@ -113,6 +146,22 @@ async function updateHubOrExplorer(module) {
                     }
                 }
             }
+
+            // Row 39: advertise a self-synced checkpoint schema for this coin/
+            // network once an indexer is actually installed for it (the
+            // checkpoint config needs the indexer's own DB host/port/user/pass)
+            // and the operator opted in. See buildCheckpointConfig above.
+            if (process.env.EXPLORER_CHECKPOINT_SELF_SYNC !== undefined && process.env.EXPLORER_CHECKPOINT_SELF_SYNC !== "" &&
+                XChainService.XCHAIN_INDEXER in lastStatus[nextCoin][nextNetwork]) {
+                const checkpointConfig = buildCheckpointConfig(defaultConfigCoinNetwork)
+                if (module === "xchain-explorer") {
+                    nextConfigObject.checkpoint = checkpointConfig
+                } else {
+                    if (!(nextCoin in jsonConfig)) jsonConfig[nextCoin] = {}
+                    if (!(nextNetwork in jsonConfig[nextCoin])) jsonConfig[nextCoin][nextNetwork] = {}
+                    jsonConfig[nextCoin][nextNetwork].checkpoint = checkpointConfig
+                }
+            }
         }
     }
 

From d55e11a341f125c0f01815e916b193cee8e4e24c Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Thu, 20 Aug 2026 09:09:26 -0700
Subject: [PATCH 15/18] fix(autoheal): never restart a container an operator
 deliberately stopped

---
 src/services/AutohealService.js   | 27 ++++++++++++++
 test/unit/AutohealService.test.js | 62 +++++++++++++++++++++++++++++--
 2 files changed, 86 insertions(+), 3 deletions(-)

diff --git a/src/services/AutohealService.js b/src/services/AutohealService.js
index b387193..10d7219 100644
--- a/src/services/AutohealService.js
+++ b/src/services/AutohealService.js
@@ -203,6 +203,33 @@ async function runAutoheal({ dryRun = false, now = Date.now() } = {}) {
             continue
         }
 
+        // Heal only what is actually RUNNING. Docker freezes State.Health.Status
+        // at its last value the moment a container stops (the probe goroutine
+        // runs only while the container is up), so a container that happened to
+        // be unhealthy when an operator stopped it keeps reporting `unhealthy`
+        // while State.Status is `exited` - and `docker restart` on a stopped
+        // container STARTS it, silently undoing the stop. Frozen health from a
+        // container that is no longer probing is not evidence of a wedge. This
+        // costs no healing either: autoheal exists for the ALIVE-but-stalled
+        // case (see the file header), because `--restart unless-stopped` already
+        // covers a service whose PID exits, and declines to fire exactly when
+        // the operator was the one who stopped it. Same guard the bootstrap gate
+        // applies at BootstrapHealthGate.evaluateContainerState.
+        const runState = status && status.State && status.State.Status
+        if (runState !== 'running') {
+            result.skipped.push({ module, coin, network, containerId, reason: `not running (state: ${runState || 'unknown'})` })
+            // Forget the onset: a container that comes back up gets a fresh grace
+            // window instead of inheriting a clock that has been stopped all along.
+            // The attempt count is deliberately NOT dropped here - it is cleared on
+            // an observed RECOVERY (below), and a pass that catches a container
+            // mid-restart must not reset the backoff a real wedge has earned.
+            if (state.unhealthySince[containerId] !== undefined) {
+                delete state.unhealthySince[containerId]
+                onsetChanged = true
+            }
+            continue
+        }
+
         const health = status && status.State && status.State.Health
         if (!health || health.Status !== 'unhealthy') {
             // Episode over (or the healthcheck is gone): forget the onset so the
diff --git a/test/unit/AutohealService.test.js b/test/unit/AutohealService.test.js
index 24858d8..6de8c9c 100644
--- a/test/unit/AutohealService.test.js
+++ b/test/unit/AutohealService.test.js
@@ -30,11 +30,14 @@ function logEntry(agoMs, exitCode) {
     }
 }
 
-// docker-inspect shape for a container in a given health state.
-function inspectStatus(healthStatus, log) {
+// docker-inspect shape for a container in a given health state. `runState` is
+// State.Status and defaults to 'running'; pass 'exited' to model what Docker
+// reports for a STOPPED container, whose Health.Status stays frozen at whatever
+// it read the moment the container went down.
+function inspectStatus(healthStatus, log, runState) {
     return {
         State: {
-            Status: 'running',
+            Status: runState || 'running',
             Health: { Status: healthStatus, FailingStreak: healthStatus === 'unhealthy' ? 5 : 0, Log: log }
         }
     }
@@ -45,6 +48,13 @@ function unhealthyPastGrace() {
     return inspectStatus('unhealthy', [logEntry(11 * 60000, 0), logEntry(10 * 60000, 1), logEntry(5 * 60000, 1), logEntry(60000, 1)])
 }
 
+// An operator stopped this container while it was unhealthy: State.Status is
+// 'exited' and Health.Status is frozen at the last value the probe read, well
+// past the grace window. Docker keeps answering `docker inspect` for it.
+function stoppedWithFrozenUnhealthy() {
+    return inspectStatus('unhealthy', [logEntry(11 * 60000, 0), logEntry(10 * 60000, 1), logEntry(5 * 60000, 1), logEntry(60000, 1)], 'exited')
+}
+
 // Unhealthy, but the failing run only started 30s ago.
 function unhealthyInsideGrace() {
     return inspectStatus('unhealthy', [logEntry(90000, 0), logEntry(30000, 1), logEntry(15000, 1)])
@@ -165,6 +175,52 @@ describe('AutohealService', () => {
         expect(result.candidates).to.have.length(0)
     })
 
+    // Docker freezes Health.Status when a container stops, so a container an
+    // operator deliberately stopped while it was unhealthy still reads
+    // `unhealthy` forever. Restarting on that reading STARTS the container the
+    // operator just pulled out of rotation.
+    it('does NOT restart a container an operator stopped, whose health is frozen at unhealthy', async () => {
+        stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'stp')])
+        stubs.getStatusFromContainer.resolves(stoppedWithFrozenUnhealthy())
+
+        const result = await service.runAutoheal({ now: NOW })
+
+        expect(stubs.restartContainer.called).to.equal(false)
+        expect(result.candidates).to.have.length(0)
+        expect(result.skipped[0].reason).to.equal('not running (state: exited)')
+    })
+
+    // The grace clock must not keep running while the container is down: a
+    // container started again after an operator stop gets a full grace window to
+    // come back, not an instant restart off a clock from before the stop.
+    it('drops the episode onset while a container is stopped, so a restarted one gets a fresh grace window', async () => {
+        stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'stp2')])
+
+        // Pass 1: unhealthy and running, inside grace - the onset gets recorded.
+        stubs.getStatusFromContainer.resolves(unhealthyInsideGrace())
+        await service.runAutoheal({ now: NOW })
+
+        // Pass 2: the operator has stopped it; health stays frozen at unhealthy.
+        stubs.getStatusFromContainer.resolves(stoppedWithFrozenUnhealthy())
+        await service.runAutoheal({ now: NOW + 60000 })
+
+        // Pass 3, an hour later: running again, and unhealthy from a probe that
+        // only started failing 30s ago. With the pre-stop onset still on file
+        // this reads as an hour-long episode and restarts immediately.
+        const at = NOW + 60 * 60000
+        const freshFailure = {
+            Start: new Date(at - 30000).toISOString(),
+            End: new Date(at - 29000).toISOString(),
+            ExitCode: 1,
+            Output: 'wget: server returned error'
+        }
+        stubs.getStatusFromContainer.resolves(inspectStatus('unhealthy', [freshFailure]))
+        const third = await service.runAutoheal({ now: at })
+
+        expect(stubs.restartContainer.called).to.equal(false)
+        expect(third.skipped[0].reason).to.equal('inside grace window')
+    })
+
     it('does NOT restart the same container twice within the cooldown window', async () => {
         stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'eee')])
         stubs.getStatusFromContainer.resolves(unhealthyPastGrace())

From 414726e756a61288211147068eb5bcd48cd29665 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Thu, 20 Aug 2026 18:58:49 -0700
Subject: [PATCH 16/18] fix(node): rotate the hub DB account on recreate and
 guard the headless config prompt

2026-08-20 xchain-platform review round sitting 3.
Findings: 5322, 5323, 5324, 5338, 5395.
Recorded in the platform review store and the round report.
---
 scripts/publish-bootstraps.sh      | 76 ++++++++++++++++++++++++++++--
 src/operations/moduleOperations.js | 33 ++++++++++++-
 src/services/AutohealService.js    | 19 ++++++--
 src/services/DatabaseService.js    | 18 +++++++
 test/unit/AutohealService.test.js  | 48 +++++++++++++++++++
 test/unit/DatabaseService.test.js  | 63 +++++++++++++++++++++++--
 test/unit/moduleOperations.test.js | 60 ++++++++++++++++++++++-
 7 files changed, 301 insertions(+), 16 deletions(-)

diff --git a/scripts/publish-bootstraps.sh b/scripts/publish-bootstraps.sh
index e05613f..9810283 100755
--- a/scripts/publish-bootstraps.sh
+++ b/scripts/publish-bootstraps.sh
@@ -36,7 +36,9 @@
 #     XCHAIN_NODE_DATA_DIR / XCHAIN_NODE_TMP_DIR - the defaults live under the
 #     repo on the small root fs and WILL fill it mid-create otherwise.
 #   - Transfers each archive + its .sig to the sync host (scp, origin->sync).
-#   - Prunes old archives locally and on the sync host (keep newest $KEEP).
+#   - Prunes old archives locally and on the sync host, keeping the newest $KEEP
+#     by filename plus the newest $KEEP that are SIGNED (see PRUNE_SCRIPT), so a
+#     prune can never evict the archive the sync host advertises as latest.
 #   - flock guard so overlapping cron runs cannot collide.
 #
 # ┌─ DOWNTIME WARNING ──────────────────────────────────────────────────────┐
@@ -111,6 +113,70 @@ done
 log() { printf '%s  %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
 die() { log "FATAL: $*"; exit 1; }
 
+# ── Retention policy ──────────────────────────────────────────────────────
+# One policy body, piped to `sh -s` for BOTH the sync host (over ssh) and the
+# local stage, so the two sides cannot drift the way the producer and the server
+# already did.
+#
+# Serving resolves "latest" as the newest archive that HAS a paired .sig,
+# ordered by the UTC timestamp in the FILENAME (sync host latest.php). A prune
+# keyed on mtime that lets unsigned archives occupy retention slots therefore
+# deletes the exact file the server advertises: with KEEP=2, two unsigned
+# strays (an --allow-unsigned run, an operator drop) fill both slots and the
+# newest SIGNED archive goes, leaving consumers a 404 and a forced full resync
+# while a perfectly good archive existed a moment earlier. mtime is the wrong
+# key besides, because a backfilled or re-uploaded archive carries upload time,
+# not its name's timestamp.
+#
+# So retain the union of the newest KEEP by filename and the newest KEEP that
+# are signed. The union is never smaller than the by-name set alone, so this
+# deletes no more than a name-keyed prune, it cannot evict the serve target, and
+# it still bounds a directory at 2*KEEP archives. That bias matters because
+# deletions here fan out to the public web tier by a downstream rsync
+# --delete-after.
+#
+# Never candidates: latest.tgz / latest.tar.gz (the hand-placed manual-publish
+# aliases, which the server lets win on their own route) and *.part uploads in
+# flight. An empty retention set aborts the directory rather than deleting
+# everything in it.
+PRUNE_SCRIPT=$(cat <<'PRUNE_EOF'
+dir="$1"; keep="$2"
+[ -n "$dir" ] && [ -d "$dir" ] || exit 0
+case "$keep" in ''|*[!0-9]*) exit 0 ;; esac
+[ "$keep" -ge 1 ] || exit 0
+cd "$dir" || exit 0
+
+all=$(ls -1 ./*.tar.gz 2>/dev/null | sed 's#^\./##' | grep -v '^latest\.tar\.gz$' | LC_ALL=C sort -r)
+[ -n "$all" ] || exit 0
+
+signed=$(printf '%s\n' "$all" | while IFS= read -r f; do
+           [ -f "$f.sig" ] && printf '%s\n' "$f"
+         done)
+retain=$({ printf '%s\n' "$all" | head -n "$keep"
+           [ -n "$signed" ] && printf '%s\n' "$signed" | head -n "$keep"
+         } | sed '/^$/d' | LC_ALL=C sort -u)
+
+# With candidates present, an empty retention set can only mean the selection
+# above failed, and "retain nothing" here means "delete every archive".
+[ -n "$retain" ] || { echo "  PRUNE ABORTED in $dir: empty retention set, nothing deleted"; exit 0; }
+
+printf '%s\n' "$all" | while IFS= read -r f; do
+  [ -n "$f" ] || continue
+  printf '%s\n' "$retain" | grep -q -x -F -- "$f" && continue
+  rm -f -- "$f" "$f.sig" "$f.sha256"
+done
+
+left=$(printf '%s\n' "$retain" | while IFS= read -r f; do
+         [ -n "$f" ] && [ -f "$f.sig" ] && printf '%s\n' "$f"
+       done | wc -l | tr -d ' ')
+if [ "$left" -gt 0 ]; then
+  echo "  prune: $left signed archive(s) retained in $dir"
+else
+  echo "  PRUNE WARNING: no signed archive remains in $dir; the latest endpoint 404s there"
+fi
+PRUNE_EOF
+)
+
 # ── Preconditions ─────────────────────────────────────────────────────────
 command -v "$XCHAIN_NODE_BIN" >/dev/null || die "xchain-node CLI not found ($XCHAIN_NODE_BIN)"
 if [ ! -f "$SIGNING_KEY" ]; then
@@ -249,16 +315,16 @@ for c in "${SELECTED[@]}"; do
     if ssh -o BatchMode=yes "$SYNC_HOST" "$mv_cmd"; then
       log "  published $a_base (+sig) to $SYNC_HOST:$dest"
       SUMMARY+=("$c: PUBLISHED")
-      # Prune remote: keep newest $KEEP archives + their sidecars.
-      ssh -o BatchMode=yes "$SYNC_HOST" "cd '$dest' && ls -t *.tar.gz 2>/dev/null | tail -n +$((KEEP+1)) | while read -r f; do rm -f \"\$f\" \"\$f.sig\" \"\$f.sha256\"; done" || log "  (remote prune warning)"
+      # Prune remote under the shared retention policy (see PRUNE_SCRIPT).
+      printf '%s\n' "$PRUNE_SCRIPT" | ssh -o BatchMode=yes "$SYNC_HOST" "sh -s -- '$dest' '$KEEP'" || log "  (remote prune warning)"
     else
       cleanup_parts
       log "  publish rename FAILED"; SUMMARY+=("$c: PUBLISH-FAIL"); fail=1; continue
     fi
   fi
 
-  # Prune local stage: keep newest $KEEP archives + sidecars to bound disk use.
-  ( cd "$out_dir" && ls -t ./*.tar.gz 2>/dev/null | tail -n +$((KEEP+1)) | while read -r f; do rm -f "$f" "$f.sig" "$f.sha256"; done ) || true
+  # Prune the local stage under the SAME policy, to bound disk use.
+  printf '%s\n' "$PRUNE_SCRIPT" | sh -s -- "$out_dir" "$KEEP" || true
 done
 
 log "── summary ──"
diff --git a/src/operations/moduleOperations.js b/src/operations/moduleOperations.js
index 1917dfd..6bab3fd 100644
--- a/src/operations/moduleOperations.js
+++ b/src/operations/moduleOperations.js
@@ -205,6 +205,19 @@ async function updateModulesOnBranch(servicesList, branch = null) {
     for (const nextCoin in servicesList) {
         for (const nextNetwork in servicesList[nextCoin]) {
             for (const nextModule of servicesList[nextCoin][nextNetwork]) {
+                if (nextModule === DB_MODULE_NAME) {
+                    // `update` cannot rebuild the database. Its container is created by
+                    // buildDatabaseModule from a pinned mariadb image, not from module
+                    // source, and the existing-container branch there does nothing at
+                    // all - yet the DB branch of installModule answered a hard `true`,
+                    // which recordInstallOutcome counts as an updated module. So
+                    // `update database` exited 0 reporting a landed upgrade over an
+                    // untouched container. Refuse it here, where the update contract
+                    // lives, and state the remediation uninstallModule already names.
+                    console.warn(`update: ${nextModule} (${nextCoin} ${nextNetwork}) is not rebuilt by update; the database container must be removed manually and reinstalled.`)
+                    outcome.skipped.push({ module: nextModule, coin: nextCoin, network: nextNetwork, reason: 'not-updatable' })
+                    continue
+                }
                 const moduleContainerId = await db.getModuleContainer(nextModule, nextCoin, nextNetwork)
                 if (nextModule === NODE_MODULE_NAME) {
                     // Tear down the existing node container before rebuilding. The node
@@ -306,10 +319,11 @@ const RECREATE_UNSUPPORTED_MODULES = [NODE_MODULE_NAME, DB_MODULE_NAME]
  */
 async function recreateModules(servicesList) {
     const { buildAndUp } = require('../services/ModuleService')
-    const { setDatabaseParameters } = require('../services/DatabaseService')
+    const { setDatabaseParameters, setHubDatabaseParameters } = require('../services/DatabaseService')
 
     const outcome = { recreated: [], skipped: [] }
     let touchedDbModule = false
+    let touchedHubModule = false
     for (const nextCoin in servicesList) {
         for (const nextNetwork in servicesList[nextCoin]) {
             for (const nextModule of servicesList[nextCoin][nextNetwork]) {
@@ -318,7 +332,13 @@ async function recreateModules(servicesList) {
                     // node and the database. What changed is that the skip is now
                     // recorded, so a run that recreated NOTHING can be reported as
                     // the failed request it is instead of exiting 0.
-                    console.log("recreate does not apply to " + nextModule + "; use `update " + nextModule + "` instead")
+                    // The database has no `update` to redirect to either: that verb
+                    // refuses it for the same reason (no container built from the
+                    // config map, no in-place image upgrade). Say the real remedy.
+                    const remedy = nextModule === DB_MODULE_NAME
+                        ? "; the database container must be removed manually and reinstalled"
+                        : "; use `update " + nextModule + "` instead"
+                    console.log("recreate does not apply to " + nextModule + remedy)
                     outcome.skipped.push({ module: nextModule, coin: nextCoin, network: nextNetwork, reason: 'not-recreatable' })
                     continue
                 }
@@ -328,6 +348,9 @@ async function recreateModules(servicesList) {
                 if (nextModule === XChainService.XCHAIN_DECODER || nextModule === XChainService.XCHAIN_INDEXER) {
                     touchedDbModule = true
                 }
+                if (nextModule === HUB_MODULE_NAME) {
+                    touchedHubModule = true
+                }
             }
         }
     }
@@ -336,6 +359,12 @@ async function recreateModules(servicesList) {
     // in setDatabaseParameters sees the state we just converged rather than the one
     // that made the recreate necessary.
     if (touchedDbModule) await setDatabaseParameters()
+    // Same rule for the SHARED hub account, and it matters most on this verb: the
+    // recreated hub starts on the config store's HUB_DB_PASS, so without rotating
+    // the live 'xchain_hub'@'%' account to match, `recreate xchain-hub` hands the
+    // hub a password MariaDB never received and it crash-loops on ER_ACCESS_DENIED.
+    // The `update` path rotates here for the same reason (ModuleService installModule).
+    if (touchedHubModule) await setHubDatabaseParameters()
     await statusChanged()
     return outcome
 }
diff --git a/src/services/AutohealService.js b/src/services/AutohealService.js
index 10d7219..e4da66f 100644
--- a/src/services/AutohealService.js
+++ b/src/services/AutohealService.js
@@ -238,10 +238,21 @@ async function runAutoheal({ dryRun = false, now = Date.now() } = {}) {
                 delete state.unhealthySince[containerId]
                 onsetChanged = true
             }
-            // Drop the attempt count too, so a container that DID recover starts its
-            // next episode at the base cooldown. Backing off is a response to a wedge
-            // restarts are not clearing; a recovery is the evidence they cleared it.
-            if (state.restartCount[containerId] !== undefined) {
+            // Drop the attempt count only on an OBSERVED `healthy`, so a container
+            // that DID recover starts its next episode at the base cooldown. Backing
+            // off is a response to a wedge restarts are not clearing; a recovery is
+            // the evidence they cleared it, and `!== 'unhealthy'` is not that
+            // evidence: `docker restart` puts the container into `starting` for the
+            // descriptor's start period plus its retry budget (60s + 3x15s for the
+            // decoder/indexer, and an operator can widen it to minutes via
+            // XCHAIN_NODE_HEALTH_START_PERIOD_<SERVICE>), so a pass landing in that
+            // window would wipe the counter the restart it had just issued earned.
+            // A wedge no restart clears then stayed at the BASE cooldown forever,
+            // which is the churn the doubling exists to end. Same principle the
+            // not-running guard above states: mid-restart is not recovery. `starting`
+            // is its health-probation form. A container with no healthcheck keeps its
+            // count too, and inertly: it can never reach the restart path below.
+            if (health && health.Status === 'healthy' && state.restartCount[containerId] !== undefined) {
                 delete state.restartCount[containerId]
                 onsetChanged = true
             }
diff --git a/src/services/DatabaseService.js b/src/services/DatabaseService.js
index 54fb409..be60241 100644
--- a/src/services/DatabaseService.js
+++ b/src/services/DatabaseService.js
@@ -189,6 +189,24 @@ async function getExternalDbConfig() {
         }
     }
 
+    // Non-interactive run (cron, ssh BatchMode, CI): the prompt loop below would
+    // block forever on a stdin that never answers, and this resolver is reached
+    // from preCheck and ensureDatabasePool INSIDE the CLI command lock, so the
+    // hang wedges every later xchain-node command on the host rather than just
+    // this one. Same fail-fast the bundled-DB path already does in
+    // askMariadbRootPassword. Placed AFTER the env and saved-credential branches
+    // so both headless success paths keep working untouched.
+    if (!process.stdin.isTTY) {
+        throw new Error(
+            'External-DB connection details are needed but there is no TTY to prompt on. ' +
+            'Set ALL FOUR of XCHAIN_NODE_EXTERNAL_DB_HOST, XCHAIN_NODE_EXTERNAL_DB_PORT, ' +
+            'XCHAIN_NODE_EXTERNAL_DB_ROOT_USER and XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD ' +
+            '(a partial set does not qualify for the headless path), or run any xchain-node ' +
+            'command once interactively so the verified details are saved to ' +
+            '~/.xchain-node/credentials.json for later runs.'
+        )
+    }
+
     // Interactive prompt
     console.log("\nExternal MariaDB configuration (XCHAIN_NODE_EXTERNAL_DB=1)")
     console.log("Provide the connection details for the host-native MariaDB this node should use.\n")
diff --git a/test/unit/AutohealService.test.js b/test/unit/AutohealService.test.js
index 6de8c9c..a95e4ef 100644
--- a/test/unit/AutohealService.test.js
+++ b/test/unit/AutohealService.test.js
@@ -299,6 +299,54 @@ describe('AutohealService', () => {
         expect(again.restarted).to.have.length(1)
     })
 
+    // `docker restart` puts the container into Docker's `starting` probation for the
+    // descriptor's start period plus its retry budget, so a pass landing in that
+    // window sees a status that is not 'unhealthy' and would read it as recovery,
+    // wiping the very counter the restart it had just issued earned. A wedge no
+    // restart clears then sat at the BASE cooldown forever.
+    it('keeps the earned backoff when a restarted container is still in Docker starting probation', async () => {
+        stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'bo3')])
+        stubs.getStatusFromContainer.resolves(unhealthyPastGrace())
+
+        await service.runAutoheal({ now: NOW })                      // restart #1
+        await service.runAutoheal({ now: NOW + 11 * 60000 })         // restart #2 -> next wait doubles
+        expect(stubs.restartContainer.callCount).to.equal(2)
+
+        // A minute after restart #2: Docker still reports `starting`.
+        stubs.getStatusFromContainer.resolves(inspectStatus('starting', [logEntry(15000, 1)]))
+        await service.runAutoheal({ now: NOW + 12 * 60000 })
+
+        const state = JSON.parse(fs.readFileSync(path.join(stateDir, 'autoheal-state.json'), 'utf8'))
+        expect(state.restartCount.bo3, 'probation is not recovery; the backoff must survive it').to.equal(2)
+        expect(state.unhealthySince, 'a restarted container still earns a fresh grace window').to.not.have.property('bo3')
+
+        // Wedge returns 11 minutes after restart #2. The doubled 20-minute window is
+        // still open, so nothing restarts; with the counter wiped it would have.
+        stubs.getStatusFromContainer.resolves(unhealthyPastGrace())
+        const throttled = await service.runAutoheal({ now: NOW + 22 * 60000 })
+        expect(stubs.restartContainer.callCount).to.equal(2)
+        expect(throttled.skipped[0].reason).to.equal('inside restart cooldown')
+    })
+
+    // The not-running guard's documented asymmetry (drop the onset, keep the count)
+    // had no test of its own, so a regression flipping it would have passed green.
+    it('keeps the earned backoff when a pass catches the container not running', async () => {
+        stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'st1')])
+        stubs.getStatusFromContainer.resolves(unhealthyPastGrace())
+
+        await service.runAutoheal({ now: NOW })                      // restart #1
+        await service.runAutoheal({ now: NOW + 11 * 60000 })         // restart #2
+        expect(stubs.restartContainer.callCount).to.equal(2)
+
+        stubs.getStatusFromContainer.resolves(stoppedWithFrozenUnhealthy())
+        const skipped = await service.runAutoheal({ now: NOW + 12 * 60000 })
+        expect(skipped.skipped[0].reason).to.match(/not running/)
+
+        const state = JSON.parse(fs.readFileSync(path.join(stateDir, 'autoheal-state.json'), 'utf8'))
+        expect(state.restartCount.st1, 'mid-restart is not recovery').to.equal(2)
+        expect(state.unhealthySince).to.not.have.property('st1')
+    })
+
     it('caps the doubled cooldown at the ceiling instead of growing without bound', () => {
         const base = service.DEFAULT_COOLDOWN_MS
         const ceiling = service.DEFAULT_COOLDOWN_CEILING_MS
diff --git a/test/unit/DatabaseService.test.js b/test/unit/DatabaseService.test.js
index 2a93ea4..e12a616 100644
--- a/test/unit/DatabaseService.test.js
+++ b/test/unit/DatabaseService.test.js
@@ -754,10 +754,65 @@ describe('DatabaseService', function () {
                 .onFirstCall().rejects(new Error('auth failed'))
                 .resolves(stubs.mariadb._fakeConn)
 
-            const ds = loadDatabaseService(stubs)
-            const result = await ds.getExternalDbConfig()
-            expect(stubs.saveExternalDbConfig.calledOnce).to.be.true
-            expect(result).to.be.an('object')
+            // The prompt path is TTY-gated, and a mocha run has no TTY on stdin
+            // under CI. Assert the interactive behaviour on an interactive stdin.
+            const savedIsTTY = Object.getOwnPropertyDescriptor(process.stdin, 'isTTY')
+            Object.defineProperty(process.stdin, 'isTTY', { value: true, configurable: true })
+            try {
+                const ds = loadDatabaseService(stubs)
+                const result = await ds.getExternalDbConfig()
+                expect(stubs.saveExternalDbConfig.calledOnce).to.be.true
+                expect(result).to.be.an('object')
+            } finally {
+                if (savedIsTTY) Object.defineProperty(process.stdin, 'isTTY', savedIsTTY)
+                else delete process.stdin.isTTY
+            }
+        })
+
+        // The prompt loop is reached from preCheck/ensureDatabasePool inside the CLI
+        // command lock, so hanging on stdin wedges every later command on the host.
+        it('fails fast instead of prompting when stdin is not a TTY', async function () {
+            const stubs = makeStubs()
+            const savedIsTTY = Object.getOwnPropertyDescriptor(process.stdin, 'isTTY')
+            Object.defineProperty(process.stdin, 'isTTY', { value: undefined, configurable: true })
+            try {
+                const ds = loadDatabaseService(stubs)
+                let threw = null
+                try { await ds.getExternalDbConfig() } catch (e) { threw = e }
+                expect(threw, 'expected a fail-fast error with no TTY').to.be.an('error')
+                expect(threw.message).to.match(/no TTY to prompt on/)
+                expect(threw.message).to.include('XCHAIN_NODE_EXTERNAL_DB_HOST')
+                expect(threw.message).to.include('XCHAIN_NODE_EXTERNAL_DB_PORT')
+                expect(threw.message).to.include('XCHAIN_NODE_EXTERNAL_DB_ROOT_USER')
+                expect(threw.message).to.include('XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD')
+                expect(stubs.saveExternalDbConfig.called).to.be.false
+            } finally {
+                if (savedIsTTY) Object.defineProperty(process.stdin, 'isTTY', savedIsTTY)
+                else delete process.stdin.isTTY
+            }
+        })
+
+        // A partial env set must NOT be treated as the headless fast path: it would
+        // silently fall back to 127.0.0.1:3306 defaults if the guard were relaxed.
+        it('fails fast on a partial env set rather than prompting headlessly', async function () {
+            process.env.XCHAIN_NODE_EXTERNAL_DB_HOST      = 'db.example.com'
+            process.env.XCHAIN_NODE_EXTERNAL_DB_PORT      = '3307'
+            process.env.XCHAIN_NODE_EXTERNAL_DB_ROOT_USER = 'admin'
+            // XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD deliberately absent
+
+            const stubs = makeStubs()
+            const savedIsTTY = Object.getOwnPropertyDescriptor(process.stdin, 'isTTY')
+            Object.defineProperty(process.stdin, 'isTTY', { value: false, configurable: true })
+            try {
+                const ds = loadDatabaseService(stubs)
+                let threw = null
+                try { await ds.getExternalDbConfig() } catch (e) { threw = e }
+                expect(threw, 'expected a fail-fast error for a partial env set').to.be.an('error')
+                expect(threw.message).to.match(/no TTY to prompt on/)
+            } finally {
+                if (savedIsTTY) Object.defineProperty(process.stdin, 'isTTY', savedIsTTY)
+                else delete process.stdin.isTTY
+            }
         })
 
         // #3143: the external-DB port must be validatePort-gated at the resolver,
diff --git a/test/unit/moduleOperations.test.js b/test/unit/moduleOperations.test.js
index ef1d842..3670eff 100644
--- a/test/unit/moduleOperations.test.js
+++ b/test/unit/moduleOperations.test.js
@@ -55,6 +55,7 @@ function makeStubs() {
         getModuleBranch: sinon.stub().resolves('master'),
         buildAndUp: sinon.stub().resolves('b'.repeat(64)),
         setDatabaseParameters: sinon.stub().resolves(true),
+        setHubDatabaseParameters: sinon.stub().resolves(true),
         installModule: sinon.stub().resolves('new-container-id'),
         uninstallModule: sinon.stub().resolves(true),
         assertHubNotBehind: sinon.stub().resolves({ checked: false, reason: 'not-hub-dependent' }),
@@ -97,7 +98,8 @@ function loadOperations(stubs) {
             resetDatabases: stubs.resetDatabases,
             clearHubPriceIngestWatermark: stubs.clearHubPriceIngestWatermark,
             getDatabaseContainerId: stubs.getDatabaseContainerId,
-            setDatabaseParameters: stubs.setDatabaseParameters
+            setDatabaseParameters: stubs.setDatabaseParameters,
+            setHubDatabaseParameters: stubs.setHubDatabaseParameters
         },
         '../services/ModuleService': {
             cloneGit: stubs.cloneGit,
@@ -375,6 +377,39 @@ describe('moduleOperations', function () {
             const outcome = await ops.updateModules({ bitcoin: { mainnet: ['node'] } })
             expect(outcome.updated).to.deep.equal([{ module: 'node', coin: 'bitcoin', network: 'mainnet' }])
         })
+
+        // The database container is built from a pinned image, not from module
+        // source, and its existing-container path changes nothing. Counting it as
+        // updated is how `update database` exited 0 over an untouched container.
+        it('refuses the database instead of reporting an untouched container as updated', async function () {
+            const stubs = makeStubs()
+            const ops = loadOperations(stubs)
+            const warn = sinon.stub(console, 'warn')
+            let outcome
+            try {
+                outcome = await ops.updateModules({ '': { '': ['database'] } })
+            } finally { warn.restore() }
+            expect(outcome.updated).to.deep.equal([])
+            expect(outcome.skipped).to.deep.equal([
+                { module: 'database', coin: '', network: '', reason: 'not-updatable' }
+            ])
+            // Refused BEFORE any rebuild machinery runs, so nothing is torn down.
+            expect(stubs.installModule.called).to.be.false
+            expect(stubs.db.getModuleContainer.called).to.be.false
+            expect(warn.calledWithMatch(/removed manually and reinstalled/)).to.be.true
+        })
+
+        it('still updates the other requested modules when the request also names the database', async function () {
+            const stubs = makeStubs()
+            const ops = loadOperations(stubs)
+            const warn = sinon.stub(console, 'warn')
+            let outcome
+            try {
+                outcome = await ops.updateModules({ bitcoin: { mainnet: ['database', 'xchain-encoder'] } })
+            } finally { warn.restore() }
+            expect(outcome.updated).to.deep.equal([{ module: 'xchain-encoder', coin: 'bitcoin', network: 'mainnet' }])
+            expect(outcome.skipped.map(s => s.module)).to.deep.equal(['database'])
+        })
     })
 
     // -------------------------------------------------------------------
@@ -421,6 +456,29 @@ describe('moduleOperations', function () {
             await ops.recreateModules({ dogecoin: { regtest: ['xchain-encoder'] } })
             expect(stubs.buildAndUp.calledOnce).to.be.true
             expect(stubs.setDatabaseParameters.called).to.be.false
+            expect(stubs.setHubDatabaseParameters.called).to.be.false
+        })
+
+        // The recreated hub starts on the config store's HUB_DB_PASS. Without
+        // rotating the live shared hub account to match, the verb that exists to
+        // REPAIR credentials is the one that locks the hub out (ER_ACCESS_DENIED).
+        it('rotates the shared hub DB account after recreating the hub', async function () {
+            const stubs = makeStubs()
+            const ops = loadOperations(stubs)
+            const result = await ops.recreateModules({ '': { '': ['xchain-hub'] } })
+            expect(result.recreated).to.deep.equal([{ module: 'xchain-hub', coin: '', network: '' }])
+            expect(stubs.setHubDatabaseParameters.calledOnce).to.be.true
+            expect(stubs.buildAndUp.firstCall.calledBefore(stubs.setHubDatabaseParameters.firstCall)).to.be.true
+            // The per-coin decoder/indexer provisioning is a different account set
+            // and must not be dragged in by a hub-only recreate.
+            expect(stubs.setDatabaseParameters.called).to.be.false
+        })
+
+        it('does not rotate the hub account when the hub was not recreated', async function () {
+            const stubs = makeStubs()
+            const ops = loadOperations(stubs)
+            await ops.recreateModules({ dogecoin: { regtest: ['xchain-indexer'] } })
+            expect(stubs.setHubDatabaseParameters.called).to.be.false
         })
 
         it('recreates a container the registry has lost rather than skipping it', async function () {

From 58e8fa9c9ecfa0ce08eff8f59b9c645ebcbf4b45 Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Fri, 21 Aug 2026 08:29:36 -0700
Subject: [PATCH 17/18] fix(node): refuse a bootstrap source that reports no
 lag or a negative lag

A status body with no lag field at all (an image that publishes none, or a
shed health POST falling back to GET /status) slipped past the lag refusal,
and a negative lag read as "ahead" when it means the service's committed tip
sits above its node's, so the rows it exports reference blocks the node no
longer recognizes. Both now refuse with a reason. The block-fetch desync
record is rendered field by field instead of as [object Object].
---
 src/services/BootstrapHealthGate.js   | 37 ++++++++++++++++++++++-----
 test/unit/BootstrapHealthGate.test.js | 35 +++++++++++++++++++++++--
 2 files changed, 64 insertions(+), 8 deletions(-)

diff --git a/src/services/BootstrapHealthGate.js b/src/services/BootstrapHealthGate.js
index 95ad10a..67ff5f7 100644
--- a/src/services/BootstrapHealthGate.js
+++ b/src/services/BootstrapHealthGate.js
@@ -173,20 +173,32 @@ function evaluateStatusPayload(payload, { maxLag = DEFAULT_MAX_LAG_BLOCKS } = {}
     if (payload.decoderReorgHalted === true)
         reasons.push('the upstream decoder carries a durable REORG_HALT marker, so this database is frozen behind it')
     if (payload.block_fetch_desync)
-        reasons.push(`the service reports a block-fetch desync: ${payload.block_fetch_desync}`)
+        reasons.push(`the service reports a block-fetch desync (${formatBlockFetchDesync(payload.block_fetch_desync)})`)
     if (payload.node_height_stale === true)
         reasons.push('the service cannot see the node tip (stale node height), so its lag is unknown')
 
-    // First lag field the service actually publishes. `null` is a real answer
-    // here and means "position unknown", which cannot be certified as caught-up.
+    // First lag field the service actually publishes, bounded on BOTH sides. `null`
+    // is a real answer and means "position unknown"; NO lag field at all means the
+    // same thing (a /status body from an image that publishes none), and neither
+    // can be certified as caught-up. A NEGATIVE lag is not "ahead": the service's
+    // committed tip sits above its node's (the node-reset/reindex regression), so
+    // the rows it would export reference blocks the node no longer recognizes. The
+    // tracker floors its own synced verdict the same way; the ceiling stays local
+    // because consumers own their own lag budget.
     const lagKeys = ['lag_blocks', 'blockLag', 'lag']
     const reported = lagKeys.find(k => Object.prototype.hasOwnProperty.call(payload, k))
-    if (reported !== undefined) {
+    if (reported === undefined) {
+        reasons.push('the service did not report how far behind it is (no lag field in its status payload), ' +
+            'so its position could not be verified')
+    } else {
         const lag = payload[reported]
         if (lag === null || lag === undefined)
             reasons.push(`the service cannot report how far behind it is (${reported} is null)`)
         else if (!Number.isFinite(Number(lag)))
             reasons.push(`the service reported an unreadable ${reported} (${lag})`)
+        else if (Number(lag) < 0)
+            reasons.push(`the service reported a negative ${reported} (${Number(lag)}): its committed tip sits above ` +
+                'its upstream node\'s, so the data it would export may reference blocks the node no longer recognizes')
         else if (Number(lag) > maxLag)
             reasons.push(`the service is ${Number(lag)} blocks behind its upstream tip (limit ${maxLag}; ` +
                 'override with XCHAIN_NODE_BOOTSTRAP_MAX_LAG_BLOCKS)')
@@ -195,9 +207,22 @@ function evaluateStatusPayload(payload, { maxLag = DEFAULT_MAX_LAG_BLOCKS } = {}
     return reasons
 }
 
+// The tracker publishes block_fetch_desync as {height, failures, lastError,
+// detectedAt}, not a string; interpolated raw it renders "[object Object]" and
+// loses the height/lastError that says the node is pruned past the cursor.
+function formatBlockFetchDesync(desync) {
+    if (!desync || typeof desync !== 'object') return String(desync)
+    const parts = []
+    if (desync.height !== undefined && desync.height !== null) parts.push(`height ${desync.height}`)
+    if (desync.failures !== undefined && desync.failures !== null) parts.push(`${desync.failures} consecutive failed fetches`)
+    if (desync.lastError) parts.push(`last error: ${desync.lastError}`)
+    return parts.length ? parts.join(', ') : JSON.stringify(desync)
+}
+
 // Ask the service itself. JSON-RPC `health` first because it is the richest
-// surface (lag + halt markers); GET /status is the fallback for an older image,
-// and is exactly what the Docker healthcheck runs.
+// surface (lag + halt markers); GET /status is the fallback for an older image
+// or a shed health POST, and is exactly what the Docker healthcheck runs. A
+// fallback body that carries no lag field still refuses in evaluateStatusPayload.
 async function probeServiceStatus(containerId, port, runner) {
     const rpcBody = JSON.stringify({ jsonrpc: '2.0', method: 'health', id: 1 })
     const attempts = [
diff --git a/test/unit/BootstrapHealthGate.test.js b/test/unit/BootstrapHealthGate.test.js
index ae285c0..025e28b 100644
--- a/test/unit/BootstrapHealthGate.test.js
+++ b/test/unit/BootstrapHealthGate.test.js
@@ -343,9 +343,40 @@ describe('BootstrapHealthGate', function () {
             expect(reasons.join(' ')).to.match(/cannot see the node tip/)
         })
 
-        it('passes a healthy, caught-up payload with no lag field at all', function () {
+        it('passes a healthy payload whose lag is zero', function () {
             const gate = loadGate()
-            expect(gate.evaluateStatusPayload({ status: 'ok', db: true })).to.deep.equal([])
+            expect(gate.evaluateStatusPayload({ status: 'ok', db: true, lag: 0 })).to.deep.equal([])
+        })
+
+        it('REFUSES a payload with no lag field at all: position unverifiable is not caught-up', function () {
+            const gate = loadGate()
+            const reasons = gate.evaluateStatusPayload({ status: 'ok', db: true })
+            expect(reasons).to.have.lengthOf(1)
+            expect(reasons[0]).to.match(/did not report how far behind it is/)
+        })
+
+        it('REFUSES a negative lag: the committed tip sits above the node tip (orphaned view)', function () {
+            const gate = loadGate()
+            const reasons = gate.evaluateStatusPayload({ status: 'healthy', lag: -100, synced: false })
+            expect(reasons).to.have.lengthOf(1)
+            expect(reasons[0]).to.match(/negative lag \(-100\)/)
+            expect(reasons[0]).to.not.match(/blocks behind/)
+            expect(gate.evaluateStatusPayload({ status: 'healthy', lag_blocks: -1 }).join(' ')).to.match(/negative lag_blocks/)
+        })
+
+        it('formats a block-fetch desync object instead of printing [object Object]', function () {
+            const gate = loadGate()
+            const reasons = gate.evaluateStatusPayload({
+                status: 'healthy', lag: 0,
+                block_fetch_desync: { height: 5342110, failures: 20, lastError: 'Block not found', detectedAt: Date.now() }
+            })
+            expect(reasons).to.have.lengthOf(1)
+            expect(reasons[0]).to.match(/height 5342110/)
+            expect(reasons[0]).to.match(/20 consecutive failed fetches/)
+            expect(reasons[0]).to.match(/last error: Block not found/)
+            expect(reasons[0]).to.not.match(/\[object Object\]/)
+            expect(gate.evaluateStatusPayload({ lag: 0, block_fetch_desync: 'node pruned' }).join(' ')).to.match(/desync \(node pruned\)/)
+            expect(gate.evaluateStatusPayload({ lag: 0, block_fetch_desync: { other: 1 } }).join(' ')).to.match(/\{"other":1\}/)
         })
     })
 

From 8ebac2e022ba9e43b04d193a49f4b9ed83239f7d Mon Sep 17 00:00:00 2001
From: J-Dog <j-dog@j-dog.net>
Date: Fri, 21 Aug 2026 20:12:51 -0700
Subject: [PATCH 18/18] chore(release): v0.10.0

Pins the twelve tagged component commits, bumps the platform version and records
the train's changes.

The manifest now covers every module the installer clones. The 0.9.0 manifest
listed eight of the twelve, so a pinned install of that train still resolved
xchain-sdk, xchain-e2e-test, xchain-contracts and xchain-regtest-miner at their
default branch, which a pinned install is supposed to rule out.
---
 CHANGELOG.md              | 25 ++++++++++++++++
 package-lock.json         |  4 +--
 package.json              |  2 +-
 src/release-manifest.json | 60 ++++++++++++++++++++++++++-------------
 4 files changed, 68 insertions(+), 23 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9daf936..7c1dd85 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.10.0] - 2026-08-22
+
+### Added
+- Pinned installs verify downloaded release artifacts against the pinned signing key.
+- The explorer can run a self-synced checkpoint mirror, with the database grants it needs, so a deployment with no colocated hub schema can still serve the checkpoint, proof and cross-chain routes.
+- A module update is refused when its source asserts a gated migration that has not been applied.
+
+### Changed
+- The release manifest pins every module the installer clones. The 0.10.0 set is twelve; the 0.9.0 manifest carried eight, so a pinned install of that train still cloned four modules at their default branch.
+
+### Fixed
+- A bootstrap source that reports no lag, or a negative lag, is refused with a reason instead of being read as healthy.
+- Autoheal no longer restarts a container an operator deliberately stopped.
+- The hub database account is rotated when the hub is recreated, and the headless config prompt is guarded.
+- Uninstall keeps a shared service that is still serving another coin.
+- Install returns only once the explorer is actually serving the new coins, and fails when the explorer never starts serving them.
+- Install tells the shared services about coins the same run just created.
+- The explorer installs on a stack that has no coins yet.
+- The hub and explorer are staged at the ref the command named.
+- A migration precondition reads its skip flag by name.
+- The block-fetch desync record renders field by field instead of as an object placeholder.
+
+### Security
+- Raised the brace-expansion and js-yaml dependency floors and the advisory guards that pin them.
+
 ## [0.9.0] - 2026-08-14
 
 First release of the XChain Platform release train. Every component in the train
diff --git a/package-lock.json b/package-lock.json
index ec056b6..452b899 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "xchain-node",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "xchain-node",
-      "version": "0.9.0",
+      "version": "0.10.0",
       "license": "AGPL-3.0-or-later",
       "dependencies": {
         "axios": "^1.18.1",
diff --git a/package.json b/package.json
index cc59b6c..7f2aa2f 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "xchain-node",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "xchain-node allows users to install, configure and run XChain platform nodes.",
   "license": "AGPL-3.0-or-later",
   "repository": {
diff --git a/src/release-manifest.json b/src/release-manifest.json
index bbfeb26..f212290 100644
--- a/src/release-manifest.json
+++ b/src/release-manifest.json
@@ -1,43 +1,63 @@
 {
   "_comment": [
-    "Pinned component set for XChain Platform v0.9.0.",
+    "Pinned component set for XChain Platform v0.10.0.",
     "Written at ceremony step 6 from the ACTUAL tagged master merge commits.",
-    "xchain-node is the carrier and is not listed: checking out its tag IS this manifest."
+    "xchain-node is the carrier and is not listed: checking out its tag IS this manifest.",
+    "Every module in constants.js modulesUrls is pinned here. The v0.9.0 manifest",
+    "carried only 8 of the 12, so a pinned install of that train still cloned",
+    "xchain-sdk, xchain-e2e-test, xchain-contracts and xchain-regtest-miner at their",
+    "default branch, which is not reproducible. All 12 are pinned from this train on."
   ],
-  "platform_version": "0.9.0",
-  "released": "2026-08-14",
+  "platform_version": "0.10.0",
+  "released": "2026-08-22",
   "components": {
     "xchain-vm": {
-      "tag": "v0.9.0",
-      "commit": "d29fba335d98cd156c5ee927a89eccdda1aaaf01"
+      "tag": "v0.10.0",
+      "commit": "4db160dd6ebcb1a7c8e8309b45ce00ffb5f7793c"
     },
     "xchain-decoder": {
-      "tag": "v0.9.0",
-      "commit": "622ce5ab616f605f990692771a6682cb730fb2f4"
+      "tag": "v0.10.0",
+      "commit": "09170e9a32c67484ece203a6cfc6f149f530f0a7"
     },
     "xchain-indexer": {
-      "tag": "v0.9.0",
-      "commit": "ea53a7f99255fa20f8bacb2aee5cbc9f0b3ad1dd"
+      "tag": "v0.10.0",
+      "commit": "698be256cb062ba3f05bd5a4e69d9d2f1cb206e7"
     },
     "xchain-hub": {
-      "tag": "v0.9.0",
-      "commit": "3a9e1b5fd3ba115f12826ea88a0868a0d10f5b5a"
+      "tag": "v0.10.0",
+      "commit": "cbb97b97e1b670a8b28b40d4abe9f664d8eb284e"
     },
     "xchain-sync": {
-      "tag": "v0.9.0",
-      "commit": "dc75649f3469a8fa32c4a1ea7749d05da4a319f7"
+      "tag": "v0.10.0",
+      "commit": "70a58033ff3b8468c26053afdfa4b8614e502bc8"
     },
     "xchain-encoder": {
-      "tag": "v0.9.0",
-      "commit": "ea785f6da4281ebc45912f7dea1e9e062a4e8f44"
+      "tag": "v0.10.0",
+      "commit": "27a2a87f892aed2b9602361ceb78f24a639050bb"
     },
     "xchain-utxo-tracker": {
-      "tag": "v0.9.0",
-      "commit": "21fdfd80660bd86d09d91402e23b873f0cee8574"
+      "tag": "v0.10.0",
+      "commit": "c48b87be04a7115f78b2ec7d3031d25104be1706"
     },
     "xchain-explorer": {
-      "tag": "v0.9.0",
-      "commit": "ffc73bf4d33716e9932a87d2405b5d42e69092ba"
+      "tag": "v0.10.0",
+      "commit": "e4ca98a6af0f153d4ec928576e1f24568c3dfa8b"
+    },
+    "xchain-sdk": {
+      "tag": "v0.10.0",
+      "commit": "7bcefac0c28544f7fcb742629fc553e3d6e2c49e"
+    },
+    "xchain-e2e-test": {
+      "tag": "v0.10.0",
+      "commit": "5db34052213ac847218844eef3c1fd2690f2c2b4"
+    },
+    "xchain-contracts": {
+      "tag": "v0.10.0",
+      "commit": "da194411bdaa43add28fb1af1d345f92c89c0d2e"
+    },
+    "xchain-regtest-miner": {
+      "tag": "v0.10.0",
+      "commit": "ba73838eeaab95c942ef3cbbae9f6efccb9726bb"
     }
   }
 }