From 108b03626964691386920404cfae31923c2b4156 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 18 Aug 2026 14:34:06 -0700 Subject: [PATCH 1/6] fix(dispenser): arm the oracle-fee capture and expiry realignment on testnet Both gates fix defects that spend a payer's native coin and give nothing back: a refill's oracle-fee output captured against the wrong dispenser, and a payment landing in the exact block a dispenser expires being dropped before the indexer sees it. A public testnet will hit both, so they are armed from genesis there. Mainnet stays disarmed pending its own ratified instant. --- src/protocol/constants.js | 12 ++++++++++-- .../dispenserExpiryRealignActivation.test.js | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/protocol/constants.js b/src/protocol/constants.js index 4e7969f..88c5c39 100644 --- a/src/protocol/constants.js +++ b/src/protocol/constants.js @@ -370,7 +370,11 @@ const ORACLE_FEE_OUTPUT_ACTIVATION = { // ORACLE_FEE_OUTPUT_ACTIVATION. const ORACLE_FEE_SET_CAPTURE_ACTIVATION = { mainnet: null, // DISARMED: awaiting the operator's ratified per-network instant - testnet: null, // DISARMED: awaiting the operator's ratified per-network instant + // ARMED AT GENESIS (instant 0 = always in force), operator-ratified 2026-08-18 under the + // pre-launch ruling that every feature must be ACTIVE on testnet. This gate fixes a defect + // that spends a payer native coin and gives nothing back, so a public testnet WILL hit it. + // Safe at 0 because testnet decoder/indexer state is REBUILT from the chain before launch. + testnet: 0, regtest: 0, }; @@ -415,7 +419,11 @@ const ORACLE_FEE_SET_CAPTURE_ACTIVATION = { // keeps the two copies in lockstep. const DISPENSER_EXPIRY_REALIGN_ACTIVATION = { mainnet: null, // DISARMED: awaiting the operator's ratified per-network instant - testnet: null, // DISARMED: awaiting the operator's ratified per-network instant + // ARMED AT GENESIS (instant 0 = always in force), operator-ratified 2026-08-18 under the + // pre-launch ruling that every feature must be ACTIVE on testnet. This gate fixes a defect + // that spends a payer native coin and gives nothing back, so a public testnet WILL hit it. + // Safe at 0 because testnet decoder/indexer state is REBUILT from the chain before launch. + testnet: 0, regtest: 0, }; diff --git a/test/unit/dispenserExpiryRealignActivation.test.js b/test/unit/dispenserExpiryRealignActivation.test.js index 68d4fae..e45a39b 100644 --- a/test/unit/dispenserExpiryRealignActivation.test.js +++ b/test/unit/dispenserExpiryRealignActivation.test.js @@ -51,11 +51,13 @@ function siblingOrSkip(ctx, file){ describe('DISPENSER_EXPIRY_REALIGN_ACTIVATION conformance', function () { - it('keeps mainnet and testnet DISARMED and regtest genesis-on', function () { - // Teeth for the ratification requirement: a number on mainnet/testnet means someone - // armed a consensus boundary without the operator's ratified instant. + it('keeps mainnet DISARMED, with testnet and regtest genesis-on', function () { + // Teeth for the ratification requirement: a number on MAINNET means someone armed a + // consensus boundary without the operator's ratified instant. Testnet was ratified at + // instant 0 on 2026-08-18 (pre-launch, every feature active on testnet), which is safe + // only because testnet decoder/indexer state is rebuilt from the chain before launch. assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet, null); - assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.testnet, null); + assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.testnet, 0); assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.regtest, 0); }); @@ -74,10 +76,16 @@ describe('DISPENSER_EXPIRY_REALIGN_ACTIVATION conformance', function () { }); it('a DISARMED network is inactive at every block time, including absurd ones', function () { + // Mainnet is the network still carrying the null sentinel. A `time >= null` coercion + // would read 0 and arm it from genesis, which is the failure this pins. assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 0), false); assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 1786060800), false); assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 4000000000), false); - assert.strictEqual(isDispenserExpiryRealignActive('testnet', 4000000000), false); + }); + + it('testnet is active from genesis, so the launch runs the realigned path', function () { + assert.strictEqual(isDispenserExpiryRealignActive('testnet', 0), true); + assert.strictEqual(isDispenserExpiryRealignActive('testnet', 4000000000), true); }); it('regtest is active from genesis so the venues exercise the realigned path', function () { From 9b077d00ee31af540e659feac76634982b7055e4 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Thu, 20 Aug 2026 18:58:48 -0700 Subject: [PATCH 2/6] fix(decoder): close the hash-comment DDL gate, count reorgs, and drop npm from PID 1 2026-08-20 xchain-platform review round sitting 3. Findings: 5312, 5352, 5365, 5366, 5369, 5400, 5401. Recorded in the platform review store and the round report. --- Dockerfile | 8 +- package-lock.json | 7 -- package.json | 1 - src/BlockchainConnector.js | 14 ++- src/XChainDecoder.js | 33 +++++- src/db.js | 103 ++++++++++++++++-- src/decoderMetrics.js | 12 +- src/protocol/constants.js | 12 +- .../blockchainConnectorReviewFixes.test.js | 32 ++++++ test/unit/db.unit.test.js | 24 ++++ test/unit/decoderTipStaleSurface.test.js | 51 +++++++++ test/unit/mempoolIsolation.test.js | 18 +-- test/unit/migration-runner.test.js | 56 ++++++++++ test/unit/protocol-constants.test.js | 14 +++ 14 files changed, 342 insertions(+), 43 deletions(-) diff --git a/Dockerfile b/Dockerfile index bf4aa4d..dc0baae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,4 +16,10 @@ COPY ./src /XChainDecoder/src COPY ./src/bufferutils.js /XChainDecoder/node_modules/bitcoinjs-lib/src/bufferutils.js COPY ./.en[v] /XChainDecoder/.env -CMD ["npm", "run", "api"] \ No newline at end of file +# Exec-form node, not `npm run api` (which is this exact command). npm builds an +# npm -> sh -c -> node tree and no wrapper forwards signals, so `docker stop` +# kills npm while node is never told anything (measured on the regtest encoder, +# xchain-encoder/Dockerfile). This image registers real drain work on SIGTERM +# (src/api.js: flip decoderRunning, then decoder.stop()), which only runs when +# node is PID 1 and receives the signal itself. +CMD ["node", "./src/api.js"] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 3ffd92a..bbb1db7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,6 @@ "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", - "binary-search": "^1.3.6", "bip32": "4.0.0", "bip39": "^3.1.0", "bitcoinjs-lib": "6.1.7", @@ -1574,12 +1573,6 @@ "node": "*" } }, - "node_modules/binary-search": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", - "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", - "license": "CC0-1.0" - }, "node_modules/bip174": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/bip174/-/bip174-2.1.1.tgz", diff --git a/package.json b/package.json index 9c37580..31d5f8a 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,6 @@ }, "dependencies": { "axios": "^1.18.1", - "binary-search": "^1.3.6", "bip32": "4.0.0", "bip39": "^3.1.0", "bitcoinjs-lib": "6.1.7", diff --git a/src/BlockchainConnector.js b/src/BlockchainConnector.js index 35310af..feb3f97 100644 --- a/src/BlockchainConnector.js +++ b/src/BlockchainConnector.js @@ -73,6 +73,14 @@ function sanitizeRpcError(error){ // with HTTP 200 and result: null, in which case the real cause (Block height // out of range, Loading block index..., auth/queue errors) must not be masked // by a hand-written placeholder. `label` is the existing per-method message. +// +// "Missing" is PRESENCE, not truthiness: a JSON-RPC success carries a `result` +// member that may legitimately be 0, false or "", and only undefined/null mean +// the node sent no result. Every method funnelled through here today answers +// with an object, an array or a non-empty hex string, so this changes nothing +// for them; it is the guard the first falsy-answering method (a count at +// genesis, a boolean) would otherwise be misread by and burned through the +// caller's retry loop as a hard RPC failure. function rpcResult(response, label) { const rpcError = response && response.data && response.data.error if (rpcError) { @@ -80,8 +88,10 @@ function rpcResult(response, label) { const message = (typeof rpcError.message === 'string') ? rpcError.message : JSON.stringify(rpcError) throw new Error(`${label}: RPC error ${code}: ${message}`) } - if (!response || !response.data || !response.data.result) throw new Error(label) - return response.data.result + if (!response || !response.data) throw new Error(label) + const result = response.data.result + if (result === undefined || result === null) throw new Error(label) + return result } // Decode a Bitcoin-style varint from `buf` at `offset`. diff --git a/src/XChainDecoder.js b/src/XChainDecoder.js index dbd744b..08695cb 100644 --- a/src/XChainDecoder.js +++ b/src/XChainDecoder.js @@ -369,6 +369,15 @@ class XChainDecoder { this.rpcErrors = 0 this.parseErrors = 0 + // Lifetime reorg counters, mirroring xchain-utxo-tracker. Each rolled-back block + // already writes a durable REORG row, but that trace is DB-only: without these a + // metrics-only deployment (no monitor plugin, indexer possibly down) has no + // scrapeable signal for a decoder thrashing through repeated shallow reorgs. + // Counted once per completed verifyReorg run, so count is reorg EVENTS and depth + // is the blocks rolled back by the most recent one. + this.reorgCount = 0 + this.lastReorgDepth = 0 + // Consecutive block-fetch failures at _fetchErrorHeight. _fetchErrorCount counts // every failure (operator visibility); _auxPowParseErrorCount counts only the // AuxPoW-header-strip content faults that may escalate to per-tx block @@ -558,7 +567,13 @@ class XChainDecoder { const status = { last_processed_block: this.lastProcessedBlockIndex, node_height: this.blockchainInfoLastBlock, - lag: this.blockchainInfoLastBlock - this.lastProcessedBlockIndex + lag: this.blockchainInfoLastBlock - this.lastProcessedBlockIndex, + // Reorg churn, additive: an operator polling /status sees how often this + // decoder has rolled back and how deep the last one went, without joining + // against the indexer. Absent from the nothing-processed-yet shape above, + // which deliberately reports unknowns rather than zeros. + reorg_count: this.reorgCount, + last_reorg_depth: this.lastReorgDepth } if (nodeHeightStale) status.node_height_stale = true return status @@ -1858,6 +1873,10 @@ class XChainDecoder { // delete (deleteBlockByIndex), so there is no separate end-of-run event to write. // This is only an ops summary of the completed reorg. this.log(`reorg: rolled back ${blocksDeleted.length} block(s): ` + JSON.stringify(blocksDeleted.map(b => b.block_index))) + // Once per RUN, never per deleted block: a per-block increment would report a + // single depth-5 reorg as five reorgs and destroy the frequency signal. + this.reorgCount++ + this.lastReorgDepth = blocksDeleted.length } return true @@ -3226,11 +3245,13 @@ class XChainDecoder { // Dedup + single O(n log n) sort. The old per-txid binary-insert // (bs + splice) was O(n^2) in mempool size every poll cycle, a CPU - // hazard under a mempool flood. ORDER CONTRACT: descending - // lexicographic, i.e. exactly what the inverted bs comparator - // `needle.localeCompare(element)` produced; db.js - // deleteAndCompareTxsNotInList binary-searches this array with - // that same comparator and silently breaks on any other order. + // hazard under a mempool flood. What the consumer needs is the DEDUP: + // db.js deleteAndCompareTxsNotInList seeds this array into a temp + // table and filters it through a Set, so a repeated txid would be + // fetched and inserted twice. The descending sort is deterministic + // poll-order only (it preserves the order the old bs comparator + // produced, which keeps logs and fixtures comparable); nothing in the + // DB layer searches this array, so no ordering is load-bearing. rawMempool = Array.from(new Set(rawMempoolUnordered)) .sort((a, b) => b.localeCompare(a)) diff --git a/src/db.js b/src/db.js index e83b7a7..74a28b9 100644 --- a/src/db.js +++ b/src/db.js @@ -21,7 +21,6 @@ const mariadb = require('mariadb'); const fs = require('fs'); const util = require('./util') -const bs = require("binary-search") const SATOSHIS_DECIMALS = 8 const DB_NAME_REGEX = /^[A-Za-z0-9_]+$/ @@ -675,20 +674,23 @@ class Database { } // Destructive-DDL scan for the auto-apply path. Given a migration file's - // statement list (already `--`-comment-stripped and ';'-split), returns the + // statement list (already line-comment-stripped and ';'-split), returns the // first statement that can lose, truncate, or rename data - or null when the // file is safe to auto-run. Pure string logic (no DB), unit-tested directly. // Byte-for-byte the same classifier as xchain-indexer/src/db.js so the two // migration runners stay legible as a pair. // // Flagged as destructive: DROP TABLE/DATABASE/SCHEMA, TRUNCATE, RENAME TABLE, - // DELETE (any form), REPLACE INTO (atomic DELETE+INSERT), UPDATE (except the + // DELETE (any form), REPLACE INTO (atomic DELETE+INSERT), INSERT ... ON DUPLICATE + // KEY UPDATE (rewrites every colliding row), LOAD DATA (rows from a file the + // scanner cannot read), UPDATE (except the // committed AUTO_INCREMENT id=0 repair), // ALTER TABLE ... DROP , // ALTER TABLE ... RENAME (except RENAME INDEX/KEY), ALTER TABLE ... CHANGE - // (rename+retype), and MODIFY ... NOT NULL (the statically detectable + // (rename+retype), MODIFY ... NOT NULL (the statically detectable // narrowing; a width reduction cannot be seen without the live schema and - // stays covered by the manual-tag convention). + // stays covered by the manual-tag convention), and any ALTER TABLE PARTITION or + // TABLESPACE clause. // // Deliberately NOT flagged (legitimate existing auto patterns): DROP INDEX/KEY, // DROP FOREIGN KEY/CONSTRAINT/CHECK/DEFAULT/PRIMARY KEY (structural, no row @@ -699,6 +701,27 @@ class Database { // Drops that remove metadata only; anything else after DROP inside an // ALTER (COLUMN, PARTITION, or a bare column identifier) loses data. const SAFE_ALTER_DROP = new Set(['INDEX', 'KEY', 'FOREIGN', 'CONSTRAINT', 'CHECK', 'DEFAULT', 'PRIMARY']); + // True when a `#` sits outside every quoted span - a line comment + // stripSqlLineComments should already have removed. Quote-aware so a `#` + // inside a string literal or a backtick identifier is not mistaken for one. + // Local rather than a method: runMigrations' callers build partial `this` + // objects, and a second prototype hop would break the guard on those. + const hasUnquotedHash = (s) => { + let q = null; + for(let i = 0; i < s.length; i++){ + const c = s[i]; + if(q){ + if(c === q){ + if(s[i + 1] === q){ i++; } + else { q = null; } + } + continue; + } + if(c === "'" || c === '"' || c === '`'){ q = c; continue; } + if(c === '#') return true; + } + return false; + }; for(const raw of (statements || [])){ // Executable (versioned) comments are the one /* */ form the server RUNS: // MariaDB/MySQL execute `/*!50000 DROP TABLE balances */` and `/*M! ... */` @@ -713,6 +736,12 @@ class Database { // gone) so a keyword inside comment prose never triggers or hides a hit. const stmt = String(raw).replace(/\/\*[\s\S]*?\*\//g, ' ').trim(); if(!stmt) continue; + // Second layer behind stripSqlLineComments: MariaDB/MySQL honour `#` to + // end-of-line as a comment, so `# note\nDROP TABLE balances` is a DROP every + // ^-anchored check below is blind to. The strip removes it upstream; if one + // ever reaches here the strip has regressed, and the only safe reading of a + // comment introducer the classifier can still see is non-auto-eligible. + if(hasUnquotedHash(stmt)) return raw; // Server-side indirection escapes a statement-prefix classifier: a mode=auto // file can smuggle destructive SQL past every keyword check below via dynamic // SQL (`SET @s = 'DROP TABLE balances'; PREPARE stmt FROM @s; EXECUTE stmt;`) @@ -744,6 +773,16 @@ class Database { // touches - the same data-loss profile as DELETE, with no non-destructive // form - so match the bare keyword like DELETE above. if(/^REPLACE\b/i.test(stmt)) return raw; + // INSERT ... ON DUPLICATE KEY UPDATE overwrites columns of every existing + // duplicate-key row it touches - the same data-rewrite profile the UPDATE arm + // below hard-blocks, reached from a keyword that arm never sees. Plain INSERT + // stays auto-eligible: with no ON DUPLICATE clause it only adds rows. + if(/^INSERT\b[\s\S]*\bON\s+DUPLICATE\s+KEY\s+UPDATE\b/i.test(stmt)) return raw; + // LOAD DATA ... REPLACE INTO TABLE is a DELETE+INSERT on every key collision, + // and the rows come from a file the classifier cannot read, so no form of it + // can be judged safe from the statement text. No committed auto migration + // loads a file; treat the whole form as non-auto-eligible. + if(/^LOAD\s+DATA\b/i.test(stmt)) return raw; // A bare UPDATE can rewrite arbitrary row data. The one committed auto // pattern is the AUTO_INCREMENT id repair (`UPDATE SET id = (...) // WHERE id = 0;` in 2026-06-10-mirror-id-autoincrement-repair.sql), which @@ -751,6 +790,15 @@ class Database { // flag every other UPDATE. if(/^UPDATE\b/i.test(stmt) && !this._isIdRepairUpdate(stmt)) return raw; if(/^ALTER\s+TABLE\b/i.test(stmt)){ + // Partition and tablespace clauses move or discard row data while carrying + // none of the keywords the checks below look for: TRUNCATE PARTITION empties + // a partition, EXCHANGE PARTITION swaps its rows out to another table, + // DISCARD TABLESPACE deletes the table's data file. The additive members of + // the class (ADD PARTITION, IMPORT TABLESPACE) are not separable from the + // destructive ones by prefix, and no committed migration partitions anything, + // so the whole class is non-auto-eligible - re-tag mode=manual to run one. + if(/\bPARTITION(?:ING)?\b/i.test(stmt)) return raw; + if(/\bTABLESPACE\b/i.test(stmt)) return raw; // Every DROP inside the ALTER must target a safe (metadata-only) object. let m; const dropRe = /\bDROP\s+([A-Za-z_]+|`[^`]+`)/gi; @@ -836,11 +884,27 @@ class Database { ); } - // Remove SQL `--` line comments while respecting quoted strings, so a ';' + // Remove SQL line comments while respecting quoted strings, so a ';' // or ',' appearing inside comment prose is never mistaken for SQL structure. // Single/double-quote and backtick spans are preserved verbatim (doubled - // quotes treated as escapes); a `--` outside any quote skips to the end of - // its line. Newlines are kept so the column-split below stays well-formed. + // quotes treated as escapes); a `--` or `#` outside any quote or block comment + // skips to the end of its line. Newlines are kept so the column-split below + // stays well-formed. + // + // `#` counts because MariaDB/MySQL honour it to end-of-line exactly like + // `--`. Missing it made a `# note` line ahead of a destructive statement + // invisible to the ^-anchored checks in _destructiveAutoStatement: the + // chunk began with `#`, matched no keyword, scored the file auto-eligible, + // and the server ran the DROP unattended at startup. A `;` inside a `#` + // comment also tore the statement in two for both the classifier and the + // apply loop. + // + // `/* ... */` spans are copied through verbatim rather than scanned: a `--` + // or `#` inside one would otherwise swallow the closing `*/` and the rest of + // that line (the server does not treat either as a comment start there), and + // an apostrophe in block-comment prose would open a bogus quote span. The + // verbatim copy also keeps `/*!...*/` executable-comment payloads intact for + // _destructiveAutoStatement to flag. stripSqlLineComments(sql){ let out = ''; let quote = null; @@ -855,7 +919,14 @@ class Database { continue; } if(ch === "'" || ch === '"' || ch === '`'){ quote = ch; out += ch; continue; } - if(ch === '-' && sql[i + 1] === '-'){ + if(ch === '/' && sql[i + 1] === '*'){ + const end = sql.indexOf('*/', i + 2); + if(end === -1){ out += sql.slice(i); break; } // unterminated: copy the rest as-is + out += sql.slice(i, end + 2); + i = end + 1; + continue; + } + if((ch === '-' && sql[i + 1] === '-') || ch === '#'){ while(i < sql.length && sql[i] !== '\n'){ i++; } if(i < sql.length){ out += '\n'; } continue; @@ -870,8 +941,8 @@ class Database { // string literal contains a semicolon (e.g. `SET data = 'a;b'`) into invalid // fragments, so no migration or seed carrying a semicolon in quoted data can // ship, and _destructiveAutoStatement ends up classifying fragments rather than - // real statements. `--` line comments are stripped first (same rule as the - // callers used); the quote model matches stripSqlLineComments exactly + // real statements. `--` and `#` line comments are stripped first (same rule as + // the callers used); the quote model matches stripSqlLineComments exactly // (single/double-quote and backtick spans, doubled quotes treated as escapes). // Returns trimmed, non-empty statements. Mirrors xchain-indexer/src/db.js. splitSqlStatements(sql){ @@ -890,6 +961,16 @@ class Database { continue; } if(ch === "'" || ch === '"' || ch === '`'){ quote = ch; current += ch; continue; } + // Block comments survive the strip (the classifier needs `/*!...*/` payloads + // intact), so carry them across whole: an apostrophe in comment prose must not + // open a quote span, and a ';' inside one must not terminate the statement. + if(ch === '/' && stripped[i + 1] === '*'){ + const end = stripped.indexOf('*/', i + 2); + if(end === -1){ current += stripped.slice(i); break; } + current += stripped.slice(i, end + 2); + i = end + 1; + continue; + } if(ch === ';'){ statements.push(current); current = ''; continue; } current += ch; } diff --git a/src/decoderMetrics.js b/src/decoderMetrics.js index 4a0ce37..4b27b06 100644 --- a/src/decoderMetrics.js +++ b/src/decoderMetrics.js @@ -37,12 +37,14 @@ const DECODER_GAUGES = [ ['last_block_advance_timestamp_seconds', 'Unix time of the last forward block advance'], ['node_height_stale', '1 when the cached node tip is frozen (two or more consecutive tip polls failed)'], ['synced', '1 when the decoder is caught up to a fresh node tip'], - ['stalled', '1 when the block loop is wedged (the /live liveness signal)'] + ['stalled', '1 when the block loop is wedged (the /live liveness signal)'], + ['last_reorg_depth', 'Blocks rolled back by the most recent reorg'] ]; const DECODER_COUNTERS = [ ['parse_errors_total', 'Transactions the decoder failed to parse since process start'], - ['rpc_errors_total', 'Node RPC errors seen since process start'] + ['rpc_errors_total', 'Node RPC errors seen since process start'], + ['reorgs_total', 'Reorgs this decoder has rolled back since process start'] ]; /** @@ -95,6 +97,12 @@ function registerDecoderMetrics(registry, decoder) { const rpcErrors = (decoder.rpcErrors || 0) + ((decoder.connector && decoder.connector.rpcErrors) || 0); counters.rpc_errors_total.setMonotonic({}, rpcErrors); counters.parse_errors_total.setMonotonic({}, decoder.parseErrors || 0); + + // Reorg churn. The durable REORG rows and the indexer's reorgsProcessed cover + // the completed handshake, but neither is scrapeable when only Prometheus is + // deployed; these read the decoder's own lifetime counters at scrape time. + setIf(gauges.last_reorg_depth, decoder.lastReorgDepth); + counters.reorgs_total.setMonotonic({}, decoder.reorgCount || 0); }); return { gauges, counters, collector }; diff --git a/src/protocol/constants.js b/src/protocol/constants.js index 88c5c39..5c8d842 100644 --- a/src/protocol/constants.js +++ b/src/protocol/constants.js @@ -568,11 +568,13 @@ const VALID_FIAT_CODES = ['USD', 'CAD', 'AUD', 'MXN', 'GBP', 'JPY', 'CNY', 'CHF' const GAS_TICK = 'XCHAIN'; // Oracle federation (xchain-hub). -// Canonical source: xchain-hub/src/constants.js. Mirrored here by hand and gated by -// NOTHING: this repo is in neither the xcall constants guard nor the hub-side oracle -// mirror list (xchain-hub/test/unit/constants-conformance.test.js), so a hub-side edit -// drifts this copy silently. Treat any change as a manual all-copies edit. Nothing -// here reads either one; they are re-exports for consumers. +// Canonical source: xchain-hub/src/constants.js, mirrored here by hand. These two are +// NOT in the GOLDEN set of the xcall constants gate, and this repo carries no copy of +// that gate at all; the guard that diffs this copy against the canonical lives in +// xchain-hub/test/unit/constants-conformance.test.js (#3886), whose MIRRORS roster +// names this repo, so a drift here reddens hub CI rather than this repo's. Treat any +// change as a manual all-copies edit. Nothing here reads either one; they are +// re-exports for consumers. // Coarse global sanity ceiling on an ingested price_snapshots value (pre-scale, // covers pairs like BTC/KRW up to ~$7M BTC with headroom); rejects diff --git a/test/unit/blockchainConnectorReviewFixes.test.js b/test/unit/blockchainConnectorReviewFixes.test.js index 4219560..610517e 100644 --- a/test/unit/blockchainConnectorReviewFixes.test.js +++ b/test/unit/blockchainConnectorReviewFixes.test.js @@ -117,4 +117,36 @@ describe('BlockchainConnector RPC error accounting and reporting', () => { ) }).timeout(5000) }) + + describe('the shared result extractor reads PRESENCE, not truthiness', () => { + // JSON-RPC 2.0: a success carries a `result` member, which may legitimately + // be 0, false or "". Only undefined/null mean the node sent no result. No + // method routed through the extractor today can answer falsy, so these pin + // the contract for the next one rather than a behaviour change. + it('returns a falsy-but-present result instead of throwing', async () => { + axiosStub.resolves({ data: { result: '' } }) + assert.strictEqual(await connector.getBlockHash(0), '') + + axiosStub.resolves({ data: { result: 0 } }) + assert.strictEqual(await connector.getBlockHash(0), 0) + + axiosStub.resolves({ data: { result: false } }) + assert.strictEqual(await connector.getBlockHash(0), false) + + assert.strictEqual(connector.rpcErrors, 0, 'a valid falsy result is not an RPC failure') + }).timeout(5000) + + it('still throws the per-method label when the result is absent', async () => { + axiosStub.resolves({ data: { result: null } }) + await assert.rejects(() => connector.getBlockHash(0), /Error getting block hash/) + + axiosStub.resolves({ data: {} }) + await assert.rejects(() => connector.getBlockHash(0), /Error getting block hash/) + }).timeout(5000) + + it('still prefers the node error object over the result member', async () => { + axiosStub.resolves({ data: { result: 0, error: { code: -8, message: 'Block height out of range' } } }) + await assert.rejects(() => connector.getBlockHash(0), /RPC error -8: Block height out of range/) + }).timeout(5000) + }) }) diff --git a/test/unit/db.unit.test.js b/test/unit/db.unit.test.js index 5ec507b..ab00bcd 100644 --- a/test/unit/db.unit.test.js +++ b/test/unit/db.unit.test.js @@ -247,6 +247,30 @@ describe('Database#stripSqlLineComments()', () => { assert.ok(!result.includes('end of file')) assert.ok(result.includes('SELECT 1')) }) + + it('should strip a # comment, which MariaDB honours to end-of-line like --', () => { + const result = db.stripSqlLineComments('SELECT 1 # this is a comment\nSELECT 2') + assert.ok(!result.includes('this is a comment')) + assert.ok(result.includes('SELECT 1')) + assert.ok(result.includes('SELECT 2')) + }) + + it('should preserve a # inside quoted strings and backtick identifiers', () => { + assert.ok(db.stripSqlLineComments("SELECT '# not a comment' FROM t").includes('# not a comment')) + assert.ok(db.stripSqlLineComments('SELECT "# not a comment" FROM t').includes('# not a comment')) + assert.ok(db.stripSqlLineComments('SELECT `col#1` FROM t').includes('`col#1`')) + }) + + it('should copy /* */ block comments through verbatim', () => { + const sql = '/* see issue #4413 -- and this */ SELECT 1' + assert.strictEqual(db.stripSqlLineComments(sql), sql) + }) + + it('should not treat an apostrophe in block-comment prose as a quote start', () => { + const result = db.stripSqlLineComments("/* don't do this */ SELECT 1 -- gone\nSELECT 2") + assert.ok(!result.includes('gone')) + assert.ok(result.includes('SELECT 2')) + }) }) describe('Database#parseExpectedColumns()', () => { diff --git a/test/unit/decoderTipStaleSurface.test.js b/test/unit/decoderTipStaleSurface.test.js index fbd119e..39837b7 100644 --- a/test/unit/decoderTipStaleSurface.test.js +++ b/test/unit/decoderTipStaleSurface.test.js @@ -218,6 +218,57 @@ describe('registerDecoderMetrics() feed-freshness gauges', function () { assert.match(body, /^xchain_decoder_node_height_stale 0$/m); }); + // Reorg churn had no decoder-side signal at all: the durable REORG rows are + // DB-only and the indexer's reorgsProcessed needs the indexer to be up, so a + // metrics-only deployment could watch a decoder thrash through shallow reorgs + // and see nothing move. + + it('exports reorg count and depth so a metrics-only deployment sees churn', function () { + const registry = new Registry(); + const decoder = makeRunningDecoder(); + decoder.reorgCount = 3; + decoder.lastReorgDepth = 5; + registerDecoderMetrics(registry, decoder); + + const body = registry.render(); + assert.match(body, /^xchain_decoder_reorgs_total 3$/m); + assert.match(body, /^xchain_decoder_last_reorg_depth 5$/m); + }); + + it('reports zero reorgs on a fresh decoder rather than no series at all', function () { + // Unlike the height gauges, absent here is NOT safe: a missing counter and a + // decoder that has never reorged look identical to a rate() query. + const registry = new Registry(); + registerDecoderMetrics(registry, makeDecoder()); + assert.match(registry.render(), /^xchain_decoder_reorgs_total 0$/m); + }); + + it('counts reorg EVENTS, incrementing once per verifyReorg run', function () { + // A per-block increment inside either delete branch would report one depth-5 + // reorg as five reorgs and destroy the frequency signal the counter exists for. + // The branches need a live node to reach, so this is a source-level guard. + const source = fs.readFileSync(require.resolve('../../src/XChainDecoder.js'), 'utf-8'); + const increments = source.match(/this\.reorgCount\+\+/g) || []; + assert.strictEqual(increments.length, 1, 'exactly one reorgCount increment site'); + assert.ok( + /if \(blocksDeleted\.length > 0\)\{[\s\S]{0,800}?this\.reorgCount\+\+/.test(source), + 'the increment must sit in verifyReorg\'s end-of-run summary block, not in a per-block delete branch' + ); + assert.ok( + /this\.lastReorgDepth = blocksDeleted\.length/.test(source), + 'depth must be the blocks rolled back by the run that just completed' + ); + }); + + it('surfaces the same counters on getSyncStatus, which /status spreads', function () { + const decoder = makeRunningDecoder(); + decoder.reorgCount = 2; + decoder.lastReorgDepth = 1; + const status = decoder.getSyncStatus(); + assert.strictEqual(status.reorg_count, 2); + assert.strictEqual(status.last_reorg_depth, 1); + }); + it('registers on the handle api.js captures, not a discarded return value', function () { const source = fs.readFileSync(require.resolve('../../src/api.js'), 'utf-8'); assert.ok( diff --git a/test/unit/mempoolIsolation.test.js b/test/unit/mempoolIsolation.test.js index 034cd60..613a330 100644 --- a/test/unit/mempoolIsolation.test.js +++ b/test/unit/mempoolIsolation.test.js @@ -70,12 +70,13 @@ describe('updateMempool DB isolation', function () { assert.strictEqual(getParseTxDbArg(), decoder.mempoolDb, 'mempool parse must use mempoolDb for pubkey capture') }) - it('hands deleteAndCompareTxsNotInList a deduped, DESCENDING-sorted txid list', async () => { - // db.js deleteAndCompareTxsNotInList binary-searches this array with the - // inverted comparator `needle.localeCompare(element)`, which requires - // descending lexicographic order. The old O(n^2) bs+splice build produced - // exactly that order (with duplicates skipped); the O(n log n) sort - // replacement must keep the same contract or the delete-diff silently breaks. + it('hands deleteAndCompareTxsNotInList a DEDUPED txid list', async () => { + // Dedup is the real contract at this seam: db.js deleteAndCompareTxsNotInList + // seeds the array into a temp table and filters it through a Set, so a repeated + // txid would be fetched and inserted twice. ORDER is deliberately NOT asserted + // here: the DB layer runs no binary search over this array (it did once, which + // is why an older version of this test pinned descending order), so the poll's + // sort is deterministic output rather than a requirement the consumer imposes. const { decoder } = buildDecoder(true) let received decoder.mempoolDb.deleteAndCompareTxsNotInList = async (list) => { @@ -84,8 +85,9 @@ describe('updateMempool DB isolation', function () { decoder.connector.getRawMempool = async () => ['bbb', 'aaa', 'ccc', 'aaa', 'bbb'] decoder.connector.getRawTransactions = async () => [] await decoder.updateMempool() - assert.deepStrictEqual(received, ['ccc', 'bbb', 'aaa'], - 'rawMempool must be deduped and sorted descending (the bs-comparator order)') + assert.strictEqual(received.length, 3, 'rawMempool must carry each txid once') + assert.deepStrictEqual(received.slice().sort(), ['aaa', 'bbb', 'ccc'], + 'rawMempool must be the deduped txid set the node reported') }) it('a mempool insert failure never rolls back or ends the block transaction', async () => { diff --git a/test/unit/migration-runner.test.js b/test/unit/migration-runner.test.js index 4519d42..fefd624 100644 --- a/test/unit/migration-runner.test.js +++ b/test/unit/migration-runner.test.js @@ -202,6 +202,47 @@ describe('Database._destructiveAutoStatement() @regression', function () { assert.strictEqual(scanSql('SET sql_mode = "STRICT_ALL_TABLES";'), null); assert.strictEqual(scanSql('SET @@session.foreign_key_checks = 0;'), null); }); + + // The indexer twin carries the same cases; keep the two suites in step. + + it('flags a DROP hidden behind a `#` line comment (the server honours `#`)', function () { + // Before the strip knew `#`, this reached the classifier as one chunk starting + // with `#`, matched no ^-anchored check, and auto-ran the DROP at startup. + const offender = scanSql('# cleanup\nDROP TABLE transactions;'); + assert.ok(offender && /DROP TABLE transactions/i.test(offender)); + }); + + it('flags a statement still carrying a `#` line comment (strip-regression guard)', function () { + assert.ok(scanOf(['# cleanup\nDROP TABLE transactions'])); + }); + + it('does not flag a `#` inside a quoted literal or a block comment', function () { + assert.strictEqual(scanSql("INSERT INTO notes (body) VALUES ('#tag');"), null); + assert.strictEqual(scanSql('/* see issue #4413 */ ALTER TABLE t ADD COLUMN y INT;'), null); + }); + + it('flags INSERT ... ON DUPLICATE KEY UPDATE but not a plain INSERT', function () { + assert.ok(scanSql("INSERT INTO dispensers (id, source) VALUES (1,'x') ON DUPLICATE KEY UPDATE source='y';")); + assert.strictEqual(scanSql("INSERT INTO dispensers (id, source) VALUES (1,'x');"), null); + }); + + it('flags LOAD DATA (rows come from a file the classifier cannot read)', function () { + assert.ok(scanSql("LOAD DATA INFILE '/tmp/x.csv' REPLACE INTO TABLE transactions;")); + assert.ok(scanSql("LOAD DATA LOCAL INFILE '/tmp/x.csv' INTO TABLE transactions;")); + }); + + it('flags ALTER TABLE partition and tablespace clauses', function () { + assert.ok(scanSql('ALTER TABLE events DROP PARTITION p2025;')); + assert.ok(scanSql('ALTER TABLE events TRUNCATE PARTITION p0;')); + assert.ok(scanSql('ALTER TABLE events EXCHANGE PARTITION p0 WITH TABLE events_old;')); + assert.ok(scanSql('ALTER TABLE events DISCARD TABLESPACE;')); + // Additive partition DDL is not separable by prefix, so it is non-auto too. + assert.ok(scanSql('ALTER TABLE events ADD PARTITION (PARTITION p2 VALUES LESS THAN (200));')); + }); + + it('does not flag an ordinary column whose name merely contains "partition"', function () { + assert.strictEqual(scanSql('ALTER TABLE t ADD COLUMN partition_id INT NULL;'), null); + }); }); describe('Database.backdatedFrontierViolation() @regression', function () { @@ -759,6 +800,21 @@ describe('Database.splitSqlStatements() @regression', function () { ['SELECT 1', 'SELECT 2']); }); + it('does not split on a ; inside a # line comment, and drops the comment', function () { + assert.deepStrictEqual(splitOf('SELECT 1; # see foo; bar\nSELECT 2;'), + ['SELECT 1', 'SELECT 2']); + assert.deepStrictEqual(splitOf('# cleanup\nDROP TABLE transactions;'), + ['DROP TABLE transactions']); + }); + + it('leaves a # or an apostrophe inside a block comment alone', function () { + // A naive #-to-end-of-line strip would eat the closing */ and the rest of the line. + assert.deepStrictEqual(splitOf('/* see issue #4413 */ SELECT 1;'), + ['/* see issue #4413 */ SELECT 1']); + assert.deepStrictEqual(splitOf("/* don't do this */ SELECT 1; SELECT 2;"), + ["/* don't do this */ SELECT 1", 'SELECT 2']); + }); + it('splits ordinary multi-statement SQL into the same statements as before', function () { assert.deepStrictEqual(splitOf('CREATE TABLE a (id INT);\nCREATE TABLE b (id INT);'), ['CREATE TABLE a (id INT)', 'CREATE TABLE b (id INT)']); diff --git a/test/unit/protocol-constants.test.js b/test/unit/protocol-constants.test.js index 25b954b..9573172 100644 --- a/test/unit/protocol-constants.test.js +++ b/test/unit/protocol-constants.test.js @@ -24,6 +24,20 @@ describe('protocol/constants', function () { }); } + // Value pins for the four constants the cross-repo xcall gate freezes. + // That gate (xchain-indexer/test/unit/xcall-constants-cross-repo.test.js) + // rosters only xchain-vm / -indexer / -sdk plus the xchain-documentation + // canonical, so this repo's mirror is tied to those values by nothing else: + // shape assertions alone let a one-sided edit here pass every suite in the + // platform. Same GOLDEN literals, same idiom as the sibling mirror suite in + // xchain-explorer. A real protocol bump edits every copy, this one included. + it('pins the gated cross-repo VM/XCALL limits to their GOLDEN values', function () { + assert.strictEqual(C.MAX_CODE_SIZE, 65536); + assert.strictEqual(C.XCALL_MAX_GAS, 200000); + assert.strictEqual(C.XCALL_MAX_HOPS, 2); + assert.strictEqual(C.XCALL_MIN_DEADLINE_BLOCKS, 10); + }); + it('XCALL gas floor does not exceed its ceiling', function () { assert.ok(C.XCALL_MIN_GAS <= C.XCALL_MAX_GAS); }); From c70c834ee629f3de5d76c82a2ed63ca7fceb4c6a Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 21 Aug 2026 08:29:36 -0700 Subject: [PATCH 3/6] fix(decoder): announce reorgs at warn, pin the DISPENSER wire offsets, arm capture on one boundary A detected reorg was announced on stdout, which warn-and-above alerting rules never read, and the block-hash retry loop that can spin through a whole node outage logged at info. Both now go to stderr at warn and error respectively, and the chaos helpers capture the three streams separately so the level is part of what the suites assert. The DISPENSER v0 and v2 expiration offsets were bare literals (14 and 4) with a comment as their only contract; they are now named constants, and a new unit suite derives all three offsets from the sibling indexer's live Dispenser formats so a format change turns red here. The sub-command capture activation test asserted capture never precedes the batch settlement ledger (>=). The canonical map states one decision, so it now asserts equality on every network: a gap in either direction is a consensus window, and ordering alone let a re-arm open it with every pin still green. --- src/XChainDecoder.js | 16 ++- src/oracleFeeOutput.js | 13 ++ .../chaos/CE04-midTransactionFailure.chaos.js | 8 +- test/chaos/CE06-chainReorg.chaos.js | 15 +- test/chaos/helpers.js | 11 +- ...hSubCommandOutputCaptureActivation.test.js | 44 ++++-- test/unit/dispenserFieldOffsets.test.js | 129 ++++++++++++++++++ test/unit/sibling-coverage.test.js | 3 +- 8 files changed, 212 insertions(+), 27 deletions(-) create mode 100644 test/unit/dispenserFieldOffsets.test.js diff --git a/src/XChainDecoder.js b/src/XChainDecoder.js index 08695cb..6cfd686 100644 --- a/src/XChainDecoder.js +++ b/src/XChainDecoder.js @@ -28,7 +28,7 @@ const ecc = require('tiny-secp256k1') const BlockchainConnector = require('./BlockchainConnector') const CryptoNetworks = require('./CryptoNetworks') const XChainBlockDecoder = require('./XChainBlockDecoder') -const { isOracleFeeCaptureActive, isOracleFeeSetCaptureActive, oracleAddressFromCreate, isCompactedOracleAddress } = require('./oracleFeeOutput') +const { isOracleFeeCaptureActive, isOracleFeeSetCaptureActive, oracleAddressFromCreate, isCompactedOracleAddress, V0_EXPIRATION_INDEX, V2_EXPIRATION_INDEX } = require('./oracleFeeOutput') const { isDispenserExpiryRealignActive } = require('./dispenserExpiryRealign') const { captureCommands, collapseDispenserRegistrations, isBatchSubCommandCaptureActive } = require('./batchSubCommandCapture') const { chainTierMismatch, chainFieldMissing, chainGenesisMismatch, chainGenesisUnpinned } = require('./chainIdentity') @@ -284,6 +284,10 @@ class XChainDecoder { // Coin/network-prefixed loggers so cadence/reorg/stall lines are self-describing // even when a log pipeline strips container labels. Reads the fields at call time. this.log = (...args) => console.log('[' + this.coinTick + '/' + this.consensusNetwork + ']', ...args) + // Warn exists so a notable-but-not-failed event (a reorg starting) can reach a + // warn-and-above alerting rule without being dressed up as an error. console.log + // writes to stdout, which those rules do not read. + this.logWarn = (...args) => console.warn('[' + this.coinTick + '/' + this.consensusNetwork + ']', ...args) this.logError = (...args) => console.error('[' + this.coinTick + '/' + this.consensusNetwork + ']', ...args) // Native-coin protocol fee destination address for this coin+network. When set (not the @@ -1804,7 +1808,7 @@ class XChainDecoder { try { blockHashFromNode = await this.connector.getBlockHash(lastBlockIndex) } catch (err){ - console.log("There was a problem trying to get a block hash from the node. Trying again...", err) + console.error("There was a problem trying to get a block hash from the node. Trying again...", err) // The node's tip may have regressed below lastBlockIndex mid-walk (node // restart onto a shorter chain, or a second reorg). Against the frozen // call-time nodeTip that makes getBlockHash(lastBlockIndex) throw "Block @@ -2458,7 +2462,7 @@ class XChainDecoder { //previousBlockHash is not the same, it must be a reorg if (previousBlockHash != previousBlock.block_hash){ await this.db.endTransaction() - console.log("A reorg has been detected at block " + nextBlockHeight + ". Cleaning blocks...") + this.logWarn("A reorg has been detected at block " + nextBlockHeight + ". Cleaning blocks...") const preReorgBlock = lastProcessedBlockIndex await this.verifyReorg(this.blockchainInfoLastBlock) // Re-clamp: same as the pre-loop guard and the node-tip regression path. @@ -2928,7 +2932,7 @@ class XChainDecoder { // Treat a missing token OR an empty-string token as an // omitted EXPIRATION and substitute the same default the // indexer uses; only a present, non-empty value is validated. - let expirationToken = decodedDataSplit[14] + let expirationToken = decodedDataSplit[V0_EXPIRATION_INDEX] let expiration if (expirationToken === undefined || expirationToken === "") { expiration = this.getDefaultExpiration(block.timestamp) @@ -2963,7 +2967,7 @@ class XChainDecoder { // loop on the same deterministic tx forever. if (!Number.isSafeInteger(expiration) || expiration < 0) { this.parseErrors++ - console.error(`Skipping dispenser in tx ${nextTransactionHash}: invalid expiration value '${decodedDataSplit[14]}'`) + console.error(`Skipping dispenser in tx ${nextTransactionHash}: invalid expiration value '${decodedDataSplit[V0_EXPIRATION_INDEX]}'`) } else if (this.dispenserOpensForThisChain(giveCoin, getCoin)){ if (getAddress && getAddress.length > 0 && getAddress.charAt(0) === "^"){ // Fail loud on a compacted `^` GET_ADDRESS. This is a @@ -3077,7 +3081,7 @@ class XChainDecoder { // failing to mirror WOULD close early. An edit that shortens one // is deliberately not mirrored. const editSource = parseResult["source"] - const editExpirationToken = decodedDataSplit[4] + const editExpirationToken = decodedDataSplit[V2_EXPIRATION_INDEX] if (editSource && editSource.length > 0 && editExpirationToken !== undefined && editExpirationToken !== ""){ const newExpiration = Number(editExpirationToken) diff --git a/src/oracleFeeOutput.js b/src/oracleFeeOutput.js index 3233cef..38e4d39 100644 --- a/src/oracleFeeOutput.js +++ b/src/oracleFeeOutput.js @@ -38,7 +38,18 @@ const { ORACLE_FEE_OUTPUT_ACTIVATION, ORACLE_FEE_SET_CAPTURE_ACTIVATION } = requ // 5 GIVE_OWNERSHIP | 6 GIVE_ESCROW | 7 GET_COIN | 8 GET_TICK | 9 GET_AMOUNT // 10 GET_ADDRESS | 11 FIAT_CODE | 12 FIAT_AMOUNT | 13 ORACLE_ADDRESS // 14 EXPIRATION | 15 ALLOW_LIST | 16 BLOCK_LIST | 17 MEMO +// Decoder offset = indexer format position + 1, because the decoder splits with the +// ACTION token ('DISPENSER') at 0 while the indexer's format string starts at VERSION. +// The comment is no longer the only contract: test/unit/dispenserFieldOffsets.test.js +// derives all three offsets from the live sibling Dispenser's this.formats. const ORACLE_ADDRESS_INDEX = 13 +const V0_EXPIRATION_INDEX = 14 + +// Field positions in the DISPENSER v2 (edit) wire format, same +1 convention +// (indexer this.formats[2]): +// 0 DISPENSER | 1 VERSION | 2 DISPENSER_ACTION_INDEX | 3 GIVE_ESCROW +// 4 EXPIRATION | 5 ALLOW_LIST | 6 BLOCK_LIST | 7 MEMO +const V2_EXPIRATION_INDEX = 4 // Is oracle-fee output capture in force for a block at `blockTime` on this network? // @@ -108,6 +119,8 @@ function isCompactedOracleAddress(fields){ module.exports = { ORACLE_ADDRESS_INDEX, + V0_EXPIRATION_INDEX, + V2_EXPIRATION_INDEX, isOracleFeeCaptureActive, isOracleFeeSetCaptureActive, oracleAddressFromCreate, diff --git a/test/chaos/CE04-midTransactionFailure.chaos.js b/test/chaos/CE04-midTransactionFailure.chaos.js index 499e007..fb74eff 100644 --- a/test/chaos/CE04-midTransactionFailure.chaos.js +++ b/test/chaos/CE04-midTransactionFailure.chaos.js @@ -146,12 +146,14 @@ describe('CE-04: Mid-Transaction Database Failure', function () { sinon.stub(decoder, 'verifyReorg').resolves(true) - const { logs } = await captureConsole(async () => { + const { warnings } = await captureConsole(async () => { await decoder.start() }) - const reorgLogs = logs.filter(l => l.includes('reorg has been detected')) - assert.ok(reorgLogs.length >= 1, 'Should detect reorg in main loop') + // Warn, not info: a reorg is the trigger for the decoder-to-indexer rollback + // handshake, and a warn-and-above alerting rule must see it start. + const reorgLogs = warnings.filter(l => l.includes('reorg has been detected')) + assert.ok(reorgLogs.length >= 1, 'Should detect reorg in main loop and announce it at warn level') assert.ok(decoder.verifyReorg.called, 'Should call verifyReorg') assert.ok(mockDb.endTransaction.called, 'Should end transaction before reorg processing') }) diff --git a/test/chaos/CE06-chainReorg.chaos.js b/test/chaos/CE06-chainReorg.chaos.js index 724a1e8..6db04f5 100644 --- a/test/chaos/CE06-chainReorg.chaos.js +++ b/test/chaos/CE06-chainReorg.chaos.js @@ -91,13 +91,15 @@ describe('CE-06: Chain Reorganization Detection and Recovery', function () { return 'old' }) - const { logs } = await captureConsole(async () => { + const { errors } = await captureConsole(async () => { await decoder.verifyReorg() }) assert.ok(hashCallCount >= 2, 'Should have retried getBlockHash') - const retryLogs = logs.filter(l => l.includes('problem trying to get a block hash')) - assert.ok(retryLogs.length >= 1, 'Should log retry message') + // Error, not info: this handler can spin for a whole node outage in the middle of + // a reorg walk, and it is the only trace that walk leaves while it is stalled. + const retryLogs = errors.filter(l => l.includes('problem trying to get a block hash')) + assert.ok(retryLogs.length >= 1, 'Should log retry message at error level') }) it('verifyReorg should stop cleanly when every processed block is invalidated', async function () { @@ -180,12 +182,13 @@ describe('CE-06: Chain Reorganization Detection and Recovery', function () { // Stub verifyReorg to just reset state sinon.stub(decoder, 'verifyReorg').resolves(true) - const { logs } = await captureConsole(async () => { + const { warnings } = await captureConsole(async () => { await decoder.start() }) - const reorgLogs = logs.filter(l => l.includes('reorg has been detected')) - assert.ok(reorgLogs.length >= 1, 'Should detect reorg in main loop') + // Warn, not info: see the level rationale on the same assertion in CE-04. + const reorgLogs = warnings.filter(l => l.includes('reorg has been detected')) + assert.ok(reorgLogs.length >= 1, 'Should detect reorg in main loop and announce it at warn level') assert.ok(decoder.verifyReorg.called, 'Should call verifyReorg') assert.ok(mockDb.endTransaction.called, 'Should end transaction before reorg processing') }) diff --git a/test/chaos/helpers.js b/test/chaos/helpers.js index b24d065..f3d83ac 100644 --- a/test/chaos/helpers.js +++ b/test/chaos/helpers.js @@ -142,21 +142,30 @@ function wait(ms) { /** * Captures console output during a function execution. + * + * All three streams are captured separately, because the LEVEL a line is emitted at is + * part of what these suites assert: an alerting rule that reads warn-and-above sees + * `warnings` and `errors` and never sees `logs`, so a test that accepts any stream would + * pass while the operator-visible signal was gone. */ async function captureConsole(fn) { const logs = [] + const warnings = [] const errors = [] const origLog = console.log + const origWarn = console.warn const origError = console.error console.log = (...args) => logs.push(args.join(' ')) + console.warn = (...args) => warnings.push(args.join(' ')) console.error = (...args) => errors.push(args.join(' ')) try { await fn() } finally { console.log = origLog + console.warn = origWarn console.error = origError } - return { logs, errors } + return { logs, warnings, errors } } /** diff --git a/test/unit/batchSubCommandOutputCaptureActivation.test.js b/test/unit/batchSubCommandOutputCaptureActivation.test.js index 4319239..dde8eec 100644 --- a/test/unit/batchSubCommandOutputCaptureActivation.test.js +++ b/test/unit/batchSubCommandOutputCaptureActivation.test.js @@ -27,9 +27,13 @@ // it out to several rows, and below that flag-day // output_fanout.collapseOutputFanout treats that as a consensus-critical // fault and HALTS the block. -// 3. LEDGER - on every ARMED network it is >= the indexer's BATCH_ISSUANCE_LIMITS instant, -// which carries the batch-cumulative settlement ledger. Capture without that -// ledger lets N COINPAY sub-commands settle N obligations from ONE payment. +// 3. LEDGER - on every network it is EQUAL to the indexer's BATCH_ISSUANCE_LIMITS instant, +// which carries the batch-cumulative settlement ledger: one decision, one +// boundary (xchain-documentation/protocol/constants.js). Capture without the +// ledger lets N COINPAY sub-commands settle N obligations from ONE payment; +// the ledger without capture makes a batched COINPAY spend the coin and +// settle nothing. Ordering alone (>=) would let a re-arm open that second +// window with every pin still green, so this tier is equality, not a floor. // 4. DOCS - it is value-identical to the canonical map in // xchain-documentation/protocol/constants.js, which must exist before mainnet // may be armed. @@ -75,6 +79,10 @@ const BELOW_MAINNET_GATE = ? BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION.mainnet - 1 : 4000000000; +// 2100-01-01, the boundary the indexer's unarmed-gate suites use to tell a scheduled date +// from the house UNARMED sentinel (9999999999): a ledger instant at or past it is unarmed. +const YEAR_2100 = 4102444800; + function siblingOrSkip(ctx, file){ if (fs.existsSync(file)) return true; if (REQUIRE_SIBLINGS) @@ -154,17 +162,33 @@ describe('BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION conformance', function () { } }); - it('never precedes the indexer BATCH_ISSUANCE_LIMITS flag-day (the settlement ledger)', function () { + it('arms at exactly the indexer BATCH_ISSUANCE_LIMITS instant on every network (one boundary)', function () { if (!siblingOrSkip(this, INDEXER_CHANGES)) return; + // Equality, not ordering. The canonical map states this as ONE decision: capture and + // the settlement ledger flip together. A gap in either direction is a consensus + // window: capture before the ledger lets N COINPAY sub-commands settle N obligations + // from one payment; the ledger before capture makes a batched COINPAY spend the coin + // and settle nothing. A >= leg plus two independent literal pins stayed green while + // a re-arm moved either side, which is exactly the edit this must refuse. const limits = indexerChangeTimes('BATCH_ISSUANCE_LIMITS'); for (const network of Object.keys(BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION)) { const gate = BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION[network]; - if (gate === null) continue; - assert.ok(gate >= limits[network], - network + ' sub-command capture (' + gate + ') must not begin before ' + - 'BATCH_ISSUANCE_LIMITS (' + limits[network] + '): that gate carries the ' + - 'batch-cumulative settlement ledger, without which N COINPAY sub-commands ' + - 'settle N obligations from one payment'); + if (gate === null) { + // A DISARMED capture gate may only sit under a DISARMED ledger: the indexer + // parks an unarmed mainnet on a far-future sentinel, never a real date. + assert.ok(limits[network] >= YEAR_2100, + network + ' sub-command capture is disarmed (null) while the indexer ' + + 'arms BATCH_ISSUANCE_LIMITS at ' + limits[network] + ': the settlement ' + + 'ledger would run with capture reading only the top-level ACTION name, so ' + + 'a batched COINPAY spends the coin and settles nothing'); + continue; + } + assert.strictEqual(gate, limits[network], + network + ' sub-command capture (' + gate + ') must equal the indexer ' + + 'BATCH_ISSUANCE_LIMITS instant (' + limits[network] + '): the canonical map ' + + 'states them as one boundary, and the window [' + + Math.min(gate, limits[network]) + ', ' + Math.max(gate, limits[network]) + + ') either double-settles one payment or settles nothing from it'); } }); diff --git a/test/unit/dispenserFieldOffsets.test.js b/test/unit/dispenserFieldOffsets.test.js new file mode 100644 index 0000000..de2ba46 --- /dev/null +++ b/test/unit/dispenserFieldOffsets.test.js @@ -0,0 +1,129 @@ +'use strict'; + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// DISPENSER wire field-offset drift guard. +// +// The decoder reads three DISPENSER fields by split offset: ORACLE_ADDRESS (the token +// oracle-fee capture keys on), the v0 create EXPIRATION and the v2 edit EXPIRATION. The +// authoritative layout is the indexer's own format strings +// (xchain-indexer/src/actions/dispenser.js this.formats), and until this guard existed the +// only thing binding the two was a prose comment, while every comparable dependency at this +// seam already had a mechanical gate (indexerBatchLimits.js vendoring, +// oracleFeeOutputActivationConformance.js). +// +// Drift is money-bearing in both directions: a field inserted ahead of ORACLE_ADDRESS makes +// capture key on the wrong token, so the indexer rejects every fee-bearing Mode B create +// with 'missing oracle fee output' after the payer's coin is spent; a shifted EXPIRATION +// diverges the decoder's open-dispenser set from the indexer's. +// +// Two tiers, so a one-sided edit fails somewhere no matter which checkout is present: +// 1. PIN - the constants equal the offsets this repo's decode path was written +// against, in this repo alone. +// 2. INDEXER - they equal the offsets derived from the LIVE sibling Dispenser's +// this.formats, read off an instance rather than scraped from source. +// Tier 2 skips when the sibling is absent (standalone deploy); XCHAIN_REQUIRE_SIBLINGS=1 +// makes absence a failure instead of a green-by-skip. +// +// The PIN tier is deliberately hard-coded rather than derived. A sibling that REORDERS the +// format string is a protocol fork that needs a human decision, so it must fail this file +// loudly rather than be adopted by re-running a generator. + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const { ORACLE_ADDRESS_INDEX, V0_EXPIRATION_INDEX, V2_EXPIRATION_INDEX, oracleAddressFromCreate } = + require('../../src/oracleFeeOutput.js'); + +// Offsets the decode path in src/XChainDecoder.js and src/oracleFeeOutput.js was written +// against. Decoder offset = indexer format position + 1: the decoder splits with the ACTION +// token ('DISPENSER') at 0, the indexer's format string starts at VERSION. +const PINNED = { ORACLE_ADDRESS: 13, V0_EXPIRATION: 14, V2_EXPIRATION: 4 }; +const ACTION_TOKEN_OFFSET = 1; + +const INDEXER_DISPENSER = process.env.XCHAIN_INDEXER_DIR + ? path.join(process.env.XCHAIN_INDEXER_DIR, 'src', 'actions', 'dispenser.js') + : path.join(__dirname, '..', '..', '..', 'xchain-indexer', 'src', 'actions', 'dispenser.js'); +const REQUIRE_SIBLINGS = process.env.XCHAIN_REQUIRE_SIBLINGS === '1'; + +function siblingOrSkip(ctx, file){ + if (fs.existsSync(file)) return true; + if (REQUIRE_SIBLINGS) + throw new Error('XCHAIN_REQUIRE_SIBLINGS=1 but sibling not found: ' + file); + ctx.skip(); + return false; +} + +// Read the formats off a REAL Dispenser instance, not a regex over its source: they are +// assigned on `this` in the constructor, so a rename or a reformat keeps working while a +// scrape would silently read nothing and pass. The constructor only stores its collaborators, +// so a stub-shaped action object is enough and no database is needed (same trick as +// test/tools/sync-batch-limits.js). +function siblingFormats(){ + const Dispenser = require(INDEXER_DISPENSER); + const dispenser = new Dispenser({ + config: {}, + decoderDb: {}, + indexerDb: {}, + util: {}, + mapper: {}, + protocolChanges: {}, + actionAliases: {}, + }); + return dispenser.formats; +} + +function offsetOf(format, field){ + const position = String(format).split('|').indexOf(field); + assert.notStrictEqual(position, -1, 'the sibling format no longer carries ' + field + ': ' + format); + return position + ACTION_TOKEN_OFFSET; +} + +describe('DISPENSER wire field offsets', function () { + + it('pins the offsets the decode path was written against', function () { + assert.strictEqual(ORACLE_ADDRESS_INDEX, PINNED.ORACLE_ADDRESS); + assert.strictEqual(V0_EXPIRATION_INDEX, PINNED.V0_EXPIRATION); + assert.strictEqual(V2_EXPIRATION_INDEX, PINNED.V2_EXPIRATION); + }); + + it('reads ORACLE_ADDRESS from the pinned slot and not a neighbouring one', function () { + // Binds the behaviour, not just the constant: a split whose ORACLE_ADDRESS slot alone + // carries an address must resolve to that address, and its neighbours must not. + const fields = new Array(18).fill(''); + fields[0] = 'DISPENSER'; + fields[PINNED.ORACLE_ADDRESS] = 'bc1qoracle'; + assert.strictEqual(oracleAddressFromCreate(fields), 'bc1qoracle'); + + const shifted = new Array(18).fill(''); + shifted[0] = 'DISPENSER'; + shifted[PINNED.ORACLE_ADDRESS - 1] = 'bc1qneighbour'; + assert.strictEqual(oracleAddressFromCreate(shifted), null); + }); + + it('matches the offsets derived from the live sibling indexer Dispenser formats', function () { + if (!siblingOrSkip(this, INDEXER_DISPENSER)) return; + const formats = siblingFormats(); + assert.ok(formats && typeof formats === 'object', + 'xchain-indexer Dispenser must expose this.formats'); + + assert.strictEqual(offsetOf(formats[0], 'ORACLE_ADDRESS'), ORACLE_ADDRESS_INDEX, + 'ORACLE_ADDRESS moved in the indexer v0 format: oracle-fee capture would key on the ' + + 'wrong token and the indexer would reject every fee-bearing Mode B create'); + assert.strictEqual(offsetOf(formats[0], 'EXPIRATION'), V0_EXPIRATION_INDEX, + 'the v0 create EXPIRATION moved in the indexer format: the decoder open-dispenser ' + + 'set would diverge from the indexer'); + assert.strictEqual(offsetOf(formats[2], 'EXPIRATION'), V2_EXPIRATION_INDEX, + 'the v2 edit EXPIRATION moved in the indexer format: the decoder would extend the ' + + 'wrong expiry, or none'); + }); +}); diff --git a/test/unit/sibling-coverage.test.js b/test/unit/sibling-coverage.test.js index f986fb0..7cffe45 100644 --- a/test/unit/sibling-coverage.test.js +++ b/test/unit/sibling-coverage.test.js @@ -68,7 +68,8 @@ const SIBLINGS = [ guards: 'vendored coins-registry byte-identity (BTC/LTC/DOGE/index/consensus_pin)' }, { repo: 'xchain-indexer', envs: ['XCHAIN_INDEXER_DIR'], marker: path.join('src', 'protocol_changes.js'), - guards: 'the FIX_OUTPUT_FANOUT registration in the indexer protocol-change table' }, + guards: 'the FIX_OUTPUT_FANOUT registration in the indexer protocol-change table, and the ' + + 'DISPENSER v0/v2 wire field offsets derived from the indexer Dispenser formats' }, { repo: 'xchain-utxo-tracker', envs: ['XCHAIN_UTXO_TRACKER_DIR'], marker: path.join('src', 'BlockchainConnector.js'), guards: 'AuxPoW strip parity and the dispenser safe-depth twin' }, From 7aff1b9a6225d03f2b8278b56e9c99b3c2e315e6 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Mon, 17 Aug 2026 19:28:01 -0700 Subject: [PATCH 4/6] chore(release): v0.10.0 --- CHANGELOG.md | 9 +++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e83c038..c141fa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.0] - 2026-08-18 + +### Fixed +- Chain identity is re-proven on the reorg tip re-read, so a wrong-chain answer cannot slip in during a reorganization. +- Code-review round fixes across the decode path (two rounds, 14 files). + +### Security +- Raised the brace-expansion and js-yaml dependency floors and the advisory guards that pin them. + ## [0.9.0] - 2026-08-14 First release of the XChain Platform release train. Every component in the train diff --git a/package-lock.json b/package-lock.json index bbb1db7..6fd9dc9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-decoder", - "version": "0.9.0", + "version": "0.10.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-decoder", - "version": "0.9.0", + "version": "0.10.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", diff --git a/package.json b/package.json index 31d5f8a..4619262 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "xchain-decoder", "description": "xchain-decoder decodes XChain platform transactions from a given blockchain and populates a database with the decoded data.", - "version": "0.9.0", + "version": "0.10.0", "license": "AGPL-3.0-or-later", "repository": { "type": "git", From c9748a4d5842cb15b5c202dd35f6d02f8214c989 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Wed, 19 Aug 2026 21:28:17 -0700 Subject: [PATCH 5/6] docs(changelog): record the dispenser gates this release ships --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c141fa5..e3bcfbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.10.0] - 2026-08-18 +Consensus-affecting changes in this release ship behind per-chain activation +points; behavior below each activation height is unchanged. + ### Fixed +- Two dispenser defects that spend a payer's own coin and give nothing back are corrected behind activation gates, armed on testnet from genesis and disarmed on mainnet. - Chain identity is re-proven on the reorg tip re-read, so a wrong-chain answer cannot slip in during a reorganization. - Code-review round fixes across the decode path (two rounds, 14 files). From 76cb7a24db5bfdd49e1cd3537f3ce0b062804b87 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 21 Aug 2026 09:19:32 -0700 Subject: [PATCH 6/6] docs(changelog): record the post-freeze work in the 0.10.0 section --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3bcfbc..f4c2ed2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ points; behavior below each activation height is unchanged. ### Fixed - Two dispenser defects that spend a payer's own coin and give nothing back are corrected behind activation gates, armed on testnet from genesis and disarmed on mainnet. - Chain identity is re-proven on the reorg tip re-read, so a wrong-chain answer cannot slip in during a reorganization. +- A detected reorg is announced on stderr at warn and the block-hash retry loop at error, so alerting rules that only read warn-and-above can see them; reorgs are also counted. +- The dispenser expiration wire offsets are named constants derived from the indexer's live formats, instead of bare literals with a comment as their only contract. +- Sub-command capture arms on exactly the same boundary as the batch settlement ledger on every network, closing the consensus window an ordering-only check allowed. +- The container no longer runs npm as its first process. - Code-review round fixes across the decode path (two rounds, 14 files). ### Security