diff --git a/CHANGELOG.md b/CHANGELOG.md index e83c038..f4c2ed2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.0] - 2026-08-18 + +Consensus-affecting changes in this release ship behind per-chain activation +points; behavior below each activation height is unchanged. + +### Fixed +- Two dispenser defects that spend a payer's own coin and give nothing back are corrected behind activation gates, armed on testnet from genesis and disarmed on mainnet. +- Chain identity is re-proven on the reorg tip re-read, so a wrong-chain answer cannot slip in during a reorganization. +- A detected reorg is announced on stderr at warn and the block-hash retry loop at error, so alerting rules that only read warn-and-above can see them; reorgs are also counted. +- The dispenser expiration wire offsets are named constants derived from the indexer's live formats, instead of bare literals with a comment as their only contract. +- Sub-command capture arms on exactly the same boundary as the batch settlement ledger on every network, closing the consensus window an ordering-only check allowed. +- The container no longer runs npm as its first process. +- Code-review round fixes across the decode path (two rounds, 14 files). + +### Security +- Raised the brace-expansion and js-yaml dependency floors and the advisory guards that pin them. + ## [0.9.0] - 2026-08-14 First release of the XChain Platform release train. Every component in the train diff --git a/Dockerfile b/Dockerfile index bf4aa4d..dc0baae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,4 +16,10 @@ COPY ./src /XChainDecoder/src COPY ./src/bufferutils.js /XChainDecoder/node_modules/bitcoinjs-lib/src/bufferutils.js COPY ./.en[v] /XChainDecoder/.env -CMD ["npm", "run", "api"] \ No newline at end of file +# Exec-form node, not `npm run api` (which is this exact command). npm builds an +# npm -> sh -c -> node tree and no wrapper forwards signals, so `docker stop` +# kills npm while node is never told anything (measured on the regtest encoder, +# xchain-encoder/Dockerfile). This image registers real drain work on SIGTERM +# (src/api.js: flip decoderRunning, then decoder.stop()), which only runs when +# node is PID 1 and receives the signal itself. +CMD ["node", "./src/api.js"] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 3ffd92a..6fd9dc9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,15 @@ { "name": "xchain-decoder", - "version": "0.9.0", + "version": "0.10.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-decoder", - "version": "0.9.0", + "version": "0.10.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", - "binary-search": "^1.3.6", "bip32": "4.0.0", "bip39": "^3.1.0", "bitcoinjs-lib": "6.1.7", @@ -1574,12 +1573,6 @@ "node": "*" } }, - "node_modules/binary-search": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", - "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", - "license": "CC0-1.0" - }, "node_modules/bip174": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/bip174/-/bip174-2.1.1.tgz", diff --git a/package.json b/package.json index 9c37580..4619262 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "xchain-decoder", "description": "xchain-decoder decodes XChain platform transactions from a given blockchain and populates a database with the decoded data.", - "version": "0.9.0", + "version": "0.10.0", "license": "AGPL-3.0-or-later", "repository": { "type": "git", @@ -9,7 +9,6 @@ }, "dependencies": { "axios": "^1.18.1", - "binary-search": "^1.3.6", "bip32": "4.0.0", "bip39": "^3.1.0", "bitcoinjs-lib": "6.1.7", diff --git a/src/BlockchainConnector.js b/src/BlockchainConnector.js index 35310af..feb3f97 100644 --- a/src/BlockchainConnector.js +++ b/src/BlockchainConnector.js @@ -73,6 +73,14 @@ function sanitizeRpcError(error){ // with HTTP 200 and result: null, in which case the real cause (Block height // out of range, Loading block index..., auth/queue errors) must not be masked // by a hand-written placeholder. `label` is the existing per-method message. +// +// "Missing" is PRESENCE, not truthiness: a JSON-RPC success carries a `result` +// member that may legitimately be 0, false or "", and only undefined/null mean +// the node sent no result. Every method funnelled through here today answers +// with an object, an array or a non-empty hex string, so this changes nothing +// for them; it is the guard the first falsy-answering method (a count at +// genesis, a boolean) would otherwise be misread by and burned through the +// caller's retry loop as a hard RPC failure. function rpcResult(response, label) { const rpcError = response && response.data && response.data.error if (rpcError) { @@ -80,8 +88,10 @@ function rpcResult(response, label) { const message = (typeof rpcError.message === 'string') ? rpcError.message : JSON.stringify(rpcError) throw new Error(`${label}: RPC error ${code}: ${message}`) } - if (!response || !response.data || !response.data.result) throw new Error(label) - return response.data.result + if (!response || !response.data) throw new Error(label) + const result = response.data.result + if (result === undefined || result === null) throw new Error(label) + return result } // Decode a Bitcoin-style varint from `buf` at `offset`. diff --git a/src/XChainDecoder.js b/src/XChainDecoder.js index dbd744b..6cfd686 100644 --- a/src/XChainDecoder.js +++ b/src/XChainDecoder.js @@ -28,7 +28,7 @@ const ecc = require('tiny-secp256k1') const BlockchainConnector = require('./BlockchainConnector') const CryptoNetworks = require('./CryptoNetworks') const XChainBlockDecoder = require('./XChainBlockDecoder') -const { isOracleFeeCaptureActive, isOracleFeeSetCaptureActive, oracleAddressFromCreate, isCompactedOracleAddress } = require('./oracleFeeOutput') +const { isOracleFeeCaptureActive, isOracleFeeSetCaptureActive, oracleAddressFromCreate, isCompactedOracleAddress, V0_EXPIRATION_INDEX, V2_EXPIRATION_INDEX } = require('./oracleFeeOutput') const { isDispenserExpiryRealignActive } = require('./dispenserExpiryRealign') const { captureCommands, collapseDispenserRegistrations, isBatchSubCommandCaptureActive } = require('./batchSubCommandCapture') const { chainTierMismatch, chainFieldMissing, chainGenesisMismatch, chainGenesisUnpinned } = require('./chainIdentity') @@ -284,6 +284,10 @@ class XChainDecoder { // Coin/network-prefixed loggers so cadence/reorg/stall lines are self-describing // even when a log pipeline strips container labels. Reads the fields at call time. this.log = (...args) => console.log('[' + this.coinTick + '/' + this.consensusNetwork + ']', ...args) + // Warn exists so a notable-but-not-failed event (a reorg starting) can reach a + // warn-and-above alerting rule without being dressed up as an error. console.log + // writes to stdout, which those rules do not read. + this.logWarn = (...args) => console.warn('[' + this.coinTick + '/' + this.consensusNetwork + ']', ...args) this.logError = (...args) => console.error('[' + this.coinTick + '/' + this.consensusNetwork + ']', ...args) // Native-coin protocol fee destination address for this coin+network. When set (not the @@ -369,6 +373,15 @@ class XChainDecoder { this.rpcErrors = 0 this.parseErrors = 0 + // Lifetime reorg counters, mirroring xchain-utxo-tracker. Each rolled-back block + // already writes a durable REORG row, but that trace is DB-only: without these a + // metrics-only deployment (no monitor plugin, indexer possibly down) has no + // scrapeable signal for a decoder thrashing through repeated shallow reorgs. + // Counted once per completed verifyReorg run, so count is reorg EVENTS and depth + // is the blocks rolled back by the most recent one. + this.reorgCount = 0 + this.lastReorgDepth = 0 + // Consecutive block-fetch failures at _fetchErrorHeight. _fetchErrorCount counts // every failure (operator visibility); _auxPowParseErrorCount counts only the // AuxPoW-header-strip content faults that may escalate to per-tx block @@ -558,7 +571,13 @@ class XChainDecoder { const status = { last_processed_block: this.lastProcessedBlockIndex, node_height: this.blockchainInfoLastBlock, - lag: this.blockchainInfoLastBlock - this.lastProcessedBlockIndex + lag: this.blockchainInfoLastBlock - this.lastProcessedBlockIndex, + // Reorg churn, additive: an operator polling /status sees how often this + // decoder has rolled back and how deep the last one went, without joining + // against the indexer. Absent from the nothing-processed-yet shape above, + // which deliberately reports unknowns rather than zeros. + reorg_count: this.reorgCount, + last_reorg_depth: this.lastReorgDepth } if (nodeHeightStale) status.node_height_stale = true return status @@ -1789,7 +1808,7 @@ class XChainDecoder { try { blockHashFromNode = await this.connector.getBlockHash(lastBlockIndex) } catch (err){ - console.log("There was a problem trying to get a block hash from the node. Trying again...", err) + console.error("There was a problem trying to get a block hash from the node. Trying again...", err) // The node's tip may have regressed below lastBlockIndex mid-walk (node // restart onto a shorter chain, or a second reorg). Against the frozen // call-time nodeTip that makes getBlockHash(lastBlockIndex) throw "Block @@ -1858,6 +1877,10 @@ class XChainDecoder { // delete (deleteBlockByIndex), so there is no separate end-of-run event to write. // This is only an ops summary of the completed reorg. this.log(`reorg: rolled back ${blocksDeleted.length} block(s): ` + JSON.stringify(blocksDeleted.map(b => b.block_index))) + // Once per RUN, never per deleted block: a per-block increment would report a + // single depth-5 reorg as five reorgs and destroy the frequency signal. + this.reorgCount++ + this.lastReorgDepth = blocksDeleted.length } return true @@ -2439,7 +2462,7 @@ class XChainDecoder { //previousBlockHash is not the same, it must be a reorg if (previousBlockHash != previousBlock.block_hash){ await this.db.endTransaction() - console.log("A reorg has been detected at block " + nextBlockHeight + ". Cleaning blocks...") + this.logWarn("A reorg has been detected at block " + nextBlockHeight + ". Cleaning blocks...") const preReorgBlock = lastProcessedBlockIndex await this.verifyReorg(this.blockchainInfoLastBlock) // Re-clamp: same as the pre-loop guard and the node-tip regression path. @@ -2909,7 +2932,7 @@ class XChainDecoder { // Treat a missing token OR an empty-string token as an // omitted EXPIRATION and substitute the same default the // indexer uses; only a present, non-empty value is validated. - let expirationToken = decodedDataSplit[14] + let expirationToken = decodedDataSplit[V0_EXPIRATION_INDEX] let expiration if (expirationToken === undefined || expirationToken === "") { expiration = this.getDefaultExpiration(block.timestamp) @@ -2944,7 +2967,7 @@ class XChainDecoder { // loop on the same deterministic tx forever. if (!Number.isSafeInteger(expiration) || expiration < 0) { this.parseErrors++ - console.error(`Skipping dispenser in tx ${nextTransactionHash}: invalid expiration value '${decodedDataSplit[14]}'`) + console.error(`Skipping dispenser in tx ${nextTransactionHash}: invalid expiration value '${decodedDataSplit[V0_EXPIRATION_INDEX]}'`) } else if (this.dispenserOpensForThisChain(giveCoin, getCoin)){ if (getAddress && getAddress.length > 0 && getAddress.charAt(0) === "^"){ // Fail loud on a compacted `^` GET_ADDRESS. This is a @@ -3058,7 +3081,7 @@ class XChainDecoder { // failing to mirror WOULD close early. An edit that shortens one // is deliberately not mirrored. const editSource = parseResult["source"] - const editExpirationToken = decodedDataSplit[4] + const editExpirationToken = decodedDataSplit[V2_EXPIRATION_INDEX] if (editSource && editSource.length > 0 && editExpirationToken !== undefined && editExpirationToken !== ""){ const newExpiration = Number(editExpirationToken) @@ -3226,11 +3249,13 @@ class XChainDecoder { // Dedup + single O(n log n) sort. The old per-txid binary-insert // (bs + splice) was O(n^2) in mempool size every poll cycle, a CPU - // hazard under a mempool flood. ORDER CONTRACT: descending - // lexicographic, i.e. exactly what the inverted bs comparator - // `needle.localeCompare(element)` produced; db.js - // deleteAndCompareTxsNotInList binary-searches this array with - // that same comparator and silently breaks on any other order. + // hazard under a mempool flood. What the consumer needs is the DEDUP: + // db.js deleteAndCompareTxsNotInList seeds this array into a temp + // table and filters it through a Set, so a repeated txid would be + // fetched and inserted twice. The descending sort is deterministic + // poll-order only (it preserves the order the old bs comparator + // produced, which keeps logs and fixtures comparable); nothing in the + // DB layer searches this array, so no ordering is load-bearing. rawMempool = Array.from(new Set(rawMempoolUnordered)) .sort((a, b) => b.localeCompare(a)) diff --git a/src/db.js b/src/db.js index e83b7a7..74a28b9 100644 --- a/src/db.js +++ b/src/db.js @@ -21,7 +21,6 @@ const mariadb = require('mariadb'); const fs = require('fs'); const util = require('./util') -const bs = require("binary-search") const SATOSHIS_DECIMALS = 8 const DB_NAME_REGEX = /^[A-Za-z0-9_]+$/ @@ -675,20 +674,23 @@ class Database { } // Destructive-DDL scan for the auto-apply path. Given a migration file's - // statement list (already `--`-comment-stripped and ';'-split), returns the + // statement list (already line-comment-stripped and ';'-split), returns the // first statement that can lose, truncate, or rename data - or null when the // file is safe to auto-run. Pure string logic (no DB), unit-tested directly. // Byte-for-byte the same classifier as xchain-indexer/src/db.js so the two // migration runners stay legible as a pair. // // Flagged as destructive: DROP TABLE/DATABASE/SCHEMA, TRUNCATE, RENAME TABLE, - // DELETE (any form), REPLACE INTO (atomic DELETE+INSERT), UPDATE (except the + // DELETE (any form), REPLACE INTO (atomic DELETE+INSERT), INSERT ... ON DUPLICATE + // KEY UPDATE (rewrites every colliding row), LOAD DATA (rows from a file the + // scanner cannot read), UPDATE (except the // committed AUTO_INCREMENT id=0 repair), // ALTER TABLE ... DROP , // ALTER TABLE ... RENAME (except RENAME INDEX/KEY), ALTER TABLE ... CHANGE - // (rename+retype), and MODIFY ... NOT NULL (the statically detectable + // (rename+retype), MODIFY ... NOT NULL (the statically detectable // narrowing; a width reduction cannot be seen without the live schema and - // stays covered by the manual-tag convention). + // stays covered by the manual-tag convention), and any ALTER TABLE PARTITION or + // TABLESPACE clause. // // Deliberately NOT flagged (legitimate existing auto patterns): DROP INDEX/KEY, // DROP FOREIGN KEY/CONSTRAINT/CHECK/DEFAULT/PRIMARY KEY (structural, no row @@ -699,6 +701,27 @@ class Database { // Drops that remove metadata only; anything else after DROP inside an // ALTER (COLUMN, PARTITION, or a bare column identifier) loses data. const SAFE_ALTER_DROP = new Set(['INDEX', 'KEY', 'FOREIGN', 'CONSTRAINT', 'CHECK', 'DEFAULT', 'PRIMARY']); + // True when a `#` sits outside every quoted span - a line comment + // stripSqlLineComments should already have removed. Quote-aware so a `#` + // inside a string literal or a backtick identifier is not mistaken for one. + // Local rather than a method: runMigrations' callers build partial `this` + // objects, and a second prototype hop would break the guard on those. + const hasUnquotedHash = (s) => { + let q = null; + for(let i = 0; i < s.length; i++){ + const c = s[i]; + if(q){ + if(c === q){ + if(s[i + 1] === q){ i++; } + else { q = null; } + } + continue; + } + if(c === "'" || c === '"' || c === '`'){ q = c; continue; } + if(c === '#') return true; + } + return false; + }; for(const raw of (statements || [])){ // Executable (versioned) comments are the one /* */ form the server RUNS: // MariaDB/MySQL execute `/*!50000 DROP TABLE balances */` and `/*M! ... */` @@ -713,6 +736,12 @@ class Database { // gone) so a keyword inside comment prose never triggers or hides a hit. const stmt = String(raw).replace(/\/\*[\s\S]*?\*\//g, ' ').trim(); if(!stmt) continue; + // Second layer behind stripSqlLineComments: MariaDB/MySQL honour `#` to + // end-of-line as a comment, so `# note\nDROP TABLE balances` is a DROP every + // ^-anchored check below is blind to. The strip removes it upstream; if one + // ever reaches here the strip has regressed, and the only safe reading of a + // comment introducer the classifier can still see is non-auto-eligible. + if(hasUnquotedHash(stmt)) return raw; // Server-side indirection escapes a statement-prefix classifier: a mode=auto // file can smuggle destructive SQL past every keyword check below via dynamic // SQL (`SET @s = 'DROP TABLE balances'; PREPARE stmt FROM @s; EXECUTE stmt;`) @@ -744,6 +773,16 @@ class Database { // touches - the same data-loss profile as DELETE, with no non-destructive // form - so match the bare keyword like DELETE above. if(/^REPLACE\b/i.test(stmt)) return raw; + // INSERT ... ON DUPLICATE KEY UPDATE overwrites columns of every existing + // duplicate-key row it touches - the same data-rewrite profile the UPDATE arm + // below hard-blocks, reached from a keyword that arm never sees. Plain INSERT + // stays auto-eligible: with no ON DUPLICATE clause it only adds rows. + if(/^INSERT\b[\s\S]*\bON\s+DUPLICATE\s+KEY\s+UPDATE\b/i.test(stmt)) return raw; + // LOAD DATA ... REPLACE INTO TABLE is a DELETE+INSERT on every key collision, + // and the rows come from a file the classifier cannot read, so no form of it + // can be judged safe from the statement text. No committed auto migration + // loads a file; treat the whole form as non-auto-eligible. + if(/^LOAD\s+DATA\b/i.test(stmt)) return raw; // A bare UPDATE can rewrite arbitrary row data. The one committed auto // pattern is the AUTO_INCREMENT id repair (`UPDATE SET id = (...) // WHERE id = 0;` in 2026-06-10-mirror-id-autoincrement-repair.sql), which @@ -751,6 +790,15 @@ class Database { // flag every other UPDATE. if(/^UPDATE\b/i.test(stmt) && !this._isIdRepairUpdate(stmt)) return raw; if(/^ALTER\s+TABLE\b/i.test(stmt)){ + // Partition and tablespace clauses move or discard row data while carrying + // none of the keywords the checks below look for: TRUNCATE PARTITION empties + // a partition, EXCHANGE PARTITION swaps its rows out to another table, + // DISCARD TABLESPACE deletes the table's data file. The additive members of + // the class (ADD PARTITION, IMPORT TABLESPACE) are not separable from the + // destructive ones by prefix, and no committed migration partitions anything, + // so the whole class is non-auto-eligible - re-tag mode=manual to run one. + if(/\bPARTITION(?:ING)?\b/i.test(stmt)) return raw; + if(/\bTABLESPACE\b/i.test(stmt)) return raw; // Every DROP inside the ALTER must target a safe (metadata-only) object. let m; const dropRe = /\bDROP\s+([A-Za-z_]+|`[^`]+`)/gi; @@ -836,11 +884,27 @@ class Database { ); } - // Remove SQL `--` line comments while respecting quoted strings, so a ';' + // Remove SQL line comments while respecting quoted strings, so a ';' // or ',' appearing inside comment prose is never mistaken for SQL structure. // Single/double-quote and backtick spans are preserved verbatim (doubled - // quotes treated as escapes); a `--` outside any quote skips to the end of - // its line. Newlines are kept so the column-split below stays well-formed. + // quotes treated as escapes); a `--` or `#` outside any quote or block comment + // skips to the end of its line. Newlines are kept so the column-split below + // stays well-formed. + // + // `#` counts because MariaDB/MySQL honour it to end-of-line exactly like + // `--`. Missing it made a `# note` line ahead of a destructive statement + // invisible to the ^-anchored checks in _destructiveAutoStatement: the + // chunk began with `#`, matched no keyword, scored the file auto-eligible, + // and the server ran the DROP unattended at startup. A `;` inside a `#` + // comment also tore the statement in two for both the classifier and the + // apply loop. + // + // `/* ... */` spans are copied through verbatim rather than scanned: a `--` + // or `#` inside one would otherwise swallow the closing `*/` and the rest of + // that line (the server does not treat either as a comment start there), and + // an apostrophe in block-comment prose would open a bogus quote span. The + // verbatim copy also keeps `/*!...*/` executable-comment payloads intact for + // _destructiveAutoStatement to flag. stripSqlLineComments(sql){ let out = ''; let quote = null; @@ -855,7 +919,14 @@ class Database { continue; } if(ch === "'" || ch === '"' || ch === '`'){ quote = ch; out += ch; continue; } - if(ch === '-' && sql[i + 1] === '-'){ + if(ch === '/' && sql[i + 1] === '*'){ + const end = sql.indexOf('*/', i + 2); + if(end === -1){ out += sql.slice(i); break; } // unterminated: copy the rest as-is + out += sql.slice(i, end + 2); + i = end + 1; + continue; + } + if((ch === '-' && sql[i + 1] === '-') || ch === '#'){ while(i < sql.length && sql[i] !== '\n'){ i++; } if(i < sql.length){ out += '\n'; } continue; @@ -870,8 +941,8 @@ class Database { // string literal contains a semicolon (e.g. `SET data = 'a;b'`) into invalid // fragments, so no migration or seed carrying a semicolon in quoted data can // ship, and _destructiveAutoStatement ends up classifying fragments rather than - // real statements. `--` line comments are stripped first (same rule as the - // callers used); the quote model matches stripSqlLineComments exactly + // real statements. `--` and `#` line comments are stripped first (same rule as + // the callers used); the quote model matches stripSqlLineComments exactly // (single/double-quote and backtick spans, doubled quotes treated as escapes). // Returns trimmed, non-empty statements. Mirrors xchain-indexer/src/db.js. splitSqlStatements(sql){ @@ -890,6 +961,16 @@ class Database { continue; } if(ch === "'" || ch === '"' || ch === '`'){ quote = ch; current += ch; continue; } + // Block comments survive the strip (the classifier needs `/*!...*/` payloads + // intact), so carry them across whole: an apostrophe in comment prose must not + // open a quote span, and a ';' inside one must not terminate the statement. + if(ch === '/' && stripped[i + 1] === '*'){ + const end = stripped.indexOf('*/', i + 2); + if(end === -1){ current += stripped.slice(i); break; } + current += stripped.slice(i, end + 2); + i = end + 1; + continue; + } if(ch === ';'){ statements.push(current); current = ''; continue; } current += ch; } diff --git a/src/decoderMetrics.js b/src/decoderMetrics.js index 4a0ce37..4b27b06 100644 --- a/src/decoderMetrics.js +++ b/src/decoderMetrics.js @@ -37,12 +37,14 @@ const DECODER_GAUGES = [ ['last_block_advance_timestamp_seconds', 'Unix time of the last forward block advance'], ['node_height_stale', '1 when the cached node tip is frozen (two or more consecutive tip polls failed)'], ['synced', '1 when the decoder is caught up to a fresh node tip'], - ['stalled', '1 when the block loop is wedged (the /live liveness signal)'] + ['stalled', '1 when the block loop is wedged (the /live liveness signal)'], + ['last_reorg_depth', 'Blocks rolled back by the most recent reorg'] ]; const DECODER_COUNTERS = [ ['parse_errors_total', 'Transactions the decoder failed to parse since process start'], - ['rpc_errors_total', 'Node RPC errors seen since process start'] + ['rpc_errors_total', 'Node RPC errors seen since process start'], + ['reorgs_total', 'Reorgs this decoder has rolled back since process start'] ]; /** @@ -95,6 +97,12 @@ function registerDecoderMetrics(registry, decoder) { const rpcErrors = (decoder.rpcErrors || 0) + ((decoder.connector && decoder.connector.rpcErrors) || 0); counters.rpc_errors_total.setMonotonic({}, rpcErrors); counters.parse_errors_total.setMonotonic({}, decoder.parseErrors || 0); + + // Reorg churn. The durable REORG rows and the indexer's reorgsProcessed cover + // the completed handshake, but neither is scrapeable when only Prometheus is + // deployed; these read the decoder's own lifetime counters at scrape time. + setIf(gauges.last_reorg_depth, decoder.lastReorgDepth); + counters.reorgs_total.setMonotonic({}, decoder.reorgCount || 0); }); return { gauges, counters, collector }; diff --git a/src/oracleFeeOutput.js b/src/oracleFeeOutput.js index 3233cef..38e4d39 100644 --- a/src/oracleFeeOutput.js +++ b/src/oracleFeeOutput.js @@ -38,7 +38,18 @@ const { ORACLE_FEE_OUTPUT_ACTIVATION, ORACLE_FEE_SET_CAPTURE_ACTIVATION } = requ // 5 GIVE_OWNERSHIP | 6 GIVE_ESCROW | 7 GET_COIN | 8 GET_TICK | 9 GET_AMOUNT // 10 GET_ADDRESS | 11 FIAT_CODE | 12 FIAT_AMOUNT | 13 ORACLE_ADDRESS // 14 EXPIRATION | 15 ALLOW_LIST | 16 BLOCK_LIST | 17 MEMO +// Decoder offset = indexer format position + 1, because the decoder splits with the +// ACTION token ('DISPENSER') at 0 while the indexer's format string starts at VERSION. +// The comment is no longer the only contract: test/unit/dispenserFieldOffsets.test.js +// derives all three offsets from the live sibling Dispenser's this.formats. const ORACLE_ADDRESS_INDEX = 13 +const V0_EXPIRATION_INDEX = 14 + +// Field positions in the DISPENSER v2 (edit) wire format, same +1 convention +// (indexer this.formats[2]): +// 0 DISPENSER | 1 VERSION | 2 DISPENSER_ACTION_INDEX | 3 GIVE_ESCROW +// 4 EXPIRATION | 5 ALLOW_LIST | 6 BLOCK_LIST | 7 MEMO +const V2_EXPIRATION_INDEX = 4 // Is oracle-fee output capture in force for a block at `blockTime` on this network? // @@ -108,6 +119,8 @@ function isCompactedOracleAddress(fields){ module.exports = { ORACLE_ADDRESS_INDEX, + V0_EXPIRATION_INDEX, + V2_EXPIRATION_INDEX, isOracleFeeCaptureActive, isOracleFeeSetCaptureActive, oracleAddressFromCreate, diff --git a/src/protocol/constants.js b/src/protocol/constants.js index 4e7969f..5c8d842 100644 --- a/src/protocol/constants.js +++ b/src/protocol/constants.js @@ -370,7 +370,11 @@ const ORACLE_FEE_OUTPUT_ACTIVATION = { // ORACLE_FEE_OUTPUT_ACTIVATION. const ORACLE_FEE_SET_CAPTURE_ACTIVATION = { mainnet: null, // DISARMED: awaiting the operator's ratified per-network instant - testnet: null, // DISARMED: awaiting the operator's ratified per-network instant + // ARMED AT GENESIS (instant 0 = always in force), operator-ratified 2026-08-18 under the + // pre-launch ruling that every feature must be ACTIVE on testnet. This gate fixes a defect + // that spends a payer native coin and gives nothing back, so a public testnet WILL hit it. + // Safe at 0 because testnet decoder/indexer state is REBUILT from the chain before launch. + testnet: 0, regtest: 0, }; @@ -415,7 +419,11 @@ const ORACLE_FEE_SET_CAPTURE_ACTIVATION = { // keeps the two copies in lockstep. const DISPENSER_EXPIRY_REALIGN_ACTIVATION = { mainnet: null, // DISARMED: awaiting the operator's ratified per-network instant - testnet: null, // DISARMED: awaiting the operator's ratified per-network instant + // ARMED AT GENESIS (instant 0 = always in force), operator-ratified 2026-08-18 under the + // pre-launch ruling that every feature must be ACTIVE on testnet. This gate fixes a defect + // that spends a payer native coin and gives nothing back, so a public testnet WILL hit it. + // Safe at 0 because testnet decoder/indexer state is REBUILT from the chain before launch. + testnet: 0, regtest: 0, }; @@ -560,11 +568,13 @@ const VALID_FIAT_CODES = ['USD', 'CAD', 'AUD', 'MXN', 'GBP', 'JPY', 'CNY', 'CHF' const GAS_TICK = 'XCHAIN'; // Oracle federation (xchain-hub). -// Canonical source: xchain-hub/src/constants.js. Mirrored here by hand and gated by -// NOTHING: this repo is in neither the xcall constants guard nor the hub-side oracle -// mirror list (xchain-hub/test/unit/constants-conformance.test.js), so a hub-side edit -// drifts this copy silently. Treat any change as a manual all-copies edit. Nothing -// here reads either one; they are re-exports for consumers. +// Canonical source: xchain-hub/src/constants.js, mirrored here by hand. These two are +// NOT in the GOLDEN set of the xcall constants gate, and this repo carries no copy of +// that gate at all; the guard that diffs this copy against the canonical lives in +// xchain-hub/test/unit/constants-conformance.test.js (#3886), whose MIRRORS roster +// names this repo, so a drift here reddens hub CI rather than this repo's. Treat any +// change as a manual all-copies edit. Nothing here reads either one; they are +// re-exports for consumers. // Coarse global sanity ceiling on an ingested price_snapshots value (pre-scale, // covers pairs like BTC/KRW up to ~$7M BTC with headroom); rejects diff --git a/test/chaos/CE04-midTransactionFailure.chaos.js b/test/chaos/CE04-midTransactionFailure.chaos.js index 499e007..fb74eff 100644 --- a/test/chaos/CE04-midTransactionFailure.chaos.js +++ b/test/chaos/CE04-midTransactionFailure.chaos.js @@ -146,12 +146,14 @@ describe('CE-04: Mid-Transaction Database Failure', function () { sinon.stub(decoder, 'verifyReorg').resolves(true) - const { logs } = await captureConsole(async () => { + const { warnings } = await captureConsole(async () => { await decoder.start() }) - const reorgLogs = logs.filter(l => l.includes('reorg has been detected')) - assert.ok(reorgLogs.length >= 1, 'Should detect reorg in main loop') + // Warn, not info: a reorg is the trigger for the decoder-to-indexer rollback + // handshake, and a warn-and-above alerting rule must see it start. + const reorgLogs = warnings.filter(l => l.includes('reorg has been detected')) + assert.ok(reorgLogs.length >= 1, 'Should detect reorg in main loop and announce it at warn level') assert.ok(decoder.verifyReorg.called, 'Should call verifyReorg') assert.ok(mockDb.endTransaction.called, 'Should end transaction before reorg processing') }) diff --git a/test/chaos/CE06-chainReorg.chaos.js b/test/chaos/CE06-chainReorg.chaos.js index 724a1e8..6db04f5 100644 --- a/test/chaos/CE06-chainReorg.chaos.js +++ b/test/chaos/CE06-chainReorg.chaos.js @@ -91,13 +91,15 @@ describe('CE-06: Chain Reorganization Detection and Recovery', function () { return 'old' }) - const { logs } = await captureConsole(async () => { + const { errors } = await captureConsole(async () => { await decoder.verifyReorg() }) assert.ok(hashCallCount >= 2, 'Should have retried getBlockHash') - const retryLogs = logs.filter(l => l.includes('problem trying to get a block hash')) - assert.ok(retryLogs.length >= 1, 'Should log retry message') + // Error, not info: this handler can spin for a whole node outage in the middle of + // a reorg walk, and it is the only trace that walk leaves while it is stalled. + const retryLogs = errors.filter(l => l.includes('problem trying to get a block hash')) + assert.ok(retryLogs.length >= 1, 'Should log retry message at error level') }) it('verifyReorg should stop cleanly when every processed block is invalidated', async function () { @@ -180,12 +182,13 @@ describe('CE-06: Chain Reorganization Detection and Recovery', function () { // Stub verifyReorg to just reset state sinon.stub(decoder, 'verifyReorg').resolves(true) - const { logs } = await captureConsole(async () => { + const { warnings } = await captureConsole(async () => { await decoder.start() }) - const reorgLogs = logs.filter(l => l.includes('reorg has been detected')) - assert.ok(reorgLogs.length >= 1, 'Should detect reorg in main loop') + // Warn, not info: see the level rationale on the same assertion in CE-04. + const reorgLogs = warnings.filter(l => l.includes('reorg has been detected')) + assert.ok(reorgLogs.length >= 1, 'Should detect reorg in main loop and announce it at warn level') assert.ok(decoder.verifyReorg.called, 'Should call verifyReorg') assert.ok(mockDb.endTransaction.called, 'Should end transaction before reorg processing') }) diff --git a/test/chaos/helpers.js b/test/chaos/helpers.js index b24d065..f3d83ac 100644 --- a/test/chaos/helpers.js +++ b/test/chaos/helpers.js @@ -142,21 +142,30 @@ function wait(ms) { /** * Captures console output during a function execution. + * + * All three streams are captured separately, because the LEVEL a line is emitted at is + * part of what these suites assert: an alerting rule that reads warn-and-above sees + * `warnings` and `errors` and never sees `logs`, so a test that accepts any stream would + * pass while the operator-visible signal was gone. */ async function captureConsole(fn) { const logs = [] + const warnings = [] const errors = [] const origLog = console.log + const origWarn = console.warn const origError = console.error console.log = (...args) => logs.push(args.join(' ')) + console.warn = (...args) => warnings.push(args.join(' ')) console.error = (...args) => errors.push(args.join(' ')) try { await fn() } finally { console.log = origLog + console.warn = origWarn console.error = origError } - return { logs, errors } + return { logs, warnings, errors } } /** diff --git a/test/unit/batchSubCommandOutputCaptureActivation.test.js b/test/unit/batchSubCommandOutputCaptureActivation.test.js index 4319239..dde8eec 100644 --- a/test/unit/batchSubCommandOutputCaptureActivation.test.js +++ b/test/unit/batchSubCommandOutputCaptureActivation.test.js @@ -27,9 +27,13 @@ // it out to several rows, and below that flag-day // output_fanout.collapseOutputFanout treats that as a consensus-critical // fault and HALTS the block. -// 3. LEDGER - on every ARMED network it is >= the indexer's BATCH_ISSUANCE_LIMITS instant, -// which carries the batch-cumulative settlement ledger. Capture without that -// ledger lets N COINPAY sub-commands settle N obligations from ONE payment. +// 3. LEDGER - on every network it is EQUAL to the indexer's BATCH_ISSUANCE_LIMITS instant, +// which carries the batch-cumulative settlement ledger: one decision, one +// boundary (xchain-documentation/protocol/constants.js). Capture without the +// ledger lets N COINPAY sub-commands settle N obligations from ONE payment; +// the ledger without capture makes a batched COINPAY spend the coin and +// settle nothing. Ordering alone (>=) would let a re-arm open that second +// window with every pin still green, so this tier is equality, not a floor. // 4. DOCS - it is value-identical to the canonical map in // xchain-documentation/protocol/constants.js, which must exist before mainnet // may be armed. @@ -75,6 +79,10 @@ const BELOW_MAINNET_GATE = ? BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION.mainnet - 1 : 4000000000; +// 2100-01-01, the boundary the indexer's unarmed-gate suites use to tell a scheduled date +// from the house UNARMED sentinel (9999999999): a ledger instant at or past it is unarmed. +const YEAR_2100 = 4102444800; + function siblingOrSkip(ctx, file){ if (fs.existsSync(file)) return true; if (REQUIRE_SIBLINGS) @@ -154,17 +162,33 @@ describe('BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION conformance', function () { } }); - it('never precedes the indexer BATCH_ISSUANCE_LIMITS flag-day (the settlement ledger)', function () { + it('arms at exactly the indexer BATCH_ISSUANCE_LIMITS instant on every network (one boundary)', function () { if (!siblingOrSkip(this, INDEXER_CHANGES)) return; + // Equality, not ordering. The canonical map states this as ONE decision: capture and + // the settlement ledger flip together. A gap in either direction is a consensus + // window: capture before the ledger lets N COINPAY sub-commands settle N obligations + // from one payment; the ledger before capture makes a batched COINPAY spend the coin + // and settle nothing. A >= leg plus two independent literal pins stayed green while + // a re-arm moved either side, which is exactly the edit this must refuse. const limits = indexerChangeTimes('BATCH_ISSUANCE_LIMITS'); for (const network of Object.keys(BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION)) { const gate = BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION[network]; - if (gate === null) continue; - assert.ok(gate >= limits[network], - network + ' sub-command capture (' + gate + ') must not begin before ' + - 'BATCH_ISSUANCE_LIMITS (' + limits[network] + '): that gate carries the ' + - 'batch-cumulative settlement ledger, without which N COINPAY sub-commands ' + - 'settle N obligations from one payment'); + if (gate === null) { + // A DISARMED capture gate may only sit under a DISARMED ledger: the indexer + // parks an unarmed mainnet on a far-future sentinel, never a real date. + assert.ok(limits[network] >= YEAR_2100, + network + ' sub-command capture is disarmed (null) while the indexer ' + + 'arms BATCH_ISSUANCE_LIMITS at ' + limits[network] + ': the settlement ' + + 'ledger would run with capture reading only the top-level ACTION name, so ' + + 'a batched COINPAY spends the coin and settles nothing'); + continue; + } + assert.strictEqual(gate, limits[network], + network + ' sub-command capture (' + gate + ') must equal the indexer ' + + 'BATCH_ISSUANCE_LIMITS instant (' + limits[network] + '): the canonical map ' + + 'states them as one boundary, and the window [' + + Math.min(gate, limits[network]) + ', ' + Math.max(gate, limits[network]) + + ') either double-settles one payment or settles nothing from it'); } }); diff --git a/test/unit/blockchainConnectorReviewFixes.test.js b/test/unit/blockchainConnectorReviewFixes.test.js index 4219560..610517e 100644 --- a/test/unit/blockchainConnectorReviewFixes.test.js +++ b/test/unit/blockchainConnectorReviewFixes.test.js @@ -117,4 +117,36 @@ describe('BlockchainConnector RPC error accounting and reporting', () => { ) }).timeout(5000) }) + + describe('the shared result extractor reads PRESENCE, not truthiness', () => { + // JSON-RPC 2.0: a success carries a `result` member, which may legitimately + // be 0, false or "". Only undefined/null mean the node sent no result. No + // method routed through the extractor today can answer falsy, so these pin + // the contract for the next one rather than a behaviour change. + it('returns a falsy-but-present result instead of throwing', async () => { + axiosStub.resolves({ data: { result: '' } }) + assert.strictEqual(await connector.getBlockHash(0), '') + + axiosStub.resolves({ data: { result: 0 } }) + assert.strictEqual(await connector.getBlockHash(0), 0) + + axiosStub.resolves({ data: { result: false } }) + assert.strictEqual(await connector.getBlockHash(0), false) + + assert.strictEqual(connector.rpcErrors, 0, 'a valid falsy result is not an RPC failure') + }).timeout(5000) + + it('still throws the per-method label when the result is absent', async () => { + axiosStub.resolves({ data: { result: null } }) + await assert.rejects(() => connector.getBlockHash(0), /Error getting block hash/) + + axiosStub.resolves({ data: {} }) + await assert.rejects(() => connector.getBlockHash(0), /Error getting block hash/) + }).timeout(5000) + + it('still prefers the node error object over the result member', async () => { + axiosStub.resolves({ data: { result: 0, error: { code: -8, message: 'Block height out of range' } } }) + await assert.rejects(() => connector.getBlockHash(0), /RPC error -8: Block height out of range/) + }).timeout(5000) + }) }) diff --git a/test/unit/db.unit.test.js b/test/unit/db.unit.test.js index 5ec507b..ab00bcd 100644 --- a/test/unit/db.unit.test.js +++ b/test/unit/db.unit.test.js @@ -247,6 +247,30 @@ describe('Database#stripSqlLineComments()', () => { assert.ok(!result.includes('end of file')) assert.ok(result.includes('SELECT 1')) }) + + it('should strip a # comment, which MariaDB honours to end-of-line like --', () => { + const result = db.stripSqlLineComments('SELECT 1 # this is a comment\nSELECT 2') + assert.ok(!result.includes('this is a comment')) + assert.ok(result.includes('SELECT 1')) + assert.ok(result.includes('SELECT 2')) + }) + + it('should preserve a # inside quoted strings and backtick identifiers', () => { + assert.ok(db.stripSqlLineComments("SELECT '# not a comment' FROM t").includes('# not a comment')) + assert.ok(db.stripSqlLineComments('SELECT "# not a comment" FROM t').includes('# not a comment')) + assert.ok(db.stripSqlLineComments('SELECT `col#1` FROM t').includes('`col#1`')) + }) + + it('should copy /* */ block comments through verbatim', () => { + const sql = '/* see issue #4413 -- and this */ SELECT 1' + assert.strictEqual(db.stripSqlLineComments(sql), sql) + }) + + it('should not treat an apostrophe in block-comment prose as a quote start', () => { + const result = db.stripSqlLineComments("/* don't do this */ SELECT 1 -- gone\nSELECT 2") + assert.ok(!result.includes('gone')) + assert.ok(result.includes('SELECT 2')) + }) }) describe('Database#parseExpectedColumns()', () => { diff --git a/test/unit/decoderTipStaleSurface.test.js b/test/unit/decoderTipStaleSurface.test.js index fbd119e..39837b7 100644 --- a/test/unit/decoderTipStaleSurface.test.js +++ b/test/unit/decoderTipStaleSurface.test.js @@ -218,6 +218,57 @@ describe('registerDecoderMetrics() feed-freshness gauges', function () { assert.match(body, /^xchain_decoder_node_height_stale 0$/m); }); + // Reorg churn had no decoder-side signal at all: the durable REORG rows are + // DB-only and the indexer's reorgsProcessed needs the indexer to be up, so a + // metrics-only deployment could watch a decoder thrash through shallow reorgs + // and see nothing move. + + it('exports reorg count and depth so a metrics-only deployment sees churn', function () { + const registry = new Registry(); + const decoder = makeRunningDecoder(); + decoder.reorgCount = 3; + decoder.lastReorgDepth = 5; + registerDecoderMetrics(registry, decoder); + + const body = registry.render(); + assert.match(body, /^xchain_decoder_reorgs_total 3$/m); + assert.match(body, /^xchain_decoder_last_reorg_depth 5$/m); + }); + + it('reports zero reorgs on a fresh decoder rather than no series at all', function () { + // Unlike the height gauges, absent here is NOT safe: a missing counter and a + // decoder that has never reorged look identical to a rate() query. + const registry = new Registry(); + registerDecoderMetrics(registry, makeDecoder()); + assert.match(registry.render(), /^xchain_decoder_reorgs_total 0$/m); + }); + + it('counts reorg EVENTS, incrementing once per verifyReorg run', function () { + // A per-block increment inside either delete branch would report one depth-5 + // reorg as five reorgs and destroy the frequency signal the counter exists for. + // The branches need a live node to reach, so this is a source-level guard. + const source = fs.readFileSync(require.resolve('../../src/XChainDecoder.js'), 'utf-8'); + const increments = source.match(/this\.reorgCount\+\+/g) || []; + assert.strictEqual(increments.length, 1, 'exactly one reorgCount increment site'); + assert.ok( + /if \(blocksDeleted\.length > 0\)\{[\s\S]{0,800}?this\.reorgCount\+\+/.test(source), + 'the increment must sit in verifyReorg\'s end-of-run summary block, not in a per-block delete branch' + ); + assert.ok( + /this\.lastReorgDepth = blocksDeleted\.length/.test(source), + 'depth must be the blocks rolled back by the run that just completed' + ); + }); + + it('surfaces the same counters on getSyncStatus, which /status spreads', function () { + const decoder = makeRunningDecoder(); + decoder.reorgCount = 2; + decoder.lastReorgDepth = 1; + const status = decoder.getSyncStatus(); + assert.strictEqual(status.reorg_count, 2); + assert.strictEqual(status.last_reorg_depth, 1); + }); + it('registers on the handle api.js captures, not a discarded return value', function () { const source = fs.readFileSync(require.resolve('../../src/api.js'), 'utf-8'); assert.ok( diff --git a/test/unit/dispenserExpiryRealignActivation.test.js b/test/unit/dispenserExpiryRealignActivation.test.js index 68d4fae..e45a39b 100644 --- a/test/unit/dispenserExpiryRealignActivation.test.js +++ b/test/unit/dispenserExpiryRealignActivation.test.js @@ -51,11 +51,13 @@ function siblingOrSkip(ctx, file){ describe('DISPENSER_EXPIRY_REALIGN_ACTIVATION conformance', function () { - it('keeps mainnet and testnet DISARMED and regtest genesis-on', function () { - // Teeth for the ratification requirement: a number on mainnet/testnet means someone - // armed a consensus boundary without the operator's ratified instant. + it('keeps mainnet DISARMED, with testnet and regtest genesis-on', function () { + // Teeth for the ratification requirement: a number on MAINNET means someone armed a + // consensus boundary without the operator's ratified instant. Testnet was ratified at + // instant 0 on 2026-08-18 (pre-launch, every feature active on testnet), which is safe + // only because testnet decoder/indexer state is rebuilt from the chain before launch. assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet, null); - assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.testnet, null); + assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.testnet, 0); assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.regtest, 0); }); @@ -74,10 +76,16 @@ describe('DISPENSER_EXPIRY_REALIGN_ACTIVATION conformance', function () { }); it('a DISARMED network is inactive at every block time, including absurd ones', function () { + // Mainnet is the network still carrying the null sentinel. A `time >= null` coercion + // would read 0 and arm it from genesis, which is the failure this pins. assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 0), false); assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 1786060800), false); assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 4000000000), false); - assert.strictEqual(isDispenserExpiryRealignActive('testnet', 4000000000), false); + }); + + it('testnet is active from genesis, so the launch runs the realigned path', function () { + assert.strictEqual(isDispenserExpiryRealignActive('testnet', 0), true); + assert.strictEqual(isDispenserExpiryRealignActive('testnet', 4000000000), true); }); it('regtest is active from genesis so the venues exercise the realigned path', function () { diff --git a/test/unit/dispenserFieldOffsets.test.js b/test/unit/dispenserFieldOffsets.test.js new file mode 100644 index 0000000..de2ba46 --- /dev/null +++ b/test/unit/dispenserFieldOffsets.test.js @@ -0,0 +1,129 @@ +'use strict'; + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// DISPENSER wire field-offset drift guard. +// +// The decoder reads three DISPENSER fields by split offset: ORACLE_ADDRESS (the token +// oracle-fee capture keys on), the v0 create EXPIRATION and the v2 edit EXPIRATION. The +// authoritative layout is the indexer's own format strings +// (xchain-indexer/src/actions/dispenser.js this.formats), and until this guard existed the +// only thing binding the two was a prose comment, while every comparable dependency at this +// seam already had a mechanical gate (indexerBatchLimits.js vendoring, +// oracleFeeOutputActivationConformance.js). +// +// Drift is money-bearing in both directions: a field inserted ahead of ORACLE_ADDRESS makes +// capture key on the wrong token, so the indexer rejects every fee-bearing Mode B create +// with 'missing oracle fee output' after the payer's coin is spent; a shifted EXPIRATION +// diverges the decoder's open-dispenser set from the indexer's. +// +// Two tiers, so a one-sided edit fails somewhere no matter which checkout is present: +// 1. PIN - the constants equal the offsets this repo's decode path was written +// against, in this repo alone. +// 2. INDEXER - they equal the offsets derived from the LIVE sibling Dispenser's +// this.formats, read off an instance rather than scraped from source. +// Tier 2 skips when the sibling is absent (standalone deploy); XCHAIN_REQUIRE_SIBLINGS=1 +// makes absence a failure instead of a green-by-skip. +// +// The PIN tier is deliberately hard-coded rather than derived. A sibling that REORDERS the +// format string is a protocol fork that needs a human decision, so it must fail this file +// loudly rather than be adopted by re-running a generator. + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const { ORACLE_ADDRESS_INDEX, V0_EXPIRATION_INDEX, V2_EXPIRATION_INDEX, oracleAddressFromCreate } = + require('../../src/oracleFeeOutput.js'); + +// Offsets the decode path in src/XChainDecoder.js and src/oracleFeeOutput.js was written +// against. Decoder offset = indexer format position + 1: the decoder splits with the ACTION +// token ('DISPENSER') at 0, the indexer's format string starts at VERSION. +const PINNED = { ORACLE_ADDRESS: 13, V0_EXPIRATION: 14, V2_EXPIRATION: 4 }; +const ACTION_TOKEN_OFFSET = 1; + +const INDEXER_DISPENSER = process.env.XCHAIN_INDEXER_DIR + ? path.join(process.env.XCHAIN_INDEXER_DIR, 'src', 'actions', 'dispenser.js') + : path.join(__dirname, '..', '..', '..', 'xchain-indexer', 'src', 'actions', 'dispenser.js'); +const REQUIRE_SIBLINGS = process.env.XCHAIN_REQUIRE_SIBLINGS === '1'; + +function siblingOrSkip(ctx, file){ + if (fs.existsSync(file)) return true; + if (REQUIRE_SIBLINGS) + throw new Error('XCHAIN_REQUIRE_SIBLINGS=1 but sibling not found: ' + file); + ctx.skip(); + return false; +} + +// Read the formats off a REAL Dispenser instance, not a regex over its source: they are +// assigned on `this` in the constructor, so a rename or a reformat keeps working while a +// scrape would silently read nothing and pass. The constructor only stores its collaborators, +// so a stub-shaped action object is enough and no database is needed (same trick as +// test/tools/sync-batch-limits.js). +function siblingFormats(){ + const Dispenser = require(INDEXER_DISPENSER); + const dispenser = new Dispenser({ + config: {}, + decoderDb: {}, + indexerDb: {}, + util: {}, + mapper: {}, + protocolChanges: {}, + actionAliases: {}, + }); + return dispenser.formats; +} + +function offsetOf(format, field){ + const position = String(format).split('|').indexOf(field); + assert.notStrictEqual(position, -1, 'the sibling format no longer carries ' + field + ': ' + format); + return position + ACTION_TOKEN_OFFSET; +} + +describe('DISPENSER wire field offsets', function () { + + it('pins the offsets the decode path was written against', function () { + assert.strictEqual(ORACLE_ADDRESS_INDEX, PINNED.ORACLE_ADDRESS); + assert.strictEqual(V0_EXPIRATION_INDEX, PINNED.V0_EXPIRATION); + assert.strictEqual(V2_EXPIRATION_INDEX, PINNED.V2_EXPIRATION); + }); + + it('reads ORACLE_ADDRESS from the pinned slot and not a neighbouring one', function () { + // Binds the behaviour, not just the constant: a split whose ORACLE_ADDRESS slot alone + // carries an address must resolve to that address, and its neighbours must not. + const fields = new Array(18).fill(''); + fields[0] = 'DISPENSER'; + fields[PINNED.ORACLE_ADDRESS] = 'bc1qoracle'; + assert.strictEqual(oracleAddressFromCreate(fields), 'bc1qoracle'); + + const shifted = new Array(18).fill(''); + shifted[0] = 'DISPENSER'; + shifted[PINNED.ORACLE_ADDRESS - 1] = 'bc1qneighbour'; + assert.strictEqual(oracleAddressFromCreate(shifted), null); + }); + + it('matches the offsets derived from the live sibling indexer Dispenser formats', function () { + if (!siblingOrSkip(this, INDEXER_DISPENSER)) return; + const formats = siblingFormats(); + assert.ok(formats && typeof formats === 'object', + 'xchain-indexer Dispenser must expose this.formats'); + + assert.strictEqual(offsetOf(formats[0], 'ORACLE_ADDRESS'), ORACLE_ADDRESS_INDEX, + 'ORACLE_ADDRESS moved in the indexer v0 format: oracle-fee capture would key on the ' + + 'wrong token and the indexer would reject every fee-bearing Mode B create'); + assert.strictEqual(offsetOf(formats[0], 'EXPIRATION'), V0_EXPIRATION_INDEX, + 'the v0 create EXPIRATION moved in the indexer format: the decoder open-dispenser ' + + 'set would diverge from the indexer'); + assert.strictEqual(offsetOf(formats[2], 'EXPIRATION'), V2_EXPIRATION_INDEX, + 'the v2 edit EXPIRATION moved in the indexer format: the decoder would extend the ' + + 'wrong expiry, or none'); + }); +}); diff --git a/test/unit/mempoolIsolation.test.js b/test/unit/mempoolIsolation.test.js index 034cd60..613a330 100644 --- a/test/unit/mempoolIsolation.test.js +++ b/test/unit/mempoolIsolation.test.js @@ -70,12 +70,13 @@ describe('updateMempool DB isolation', function () { assert.strictEqual(getParseTxDbArg(), decoder.mempoolDb, 'mempool parse must use mempoolDb for pubkey capture') }) - it('hands deleteAndCompareTxsNotInList a deduped, DESCENDING-sorted txid list', async () => { - // db.js deleteAndCompareTxsNotInList binary-searches this array with the - // inverted comparator `needle.localeCompare(element)`, which requires - // descending lexicographic order. The old O(n^2) bs+splice build produced - // exactly that order (with duplicates skipped); the O(n log n) sort - // replacement must keep the same contract or the delete-diff silently breaks. + it('hands deleteAndCompareTxsNotInList a DEDUPED txid list', async () => { + // Dedup is the real contract at this seam: db.js deleteAndCompareTxsNotInList + // seeds the array into a temp table and filters it through a Set, so a repeated + // txid would be fetched and inserted twice. ORDER is deliberately NOT asserted + // here: the DB layer runs no binary search over this array (it did once, which + // is why an older version of this test pinned descending order), so the poll's + // sort is deterministic output rather than a requirement the consumer imposes. const { decoder } = buildDecoder(true) let received decoder.mempoolDb.deleteAndCompareTxsNotInList = async (list) => { @@ -84,8 +85,9 @@ describe('updateMempool DB isolation', function () { decoder.connector.getRawMempool = async () => ['bbb', 'aaa', 'ccc', 'aaa', 'bbb'] decoder.connector.getRawTransactions = async () => [] await decoder.updateMempool() - assert.deepStrictEqual(received, ['ccc', 'bbb', 'aaa'], - 'rawMempool must be deduped and sorted descending (the bs-comparator order)') + assert.strictEqual(received.length, 3, 'rawMempool must carry each txid once') + assert.deepStrictEqual(received.slice().sort(), ['aaa', 'bbb', 'ccc'], + 'rawMempool must be the deduped txid set the node reported') }) it('a mempool insert failure never rolls back or ends the block transaction', async () => { diff --git a/test/unit/migration-runner.test.js b/test/unit/migration-runner.test.js index 4519d42..fefd624 100644 --- a/test/unit/migration-runner.test.js +++ b/test/unit/migration-runner.test.js @@ -202,6 +202,47 @@ describe('Database._destructiveAutoStatement() @regression', function () { assert.strictEqual(scanSql('SET sql_mode = "STRICT_ALL_TABLES";'), null); assert.strictEqual(scanSql('SET @@session.foreign_key_checks = 0;'), null); }); + + // The indexer twin carries the same cases; keep the two suites in step. + + it('flags a DROP hidden behind a `#` line comment (the server honours `#`)', function () { + // Before the strip knew `#`, this reached the classifier as one chunk starting + // with `#`, matched no ^-anchored check, and auto-ran the DROP at startup. + const offender = scanSql('# cleanup\nDROP TABLE transactions;'); + assert.ok(offender && /DROP TABLE transactions/i.test(offender)); + }); + + it('flags a statement still carrying a `#` line comment (strip-regression guard)', function () { + assert.ok(scanOf(['# cleanup\nDROP TABLE transactions'])); + }); + + it('does not flag a `#` inside a quoted literal or a block comment', function () { + assert.strictEqual(scanSql("INSERT INTO notes (body) VALUES ('#tag');"), null); + assert.strictEqual(scanSql('/* see issue #4413 */ ALTER TABLE t ADD COLUMN y INT;'), null); + }); + + it('flags INSERT ... ON DUPLICATE KEY UPDATE but not a plain INSERT', function () { + assert.ok(scanSql("INSERT INTO dispensers (id, source) VALUES (1,'x') ON DUPLICATE KEY UPDATE source='y';")); + assert.strictEqual(scanSql("INSERT INTO dispensers (id, source) VALUES (1,'x');"), null); + }); + + it('flags LOAD DATA (rows come from a file the classifier cannot read)', function () { + assert.ok(scanSql("LOAD DATA INFILE '/tmp/x.csv' REPLACE INTO TABLE transactions;")); + assert.ok(scanSql("LOAD DATA LOCAL INFILE '/tmp/x.csv' INTO TABLE transactions;")); + }); + + it('flags ALTER TABLE partition and tablespace clauses', function () { + assert.ok(scanSql('ALTER TABLE events DROP PARTITION p2025;')); + assert.ok(scanSql('ALTER TABLE events TRUNCATE PARTITION p0;')); + assert.ok(scanSql('ALTER TABLE events EXCHANGE PARTITION p0 WITH TABLE events_old;')); + assert.ok(scanSql('ALTER TABLE events DISCARD TABLESPACE;')); + // Additive partition DDL is not separable by prefix, so it is non-auto too. + assert.ok(scanSql('ALTER TABLE events ADD PARTITION (PARTITION p2 VALUES LESS THAN (200));')); + }); + + it('does not flag an ordinary column whose name merely contains "partition"', function () { + assert.strictEqual(scanSql('ALTER TABLE t ADD COLUMN partition_id INT NULL;'), null); + }); }); describe('Database.backdatedFrontierViolation() @regression', function () { @@ -759,6 +800,21 @@ describe('Database.splitSqlStatements() @regression', function () { ['SELECT 1', 'SELECT 2']); }); + it('does not split on a ; inside a # line comment, and drops the comment', function () { + assert.deepStrictEqual(splitOf('SELECT 1; # see foo; bar\nSELECT 2;'), + ['SELECT 1', 'SELECT 2']); + assert.deepStrictEqual(splitOf('# cleanup\nDROP TABLE transactions;'), + ['DROP TABLE transactions']); + }); + + it('leaves a # or an apostrophe inside a block comment alone', function () { + // A naive #-to-end-of-line strip would eat the closing */ and the rest of the line. + assert.deepStrictEqual(splitOf('/* see issue #4413 */ SELECT 1;'), + ['/* see issue #4413 */ SELECT 1']); + assert.deepStrictEqual(splitOf("/* don't do this */ SELECT 1; SELECT 2;"), + ["/* don't do this */ SELECT 1", 'SELECT 2']); + }); + it('splits ordinary multi-statement SQL into the same statements as before', function () { assert.deepStrictEqual(splitOf('CREATE TABLE a (id INT);\nCREATE TABLE b (id INT);'), ['CREATE TABLE a (id INT)', 'CREATE TABLE b (id INT)']); diff --git a/test/unit/protocol-constants.test.js b/test/unit/protocol-constants.test.js index 25b954b..9573172 100644 --- a/test/unit/protocol-constants.test.js +++ b/test/unit/protocol-constants.test.js @@ -24,6 +24,20 @@ describe('protocol/constants', function () { }); } + // Value pins for the four constants the cross-repo xcall gate freezes. + // That gate (xchain-indexer/test/unit/xcall-constants-cross-repo.test.js) + // rosters only xchain-vm / -indexer / -sdk plus the xchain-documentation + // canonical, so this repo's mirror is tied to those values by nothing else: + // shape assertions alone let a one-sided edit here pass every suite in the + // platform. Same GOLDEN literals, same idiom as the sibling mirror suite in + // xchain-explorer. A real protocol bump edits every copy, this one included. + it('pins the gated cross-repo VM/XCALL limits to their GOLDEN values', function () { + assert.strictEqual(C.MAX_CODE_SIZE, 65536); + assert.strictEqual(C.XCALL_MAX_GAS, 200000); + assert.strictEqual(C.XCALL_MAX_HOPS, 2); + assert.strictEqual(C.XCALL_MIN_DEADLINE_BLOCKS, 10); + }); + it('XCALL gas floor does not exceed its ceiling', function () { assert.ok(C.XCALL_MIN_GAS <= C.XCALL_MAX_GAS); }); diff --git a/test/unit/sibling-coverage.test.js b/test/unit/sibling-coverage.test.js index f986fb0..7cffe45 100644 --- a/test/unit/sibling-coverage.test.js +++ b/test/unit/sibling-coverage.test.js @@ -68,7 +68,8 @@ const SIBLINGS = [ guards: 'vendored coins-registry byte-identity (BTC/LTC/DOGE/index/consensus_pin)' }, { repo: 'xchain-indexer', envs: ['XCHAIN_INDEXER_DIR'], marker: path.join('src', 'protocol_changes.js'), - guards: 'the FIX_OUTPUT_FANOUT registration in the indexer protocol-change table' }, + guards: 'the FIX_OUTPUT_FANOUT registration in the indexer protocol-change table, and the ' + + 'DISPENSER v0/v2 wire field offsets derived from the indexer Dispenser formats' }, { repo: 'xchain-utxo-tracker', envs: ['XCHAIN_UTXO_TRACKER_DIR'], marker: path.join('src', 'BlockchainConnector.js'), guards: 'AuxPoW strip parity and the dispenser safe-depth twin' },