diff --git a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts index c9930f80982..0503d090176 100644 --- a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts +++ b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts @@ -1,5 +1,6 @@ import { PassThrough, Readable } from 'stream'; import { LRUCache } from 'lru-cache'; +import { EthDater } from '../../../utils/ethDater'; import logger from '../../../logger'; import { CacheStorage } from '../../../models/cache'; import { WalletAddressStorage } from '../../../models/walletAddress'; @@ -40,6 +41,7 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { private providersByNetwork: Map = new Map(); blockAtTimeCache: { [key: string]: LRUCache } = {}; private localTipCache: Map = new Map(); + private daters: Map = new Map(); private static readonly LOCAL_TIP_TTL_MS = 5_000; constructor(chain: string = 'ETH') { @@ -435,8 +437,9 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { } if (Number(block.timestamp) > targetTimestamp) { - logger.warn(`MultiProvider: block verification exceeded ${MAX_ADJUSTMENTS} adjustments, falling back to binary search`); - return this._binarySearchBlockByTimestamp(web3, targetTimestamp); + logger.warn(`MultiProvider: block verification exceeded ${MAX_ADJUSTMENTS} adjustments, falling back to EthDater`); + const result = await this._getDater(network, web3).getDate(date, false); + return result.block; } adjustments = 0; @@ -450,24 +453,13 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { return blockNum; } - private async _binarySearchBlockByTimestamp(web3: any, targetTimestamp: number): Promise { - const latestBlock = await web3.eth.getBlock('latest'); - let high = Number(latestBlock.number); - let low = 0; - const MAX_ITERATIONS = 64; - let iterations = 0; - - while (low < high && iterations < MAX_ITERATIONS) { - const mid = Math.floor((low + high + 1) / 2); - const block = await web3.eth.getBlock(mid); - if (Number(block.timestamp) <= targetTimestamp) { - low = mid; - } else { - high = mid - 1; - } - iterations++; + private _getDater(network: string, web3: any): EthDater { + let dater = this.daters.get(network); + if (!dater) { + dater = new EthDater(web3); + this.daters.set(network, dater); } - return low; + return dater; } // @override diff --git a/packages/bitcore-node/src/providers/chain-state/external/adapters/alchemy.ts b/packages/bitcore-node/src/providers/chain-state/external/adapters/alchemy.ts index a6735b3e3fc..95a53e253aa 100644 --- a/packages/bitcore-node/src/providers/chain-state/external/adapters/alchemy.ts +++ b/packages/bitcore-node/src/providers/chain-state/external/adapters/alchemy.ts @@ -3,6 +3,7 @@ import axios from 'axios'; import config from '../../../../config'; import logger from '../../../../logger'; import { EVMTransactionStorage } from '../../evm/models/transaction'; +import { EthDater } from '../../../../utils/ethDater'; import { ExternalApiStream } from '../streams/apiStream'; import { type AdapterBlockByDateParams, @@ -26,6 +27,7 @@ export class AlchemyAdapter implements IIndexedAPIAdapter { private apiKey: string; private requestTimeout: number; + private daters = new Map(); constructor(providerConfig: IMultiProviderConfig) { const apiKey = config.externalProviders?.alchemy?.apiKey; @@ -34,6 +36,17 @@ export class AlchemyAdapter implements IIndexedAPIAdapter { this.requestTimeout = providerConfig.requestTimeout ?? 30000; } + private getDater(chain: string, network: string): EthDater { + const key = `${chain}:${network}`; + let dater = this.daters.get(key); + if (!dater) { + const web3 = new Web3(new Web3.providers.HttpProvider(this.getBaseUrl(chain, network))); + dater = new EthDater(web3); + this.daters.set(key, dater); + } + return dater; + } + private getNetworkUrlString(chain: string, network: string): string { const normalizedChain = ALCHEMY_CHAIN_ALIASES[chain.toUpperCase()] || chain.toLowerCase(); return `${normalizedChain}-${network.toLowerCase()}`; @@ -111,35 +124,13 @@ export class AlchemyAdapter implements IIndexedAPIAdapter { async getBlockNumberByDate(params: AdapterBlockByDateParams): Promise { const { chain, network, date } = params; - const url = this.getBaseUrl(chain, network); - const targetTimestamp = Math.floor(new Date(date).getTime() / 1000); - - const MAX_ITERATIONS = 64; - const latestBlockResp = await this._jsonRpc(url, 'eth_blockNumber', []); - let high = parseInt(latestBlockResp.result, 16); - let low = 0; - - const latestBlock = await this._jsonRpc(url, 'eth_getBlockByNumber', [latestBlockResp.result, false]); - const latestTimestamp = parseInt(latestBlock.result.timestamp, 16); - if (targetTimestamp >= latestTimestamp) return high; // Target in the future - if (targetTimestamp <= 0) return 0; // Target before genesis - - // Binary search: largest block with timestamp <= target - let iterations = 0; - while (low < high && iterations < MAX_ITERATIONS) { - const mid = Math.floor((low + high + 1) / 2); - const blockResp = await this._jsonRpc(url, 'eth_getBlockByNumber', [`0x${mid.toString(16)}`, false]); - const blockTimestamp = parseInt(blockResp.result.timestamp, 16); - - if (blockTimestamp <= targetTimestamp) { - low = mid; - } else { - high = mid - 1; - } - iterations++; + try { + // `false` = "block before" semantics: largest block whose timestamp < target. + const result = await this.getDater(chain, network).getDate(new Date(date), false); + return result.block; + } catch (error) { + this._classifyError(error); } - - return low; } async healthCheck(): Promise { diff --git a/packages/bitcore-node/src/utils/ethDater.ts b/packages/bitcore-node/src/utils/ethDater.ts new file mode 100644 index 00000000000..1e9026b893c --- /dev/null +++ b/packages/bitcore-node/src/utils/ethDater.ts @@ -0,0 +1,138 @@ +/** + * Block-by-date resolution via average-block-time interpolation. + * + * Ported from monosux/ethereum-block-by-date (MIT) and trimmed to what we use: + * - getDate(date, after) only (no getEvery / period iteration) + * - web3 provider only (no ethers / viem) + * - no moment.js (native Date + unix-second arithmetic) + * + * Algorithm: + * 1. Probe genesis and latest to compute average block time across the chain. + * 2. Interpolate a first guess by dividing the date's offset from genesis by avg block time. + * 3. Refine recursively: re-estimate local block time between the guess and a neighbor, + * step toward the target, repeat until the guess straddles the target. + * 4. Cache every probed block within the call so the recursion doesn't re-fetch. + * + * "after" semantics mirror the upstream library: + * - true: smallest block whose timestamp >= target + * - false: largest block whose timestamp < target (a.k.a. "block before") + */ + +export interface BlockResult { + block: number; + timestamp: number; + date: string; +} + +interface SavedBlock { + number: number; + timestamp: number; +} + +function toUnixSeconds(ts: bigint | number | string): number { + if (typeof ts === 'bigint') return Number(ts); + if (typeof ts === 'string') { + return ts.startsWith('0x') ? parseInt(ts, 16) : parseInt(ts, 10); + } + return Number(ts); +} + +export class EthDater { + private web3: any; + private latestBlock?: SavedBlock; + private firstBlock?: SavedBlock; + private blockTime?: number; + private checkedBlocks: Record = {}; + private savedBlocks: Record = {}; + public requests = 0; + + constructor(web3: any) { + this.web3 = typeof web3?.eth !== 'undefined' ? web3 : { eth: web3 }; + } + + async getDate(input: Date | string | number, after: boolean = true, refresh: boolean = false): Promise { + const targetUnix = Math.floor(new Date(input).getTime() / 1000); + const dateIso = new Date(targetUnix * 1000).toISOString(); + + if (!this.firstBlock || !this.latestBlock || this.blockTime === undefined || refresh) { + await this.getBoundaries(); + } + + if (targetUnix < this.firstBlock!.timestamp) { + return this.returnWrapper(dateIso, 1); + } + if (targetUnix >= this.latestBlock!.timestamp) { + return this.returnWrapper(dateIso, this.latestBlock!.number); + } + + this.checkedBlocks[targetUnix] = []; + const guess = Math.ceil((targetUnix - this.firstBlock!.timestamp) / this.blockTime!); + const predictedBlock = await this.getBlockWrapper(guess); + const found = await this.findBetter(targetUnix, predictedBlock, after); + return this.returnWrapper(dateIso, found); + } + + private async getBoundaries(): Promise { + this.latestBlock = await this.getBlockWrapper('latest'); + this.firstBlock = await this.getBlockWrapper(1); + const span = this.latestBlock.number - 1; + this.blockTime = span > 0 ? (this.latestBlock.timestamp - this.firstBlock.timestamp) / span : 0; + } + + private async findBetter(targetUnix: number, predicted: SavedBlock, after: boolean, blockTime: number = this.blockTime!): Promise { + if (await this.isBetterBlock(targetUnix, predicted, after)) return predicted.number; + + const difference = targetUnix - predicted.timestamp; + let skip = Math.ceil(difference / (blockTime === 0 ? 1 : blockTime)); + if (skip === 0) skip = difference < 0 ? -1 : 1; + + const nextNum = this.getNextBlock(targetUnix, predicted.number, skip); + const next = await this.getBlockWrapper(nextNum); + + const localBlockTime = Math.abs( + (predicted.timestamp - next.timestamp) / + (predicted.number - next.number || 1) + ); + return this.findBetter(targetUnix, next, after, localBlockTime); + } + + private async isBetterBlock(targetUnix: number, predicted: SavedBlock, after: boolean): Promise { + const ts = predicted.timestamp; + if (after) { + if (ts < targetUnix) return false; + const previous = await this.getBlockWrapper(predicted.number - 1); + return ts >= targetUnix && previous.timestamp < targetUnix; + } else { + if (ts >= targetUnix) return false; + const next = await this.getBlockWrapper(predicted.number + 1); + return ts < targetUnix && next.timestamp >= targetUnix; + } + } + + private getNextBlock(targetUnix: number, currentBlock: number, skip: number): number { + let next = currentBlock + skip; + if (next > this.latestBlock!.number) next = this.latestBlock!.number; + if (this.checkedBlocks[targetUnix].includes(next)) { + return this.getNextBlock(targetUnix, currentBlock, skip < 0 ? --skip : ++skip); + } + this.checkedBlocks[targetUnix].push(next); + return next < 1 ? 1 : next; + } + + private returnWrapper(dateIso: string, block: number): BlockResult { + return { date: dateIso, block, timestamp: this.savedBlocks[block].timestamp }; + } + + private async getBlockWrapper(block: number | 'latest'): Promise { + if (typeof block === 'number' && this.savedBlocks[block]) return this.savedBlocks[block]; + + const raw = await this.web3.eth.getBlock(block); + const number = toUnixSeconds(raw.number); + const timestamp = toUnixSeconds(raw.timestamp); + this.savedBlocks[number] = { number, timestamp }; + this.requests++; + return this.savedBlocks[number]; + } +} + +export default EthDater; diff --git a/packages/bitcore-node/test/integration/routes/address.test.ts b/packages/bitcore-node/test/integration/routes/address.test.ts index 1837bec630b..f3d8ad7a315 100644 --- a/packages/bitcore-node/test/integration/routes/address.test.ts +++ b/packages/bitcore-node/test/integration/routes/address.test.ts @@ -330,7 +330,7 @@ describe('Address Routes', function () { this.timeout(10000); const csp = new BaseEVMStateProvider('BASE'); sandbox.stub(ChainStateProvider, 'get').returns(csp); - request.get('/api/BASE/testnet/address/0x9bb6f7fdf81afbd8876d37f3e5e37df416bf8da1/txs?tokenAddress=0x036CbD53842c5426634e7929541eC2318f3dCF7e&startBlock=14035000&endBlock=14045000') + request.get('/api/BASE/testnet/address/0x9bb6f7fdf81afbd8876d37f3e5e37df416bf8da1/txs?tokenAddress=0x036CbD53842c5426634e7929541eC2318f3dCF7e&startBlock=14035000&endBlock=14035600') .expect(200, (err, res) => { if (err) return done(err); expect(res.body).to.deep.equal([{ diff --git a/packages/bitcore-node/test/unit/adapters/alchemy.test.ts b/packages/bitcore-node/test/unit/adapters/alchemy.test.ts index 8d45d448676..3e2cf851ee2 100644 --- a/packages/bitcore-node/test/unit/adapters/alchemy.test.ts +++ b/packages/bitcore-node/test/unit/adapters/alchemy.test.ts @@ -4,6 +4,7 @@ import axios from 'axios'; import { AdapterError, AdapterErrorCode } from '../../../src/providers/chain-state/external/adapters/errors'; import { AlchemyAdapter, AlchemyAssetTransferStream } from '../../../src/providers/chain-state/external/adapters/alchemy'; import { EVMTransactionStorage } from '../../../src/providers/chain-state/evm/models/transaction'; +import { EthDater } from '../../../src/utils/ethDater'; import config from '../../../src/config'; // --- Mock data --- @@ -336,26 +337,36 @@ describe('AlchemyAdapter', function() { // --- getBlockNumberByDate --- describe('getBlockNumberByDate', function() { - it('should binary search for block closest to target date', async function() { - axiosPostStub.callsFake(async (_url: string, body: any) => { - if (body.method === 'eth_blockNumber') return rpcOk('0x64'); - if (body.method === 'eth_getBlockByNumber') { - const num = parseInt(body.params[0], 16); - return rpcOk({ timestamp: `0x${num.toString(16)}`, number: body.params[0] }); - } - return rpcOk(null); - }); + it('should delegate to EthDater and return its block', async function() { + const getDateStub = sandbox.stub(EthDater.prototype, 'getDate') + .resolves({ block: 50, timestamp: 50, date: '1970-01-01T00:00:50Z' }); const result = await adapter.getBlockNumberByDate({ chain: 'ETH', network: 'mainnet', chainId: '1', date: new Date(50000) }); + expect(result).to.equal(50); + // `false` = "block before" semantics: largest block whose timestamp < target + expect(getDateStub.calledOnce).to.be.true; + expect(getDateStub.firstCall.args[1]).to.equal(false); + }); + + it('should cache the dater per chain:network across calls', async function() { + sandbox.stub(EthDater.prototype, 'getDate') + .resolves({ block: 50, timestamp: 50, date: '1970-01-01T00:00:50Z' }); + + await adapter.getBlockNumberByDate({ chain: 'ETH', network: 'mainnet', chainId: '1', date: new Date(50000) }); + await adapter.getBlockNumberByDate({ chain: 'ETH', network: 'mainnet', chainId: '1', date: new Date(60000) }); + + expect((adapter as any).daters.size).to.equal(1); }); - it('should return latest block if target is in the future', async function() { - axiosPostStub.onCall(0).resolves(rpcOk('0x64')); - axiosPostStub.onCall(1).resolves(rpcOk({ timestamp: '0x64', number: '0x64' })); + it('should build separate daters for different chain:network combinations', async function() { + sandbox.stub(EthDater.prototype, 'getDate') + .resolves({ block: 50, timestamp: 50, date: '1970-01-01T00:00:50Z' }); + + await adapter.getBlockNumberByDate({ chain: 'ETH', network: 'mainnet', chainId: '1', date: new Date(50000) }); + await adapter.getBlockNumberByDate({ chain: 'MATIC', network: 'mainnet', chainId: '137', date: new Date(50000) }); - const result = await adapter.getBlockNumberByDate({ chain: 'ETH', network: 'mainnet', chainId: '1', date: new Date(200000) }); - expect(result).to.equal(100); + expect((adapter as any).daters.size).to.equal(2); }); }); diff --git a/packages/bitcore-node/test/unit/modules/base/csp.test.ts b/packages/bitcore-node/test/unit/modules/base/csp.test.ts index 47c1f9ee21f..37aa6b6d5aa 100644 --- a/packages/bitcore-node/test/unit/modules/base/csp.test.ts +++ b/packages/bitcore-node/test/unit/modules/base/csp.test.ts @@ -1,6 +1,7 @@ import { expect } from 'chai'; import * as sinon from 'sinon'; import { CryptoRpc } from '@bitpay-labs/crypto-rpc'; +import { EthDater } from '../../../../src/utils/ethDater'; import { MultiProviderEVMStateProvider } from '../../../../src/modules/multiProvider/api/csp'; import { MoralisStateProvider } from '../../../../src/modules/moralis/api/csp'; import { CacheStorage } from '../../../../src/models/cache'; @@ -530,4 +531,47 @@ describe('MultiProviderEVMStateProvider: _buildWalletTransactionsStream tokenAdd expect(adapter.streamAddressTransactions.callCount).to.equal(1); expect(adapter.streamERC20Transfers.callCount).to.equal(0); }); +}); + +describe('MultiProviderEVMStateProvider: _verifyBlockBeforeDate EthDater fallback', function() { + let cfgStub: sinon.SinonStub; + let sandbox: sinon.SinonSandbox; + + before(function() { + cfgStub = sinon.stub(Config, 'get').returns({ chains: { ETH: {} } } as any); + (BaseEVMStateProvider as any).rpcInitialized = { ETH: true }; + }); + after(function() { cfgStub.restore(); }); + beforeEach(function() { sandbox = sinon.createSandbox(); }); + afterEach(function() { sandbox.restore(); }); + + const ALWAYS_AHEAD_TS = '0xffffffff'; + const targetDate = new Date(1700000000 * 1000); + + it('falls back to EthDater when the bounded walk exceeds MAX_ADJUSTMENTS', async function() { + const provider = new MultiProviderEVMStateProvider('ETH'); + const fakeWeb3 = { eth: { getBlock: sandbox.stub().resolves({ timestamp: ALWAYS_AHEAD_TS }) } }; + (provider as any).getWeb3 = async () => ({ web3: fakeWeb3 }); + const getDateStub = sandbox.stub(EthDater.prototype, 'getDate') + .resolves({ block: 12345, timestamp: 1700000000, date: '2023-11-14T22:13:20Z' }); + + const result = await (provider as any)._verifyBlockBeforeDate('mainnet', 100, targetDate); + + expect(result).to.equal(12345); + expect(getDateStub.calledOnce).to.be.true; + expect(getDateStub.firstCall.args[1]).to.equal(false); + }); + + it('caches the dater per network across calls', async function() { + const provider = new MultiProviderEVMStateProvider('ETH'); + const fakeWeb3 = { eth: { getBlock: sandbox.stub().resolves({ timestamp: ALWAYS_AHEAD_TS }) } }; + (provider as any).getWeb3 = async () => ({ web3: fakeWeb3 }); + sandbox.stub(EthDater.prototype, 'getDate') + .resolves({ block: 1, timestamp: 1, date: '1970-01-01T00:00:01Z' }); + + await (provider as any)._verifyBlockBeforeDate('mainnet', 100, targetDate); + await (provider as any)._verifyBlockBeforeDate('mainnet', 100, targetDate); + + expect((provider as any).daters.size).to.equal(1); + }); }); \ No newline at end of file diff --git a/packages/bitcore-node/test/unit/utils/ethDater.test.ts b/packages/bitcore-node/test/unit/utils/ethDater.test.ts new file mode 100644 index 00000000000..0acdd0bd49d --- /dev/null +++ b/packages/bitcore-node/test/unit/utils/ethDater.test.ts @@ -0,0 +1,116 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { EthDater } from '../../../src/utils/ethDater'; + +/** + * Fake web3 backed by a fixed map of block_number -> timestamp. + * `latest` resolves to the largest block in the map. + */ +function fakeWeb3(blocks: Record) { + const latestNum = Math.max(...Object.keys(blocks).map(Number)); + return { + eth: { + getBlock: sinon.stub().callsFake(async (id: number | 'latest') => { + const num = id === 'latest' ? latestNum : id; + if (blocks[num] === undefined) throw new Error(`no fake block ${num}`); + return { number: num, timestamp: blocks[num] }; + }) + } + }; +} + +describe('EthDater', function() { + describe('boundary edges', function() { + it('returns block 1 for a date before genesis', async function() { + // genesis (block 1) at ts=1000; latest (block 100) at ts=2000 → avg block time 1010/99 + const blocks: Record = { 1: 1000, 100: 2000 }; + const dater = new EthDater(fakeWeb3(blocks)); + const result = await dater.getDate(new Date(500 * 1000), false); + expect(result.block).to.equal(1); + }); + + it('returns the latest block for a future date', async function() { + const blocks: Record = { 1: 1000, 100: 2000 }; + const dater = new EthDater(fakeWeb3(blocks)); + const result = await dater.getDate(new Date(5000 * 1000), false); + expect(result.block).to.equal(100); + }); + }); + + describe('interpolation', function() { + it('finds the block whose timestamp is at or before target ("after=false")', async function() { + // Linear chain: block N at timestamp N*10. block 5 = ts 50, block 6 = ts 60, etc. + const blocks: Record = {}; + for (let i = 1; i <= 100; i++) blocks[i] = i * 10; + const dater = new EthDater(fakeWeb3(blocks)); + + // Target ts=555 → largest block with ts < 555 is block 55 (ts=550). after=false returns it. + const result = await dater.getDate(new Date(555 * 1000), false); + expect(result.block).to.equal(55); + expect(result.timestamp).to.equal(550); + }); + + it('finds the block whose timestamp is at or after target ("after=true")', async function() { + const blocks: Record = {}; + for (let i = 1; i <= 100; i++) blocks[i] = i * 10; + const dater = new EthDater(fakeWeb3(blocks)); + + // Target ts=555 → smallest block with ts >= 555 is block 56 (ts=560). after=true returns it. + const result = await dater.getDate(new Date(555 * 1000), true); + expect(result.block).to.equal(56); + expect(result.timestamp).to.equal(560); + }); + }); + + describe('per-call probe cache', function() { + it('does not refetch the same block within a single getDate call', async function() { + const blocks: Record = {}; + for (let i = 1; i <= 100; i++) blocks[i] = i * 10; + const web3 = fakeWeb3(blocks); + const dater = new EthDater(web3); + + await dater.getDate(new Date(555 * 1000), false); + const firstCallCount = web3.eth.getBlock.callCount; + + // Reset the stub history for clarity, then call again with same boundaries + // (no `refresh`) — boundaries reused, only refinement probes. + web3.eth.getBlock.resetHistory(); + await dater.getDate(new Date(555 * 1000), false); + const secondCallCount = web3.eth.getBlock.callCount; + + // First call probes boundaries (2) + refinement; second call skips boundaries and + // benefits from the savedBlocks cache → strictly fewer probes. + expect(secondCallCount).to.be.lessThan(firstCallCount); + }); + }); + + describe('result shape', function() { + it('returns { block, timestamp, date } where date is the requested ISO string', async function() { + const blocks: Record = {}; + for (let i = 1; i <= 100; i++) blocks[i] = i * 10; + const dater = new EthDater(fakeWeb3(blocks)); + + const target = new Date(555 * 1000); + const result = await dater.getDate(target, false); + expect(result).to.have.keys('block', 'timestamp', 'date'); + expect(result.date).to.equal(target.toISOString()); + }); + }); + + describe('bigint timestamps (web3 v4)', function() { + it('handles getBlock returning bigint number/timestamp', async function() { + const web3 = { + eth: { + getBlock: sinon.stub().callsFake(async (id: number | 'latest') => { + const num = id === 'latest' ? 100 : (id as number); + return { number: BigInt(num), timestamp: BigInt(num * 10) }; + }) + } + }; + const dater = new EthDater(web3); + const result = await dater.getDate(new Date(555 * 1000), false); + expect(result.block).to.equal(55); + expect(result.timestamp).to.equal(550); + }); + }); +});