From e20dc15391d7e88bb9cd4b648f843df4de241815 Mon Sep 17 00:00:00 2001 From: Denis <61563365+dnsi0@users.noreply.github.com> Date: Thu, 13 Aug 2026 22:02:56 +0300 Subject: [PATCH 1/6] add database retry --- docs/env.md | 3 +++ src/index.ts | 50 +++++++++++++++++++++++++++++++++++++++++-- src/utils/database.ts | 7 ++++-- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/env.md b/docs/env.md index ab1995d92..eb60b9f42 100644 --- a/docs/env.md +++ b/docs/env.md @@ -49,6 +49,9 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `ELASTICSEARCH_SNIFF_INTERVAL`: Interval in milliseconds for periodic cluster health monitoring and node discovery. Set to 'false' to disable. Default is `30000`. Example: `30000` - `ELASTICSEARCH_SNIFF_ON_CONNECTION_FAULT`: Enable automatic cluster node discovery when connection faults occur. Default is `true`. Example: `true` - `ELASTICSEARCH_HEALTH_CHECK_INTERVAL`: Interval in milliseconds for proactive connection health monitoring. Default is `60000`. Example: `60000` +- `DB_INIT_MAX_ATTEMPTS`: Maximum number of database initialization attempts at node startup. Raise it when the database container/pod starts slower than the node. Set to `1` to disable retrying. Default is `10`. Example: `10` +- `DB_INIT_RETRY_DELAY`: Initial delay in milliseconds before retrying a failed database initialization. Doubles after each attempt. Default is `2000`. Example: `2000` +- `DB_INIT_MAX_RETRY_DELAY`: Upper bound in milliseconds for the database initialization retry backoff. Default is `30000`. Example: `30000` ## Payments diff --git a/src/index.ts b/src/index.ts index 3c9892994..ed004bcbd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,10 +22,56 @@ import { fileURLToPath } from 'url' import cors from 'cors' import { scheduleCronJobs } from './utils/cronjobs/scheduleCronJobs.js' import { requestValidator } from './components/httpRoutes/requestValidator.js' -import { hasValidDBConfiguration } from './utils/database.js' +import { hasValidDBConfiguration, isReachableConnection } from './utils/database.js' +import type { OceanNodeDBConfig } from './@types/OceanNode.js' const app: Express = express() +// Database services (Elasticsearch/Typesense) frequently take longer to accept +// connections than the node itself takes to boot. Database.init() gives +// up after a single failed attempt, which leaves the node running permanently +// without Indexer and without C2D. +const DB_INIT_MAX_ATTEMPTS = parseInt(process.env.DB_INIT_MAX_ATTEMPTS || '10') +const DB_INIT_RETRY_DELAY = parseInt(process.env.DB_INIT_RETRY_DELAY || '2000') +const DB_INIT_MAX_RETRY_DELAY = parseInt(process.env.DB_INIT_MAX_RETRY_DELAY || '30000') + +async function initDatabaseWithRetry( + dbConfig: OceanNodeDBConfig, + maxAttempts: number = DB_INIT_MAX_ATTEMPTS +): Promise { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const isLastAttempt = attempt === maxAttempts + const notReachable = + !isLastAttempt && + hasValidDBConfiguration(dbConfig) && + !(await isReachableConnection(dbConfig.url)) + + if (!notReachable) { + const database = await Database.init(dbConfig) + if (database) { + if (attempt > 1) { + OCEAN_NODE_LOGGER.info(`Database initialized after ${attempt} attempts`) + } + return database + } + } + if (isLastAttempt) { + break + } + const delay = Math.min( + DB_INIT_RETRY_DELAY * 2 ** (attempt - 1), + DB_INIT_MAX_RETRY_DELAY + ) + OCEAN_NODE_LOGGER.warn( + `Database ${ + notReachable ? 'not reachable yet' : 'initialization failed' + } (attempt ${attempt}/${maxAttempts}), retrying in ${delay}ms` + ) + await new Promise((resolve) => setTimeout(resolve, delay)) + } + return null +} + process.on('uncaughtException', (err) => { OCEAN_NODE_LOGGER.error(`Uncaught exception: ${err.message}`) process.exit(1) @@ -77,7 +123,7 @@ let node: OceanP2P = null let indexer = null let provider = null // If there is no DB URL only the nonce database will be available -const dbconn: Database = await Database.init(config.dbConfig) +const dbconn: Database = await initDatabaseWithRetry(config.dbConfig) if (!dbconn) { OCEAN_NODE_LOGGER.error('Database failed to initialize') } diff --git a/src/utils/database.ts b/src/utils/database.ts index cad5d5745..108f7e530 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -24,9 +24,12 @@ export function hasValidDBConfiguration(configuration: OceanNodeDBConfig): boole } // we can use this to check if DB connection is available -export async function isReachableConnection(url: string): Promise { +export async function isReachableConnection( + url: string, + timeout: number = 3000 +): Promise { try { - await fetch(url) + await fetch(url, { signal: AbortSignal.timeout(timeout) }) return true } catch (error) { return false From c34ad6ffa6945c32889cd3d8b25983e97fd764c4 Mon Sep 17 00:00:00 2001 From: Denis <61563365+dnsi0@users.noreply.github.com> Date: Fri, 14 Aug 2026 00:10:02 +0300 Subject: [PATCH 2/6] add envs, validations, tests --- docs/env.md | 2 +- src/@types/OceanNode.ts | 4 ++ src/index.ts | 19 +++---- src/test/unit/config.test.ts | 96 ++++++++++++++++++++++++++++++++++- src/utils/config/constants.ts | 10 ++++ src/utils/config/schemas.ts | 29 ++++++++++- src/utils/constants.ts | 15 ++++++ 7 files changed, 163 insertions(+), 12 deletions(-) diff --git a/docs/env.md b/docs/env.md index eb60b9f42..86bed4d1c 100644 --- a/docs/env.md +++ b/docs/env.md @@ -49,7 +49,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `ELASTICSEARCH_SNIFF_INTERVAL`: Interval in milliseconds for periodic cluster health monitoring and node discovery. Set to 'false' to disable. Default is `30000`. Example: `30000` - `ELASTICSEARCH_SNIFF_ON_CONNECTION_FAULT`: Enable automatic cluster node discovery when connection faults occur. Default is `true`. Example: `true` - `ELASTICSEARCH_HEALTH_CHECK_INTERVAL`: Interval in milliseconds for proactive connection health monitoring. Default is `60000`. Example: `60000` -- `DB_INIT_MAX_ATTEMPTS`: Maximum number of database initialization attempts at node startup. Raise it when the database container/pod starts slower than the node. Set to `1` to disable retrying. Default is `10`. Example: `10` +- `DB_INIT_MAX_ATTEMPTS`: Maximum number of database initialization attempts at node startup. Raise it when the database container/pod starts slower than the node. Set to `1` to disable retrying. All three `DB_INIT_*` variables are validated as integers `>= 1`; a non-numeric or out-of-range value fails configuration validation and the node refuses to start. Default is `10`. Example: `10` - `DB_INIT_RETRY_DELAY`: Initial delay in milliseconds before retrying a failed database initialization. Doubles after each attempt. Default is `2000`. Example: `2000` - `DB_INIT_MAX_RETRY_DELAY`: Upper bound in milliseconds for the database initialization retry backoff. Default is `30000`. Example: `30000` diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index 1cb2a5f31..b6a25dc54 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -120,6 +120,10 @@ export interface OceanNodeConfig { hasIndexer: boolean hasHttp: boolean dbConfig?: OceanNodeDBConfig + // startup database-init retry: attempts, initial backoff (ms) and backoff ceiling (ms) + dbInitMaxAttempts: number + dbInitRetryDelay: number + dbInitMaxRetryDelay: number httpPort: number feeStrategy: FeeStrategy ipfsGateway?: string | null diff --git a/src/index.ts b/src/index.ts index ed004bcbd..b983425cc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,13 +31,12 @@ const app: Express = express() // connections than the node itself takes to boot. Database.init() gives // up after a single failed attempt, which leaves the node running permanently // without Indexer and without C2D. -const DB_INIT_MAX_ATTEMPTS = parseInt(process.env.DB_INIT_MAX_ATTEMPTS || '10') -const DB_INIT_RETRY_DELAY = parseInt(process.env.DB_INIT_RETRY_DELAY || '2000') -const DB_INIT_MAX_RETRY_DELAY = parseInt(process.env.DB_INIT_MAX_RETRY_DELAY || '30000') async function initDatabaseWithRetry( dbConfig: OceanNodeDBConfig, - maxAttempts: number = DB_INIT_MAX_ATTEMPTS + maxAttempts: number, + retryDelay: number, + maxRetryDelay: number ): Promise { for (let attempt = 1; attempt <= maxAttempts; attempt++) { const isLastAttempt = attempt === maxAttempts @@ -58,10 +57,7 @@ async function initDatabaseWithRetry( if (isLastAttempt) { break } - const delay = Math.min( - DB_INIT_RETRY_DELAY * 2 ** (attempt - 1), - DB_INIT_MAX_RETRY_DELAY - ) + const delay = Math.min(retryDelay * 2 ** (attempt - 1), maxRetryDelay) OCEAN_NODE_LOGGER.warn( `Database ${ notReachable ? 'not reachable yet' : 'initialization failed' @@ -123,7 +119,12 @@ let node: OceanP2P = null let indexer = null let provider = null // If there is no DB URL only the nonce database will be available -const dbconn: Database = await initDatabaseWithRetry(config.dbConfig) +const dbconn: Database = await initDatabaseWithRetry( + config.dbConfig, + config.dbInitMaxAttempts, + config.dbInitRetryDelay, + config.dbInitMaxRetryDelay +) if (!dbconn) { OCEAN_NODE_LOGGER.error('Database failed to initialize') } diff --git a/src/test/unit/config.test.ts b/src/test/unit/config.test.ts index 8a2b028ad..ce96c02fe 100644 --- a/src/test/unit/config.test.ts +++ b/src/test/unit/config.test.ts @@ -5,9 +5,15 @@ import { OverrideEnvConfig, TEST_ENV_CONFIG_PATH, buildEnvOverrideConfig, - setupEnvironment + setupEnvironment, + tearDownEnvironment } from '../utils/utils.js' import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' +import { + DEFAULT_DB_INIT_MAX_ATTEMPTS, + DEFAULT_DB_INIT_MAX_RETRY_DELAY, + DEFAULT_DB_INIT_RETRY_DELAY +} from '../../utils/config/constants.js' let config: OceanNodeConfig describe('Should validate configuration from JSON', () => { @@ -61,6 +67,94 @@ describe('Should validate configuration from JSON', () => { }) }) +describe('Should validate database init retry configuration', () => { + const DB_ENV_VARS = [ENVIRONMENT_VARIABLES.DB_TYPE, ENVIRONMENT_VARIABLES.DB_URL] + const DB_ENV_VALUES = ['typesense', 'http://localhost:8108/?apiKey=xyz'] + + // returns the config built with the given DB_INIT_* values, or the thrown error + async function configWith(values: { + attempts?: string + delay?: string + maxDelay?: string + }): Promise<{ config?: OceanNodeConfig; error?: Error }> { + const envVars = [...DB_ENV_VARS] + const envValues = [...DB_ENV_VALUES] + if (values.attempts !== undefined) { + envVars.push(ENVIRONMENT_VARIABLES.DB_INIT_MAX_ATTEMPTS) + envValues.push(values.attempts) + } + if (values.delay !== undefined) { + envVars.push(ENVIRONMENT_VARIABLES.DB_INIT_RETRY_DELAY) + envValues.push(values.delay) + } + if (values.maxDelay !== undefined) { + envVars.push(ENVIRONMENT_VARIABLES.DB_INIT_MAX_RETRY_DELAY) + envValues.push(values.maxDelay) + } + // setupEnvironment() reloads the configuration itself, so an invalid value already throws + // there — it must be inside the try. The override array records the original values as it + // goes, so it is still usable for the teardown after a throw. + const overrides = buildEnvOverrideConfig(envVars, envValues) + try { + await setupEnvironment(TEST_ENV_CONFIG_PATH, overrides) + return { config: await getConfiguration(true) } + } catch (error) { + return { error } + } finally { + await tearDownEnvironment(overrides) + } + } + + it('should apply the documented defaults when the variables are not set', async () => { + const { config: conf, error } = await configWith({}) + expect(error).to.be.equal(undefined) + expect(conf.dbInitMaxAttempts).to.be.equal(DEFAULT_DB_INIT_MAX_ATTEMPTS) + expect(conf.dbInitRetryDelay).to.be.equal(DEFAULT_DB_INIT_RETRY_DELAY) + expect(conf.dbInitMaxRetryDelay).to.be.equal(DEFAULT_DB_INIT_MAX_RETRY_DELAY) + }) + + it('should coerce the environment variables to numbers', async () => { + const { config: conf, error } = await configWith({ + attempts: '3', + delay: '500', + maxDelay: '5000' + }) + expect(error).to.be.equal(undefined) + expect(conf.dbInitMaxAttempts).to.be.equal(3) + expect(conf.dbInitRetryDelay).to.be.equal(500) + expect(conf.dbInitMaxRetryDelay).to.be.equal(5000) + }) + + // 0 attempts would never enter the retry loop, so Database.init() would not be called at + // all and the node would silently boot without any database + it('should refuse to start when the number of attempts is zero or negative', async () => { + for (const attempts of ['0', '-1']) { + const { config: conf, error } = await configWith({ attempts }) + expect(conf, `attempts=${attempts} should not produce a config`).to.be.equal( + undefined + ) + expect(error?.message).to.be.equal('Configuration validation failed') + } + }) + + it('should refuse to start on a non-numeric value', async () => { + const { config: conf, error } = await configWith({ attempts: 'abc' }) + expect(conf).to.be.equal(undefined) + expect(error?.message).to.be.equal('Configuration validation failed') + }) + + it('should refuse to start when a delay is zero', async () => { + const { config: conf, error } = await configWith({ delay: '0' }) + expect(conf).to.be.equal(undefined) + expect(error?.message).to.be.equal('Configuration validation failed') + }) + + after(() => { + delete process.env.CONFIG_PATH + delete process.env.PRIVATE_KEY + }) +}) + describe('Should validate P2P config from environment variables', () => { let config: OceanNodeConfig let envOverrides: OverrideEnvConfig[] diff --git a/src/utils/config/constants.ts b/src/utils/config/constants.ts index 7ab77eba4..894f65348 100644 --- a/src/utils/config/constants.ts +++ b/src/utils/config/constants.ts @@ -5,6 +5,11 @@ export const ENV_TO_CONFIG_MAPPING = { DB_USERNAME: 'DB_USERNAME', DB_PASSWORD: 'DB_PASSWORD', DB_TYPE: 'DB_TYPE', + // NOTE: deliberately flat (not under dbConfig.*) — preprocessConfigData() rebuilds + // data.dbConfig from scratch when DB_URL is set, which would drop anything nested there. + DB_INIT_MAX_ATTEMPTS: 'dbInitMaxAttempts', + DB_INIT_RETRY_DELAY: 'dbInitRetryDelay', + DB_INIT_MAX_RETRY_DELAY: 'dbInitMaxRetryDelay', FEE_AMOUNT: 'FEE_AMOUNT', FEE_TOKENS: 'FEE_TOKENS', HTTP_API_PORT: 'httpPort', @@ -76,6 +81,11 @@ export const ENV_TO_CONFIG_MAPPING = { // Configuration defaults export const DEFAULT_RATE_LIMIT_PER_MINUTE = 30 +// Database init retry at node startup. Worst case wait before giving up with these defaults is +// 2 + 4 + 8 + 16 + 30 * 5 = 180 seconds, so a container health probe must tolerate that. +export const DEFAULT_DB_INIT_MAX_ATTEMPTS = 10 +export const DEFAULT_DB_INIT_RETRY_DELAY = 2000 +export const DEFAULT_DB_INIT_MAX_RETRY_DELAY = 30000 export const DEFAULT_MAX_CONNECTIONS_PER_MINUTE = 60 * 2 // 120 requests per minute export const SEPOLIA_CHAIN_ID = '11155111' export const BASE_CHAIN_ID = '8453' diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index 5b4820384..250c4ee4c 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -8,7 +8,10 @@ import { DEFAULT_BOOTSTRAP_ADDRESSES, DEFAULT_RATE_LIMIT_PER_MINUTE, DEFAULT_UNSAFE_URLS, - DEFAULT_FILTER_ANNOUNCED_ADDRESSES + DEFAULT_FILTER_ANNOUNCED_ADDRESSES, + DEFAULT_DB_INIT_MAX_ATTEMPTS, + DEFAULT_DB_INIT_RETRY_DELAY, + DEFAULT_DB_INIT_MAX_RETRY_DELAY } from './constants.js' function isValidUrl(urlString: string): boolean { @@ -778,6 +781,30 @@ export const OceanNodeConfigSchema = z httpPort: z.coerce.number().optional().default(3000), rateLimit: z.coerce.number().optional().default(DEFAULT_RATE_LIMIT_PER_MINUTE), + // Startup database-init retry knobs. int().min(1) is load-bearing, not decoration: + // maxAttempts of 0 would skip Database.init() altogether and boot the node with no + // database, and a 0 delay would turn the backoff into a hot loop. A non-numeric or + // out-of-range value fails validation, so the node refuses to start instead of silently + // running with retrying disabled. + dbInitMaxAttempts: z.coerce + .number() + .int() + .min(1) + .optional() + .default(DEFAULT_DB_INIT_MAX_ATTEMPTS), + dbInitRetryDelay: z.coerce + .number() + .int() + .min(1) + .optional() + .default(DEFAULT_DB_INIT_RETRY_DELAY), + dbInitMaxRetryDelay: z.coerce + .number() + .int() + .min(1) + .optional() + .default(DEFAULT_DB_INIT_MAX_RETRY_DELAY), + ipfsGateway: z.string().nullable().optional(), arweaveGateway: z.string().nullable().optional(), diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 507b1b1bb..6a01bac96 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -481,6 +481,21 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.DB_TYPE, required: false }, + DB_INIT_MAX_ATTEMPTS: { + name: 'DB_INIT_MAX_ATTEMPTS', + value: process.env.DB_INIT_MAX_ATTEMPTS, + required: false + }, + DB_INIT_RETRY_DELAY: { + name: 'DB_INIT_RETRY_DELAY', + value: process.env.DB_INIT_RETRY_DELAY, + required: false + }, + DB_INIT_MAX_RETRY_DELAY: { + name: 'DB_INIT_MAX_RETRY_DELAY', + value: process.env.DB_INIT_MAX_RETRY_DELAY, + required: false + }, CRON_DELETE_DB_LOGS: { name: 'CRON_DELETE_DB_LOGS', value: process.env.CRON_DELETE_DB_LOGS, From d85d002140b8eee146543bb2cbb5185bb3cadced Mon Sep 17 00:00:00 2001 From: Denis <61563365+dnsi0@users.noreply.github.com> Date: Fri, 14 Aug 2026 00:21:38 +0300 Subject: [PATCH 3/6] fix return type --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index b983425cc..704ceb6af 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,7 +37,7 @@ async function initDatabaseWithRetry( maxAttempts: number, retryDelay: number, maxRetryDelay: number -): Promise { +): Promise { for (let attempt = 1; attempt <= maxAttempts; attempt++) { const isLastAttempt = attempt === maxAttempts const notReachable = From b7647613ffb6614c29f1431d5a4833c069fe4f32 Mon Sep 17 00:00:00 2001 From: Denis <61563365+dnsi0@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:27:21 +0300 Subject: [PATCH 4/6] remove timeout --- src/utils/database.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/utils/database.ts b/src/utils/database.ts index 108f7e530..cad5d5745 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -24,12 +24,9 @@ export function hasValidDBConfiguration(configuration: OceanNodeDBConfig): boole } // we can use this to check if DB connection is available -export async function isReachableConnection( - url: string, - timeout: number = 3000 -): Promise { +export async function isReachableConnection(url: string): Promise { try { - await fetch(url, { signal: AbortSignal.timeout(timeout) }) + await fetch(url) return true } catch (error) { return false From 3559492470b7e923403ad63c2b96c974a604e9d0 Mon Sep 17 00:00:00 2001 From: Denis <61563365+dnsi0@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:34:48 +0300 Subject: [PATCH 5/6] fix review --- src/test/utils/utils.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/utils/utils.ts b/src/test/utils/utils.ts index b3bf0917b..d3153f40b 100644 --- a/src/test/utils/utils.ts +++ b/src/test/utils/utils.ts @@ -129,6 +129,9 @@ export async function tearDownEnvironment(overrideVars?: OverrideEnvConfig[]) { } else { delete process.env[element.name] } + if (ENVIRONMENT_VARIABLES[element.name]) { + ENVIRONMENT_VARIABLES[element.name].value = element.originalValue + } forceReload = true } }) From 682a59dd47188fc286f6195577c5248d205fda85 Mon Sep 17 00:00:00 2001 From: Denis <61563365+dnsi0@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:12:53 +0300 Subject: [PATCH 6/6] fix type --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 704ceb6af..f0f112db0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -119,7 +119,7 @@ let node: OceanP2P = null let indexer = null let provider = null // If there is no DB URL only the nonce database will be available -const dbconn: Database = await initDatabaseWithRetry( +const dbconn: Database | null = await initDatabaseWithRetry( config.dbConfig, config.dbInitMaxAttempts, config.dbInitRetryDelay,