add database retry

docs/env.md

@@ -49,6 +49,9 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
 - `ELASTICSEARCH_SNIFF_INTERVAL`: Interval in milliseconds for periodic cluster health monitoring and node discovery. Set to 'false' to disable. Default is `30000`. Example: `30000`
 - `ELASTICSEARCH_SNIFF_ON_CONNECTION_FAULT`: Enable automatic cluster node discovery when connection faults occur. Default is `true`. Example: `true`
 - `ELASTICSEARCH_HEALTH_CHECK_INTERVAL`: Interval in milliseconds for proactive connection health monitoring. Default is `60000`. Example: `60000`
+- `DB_INIT_MAX_ATTEMPTS`: Maximum number of database initialization attempts at node startup. Raise it when the database container/pod starts slower than the node. Set to `1` to disable retrying. All three `DB_INIT_*` variables are validated as integers `>= 1`; a non-numeric or out-of-range value fails configuration validation and the node refuses to start. Default is `10`. Example: `10`
+- `DB_INIT_RETRY_DELAY`: Initial delay in milliseconds before retrying a failed database initialization. Doubles after each attempt. Default is `2000`. Example: `2000`
+- `DB_INIT_MAX_RETRY_DELAY`: Upper bound in milliseconds for the database initialization retry backoff. Default is `30000`. Example: `30000`
 
 ## Payments
 

src/@types/OceanNode.ts

@@ -120,6 +120,10 @@ export interface OceanNodeConfig {
   hasIndexer: boolean
   hasHttp: boolean
   dbConfig?: OceanNodeDBConfig
+  // startup database-init retry: attempts, initial backoff (ms) and backoff ceiling (ms)
+  dbInitMaxAttempts: number
+  dbInitRetryDelay: number
+  dbInitMaxRetryDelay: number
   httpPort: number
   feeStrategy: FeeStrategy
   ipfsGateway?: string | null

src/index.ts

@@ -22,10 +22,52 @@ import { fileURLToPath } from 'url'
 import cors from 'cors'
 import { scheduleCronJobs } from './utils/cronjobs/scheduleCronJobs.js'
 import { requestValidator } from './components/httpRoutes/requestValidator.js'
-import { hasValidDBConfiguration } from './utils/database.js'
+import { hasValidDBConfiguration, isReachableConnection } from './utils/database.js'
+import type { OceanNodeDBConfig } from './@types/OceanNode.js'
 
 const app: Express = express()
 
+// Database services (Elasticsearch/Typesense) frequently take longer to accept
+// connections than the node itself takes to boot. Database.init() gives
+// up after a single failed attempt, which leaves the node running permanently
+// without Indexer and without C2D.
+
+async function initDatabaseWithRetry(
+  dbConfig: OceanNodeDBConfig,
+  maxAttempts: number,
+  retryDelay: number,
+  maxRetryDelay: number
+): Promise<Database | null> {
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    const isLastAttempt = attempt === maxAttempts
+    const notReachable =
+      !isLastAttempt &&
+      hasValidDBConfiguration(dbConfig) &&
+      !(await isReachableConnection(dbConfig.url))
+
+    if (!notReachable) {
+      const database = await Database.init(dbConfig)
+      if (database) {
+        if (attempt > 1) {
+          OCEAN_NODE_LOGGER.info(`Database initialized after ${attempt} attempts`)
+        }
+        return database
+      }
+    }
+    if (isLastAttempt) {
+      break
+    }
+    const delay = Math.min(retryDelay * 2 ** (attempt - 1), maxRetryDelay)
+    OCEAN_NODE_LOGGER.warn(
+      `Database ${
+        notReachable ? 'not reachable yet' : 'initialization failed'
+      } (attempt ${attempt}/${maxAttempts}), retrying in ${delay}ms`
+    )
+    await new Promise((resolve) => setTimeout(resolve, delay))
+  }
+  return null
+}
+
 process.on('uncaughtException', (err) => {
   OCEAN_NODE_LOGGER.error(`Uncaught exception: ${err.message}`)
   process.exit(1)
@@ -77,7 +119,12 @@ let node: OceanP2P = null
 let indexer = null
 let provider = null
 // If there is no DB URL only the nonce database will be available
-const dbconn: Database = await Database.init(config.dbConfig)
+const dbconn: Database | null = await initDatabaseWithRetry(
+  config.dbConfig,
+  config.dbInitMaxAttempts,
+  config.dbInitRetryDelay,
+  config.dbInitMaxRetryDelay
+)
 if (!dbconn) {
   OCEAN_NODE_LOGGER.error('Database failed to initialize')
 }

src/test/unit/config.test.ts

@@ -5,9 +5,15 @@ import {
   OverrideEnvConfig,
   TEST_ENV_CONFIG_PATH,
   buildEnvOverrideConfig,
-  setupEnvironment
+  setupEnvironment,
+  tearDownEnvironment
 } from '../utils/utils.js'
 import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js'
+import {
+  DEFAULT_DB_INIT_MAX_ATTEMPTS,
+  DEFAULT_DB_INIT_MAX_RETRY_DELAY,
+  DEFAULT_DB_INIT_RETRY_DELAY
+} from '../../utils/config/constants.js'
 
 let config: OceanNodeConfig
 describe('Should validate configuration from JSON', () => {
@@ -61,6 +67,94 @@ describe('Should validate configuration from JSON', () => {
   })
 })
 
+describe('Should validate database init retry configuration', () => {
+  const DB_ENV_VARS = [ENVIRONMENT_VARIABLES.DB_TYPE, ENVIRONMENT_VARIABLES.DB_URL]
+  const DB_ENV_VALUES = ['typesense', 'http://localhost:8108/?apiKey=xyz']
+
+  // returns the config built with the given DB_INIT_* values, or the thrown error
+  async function configWith(values: {
+    attempts?: string
+    delay?: string
+    maxDelay?: string
+  }): Promise<{ config?: OceanNodeConfig; error?: Error }> {
+    const envVars = [...DB_ENV_VARS]
+    const envValues = [...DB_ENV_VALUES]
+    if (values.attempts !== undefined) {
+      envVars.push(ENVIRONMENT_VARIABLES.DB_INIT_MAX_ATTEMPTS)
+      envValues.push(values.attempts)
+    }
+    if (values.delay !== undefined) {
+      envVars.push(ENVIRONMENT_VARIABLES.DB_INIT_RETRY_DELAY)
+      envValues.push(values.delay)
+    }
+    if (values.maxDelay !== undefined) {
+      envVars.push(ENVIRONMENT_VARIABLES.DB_INIT_MAX_RETRY_DELAY)
+      envValues.push(values.maxDelay)
+    }
+    // setupEnvironment() reloads the configuration itself, so an invalid value already throws
+    // there — it must be inside the try. The override array records the original values as it
+    // goes, so it is still usable for the teardown after a throw.
+    const overrides = buildEnvOverrideConfig(envVars, envValues)
+    try {
+      await setupEnvironment(TEST_ENV_CONFIG_PATH, overrides)
+      return { config: await getConfiguration(true) }
+    } catch (error) {
+      return { error }
+    } finally {
+      await tearDownEnvironment(overrides)
+    }
+  }
+
+  it('should apply the documented defaults when the variables are not set', async () => {
+    const { config: conf, error } = await configWith({})
+    expect(error).to.be.equal(undefined)
+    expect(conf.dbInitMaxAttempts).to.be.equal(DEFAULT_DB_INIT_MAX_ATTEMPTS)
+    expect(conf.dbInitRetryDelay).to.be.equal(DEFAULT_DB_INIT_RETRY_DELAY)
+    expect(conf.dbInitMaxRetryDelay).to.be.equal(DEFAULT_DB_INIT_MAX_RETRY_DELAY)
+  })
+
+  it('should coerce the environment variables to numbers', async () => {
+    const { config: conf, error } = await configWith({
+      attempts: '3',
+      delay: '500',
+      maxDelay: '5000'
+    })
+    expect(error).to.be.equal(undefined)
+    expect(conf.dbInitMaxAttempts).to.be.equal(3)
+    expect(conf.dbInitRetryDelay).to.be.equal(500)
+    expect(conf.dbInitMaxRetryDelay).to.be.equal(5000)
+  })
+
+  // 0 attempts would never enter the retry loop, so Database.init() would not be called at
+  // all and the node would silently boot without any database
+  it('should refuse to start when the number of attempts is zero or negative', async () => {
+    for (const attempts of ['0', '-1']) {
+      const { config: conf, error } = await configWith({ attempts })
+      expect(conf, `attempts=${attempts} should not produce a config`).to.be.equal(
+        undefined
+      )
+      expect(error?.message).to.be.equal('Configuration validation failed')
+    }
+  })
+
+  it('should refuse to start on a non-numeric value', async () => {
+    const { config: conf, error } = await configWith({ attempts: 'abc' })
+    expect(conf).to.be.equal(undefined)
+    expect(error?.message).to.be.equal('Configuration validation failed')
+  })
+
+  it('should refuse to start when a delay is zero', async () => {
+    const { config: conf, error } = await configWith({ delay: '0' })
+    expect(conf).to.be.equal(undefined)
+    expect(error?.message).to.be.equal('Configuration validation failed')
+  })
+
+  after(() => {
+    delete process.env.CONFIG_PATH
+    delete process.env.PRIVATE_KEY
+  })
+})
+
 describe('Should validate P2P config from environment variables', () => {
   let config: OceanNodeConfig
   let envOverrides: OverrideEnvConfig[]

src/test/utils/utils.ts

@@ -129,6 +129,9 @@ export async function tearDownEnvironment(overrideVars?: OverrideEnvConfig[]) {
         } else {
           delete process.env[element.name]
         }
+        if (ENVIRONMENT_VARIABLES[element.name]) {
+          ENVIRONMENT_VARIABLES[element.name].value = element.originalValue
+        }
         forceReload = true
       }
     })

src/utils/config/constants.ts

@@ -5,6 +5,11 @@ export const ENV_TO_CONFIG_MAPPING = {
   DB_USERNAME: 'DB_USERNAME',
   DB_PASSWORD: 'DB_PASSWORD',
   DB_TYPE: 'DB_TYPE',
+  // NOTE: deliberately flat (not under dbConfig.*) — preprocessConfigData() rebuilds
+  // data.dbConfig from scratch when DB_URL is set, which would drop anything nested there.
+  DB_INIT_MAX_ATTEMPTS: 'dbInitMaxAttempts',
+  DB_INIT_RETRY_DELAY: 'dbInitRetryDelay',
+  DB_INIT_MAX_RETRY_DELAY: 'dbInitMaxRetryDelay',
   FEE_AMOUNT: 'FEE_AMOUNT',
   FEE_TOKENS: 'FEE_TOKENS',
   HTTP_API_PORT: 'httpPort',
@@ -76,6 +81,11 @@ export const ENV_TO_CONFIG_MAPPING = {
 
 // Configuration defaults
 export const DEFAULT_RATE_LIMIT_PER_MINUTE = 30
+// Database init retry at node startup. Worst case wait before giving up with these defaults is
+// 2 + 4 + 8 + 16 + 30 * 5 = 180 seconds, so a container health probe must tolerate that.
+export const DEFAULT_DB_INIT_MAX_ATTEMPTS = 10
+export const DEFAULT_DB_INIT_RETRY_DELAY = 2000
+export const DEFAULT_DB_INIT_MAX_RETRY_DELAY = 30000
 export const DEFAULT_MAX_CONNECTIONS_PER_MINUTE = 60 * 2 // 120 requests per minute
 export const SEPOLIA_CHAIN_ID = '11155111'
 export const BASE_CHAIN_ID = '8453'

src/utils/config/schemas.ts

@@ -8,7 +8,10 @@ import {
   DEFAULT_BOOTSTRAP_ADDRESSES,
   DEFAULT_RATE_LIMIT_PER_MINUTE,
   DEFAULT_UNSAFE_URLS,
-  DEFAULT_FILTER_ANNOUNCED_ADDRESSES
+  DEFAULT_FILTER_ANNOUNCED_ADDRESSES,
+  DEFAULT_DB_INIT_MAX_ATTEMPTS,
+  DEFAULT_DB_INIT_RETRY_DELAY,
+  DEFAULT_DB_INIT_MAX_RETRY_DELAY
 } from './constants.js'
 
 function isValidUrl(urlString: string): boolean {
@@ -778,6 +781,30 @@ export const OceanNodeConfigSchema = z
     httpPort: z.coerce.number().optional().default(3000),
     rateLimit: z.coerce.number().optional().default(DEFAULT_RATE_LIMIT_PER_MINUTE),
 
+    // Startup database-init retry knobs. int().min(1) is load-bearing, not decoration:
+    // maxAttempts of 0 would skip Database.init() altogether and boot the node with no
+    // database, and a 0 delay would turn the backoff into a hot loop. A non-numeric or
+    // out-of-range value fails validation, so the node refuses to start instead of silently
+    // running with retrying disabled.
+    dbInitMaxAttempts: z.coerce
+      .number()
+      .int()
+      .min(1)
+      .optional()
+      .default(DEFAULT_DB_INIT_MAX_ATTEMPTS),
+    dbInitRetryDelay: z.coerce
+      .number()
+      .int()
+      .min(1)
+      .optional()
+      .default(DEFAULT_DB_INIT_RETRY_DELAY),
+    dbInitMaxRetryDelay: z.coerce
+      .number()
+      .int()
+      .min(1)
+      .optional()
+      .default(DEFAULT_DB_INIT_MAX_RETRY_DELAY),
+
     ipfsGateway: z.string().nullable().optional(),
     arweaveGateway: z.string().nullable().optional(),
 

src/utils/constants.ts

@@ -481,6 +481,21 @@ export const ENVIRONMENT_VARIABLES: Record<any, EnvVariable> = {
     value: process.env.DB_TYPE,
     required: false
   },
+  DB_INIT_MAX_ATTEMPTS: {
+    name: 'DB_INIT_MAX_ATTEMPTS',
+    value: process.env.DB_INIT_MAX_ATTEMPTS,
+    required: false
+  },
+  DB_INIT_RETRY_DELAY: {
+    name: 'DB_INIT_RETRY_DELAY',
+    value: process.env.DB_INIT_RETRY_DELAY,
+    required: false
+  },
+  DB_INIT_MAX_RETRY_DELAY: {
+    name: 'DB_INIT_MAX_RETRY_DELAY',
+    value: process.env.DB_INIT_MAX_RETRY_DELAY,
+    required: false
+  },
   CRON_DELETE_DB_LOGS: {
     name: 'CRON_DELETE_DB_LOGS',
     value: process.env.CRON_DELETE_DB_LOGS,