feat(database): complete Phase 4.3 PostgreSQL and migration foundation

.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
       - name: "Install Build & Runtime Dependencies"
         run: |
           python -m pip install --upgrade pip
-          pip install pydantic pydantic-settings networkx sqlglot fastapi uvicorn httpx ruff bandit build setuptools
+          pip install pydantic pydantic-settings networkx sqlglot fastapi uvicorn httpx ruff bandit build setuptools sqlalchemy alembic asyncpg greenlet aiosqlite
           pip install -e .
 
       - name: "Ruff Linting & Format Verification"
@@ -186,7 +186,7 @@ jobs:
       - name: "Install Core & Connector Dependencies"
         run: |
           python -m pip install --upgrade pip
-          pip install pydantic pydantic-settings networkx sqlglot fastapi uvicorn
+          pip install pydantic pydantic-settings networkx sqlglot fastapi uvicorn httpx sqlalchemy alembic asyncpg greenlet aiosqlite
           pip install -e .
 
       - name: "Verify Schema Contract File Integrity"
@@ -237,21 +237,42 @@ jobs:
           python-version: "3.13"
           cache: "pip"
 
-      - name: "Install Connector Dependencies"
+      - name: "Install Database & Platform Dependencies"
         run: |
           python -m pip install --upgrade pip
-          pip install pydantic networkx sqlglot
+          pip install pydantic pydantic-settings networkx sqlglot fastapi uvicorn httpx sqlalchemy alembic asyncpg greenlet aiosqlite
           pip install -e .
 
       - name: "Verify PostgreSQL 16 Service Container Readiness"
         run: |
           nc -z 127.0.0.1 5432 || (sleep 2 && nc -z 127.0.0.1 5432)
-          echo "PostgreSQL 16 ephemeral service container is healthy and ready for connector testing."
+          echo "PostgreSQL 16 ephemeral service container is healthy and ready for connector and platform testing."
 
       - name: "Execute SQLite Connector Integration Suite"
         run: |
           python -m unittest tests/integration/test_sqlite_connector.py
 
+      - name: "Apply Platform Database Migrations (PostgreSQL 16 Head)"
+        env:
+          DERA_PLATFORM_DATABASE_URL: "postgresql+asyncpg://dera_test_user:dera_ephemeral_test_password@127.0.0.1:5432/dera_test_db"
+        run: |
+          python -m alembic upgrade head
+          python -m alembic current
+
+      - name: "Execute Platform Database & Migration Test Suite (PostgreSQL 16)"
+        env:
+          DERA_PLATFORM_DATABASE_URL: "postgresql+asyncpg://dera_test_user:dera_ephemeral_test_password@127.0.0.1:5432/dera_test_db"
+        run: |
+          python -m unittest discover -s tests/database -p "test_*.py"
+          python -m unittest tests/integration/test_platform_database.py
+
+      - name: "Verify Migration Reversibility (Downgrade & Re-Upgrade)"
+        env:
+          DERA_PLATFORM_DATABASE_URL: "postgresql+asyncpg://dera_test_user:dera_ephemeral_test_password@127.0.0.1:5432/dera_test_db"
+        run: |
+          python -m alembic downgrade base
+          python -m alembic upgrade head
+
   # ----------------------------------------------------------
   # 5. Full Documentation Quality & Navigation Gate
   # ----------------------------------------------------------

.github/workflows/dependency-review.yml

@@ -44,4 +44,4 @@ jobs:
         uses: actions/dependency-review-action@v5
         with:
           fail-on-severity: high
-          allow-licenses: "MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, Python-2.0, 0BSD"
+          allow-licenses: "MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, Python-2.0, PSF-2.0, 0BSD"

README.md

@@ -211,6 +211,7 @@ For comprehensive technical specifications:
 - [Architecture & Blueprint Guide](docs/architecture/README_ARCHITECTURE.md)
 - [Module Boundaries & Dependency Rules](docs/architecture/MODULE_BOUNDARIES.md)
 - [Monorepo Directory Layout](docs/architecture/MONOREPO_STRUCTURE.md)
+- [Platform Database & Migrations](docs/database/README.md)
 - [Architecture Decision Record (ADR 0001)](docs/architecture/adr/0001-monorepo-architecture.md)
 
 ---

alembic.ini

@@ -0,0 +1,66 @@
+# DERA — Authoritative Alembic Migration Configuration
+# For DERA Platform Database (PostgreSQL 16+)
+# Deterministic, environment-driven configuration without committed credentials.
+
+[alembic]
+# Path to migration environment scripts
+script_location = migrations
+
+# Template used to generate migration file names
+file_template = %%(rev)s_%%(slug)s
+
+# Sys.path prepends
+prepend_sys_path = apps/api/src:packages/core/src:.
+
+# Timezone for revision generation timestamps
+timezone = UTC
+
+# Truncate slug to 40 characters
+truncate_slug_length = 40
+
+# Max length of characters for revision IDs
+version_locations = migrations/versions
+
+# Output encoding
+output_encoding = utf-8
+
+# sqlalchemy.url is intentionally left unset here.
+# Connection URL is dynamically injected from DERA_PLATFORM_DATABASE_URL
+# or dera_api.core.config.ApiSettings in migrations/env.py.
+
+[post_write_hooks]
+# Post-write hooks for formatting generated migrations
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

apps/api/.env.example

@@ -24,3 +24,16 @@ APP_CORS_ORIGINS="http://localhost:3000,http://127.0.0.1:3000"
 # Architectural Safeguards (Immutable / Deterministic Non-AI Guarantees)
 READ_ONLY_ENFORCED=true
 ZERO_AI_POLICY_ENFORCED=true
+
+# Platform Database Configuration (PostgreSQL 16+ via SQLAlchemy Asyncpg)
+# NOTE: Platform database stores internal DERA operational metadata.
+# Strictly separated from customer target databases inspected in Phase 5+.
+DERA_PLATFORM_DATABASE_URL="postgresql+asyncpg://dera_user:dera_password@localhost:5432/dera_platform"
+DERA_DB_POOL_SIZE=10
+DERA_DB_MAX_OVERFLOW=5
+DERA_DB_POOL_TIMEOUT=30.0
+DERA_DB_POOL_RECYCLE=1800
+DERA_DB_CONNECT_TIMEOUT=10.0
+DERA_DB_STATEMENT_TIMEOUT_MS=30000
+DERA_DB_SSL_MODE="prefer"
+DERA_DB_ECHO=false

apps/api/src/dera_api/core/config.py

@@ -4,15 +4,18 @@
 and pydantic-settings, supporting development, testing, and production profiles.
 """
 
+import re
 from functools import lru_cache
 from typing import Any
+from urllib.parse import urlsplit, urlunsplit
 
 import dera
 from pydantic import AliasChoices, Field, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 VALID_ENVIRONMENTS = {"development", "testing", "production"}
 VALID_LOG_LEVELS = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}
+VALID_SSL_MODES = {"disable", "allow", "prefer", "require", "verify-ca", "verify-full"}
 
 
 class ApiSettings(BaseSettings):
@@ -92,6 +95,86 @@ class ApiSettings(BaseSettings):
         description="Deterministic zero-AI architectural constraint guarantee.",
     )
 
+    # ------------------------------------------------------------------
+    # Phase 4.3: Internal Platform PostgreSQL Database Settings
+    # ------------------------------------------------------------------
+    database_url: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("DERA_PLATFORM_DATABASE_URL", "DATABASE_URL"),
+        description="Async connection URL for the internal DERA platform PostgreSQL database.",
+    )
+    db_pool_size: int = Field(
+        default=10,
+        ge=1,
+        le=100,
+        validation_alias=AliasChoices("DERA_DB_POOL_SIZE", "DB_POOL_SIZE"),
+        description="Base connection pool size for application database operations.",
+    )
+    db_max_overflow: int = Field(
+        default=5,
+        ge=0,
+        le=50,
+        validation_alias=AliasChoices("DERA_DB_MAX_OVERFLOW", "DB_MAX_OVERFLOW"),
+        description="Maximum transient overflow connections beyond base pool.",
+    )
+    db_pool_timeout: float = Field(
+        default=30.0,
+        ge=1.0,
+        le=300.0,
+        validation_alias=AliasChoices("DERA_DB_POOL_TIMEOUT", "DB_POOL_TIMEOUT"),
+        description="Timeout in seconds to wait when acquiring a connection from the pool.",
+    )
+    db_pool_recycle: int = Field(
+        default=1800,
+        ge=60,
+        le=86400,
+        validation_alias=AliasChoices("DERA_DB_POOL_RECYCLE", "DB_POOL_RECYCLE"),
+        description="Recycle connections after this duration in seconds (default 30 minutes).",
+    )
+    db_connect_timeout: float = Field(
+        default=10.0,
+        ge=1.0,
+        le=60.0,
+        validation_alias=AliasChoices("DERA_DB_CONNECT_TIMEOUT", "DB_CONNECT_TIMEOUT"),
+        description="Connection acquisition timeout in seconds for new network sockets.",
+    )
+    db_statement_timeout_ms: int = Field(
+        default=30000,
+        ge=100,
+        le=3600000,
+        validation_alias=AliasChoices("DERA_DB_STATEMENT_TIMEOUT_MS", "DB_STATEMENT_TIMEOUT_MS"),
+        description="Server-side PostgreSQL statement timeout in milliseconds.",
+    )
+    db_ssl_mode: str = Field(
+        default="prefer",
+        validation_alias=AliasChoices("DERA_DB_SSL_MODE", "DB_SSL_MODE"),
+        description="SSL transport security mode for PostgreSQL connections.",
+    )
+    db_echo: bool = Field(
+        default=False,
+        validation_alias=AliasChoices("DERA_DB_ECHO", "DB_ECHO"),
+        description="Enable SQLAlchemy SQL query echo logging (strictly non-production).",
+    )
+
+    @property
+    def has_database_configured(self) -> bool:
+        """Returns True if a platform database URL has been explicitly specified."""
+        return bool(self.database_url and self.database_url.strip())
+
+    @property
+    def sanitized_database_url(self) -> str:
+        """Returns the database URL with authentication credentials securely masked."""
+        if not self.database_url:
+            return "not-configured"
+        try:
+            parts = urlsplit(self.database_url)
+            if parts.password:
+                netloc = parts.netloc.replace(f":{parts.password}@", ":***@")
+                return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
+            return self.database_url
+        except Exception:
+            return re.sub(r"://([^:]+):([^@]+)@", r"://\1:***@", self.database_url)
+
     @field_validator("environment")
     @classmethod
     def validate_environment(cls, v: str) -> str:
@@ -127,6 +210,14 @@ def parse_cors_origins(cls, v: Any) -> list[str]:
             return [str(item).strip() for item in v if str(item).strip()]
         return ["http://localhost:3000"]
 
+    @field_validator("db_ssl_mode")
+    @classmethod
+    def validate_db_ssl_mode(cls, v: str) -> str:
+        mode = v.strip().lower()
+        if mode not in VALID_SSL_MODES:
+            raise ValueError(f"Invalid db_ssl_mode '{v}'. Allowed modes: {sorted(VALID_SSL_MODES)}")
+        return mode
+
     @model_validator(mode="after")
     def validate_production_restrictions(self) -> "ApiSettings":
         """Enforces defensive production security rules."""
@@ -137,6 +228,10 @@ def validate_production_restrictions(self) -> "ApiSettings":
                 raise ValueError(
                     "Wildcard CORS origins ('*') are strictly prohibited in production"
                 )
+            if self.db_echo:
+                raise ValueError(
+                    "Database SQL query echo (db_echo) is strictly prohibited in production"
+                )
             if self.docs_enabled is None:
                 self.docs_enabled = False
         else:

apps/api/src/dera_api/core/lifespan.py

@@ -2,6 +2,7 @@
 
 Manages application initialization, state readiness, and graceful shutdown
 using contextlib.asynccontextmanager without deprecated startup/shutdown events.
+Integrates platform database engine lifecycle, pooling, and disposal.
 """
 
 from collections.abc import AsyncIterator
@@ -11,6 +12,8 @@
 
 from dera_api.core.config import get_settings
 from dera_api.core.logging import get_logger
+from dera_api.db.engine import create_database_engine, dispose_database_engine
+from dera_api.db.session import create_session_factory
 from fastapi import FastAPI
 
 logger = get_logger("dera_api.lifespan")
@@ -28,6 +31,30 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     app.state.service_name = "dera-api"
     app.state.version = settings.version
 
+    # Platform database initialization
+    if settings.has_database_configured:
+        try:
+            engine = create_database_engine(settings)
+            session_factory = create_session_factory(engine)
+            app.state.db_engine = engine
+            app.state.db_session_factory = session_factory
+            logger.info(
+                "Platform database engine initialized: [url=%s, pool_size=%d, max_overflow=%d]",
+                settings.sanitized_database_url,
+                settings.db_pool_size,
+                settings.db_max_overflow,
+            )
+        except Exception as exc:
+            logger.error("Failed to initialize platform database engine: %s", exc)
+            app.state.db_engine = None
+            app.state.db_session_factory = None
+            app.state.is_ready = False
+            raise
+    else:
+        app.state.db_engine = None
+        app.state.db_session_factory = None
+        logger.info("Platform database not configured; operating in stateless mode.")
+
     logger.info(
         "Application startup complete: %s v%s [env=%s, docs=%s]",
         settings.app_name,
@@ -42,13 +69,22 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         # Graceful shutdown sequence
         app.state.is_ready = False
         logger.info("Application shutdown initiated; cleaning up resources.")
+
+        db_engine = getattr(app.state, "db_engine", None)
+        if db_engine is not None:
+            await dispose_database_engine(db_engine)
+            logger.info("Platform database engine cleanly disposed.")
+
         logger.info("Application shutdown complete.")
 
 
 def get_application_readiness(app: FastAPI) -> dict[str, Any]:
     """Returns truthful readiness status and system probe states."""
     is_ready = getattr(app.state, "is_ready", False)
     startup_time = getattr(app.state, "startup_time", None)
+    db_engine = getattr(app.state, "db_engine", None)
+
+    db_status = "operational" if db_engine is not None else "not_configured"
 
     return {
         "ready": bool(is_ready),
@@ -57,5 +93,6 @@ def get_application_readiness(app: FastAPI) -> dict[str, Any]:
             "application_state": "operational" if is_ready else "not_ready",
             "database_connectors": "none_required_in_phase_4_1",
             "background_workers": "none_required_in_phase_4_1",
+            "platform_database": db_status,
         },
     }