ci: run ScalaTests against SQL Server 2022 and resolve dialect incompatibilities

.github/actions/setup-test-database/action.yml

@@ -0,0 +1,98 @@
+name: Set up the test database
+description: >
+  Selects the database the shards run against (H2 unless a manual run or a pull request label asks for
+  SQL Server or PostgreSQL), starts its container when one is needed and exports the settings the later
+  steps use: TEST_DB, TEST_DB_DRIVER, TEST_DB_URL and TEST_TIMEOUT_SECONDS.
+
+inputs:
+  database:
+    description: Database chosen on a manual run (h2, sqlserver, sqlserver2017 or postgres); empty on other triggers.
+    required: false
+    default: ''
+  pr-labels:
+    description: JSON list of the pull request labels; a test-sqlserver or test-postgres label opts in.
+    required: false
+    default: '[]'
+  commit-message:
+    description: Head commit message of a push; [test-sqlserver] or [test-postgres] in it opts in.
+    required: false
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Start the selected database
+      shell: bash
+      env:
+        INPUT_DB: ${{ inputs.database }}
+        PR_LABELS_JSON: ${{ inputs.pr-labels }}
+        COMMIT_MSG: ${{ inputs.commit-message }}
+      run: |
+        # Test-only credentials of a throwaway container that lives for the length of the job.
+        SA_PASSWORD='ObpTests2026!Ci'
+        MSSQL_IMAGE='mcr.microsoft.com/mssql/server:2022-CU14-ubuntu-22.04'
+        POSTGRES_IMAGE='postgres:16-alpine'
+
+        DB_CHOICE="${INPUT_DB:-h2}"
+        if [ "$DB_CHOICE" = "h2" ]; then
+          if echo "$PR_LABELS_JSON" | grep -q "test-sqlserver" || echo "$COMMIT_MSG" | grep -q "\[test-sqlserver\]"; then
+            DB_CHOICE="sqlserver"
+          elif echo "$PR_LABELS_JSON" | grep -q "test-postgres" || echo "$COMMIT_MSG" | grep -q "\[test-postgres\]"; then
+            DB_CHOICE="postgres"
+          fi
+        fi
+
+        # sqlserver2017 is SQL Server 2017 (CU20, the build some deployments still run): the same
+        # tests and properties as sqlserver, a different image, and the 2017 image's own sqlcmd.
+        SQLCMD_BIN=/opt/mssql-tools18/bin/sqlcmd
+        SQLCMD_TRUST=(-No)
+        if [ "$DB_CHOICE" = "sqlserver2017" ]; then
+          DB_CHOICE="sqlserver"
+          MSSQL_IMAGE='mcr.microsoft.com/mssql/server:2017-CU20-ubuntu'
+          SQLCMD_BIN=/opt/mssql-tools/bin/sqlcmd
+          SQLCMD_TRUST=()
+        fi
+
+        echo "TEST_DB=$DB_CHOICE" >> "$GITHUB_ENV"
+        echo "Selected test database: $DB_CHOICE"
+
+        wait_until_ready() {
+          # $1 = name, $2 = seconds between tries, remaining args = readiness command
+          local name="$1" pause="$2"; shift 2
+          for i in $(seq 1 30); do
+            if "$@" >/dev/null 2>&1; then
+              echo "$name is ready"
+              return 0
+            fi
+            echo "Waiting for $name ($i/30)..."
+            sleep "$pause"
+          done
+          echo "::error::$name did not become ready"
+          docker logs "$name" 2>&1 | tail -50 || true
+          return 1
+        }
+
+        if [ "$DB_CHOICE" = "sqlserver" ]; then
+          docker run -d --name sqlserver -p 1433:1433 \
+            -e ACCEPT_EULA=Y -e MSSQL_PID=Developer -e "MSSQL_SA_PASSWORD=$SA_PASSWORD" \
+            "$MSSQL_IMAGE"
+          SQLCMD=(docker exec sqlserver "$SQLCMD_BIN" -S localhost -U sa -P "$SA_PASSWORD" "${SQLCMD_TRUST[@]}" -b)
+          wait_until_ready sqlserver 5 "${SQLCMD[@]}" -Q 'SELECT 1'
+          "${SQLCMD[@]}" -Q "CREATE DATABASE obp_test_only"
+          echo "TEST_DB_DRIVER=com.microsoft.sqlserver.jdbc.SQLServerDriver" >> "$GITHUB_ENV"
+          echo "TEST_DB_URL=jdbc:sqlserver://localhost:1433;databaseName=obp_test_only;user=sa;password=$SA_PASSWORD;encrypt=false" >> "$GITHUB_ENV"
+          # Shards on a server database run longer than on H2, and shard 6 (v7 + UK Open Banking) has taken
+          # from 30 to over 50 minutes on SQL Server depending on the runner, so it gets 75 minutes.
+          echo "TEST_TIMEOUT_SECONDS=4500" >> "$GITHUB_ENV"
+        elif [ "$DB_CHOICE" = "postgres" ]; then
+          docker run -d --name postgres -p 5432:5432 \
+            -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=password -e POSTGRES_DB=obp_test \
+            "$POSTGRES_IMAGE"
+          wait_until_ready postgres 2 docker exec postgres pg_isready -U postgres
+          echo "TEST_DB_DRIVER=org.postgresql.Driver" >> "$GITHUB_ENV"
+          echo "TEST_DB_URL=jdbc:postgresql://localhost:5432/obp_test?user=postgres&password=password" >> "$GITHUB_ENV"
+          echo "TEST_TIMEOUT_SECONDS=4500" >> "$GITHUB_ENV"
+        else
+          echo "Running with the default H2 in-memory database; no container is started."
+          echo "TEST_TIMEOUT_SECONDS=1500" >> "$GITHUB_ENV"
+        fi

.github/workflows/build_container.yml

@@ -2,7 +2,20 @@ name: Build and publish container develop
 
 # read-write repo token
 # access to secrets
-on: [push]
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      database:
+        description: 'Database engine for tests (default: h2)'
+        required: false
+        default: 'h2'
+        type: choice
+        options:
+          - h2
+          - sqlserver
+          - sqlserver2017
+          - postgres
 
 env:
   DOCKER_HUB_ORGANIZATION: ${{ vars.DOCKER_HUB_ORGANIZATION }}
@@ -127,7 +140,7 @@ jobs:
   test:
     needs: compile
     runs-on: ubuntu-latest
-    timeout-minutes: 35
+    timeout-minutes: ${{ (inputs.database == 'sqlserver' || inputs.database == 'sqlserver2017' || inputs.database == 'postgres' || contains(github.event.pull_request.labels.*.name, 'test-sqlserver') || contains(github.event.pull_request.labels.*.name, 'test-postgres') || contains(github.event.head_commit.message, '[test-sqlserver]') || contains(github.event.head_commit.message, '[test-postgres]')) && 90 || 35 }}
     strategy:
       fail-fast: false
       matrix:
@@ -253,6 +266,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up the test database
+        uses: ./.github/actions/setup-test-database
+        with:
+          database: ${{ inputs.database }}
+          pr-labels: ${{ toJson(github.event.pull_request.labels.*.name) }}
+          commit-message: ${{ github.event.head_commit.message }}
+
       - name: Set up JDK 25
         uses: actions/setup-java@v4
         with:
@@ -301,6 +321,15 @@ jobs:
           cp obp-api/src/main/resources/props/sample.props.template \
              obp-api/src/main/resources/props/production.default.props
           echo connector=star > obp-api/src/main/resources/props/test.default.props
+
+          if [ -n "$TEST_DB_URL" ]; then
+            echo "Configuring $TEST_DB connection properties..."
+            echo "db.driver=$TEST_DB_DRIVER" >> obp-api/src/main/resources/props/test.default.props
+            echo "db.url=$TEST_DB_URL" >> obp-api/src/main/resources/props/test.default.props
+          else
+            echo "Using default H2 database."
+          fi
+
           echo starConnector_supported_types=mapped,internal >> obp-api/src/main/resources/props/test.default.props
           echo hostname=http://localhost:8016 >> obp-api/src/main/resources/props/test.default.props
           echo tests.port=8016 >> obp-api/src/main/resources/props/test.default.props
@@ -433,14 +462,14 @@ jobs:
           # -pl obp-commons,obp-api: obp-commons' own 5 util suites run on whichever
           # shard's filter matches com.openbankproject.* (the catch-all shard); on every
           # other shard the filter matches nothing in obp-commons → 0 tests there.
-          # timeout 1500: hard-kill after 25 min to prevent Pekko non-daemon threads
+          # timeout $TEST_TIMEOUT_SECONDS (25 min on H2, 75 on SQL Server and PostgreSQL): hard-kill to prevent Pekko non-daemon threads
           # (ConsentScheduler etc.) from keeping the JVM alive after tests complete.
           # Exit code 124 (timeout) is treated as success — tests are done, JVM just hung.
           # set +e: GitHub Actions uses -eo pipefail by default; without it, a 124 exit from
           # timeout would abort the step before the rc check below can run.
           set +e
           MAVEN_OPTS="-Xmx3G -Xss2m -XX:MaxMetaspaceSize=1G" \
-            timeout 1500 mvn process-resources scalatest:test -pl obp-commons,obp-api -DfailIfNoTests=false \
+            timeout "$TEST_TIMEOUT_SECONDS" mvn process-resources scalatest:test -pl obp-commons,obp-api -DfailIfNoTests=false \
               -DwildcardSuites="$FILTER" $EXTRA_MVN_ARGS \
             > maven-build-shard${{ matrix.shard }}.log 2>&1
           rc=$?

.github/workflows/build_pull_request.yml

@@ -2,8 +2,21 @@ name: Build on Pull Request
 
 on:
   pull_request:
+    types: [opened, synchronize, reopened, labeled]
     branches:
       - "**"
+  workflow_dispatch:
+    inputs:
+      database:
+        description: 'Database engine for tests (default: h2)'
+        required: false
+        default: 'h2'
+        type: choice
+        options:
+          - h2
+          - sqlserver
+          - sqlserver2017
+          - postgres
 
 env:
   DOCKER_HUB_ORGANIZATION: ${{ vars.DOCKER_HUB_ORGANIZATION }}
@@ -121,7 +134,7 @@ jobs:
   test:
     needs: compile
     runs-on: ubuntu-latest
-    timeout-minutes: 35
+    timeout-minutes: ${{ (inputs.database == 'sqlserver' || inputs.database == 'sqlserver2017' || inputs.database == 'postgres' || contains(github.event.pull_request.labels.*.name, 'test-sqlserver') || contains(github.event.pull_request.labels.*.name, 'test-postgres') || contains(github.event.head_commit.message, '[test-sqlserver]') || contains(github.event.head_commit.message, '[test-postgres]')) && 90 || 35 }}
     strategy:
       fail-fast: false
       matrix:
@@ -247,6 +260,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up the test database
+        uses: ./.github/actions/setup-test-database
+        with:
+          database: ${{ inputs.database }}
+          pr-labels: ${{ toJson(github.event.pull_request.labels.*.name) }}
+          commit-message: ${{ github.event.head_commit.message }}
+
       - name: Set up JDK 25
         uses: actions/setup-java@v4
         with:
@@ -295,6 +315,15 @@ jobs:
           cp obp-api/src/main/resources/props/sample.props.template \
              obp-api/src/main/resources/props/production.default.props
           echo connector=star > obp-api/src/main/resources/props/test.default.props
+
+          if [ -n "$TEST_DB_URL" ]; then
+            echo "Configuring $TEST_DB connection properties..."
+            echo "db.driver=$TEST_DB_DRIVER" >> obp-api/src/main/resources/props/test.default.props
+            echo "db.url=$TEST_DB_URL" >> obp-api/src/main/resources/props/test.default.props
+          else
+            echo "Using default H2 database."
+          fi
+
           echo starConnector_supported_types=mapped,internal >> obp-api/src/main/resources/props/test.default.props
           echo hostname=http://localhost:8016 >> obp-api/src/main/resources/props/test.default.props
           echo tests.port=8016 >> obp-api/src/main/resources/props/test.default.props
@@ -437,14 +466,14 @@ jobs:
           # -pl obp-commons,obp-api: obp-commons' own 5 util suites run on whichever
           # shard's filter matches com.openbankproject.* (the catch-all shard); on every
           # other shard the filter matches nothing in obp-commons → 0 tests there.
-          # timeout 1500: hard-kill after 25 min to prevent Pekko non-daemon threads
+          # timeout $TEST_TIMEOUT_SECONDS (25 min on H2, 75 on SQL Server and PostgreSQL): hard-kill to prevent Pekko non-daemon threads
           # (ConsentScheduler etc.) from keeping the JVM alive after tests complete.
           # Exit code 124 (timeout) is treated as success — tests are done, JVM just hung.
           # set +e: GitHub Actions uses -eo pipefail by default; without it, a 124 exit from
           # timeout would abort the step before the rc check below can run.
           set +e
           MAVEN_OPTS="-Xmx3G -Xss2m -XX:MaxMetaspaceSize=1G" \
-            timeout 1500 mvn process-resources scalatest:test -pl obp-commons,obp-api -DfailIfNoTests=false \
+            timeout "$TEST_TIMEOUT_SECONDS" mvn process-resources scalatest:test -pl obp-commons,obp-api -DfailIfNoTests=false \
               -DwildcardSuites="$FILTER" $EXTRA_MVN_ARGS \
             > maven-build-shard${{ matrix.shard }}.log 2>&1
           rc=$?

CLAUDE.md

@@ -15,7 +15,7 @@
 
 The goal is a full http4s migration — replace Lift Web across all version files and remove it entirely. **API versions are tech-agnostic**: a version bump means a changed/new API signature, never a framework change. Framework migration happens in-place inside the existing version file. v7.0.0 currently registers 117 ResourceDocs; most arrived there for historical reasons and stay as-is.
 
-**Request priority chain** (`Http4sApp.baseServices`): `corsHandler` (OPTIONS short-circuit) → `AppsPage` → `StatusPage` → `Http4sResourceDocs` → v510 → v600 → v500 → v700 → Berlin Group v2 → UK v2.0 → UK v3.1 → Berlin Group v1.3 (+Alias) → v400 → v310 → v300 → v220 → v210 → v200 → v140 → v130 → v121 → `dynamicEntityRoutes` → `dynamicEndpointRoutes` → DirectLogin → OpenIdConnect → AliveCheck → `notFoundCatchAll` (JSON 404). There is no Lift fallback — `Http4sLiftWebBridge` has been removed. Any unhandled `/obp/*` path returns a JSON 404 from `notFoundCatchAll`; it does not fall through to Lift.
+**Request priority chain** (`Http4sApp.baseServices`): `corsHandler` (OPTIONS short-circuit) → `AppsPage` → `StatusPage` → `Http4sResourceDocs` → v510 → v600 → v500 → v700 → Berlin Group v2 → UK v2.0 → UK v3.1 → Berlin Group v1.3 (+Alias) → v400 → v310 → v300 → v220 → v210 → v200 → v140 → v130 → v121 → `dynamicEntityRoutes` → `dynamicEndpointRoutes` → DirectLogin → OpenIdConnect → AliveCheck → `notFoundCatchAll` (JSON 404). There is no Lift fallback. Any unhandled `/obp/*` path returns a JSON 404 from `notFoundCatchAll`; it does not fall through to Lift.
 
 **Key files**: `Http4s700.scala` (v7.0.0 endpoints), `Http4s200.scala` (v2.0.0 endpoints — 37 own + path-rewriting bridge to Http4s140), `Http4s140.scala` (v1.4.0 endpoints — 11 own + path-rewriting bridge to Http4s130), `Http4s130.scala` (v1.3.0 endpoints — 3 own + path-rewriting bridge to Http4s121), `Http4s121.scala` (v1.2.1 endpoints — all 323 API1_2_1Test scenarios), `Http4sSupport.scala` (EndpointHelpers + recordMetric), `ResourceDocMiddleware.scala` (auth, entity resolution, transaction wrapper), `IdempotencyMiddleware.scala` (Redis-backed idempotency, opt-in via `Idempotency-Key` header, nested inside ResourceDocMiddleware), `RequestScopeConnection.scala` (DB transaction propagation to Futures).
 
@@ -261,7 +261,7 @@ POST /obp/v4.0.0/banks
   → Http4s220              (HAS POST /banks → executes v2.2.0 createBank ✗)
 ```
 
-Before `Http4sLiftWebBridge` was removed, an un-migrated v4 override fell all the way through to the Lift bridge, which honoured the `collectResourceDocs` URL+verb dedup that keeps the highest-version handler for each route — so Lift's v4 createBank ran and the test passed. **That safety net is gone**: the chain now terminates in `notFoundCatchAll`, so a v4 path not matched by `Http4s400`'s own routes cascades down the http4s version bridges to an older handler (or 404s) — it never reaches a Lift v4 handler. Cure: before flipping a new version's `wrappedRoutesVxxxServices` into `Http4sApp.baseServices`, audit the version's overrides (Lift's `excludeEndpoints` is *not* the right list — it only names *removed* endpoints, not overrides) and migrate them too.
+Under the old Lift fallback, an un-migrated v4 override fell all the way through to Lift, which honoured the `collectResourceDocs` URL+verb dedup that keeps the highest-version handler for each route — so Lift's v4 createBank ran and the test passed. **That safety net is gone**: the chain now terminates in `notFoundCatchAll`, so a v4 path not matched by `Http4s400`'s own routes cascades down the http4s version bridges to an older handler (or 404s) — it never reaches a Lift v4 handler. Cure: before flipping a new version's `wrappedRoutesVxxxServices` into `Http4sApp.baseServices`, audit the version's overrides (Lift's `excludeEndpoints` is *not* the right list — it only names *removed* endpoints, not overrides) and migrate them too.
 
 How to find overrides for a version: grep `lazy val (\w+)` in the target `APIMethods*.scala`, then check whether the same URL + verb also appears in any older `APIMethods*.scala`. The intersection is the override set. Migrate that set as part of the same PR that introduces the bridge; otherwise reviewers will see test failures whose proximate cause (a downstream version's handler running) doesn't match the file the migration touches.
 

ideas/lift_mapper_in_the_future.md

@@ -15,7 +15,7 @@ This document is the menu of responses to that staleness, cheapest to heaviest:
 
 ## Dependency-surface coupling: `lift-webkit` is hostage to Mapper (verified 2026-06-05)
 
-This is the clearest concrete symptom of the staleness problem. The Lift Web teardown (PR #2828) removed every `net.liftweb.http` import from OBP source (128 files → 0) and deleted `Http4sLiftWebBridge`. That shrank the *reachable* attack surface, but it did **not** shrink the dependency/CVE-scan surface: `lift-webkit_2.12-3.5.0.jar` (the jar that *contains* `net.liftweb.http`) is still on the classpath, pulled in transitively:
+This is the clearest concrete symptom of the staleness problem. The Lift Web teardown (PR #2828) removed every `net.liftweb.http` import from OBP source (128 files → 0) and deleted the Lift request fallback. That shrank the *reachable* attack surface, but it did **not** shrink the dependency/CVE-scan surface: `lift-webkit_2.12-3.5.0.jar` (the jar that *contains* `net.liftweb.http`) is still on the classpath, pulled in transitively:
 
 ```
 obp-api → lift-mapper → lift-db   → lift-webkit   (3.5.0)

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -666,10 +666,20 @@ class Boot extends MdcLoggable {
 
   def schemifyAll() = {
     Schemifier.schemify(true, Schemifier.infoF _, ToSchemify.models: _*)
+    if (code.api.util.DBUtil.isSqlServer) {
+      upgradeSqlServerDateTimeColumns()
+    }
     // Create default system-level "general" chat room (is_open_room = true)
     code.chat.ChatRoomTrait.chatRoomProvider.vend.getOrCreateDefaultRoom()
   }
 
+  private def upgradeSqlServerDateTimeColumns(): Unit = {
+    val result = code.api.util.SqlServerDateTimeUpgrade.upgradeAll()
+    if (result.failed.nonEmpty) {
+      logger.warn(s"upgradeSqlServerDateTimeColumns: ${result.failed.size} DATETIME columns could not be altered to DATETIME2 and keep the 1753 date floor: ${result.failed.mkString("; ")}")
+    }
+  }
+
   /**
    *  there will be a default bank and two default accounts in obp mapped mode.
    *  These bank and accounts will be used for the payments.

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -2294,7 +2294,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
    * this, so that SYS is accepted in all of them and not only in the versions written after it.
    */
   def isBankIdWhereRolesCanBeHeld(bankId: String, callContext: Option[CallContext]): Boolean =
-    bankId.isEmpty || bankId == DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID || BankX(BankId(bankId), callContext).map(_._1).isDefined
+    bankId.isEmpty || bankId == DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID || BankX(BankId(bankId), callContext).map(_._1).exists(_.bankId.value == bankId)
 
   def hasEntitlement(bankId: String, userId: String, apiRole: ApiRole): Boolean = apiRole match {
     case RoleCombination(roles) => roles.forall(hasEntitlement(bankId, userId, _))

obp-api/src/main/scala/code/api/util/DBUtil.scala

@@ -37,7 +37,12 @@ import java.sql.{ResultSet, Types}
 object DBUtil {
   def dbUrl: String = APIUtil.getPropsValue("db.url") openOr Constant.h2DatabaseDefaultUrlValue
 
-  def isSqlServer: Boolean = dbUrl.contains("sqlserver")
+  /**
+   * Whether the configured database is SQL Server. It cannot change while the instance runs, and it is asked
+   * on every paginated or locking query, so it is worked out once.
+   */
+  lazy val isSqlServer: Boolean =
+    dbUrl.contains("sqlserver") || APIUtil.getPropsValue("db.driver").exists(_.contains("sqlserver"))
 
   /** The endpoint timeout in whole seconds, rounded up, which is the unit JDBC takes. */
   private[code] def queryTimeoutSeconds: Int =