diff --git a/examples/conformance/README.md b/examples/conformance/README.md index 23ac7cbe..83fe762d 100644 --- a/examples/conformance/README.md +++ b/examples/conformance/README.md @@ -7,8 +7,12 @@ It also proves that optimistic versions remain isolated when different entity types use the same numeric ID and their mutation ledgers are merged. ```bash -make run +examples/verify-runtime-examples.sh ``` +Run the command from the repository root. It builds against current reactor +sources and verifies this example together with the School bootstrap example, +using an isolated temporary SQLite database for each run. + The generated Runtime Module is installed at application startup but remains a passive manifest. Schema reconciliation is invoked separately and explicitly. diff --git a/examples/school-management/README.md b/examples/school-management/README.md index ef56460f..e7773bfe 100644 --- a/examples/school-management/README.md +++ b/examples/school-management/README.md @@ -19,3 +19,8 @@ related entities as `NotLoaded`. Before publication, install the repository's local runtime and then run the generated workspace. The portable SQL runtime test separately changes a constant and verifies optimistic, single-version reconciliation. + +From the repository root, run `examples/verify-runtime-examples.sh`. The gate +builds both retained examples against the current reactor sources, assigns each +run an isolated temporary SQLite database, waits for its acceptance marker, and +exits non-zero if either application fails or times out. diff --git a/examples/verify-runtime-examples.sh b/examples/verify-runtime-examples.sh new file mode 100755 index 00000000..9a28d0d9 --- /dev/null +++ b/examples/verify-runtime-examples.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +run_dir="$(mktemp -d)" +active_pid="" + +cleanup() { + if [[ -n "$active_pid" ]] && kill -0 "$active_pid" 2>/dev/null; then + kill "$active_pid" 2>/dev/null || true + wait "$active_pid" 2>/dev/null || true + fi + rm -rf "$run_dir" +} +trap cleanup EXIT + +mvn -q -f "$repo_dir/pom.xml" \ + -Pruntime-examples \ + -pl examples/conformance,examples/school-management \ + -am package -DskipTests + +run_example() { + local name="$1" + local jar="$2" + local marker="$3" + local log="$run_dir/$name.log" + local database="$run_dir/$name.db" + + java -jar "$jar" \ + "--spring.datasource.url=jdbc:sqlite:$database" \ + --server.port=0 >"$log" 2>&1 & + active_pid=$! + + for _ in $(seq 1 240); do + if grep -Fq "$marker" "$log"; then + kill "$active_pid" 2>/dev/null || true + wait "$active_pid" 2>/dev/null || true + active_pid="" + printf 'PASS %s\n' "$name" + return 0 + fi + if ! kill -0 "$active_pid" 2>/dev/null; then + wait "$active_pid" || true + active_pid="" + printf 'FAIL %s exited before its acceptance marker\n' "$name" >&2 + sed -n '1,240p' "$log" >&2 + return 1 + fi + sleep 0.25 + done + + printf 'FAIL %s did not emit its acceptance marker within 60 seconds\n' "$name" >&2 + sed -n '1,240p' "$log" >&2 + return 1 +} + +run_example \ + "conformance" \ + "$repo_dir/examples/conformance/target/deploy/runtime-example-conformance-service-0.0.1-SNAPSHOT.jar" \ + "PASS Java minimum runtime conformance: 8/8" + +run_example \ + "school-management" \ + "$repo_dir/examples/school-management/target/deploy/school-management-service-0.0.1-SNAPSHOT.jar" \ + "PASS Java School bootstrap, portable Query, and native SQLite Facet parity" + +printf 'PASS Java runtime examples: 2/2\n' diff --git a/pom.xml b/pom.xml index 768c23a8..559e5a92 100644 --- a/pom.xml +++ b/pom.xml @@ -359,6 +359,16 @@ + + + runtime-examples + + examples/school-management/lib + examples/school-management + + + + TEAQL diff --git a/teaql-duckdb/README.md b/teaql-duckdb/README.md index faa7d67d..8ddbc45e 100644 --- a/teaql-duckdb/README.md +++ b/teaql-duckdb/README.md @@ -2,5 +2,22 @@ This module provides DuckDB support for TeaQL. +The executable compatibility gate defaults to DuckDB JDBC `1.5.5.1`: + +```bash +mvn -pl teaql-duckdb test +``` + +An older driver can be checked with the same tests, for example: + +```bash +mvn -pl teaql-duckdb clean test -Dduckdb.version=1.0.0 +``` + +Compatibility claims refer to the Java JDBC driver exercised by this gate, +not to a separately installed DuckDB CLI. + ## Test Report -A complete testing phase was successfully run with our core Vending Machine logic on DuckDB `1.0.0`. Read the full [TEST_REPORT.md](./TEST_REPORT.md) for details. +The historical Vending Machine report used DuckDB `1.0.0`. The retained Maven +gate verifies current JDBC compatibility; read [TEST_REPORT.md](./TEST_REPORT.md) +for the earlier test scope. diff --git a/teaql-duckdb/pom.xml b/teaql-duckdb/pom.xml index 7aba3eba..96bef910 100644 --- a/teaql-duckdb/pom.xml +++ b/teaql-duckdb/pom.xml @@ -10,6 +10,10 @@ teaql-duckdb teaql-duckdb + + + 1.5.5.1 + io.teaql @@ -19,5 +23,44 @@ io.teaql teaql-utils + + org.duckdb + duckdb_jdbc + ${duckdb.version} + test + + + io.teaql + teaql-provider-jdbc + ${project.version} + test + + + io.teaql + teaql-runtime + ${project.version} + test + + + junit + junit + test + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + --add-reads io.teaql.duck=io.teaql.runtime + --add-reads io.teaql.duck=java.sql + --add-opens io.teaql.duck/io.teaql.duckdb=io.teaql.utils + --add-opens io.teaql.core/io.teaql.core=io.teaql.utils + + + + + diff --git a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java index 5decc0f1..573f5f55 100644 --- a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java +++ b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java @@ -15,6 +15,7 @@ public class DuckDataServiceExecutor extends SqlDataServiceExecutor { public DuckDataServiceExecutor(String name, SqlExecutionAdapter executionAdapter) { super(name, executionAdapter); + this.debugDatabaseKind = "duckdb"; } @Override @@ -50,7 +51,11 @@ public void executeInTransaction(Runnable action) { @Override public List> getTableColumns(String tableName) { - String sql = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?"; + String sql = "SELECT column_name, data_type, data_type AS type_name, " + + "character_maximum_length AS column_size, " + + "numeric_precision, numeric_scale AS decimal_digits, " + + "CASE WHEN is_nullable = 'YES' THEN 1 ELSE 0 END AS nullable " + + "FROM information_schema.columns WHERE table_name = ?"; return getExecutionAdapter().queryForList(sql, new Object[] {tableName}); } }; diff --git a/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java b/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java new file mode 100644 index 00000000..00fae153 --- /dev/null +++ b/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java @@ -0,0 +1,306 @@ +package io.teaql.duckdb; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import io.teaql.core.BaseEntity; +import io.teaql.core.BaseRequest; +import io.teaql.core.EntityStatus; +import io.teaql.core.InternalIdGenerationService; +import io.teaql.core.SmartList; +import io.teaql.core.UserContext; +import io.teaql.core.criteria.Operator; +import io.teaql.core.duck.DuckDataServiceExecutor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.io.PrintWriter; +import java.math.BigDecimal; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.sql.Statement; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DuckDbIntegrationTest { + + private static UserContext context; + private static JdbcSqlExecutor sql; + private static InspectableDuckDataServiceExecutor executor; + private static Path databaseDirectory; + private static Path databaseFile; + private static String url; + + public static class Task extends BaseEntity { + private String title; + private String status; + private BigDecimal amount; + + public String getTitle() { + return title; + } + + public Task updateTitle(String value) { + handleUpdate("title", title, value); + title = value; + return this; + } + + public Task updateStatus(String value) { + handleUpdate("status", status, value); + status = value; + return this; + } + + @Override + public String typeName() { + return "Task"; + } + + @Override + public void __internalSet(String property, Object value) { + switch (property) { + case "title" -> title = (String) value; + case "status" -> status = (String) value; + case "amount" -> amount = (BigDecimal) value; + default -> super.__internalSet(property, value); + } + } + + @Override + public Object __internalGet(String property) { + return switch (property) { + case "title" -> title; + case "status" -> status; + case "amount" -> amount; + default -> super.__internalGet(property); + }; + } + } + + public static class TaskRequest extends BaseRequest { + public TaskRequest() { + super(Task.class); + } + + @Override + public String getTypeName() { + return "Task"; + } + + public TaskRequest withStatusIs(String value) { + appendSearchCriteria(createBasicSearchCriteria("status", Operator.EQUAL, value)); + return this; + } + + public TaskRequest comment(String value) { + internalComment(value); + return this; + } + } + + private static final class DriverManagerDataSource implements DataSource { + private final String connectionUrl; + + private DriverManagerDataSource(String connectionUrl) { + this.connectionUrl = connectionUrl; + } + + @Override + public Connection getConnection() throws SQLException { + return DriverManager.getConnection(connectionUrl); + } + + @Override + public Connection getConnection(String username, String password) throws SQLException { + return getConnection(); + } + + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter out) {} + @Override public void setLoginTimeout(int seconds) {} + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class iface) { return null; } + @Override public boolean isWrapperFor(Class iface) { return false; } + } + + private static final class InspectableDuckDataServiceExecutor + extends DuckDataServiceExecutor { + private InspectableDuckDataServiceExecutor( + String name, JdbcSqlExecutor executionAdapter) { + super(name, executionAdapter); + } + + private String debugDatabaseKind() { + return debugDatabaseKind; + } + } + + @BeforeClass + public static void setUp() throws Exception { + databaseDirectory = Files.createTempDirectory("teaql-duckdb-"); + databaseFile = databaseDirectory.resolve("runtime-" + UUID.randomUUID() + ".db"); + url = "jdbc:duckdb:" + databaseFile; + + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + SQLEntityDescriptor descriptor = new SQLEntityDescriptor(); + descriptor.setType("Task"); + descriptor.setTargetType(Task.class); + descriptor.setEntitySupplier(Task::new); + descriptor.setDataService("duckdb"); + ((GenericSQLProperty) descriptor.addSimpleProperty("id", Long.class)) + .setColumnType("BIGINT"); + ((GenericSQLProperty) descriptor.addSimpleProperty("version", Long.class)) + .setColumnType("BIGINT"); + ((GenericSQLProperty) descriptor.addSimpleProperty("title", String.class)) + .setColumnType("VARCHAR(200)"); + ((GenericSQLProperty) descriptor.addSimpleProperty("status", String.class)) + .setColumnType("VARCHAR(50)"); + ((GenericSQLProperty) descriptor.addSimpleProperty("amount", BigDecimal.class)) + .setColumnType("DECIMAL(19,7)"); + descriptor.with("table_name", "task_data"); + metadata.register(descriptor); + + sql = new JdbcSqlExecutor(new DriverManagerDataSource(url)); + executor = new InspectableDuckDataServiceExecutor("duckdb", sql); + AtomicLong ids = new AtomicLong(1); + InternalIdGenerationService idService = (ignored, entity) -> ids.getAndIncrement(); + TeaQLRuntime runtime = TeaQLRuntime.builder() + .metadata(metadata) + .dataService("duckdb", executor) + .idGenerationService(idService) + .build(); + context = new DefaultUserContext(runtime); + context.ensureSchema(); + } + + @AfterClass + public static void tearDown() throws Exception { + try (Connection connection = DriverManager.getConnection(url); + Statement statement = connection.createStatement()) { + statement.execute("CHECKPOINT"); + } finally { + Files.deleteIfExists(Path.of(databaseFile + ".wal")); + Files.deleteIfExists(databaseFile); + Files.deleteIfExists(databaseDirectory); + } + } + + @Test + public void exposesExpectedEngineVersion() throws Exception { + assertEquals("duckdb", executor.debugDatabaseKind()); + try (Connection connection = DriverManager.getConnection(url); + Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery("SELECT version()")) { + assertTrue(rows.next()); + assertTrue(rows.getString(1).matches("v?[0-9]+\\.[0-9]+.*")); + } + } + + @Test + public void executesTeaQlCrudAndTypedQuery() { + Task task = new Task().updateTitle("Verify DuckDB").updateStatus("OPEN"); + Task created = task.auditAs("create DuckDB compatibility fixture").save(context); + assertSame(task, created); + assertNotNull(created.getId()); + assertEquals(Long.valueOf(1), created.getVersion()); + + SmartList open = new TaskRequest() + .withStatusIs("OPEN") + .comment("what: load the DuckDB compatibility fixture") + .purpose("why: verify typed TeaQL query execution") + .executeForList(context); + assertEquals(1, open.size()); + assertEquals("Verify DuckDB", open.get(0).getTitle()); + + task.updateStatus("DONE") + .auditAs("complete DuckDB compatibility fixture") + .save(context); + assertEquals(Long.valueOf(2), task.getVersion()); + task.markForDeletion() + .auditAs("delete DuckDB compatibility fixture") + .save(context); + assertEquals(EntityStatus.PERSISTED_DELETED, task.get$status()); + assertTrue(new TaskRequest() + .withStatusIs("DONE") + .comment("what: confirm the deleted fixture is hidden") + .purpose("why: verify TeaQL deletion semantics on DuckDB") + .executeForList(context) + .isEmpty()); + } + + @Test + public void supportsFacetWindowAndTransactionSql() throws Exception { + try (Connection connection = DriverManager.getConnection(url); + Statement statement = connection.createStatement()) { + connection.setAutoCommit(false); + statement.executeUpdate( + "INSERT INTO task_data(id, version, title, status) VALUES " + + "(101, 1, 'A', 'GROUPED'), " + + "(102, 1, 'B', 'GROUPED'), " + + "(103, 1, 'C', 'OTHER')"); + connection.commit(); + + try (ResultSet facet = statement.executeQuery( + "SELECT status, count(*) FROM task_data GROUP BY status ORDER BY status")) { + assertTrue(facet.next()); + } + try (ResultSet window = statement.executeQuery( + "SELECT id, row_number() OVER (PARTITION BY status ORDER BY id) rn " + + "FROM task_data WHERE status = 'GROUPED' ORDER BY id")) { + assertTrue(window.next()); + assertEquals(1, window.getInt("rn")); + assertTrue(window.next()); + assertEquals(2, window.getInt("rn")); + assertFalse(window.next()); + } + } + } + + @Test + public void validatesSchemaValueDomainAndNullability() throws Exception { + try { + sql.execute("ALTER TABLE task_data ALTER COLUMN amount SET DATA TYPE DECIMAL(38,10)"); + context.ensureSchema(); + + sql.execute("ALTER TABLE task_data ALTER COLUMN amount SET DATA TYPE DECIMAL(18,2)"); + assertSchemaFailure("required precision=19, scale=7"); + sql.execute("ALTER TABLE task_data ALTER COLUMN amount SET DATA TYPE DECIMAL(38,10)"); + + sql.execute("ALTER TABLE task_data ALTER COLUMN status SET NOT NULL"); + assertSchemaFailure("expected nullable=true"); + } finally { + sql.execute("ALTER TABLE task_data ALTER COLUMN amount SET DATA TYPE DECIMAL(19,7)"); + sql.execute("ALTER TABLE task_data ALTER COLUMN status DROP NOT NULL"); + } + context.ensureSchema(); + } + + private static void assertSchemaFailure(String expectedMessage) { + IllegalStateException error = org.junit.Assert.assertThrows( + IllegalStateException.class, + () -> context.ensureSchema()); + assertTrue(error.getMessage(), error.getMessage().contains(expectedMessage)); + assertTrue(error.getMessage(), error.getMessage().contains("Task")); + assertTrue(error.getMessage(), error.getMessage().contains("task_data")); + } +} diff --git a/teaql-mysql/src/main/java/io/teaql/core/mysql/MysqlDataServiceExecutor.java b/teaql-mysql/src/main/java/io/teaql/core/mysql/MysqlDataServiceExecutor.java index 850d135e..f679cd12 100644 --- a/teaql-mysql/src/main/java/io/teaql/core/mysql/MysqlDataServiceExecutor.java +++ b/teaql-mysql/src/main/java/io/teaql/core/mysql/MysqlDataServiceExecutor.java @@ -57,7 +57,12 @@ public void executeInTransaction(Runnable action) { @Override public List> getTableColumns(String tableName) { - String sql = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ? AND table_schema = DATABASE()"; + String sql = "SELECT column_name, data_type, data_type AS type_name, " + + "character_maximum_length AS column_size, " + + "numeric_precision, numeric_scale AS decimal_digits, " + + "CASE WHEN is_nullable = 'YES' THEN 1 ELSE 0 END AS nullable " + + "FROM information_schema.columns " + + "WHERE table_name = ? AND table_schema = DATABASE()"; return getExecutionAdapter().queryForList(sql, new Object[] {tableName}); } diff --git a/teaql-postgres/src/main/java/io/teaql/core/postgres/PostgresDataServiceExecutor.java b/teaql-postgres/src/main/java/io/teaql/core/postgres/PostgresDataServiceExecutor.java index 730e2a21..01923c8c 100644 --- a/teaql-postgres/src/main/java/io/teaql/core/postgres/PostgresDataServiceExecutor.java +++ b/teaql-postgres/src/main/java/io/teaql/core/postgres/PostgresDataServiceExecutor.java @@ -50,7 +50,12 @@ public void executeInTransaction(Runnable action) { @Override public List> getTableColumns(String tableName) { - String sql = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ? AND table_schema = 'public'"; + String sql = "SELECT column_name, data_type, data_type AS type_name, " + + "character_maximum_length AS column_size, " + + "numeric_precision, numeric_scale AS decimal_digits, " + + "CASE WHEN is_nullable = 'YES' THEN 1 ELSE 0 END AS nullable " + + "FROM information_schema.columns " + + "WHERE table_name = ? AND table_schema = 'public'"; return getExecutionAdapter().queryForList(sql, new Object[] {tableName.toLowerCase()}); } diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/GenericSQLProperty.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/GenericSQLProperty.java index 499cfbb7..b76ceaa8 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/GenericSQLProperty.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/GenericSQLProperty.java @@ -40,6 +40,7 @@ public void setName(String name) { public List columns() { SQLColumn sqlColumn = new SQLColumn(tableName, columnName); sqlColumn.setType(columnType); + sqlColumn.setRequired(getBoolean("required", false)); return ListUtil.of(sqlColumn); } diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/dialect/SqlDialect.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/dialect/SqlDialect.java index 024ff473..cb47ba32 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/dialect/SqlDialect.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/dialect/SqlDialect.java @@ -2,6 +2,8 @@ import io.teaql.core.SearchRequest; import java.util.List; +import java.util.Locale; +import java.util.Map; public interface SqlDialect { /** @@ -51,4 +53,40 @@ default String prepareParameterizedLimit( default String mapColumnType(String type) { return type; } + + /** + * Compare the storage families reported by the provider with the type selected for the model. + * Providers may override this when their storage type system has wider compatibility rules. + */ + default boolean isCompatibleColumnType(String expected, String actual) { + String expectedFamily = normalizedTypeFamily(expected); + String actualFamily = normalizedTypeFamily(actual); + return expectedFamily.isEmpty() + || actualFamily.isEmpty() + || expectedFamily.equals(actualFamily); + } + + private static String normalizedTypeFamily(String type) { + if (type == null) return ""; + String family = type.trim().toUpperCase(Locale.ROOT) + .replaceFirst("\\s*\\(.*", "") + .replaceAll("\\s+", " "); + return Map.ofEntries( + Map.entry("CHARACTER VARYING", "VARCHAR"), + Map.entry("VARCHAR2", "VARCHAR"), + Map.entry("NVARCHAR2", "NVARCHAR"), + Map.entry("CHARACTER", "CHAR"), + Map.entry("INT", "INTEGER"), + Map.entry("INT4", "INTEGER"), + Map.entry("INT8", "BIGINT"), + Map.entry("DEC", "DECIMAL"), + Map.entry("NUMERIC", "DECIMAL"), + Map.entry("DOUBLE PRECISION", "DOUBLE"), + Map.entry("FLOAT8", "DOUBLE"), + Map.entry("FLOAT4", "REAL"), + Map.entry("BOOL", "BOOLEAN"), + Map.entry("TIMESTAMP WITHOUT TIME ZONE", "TIMESTAMP"), + Map.entry("TIMESTAMP WITH TIME ZONE", "TIMESTAMPTZ")) + .getOrDefault(family, family); + } } diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index cac8c1cb..635f05ee 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -1537,10 +1537,170 @@ protected void ensure(UserContext context, List> tableInfo, String dbColumnName = column.getColumnName().toLowerCase(); if (!fields.containsKey(dbColumnName)) { addColumn(context, column); + } else { + ensureCompatibleColumn(table, column, fields.get(dbColumnName)); } } } + private void ensureCompatibleColumn( + String table, SQLColumn column, Map databaseColumn) { + Object actualTypeValue = metadataValue(databaseColumn, "type_name", "data_type"); + if (actualTypeValue == null || actualTypeValue instanceof Number) return; + + String expectedType = dialect.mapColumnType(column.getType()); + String actualType = String.valueOf(actualTypeValue); + if (!dialect.isCompatibleColumnType(expectedType, actualType)) { + throw incompatibleColumn(table, column, + "expected type " + expectedType + " but database reports " + actualType); + } + + ensureCompatibleNullability(table, column, databaseColumn); + ensureCompatibleValueDomain(table, column, expectedType, actualType, databaseColumn); + } + + private void ensureCompatibleNullability( + String table, SQLColumn column, Map databaseColumn) { + Object nullable = metadataValue(databaseColumn, "nullable", "is_nullable"); + if (nullable == null) return; + boolean expectedNullable = !column.isIdColumn() && !column.isRequired(); + boolean actualNullable = nullableMetadata(nullable); + if (expectedNullable != actualNullable) { + throw incompatibleColumn(table, column, + "expected nullable=" + expectedNullable + + " but database reports nullable=" + actualNullable); + } + } + + private void ensureCompatibleValueDomain( + String table, + SQLColumn column, + String expectedType, + String actualType, + Map databaseColumn) { + List expectedArguments = declaredTypeArguments(expectedType); + String expectedFamily = declaredTypeFamily(expectedType); + String actualFamily = declaredTypeFamily(actualType); + + if (isTextFamily(expectedFamily) && !expectedArguments.isEmpty() + && !isUnboundedTextFamily(actualFamily)) { + Integer actualLength = integerMetadata(metadataValue( + databaseColumn, "column_size", "character_maximum_length")); + if (actualLength == null) { + List actualArguments = declaredTypeArguments(actualType); + actualLength = actualArguments.isEmpty() ? null : actualArguments.get(0); + } + int requiredLength = expectedArguments.get(0); + if (actualLength != null && actualLength < requiredLength) { + throw incompatibleColumn(table, column, + "required max length=" + requiredLength + + " but database reports max length=" + actualLength); + } + } + + if (isDecimalFamily(expectedFamily) && expectedArguments.size() >= 2) { + int expectedPrecision = expectedArguments.get(0); + int expectedScale = expectedArguments.get(1); + Integer actualPrecision = integerMetadata(metadataValue( + databaseColumn, "numeric_precision", "column_size")); + Integer actualScale = integerMetadata(metadataValue( + databaseColumn, "numeric_scale", "decimal_digits")); + if (actualPrecision == null || actualScale == null) { + List actualArguments = declaredTypeArguments(actualType); + if (actualArguments.size() >= 2) { + actualPrecision = actualArguments.get(0); + actualScale = actualArguments.get(1); + } + } + if (actualPrecision == null && actualScale == null) return; + boolean covers = actualPrecision != null + && actualScale != null + && actualScale >= expectedScale + && actualPrecision - actualScale >= expectedPrecision - expectedScale; + if (!covers) { + throw incompatibleColumn(table, column, + "required precision=" + expectedPrecision + ", scale=" + expectedScale + + " but database reports precision=" + actualPrecision + + ", scale=" + actualScale); + } + } + } + + private IllegalStateException incompatibleColumn( + String table, SQLColumn column, String detail) { + return new IllegalStateException( + "Ensure Schema incompatible existing column for entity '" + + entityDescriptor.getType() + "' on table '" + table + + "', column '" + column.getColumnName() + "', dialect '" + + dialect.getClass().getSimpleName() + "': " + detail); + } + + private static Object metadataValue(Map metadata, String... names) { + for (String name : names) { + for (Map.Entry entry : metadata.entrySet()) { + if (name.equalsIgnoreCase(entry.getKey())) return entry.getValue(); + } + } + return null; + } + + private static boolean nullableMetadata(Object value) { + if (value instanceof Boolean bool) return bool; + if (value instanceof Number number) return number.intValue() != 0; + String text = String.valueOf(value).trim(); + return "YES".equalsIgnoreCase(text) + || "Y".equalsIgnoreCase(text) + || "TRUE".equalsIgnoreCase(text) + || "1".equals(text); + } + + private static Integer integerMetadata(Object value) { + if (value == null) return null; + if (value instanceof Number number) return number.intValue(); + try { + return Integer.valueOf(String.valueOf(value)); + } catch (NumberFormatException ignored) { + return null; + } + } + + private static String declaredTypeFamily(String type) { + int argumentStart = type.indexOf('('); + return (argumentStart < 0 ? type : type.substring(0, argumentStart)) + .trim().toUpperCase(java.util.Locale.ROOT); + } + + private static List declaredTypeArguments(String type) { + int start = type.indexOf('('); + int end = type.indexOf(')', start + 1); + if (start < 0 || end < 0) return List.of(); + return Arrays.stream(type.substring(start + 1, end).split(",")) + .map(String::trim) + .map(value -> { + try { + return Integer.valueOf(value); + } catch (NumberFormatException ignored) { + return null; + } + }) + .filter(ObjectUtil::isNotNull) + .collect(Collectors.toList()); + } + + private static boolean isTextFamily(String family) { + return family.contains("CHAR") || family.contains("TEXT") || family.contains("CLOB"); + } + + private static boolean isUnboundedTextFamily(String family) { + return family.contains("TEXT") || family.contains("CLOB"); + } + + private static boolean isDecimalFamily(String family) { + return "DECIMAL".equals(family) + || "NUMERIC".equals(family) + || "NUMBER".equals(family); + } + private String metadataColumnName(Map column, String table) { for (Map.Entry entry : column.entrySet()) { if ("column_name".equalsIgnoreCase(entry.getKey()) diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/IdSpaceIdGeneratorTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/IdSpaceIdGeneratorTest.java index 77e0894a..cda450fa 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/IdSpaceIdGeneratorTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/IdSpaceIdGeneratorTest.java @@ -68,12 +68,44 @@ public void failsClosedWhenPersistentIdSpaceTableCannotBeEnsured() { assertSame(providerFailure, failure.getCause()); } + @Test + public void reportsBoundedAllocationContentionWithTypeAndAttemptCount() { + RecordingDatabase database = new RecordingDatabase(); + database.levels.put("Order", 7L); + database.alwaysCompareAndSetConflict = true; + + IllegalStateException failure = assertThrows( + IllegalStateException.class, + () -> new IdSpaceIdGenerator(database).nextId("Order")); + + assertTrue(failure.getMessage().contains("Order")); + assertTrue(failure.getMessage().contains("100 optimistic-lock attempts")); + assertEquals(100, database.updates.size()); + } + + @Test + public void reportsBoundedFloorContentionWithoutMovingFloorBackwards() { + RecordingDatabase database = new RecordingDatabase(); + database.levels.put("SchoolType", 1L); + database.alwaysCompareAndSetConflict = true; + + IllegalStateException failure = assertThrows( + IllegalStateException.class, + () -> new IdSpaceIdGenerator(database).ensureFloor("SchoolType", 1002L)); + + assertTrue(failure.getMessage().contains("SchoolType")); + assertTrue(failure.getMessage().contains("100 optimistic-lock attempts")); + assertEquals(Long.valueOf(1L), database.levels.get("SchoolType")); + assertEquals(100, database.updates.size()); + } + private static final class RecordingDatabase implements TeaQLDatabase { private final Map levels = new HashMap<>(); private final List queries = new ArrayList<>(); private final List updates = new ArrayList<>(); private boolean failFirstInsertAsRace; private boolean failFirstCompareAndSetAsRace; + private boolean alwaysCompareAndSetConflict; private RuntimeException executeFailure; @Override @@ -100,6 +132,9 @@ public int executeUpdate(String sql, Object[] args) { updates.add(sql); String typeName = String.valueOf(args[1]); long expected = ((Number) args[2]).longValue(); + if (alwaysCompareAndSetConflict) { + return 0; + } if (failFirstCompareAndSetAsRace) { failFirstCompareAndSetAsRace = false; levels.put(typeName, expected + 1); // Simulates a competing successful CAS. diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLSchemaCompatibilityTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLSchemaCompatibilityTest.java new file mode 100644 index 00000000..2b60217a --- /dev/null +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLSchemaCompatibilityTest.java @@ -0,0 +1,201 @@ +package io.teaql.core.sql.portable; + +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import io.teaql.core.UserContext; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.meta.SimplePropertyType; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.Test; + +public class PortableSQLSchemaCompatibilityTest { + + @Test + public void acceptsAliasesAndStorageThatCoversTheModelValueDomain() { + MetadataDatabase database = compatibleDatabase(); + + repository(database).ensurePhysicalSchema(context()); + + assertTrue("compatible metadata must not cause DDL", database.executedSql.isEmpty()); + } + + @Test + public void rejectsAnIncompatibleExistingColumnType() { + MetadataDatabase database = compatibleDatabase(); + database.replaceColumn(column("name", "INTEGER", 1)); + + IllegalStateException failure = assertThrows( + IllegalStateException.class, + () -> repository(database).ensurePhysicalSchema(context())); + + assertTrue(failure.getMessage().contains("SchemaCompatibilityFixture")); + assertTrue(failure.getMessage().contains("schema_compat_data")); + assertTrue(failure.getMessage().contains("name")); + assertTrue(failure.getMessage().contains("VARCHAR(100)")); + assertTrue(failure.getMessage().contains("INTEGER")); + } + + @Test + public void rejectsTextAndDecimalStorageThatIsTooNarrow() { + MetadataDatabase textDatabase = compatibleDatabase(); + textDatabase.replaceColumn(column("name", "VARCHAR", 1, "column_size", 32)); + + IllegalStateException textFailure = assertThrows( + IllegalStateException.class, + () -> repository(textDatabase).ensurePhysicalSchema(context())); + assertTrue(textFailure.getMessage().contains("required max length=100")); + assertTrue(textFailure.getMessage().contains("max length=32")); + + MetadataDatabase decimalDatabase = compatibleDatabase(); + decimalDatabase.replaceColumn(column( + "amount", "NUMERIC", 1, + "numeric_precision", 18, + "decimal_digits", 2)); + + IllegalStateException decimalFailure = assertThrows( + IllegalStateException.class, + () -> repository(decimalDatabase).ensurePhysicalSchema(context())); + assertTrue(decimalFailure.getMessage().contains("required precision=19, scale=7")); + assertTrue(decimalFailure.getMessage().contains("precision=18, scale=2")); + } + + @Test + public void rejectsNullabilityThatDoesNotMatchTheModel() { + MetadataDatabase database = compatibleDatabase(); + database.replaceColumn(column("required_code", "VARCHAR", 1, "column_size", 100)); + + IllegalStateException failure = assertThrows( + IllegalStateException.class, + () -> repository(database).ensurePhysicalSchema(context())); + + assertTrue(failure.getMessage().contains("required_code")); + assertTrue(failure.getMessage().contains("expected nullable=false")); + assertTrue(failure.getMessage().contains("nullable=true")); + } + + @Test + public void preservesCompatibilityWithProvidersThatOnlyReportColumnNames() { + MetadataDatabase database = compatibleDatabase(); + database.columns.replaceAll((name, ignored) -> Map.of("COLUMN_NAME", name)); + + repository(database).ensurePhysicalSchema(context()); + + assertTrue("missing optional shape metadata must not cause DDL", database.executedSql.isEmpty()); + } + + private static PortableSQLRepository repository( + MetadataDatabase database) { + EntityDescriptor descriptor = new EntityDescriptor(); + descriptor.setType("SchemaCompatibilityFixture"); + descriptor.setTargetType(PortableSQLDatabaseTest.Task.class); + descriptor.setEntitySupplier(PortableSQLDatabaseTest.Task::new); + descriptor.setParent(new EntityDescriptor()); + + GenericSQLProperty id = property(descriptor, "id", "BIGINT", Long.class); + GenericSQLProperty version = property(descriptor, "version", "BIGINT", Long.class); + GenericSQLProperty name = property(descriptor, "name", "VARCHAR(100)", String.class); + GenericSQLProperty amount = property( + descriptor, "amount", "NUMERIC(19,7)", BigDecimal.class); + GenericSQLProperty requiredCode = property( + descriptor, "required_code", "VARCHAR(100)", String.class); + requiredCode.with("required", "true"); + descriptor.setProperties(List.of(id, version, name, amount, requiredCode)); + return new PortableSQLRepository<>(descriptor, database, null); + } + + private static GenericSQLProperty property( + EntityDescriptor owner, String name, String sqlType, Class javaType) { + GenericSQLProperty property = new GenericSQLProperty("schema_compat_data", name, sqlType); + property.setName(name); + property.setOwner(owner); + property.setType(new SimplePropertyType(javaType)); + return property; + } + + private static UserContext context() { + return new DefaultUserContext(TeaQLRuntime.builder() + .metadata(new SimpleEntityMetaFactory()) + .build()); + } + + private static MetadataDatabase compatibleDatabase() { + return new MetadataDatabase(List.of( + column("id", "INT8", 0), + column("version", "BIGINT", 1), + column("name", "CHARACTER VARYING", 1, "column_size", 255), + column( + "amount", "DECIMAL", 1, + "numeric_precision", 38, + "decimal_digits", 10), + column("required_code", "VARCHAR", 0, "column_size", 100))); + } + + private static Map column( + String name, String type, int nullable, Object... shape) { + Map column = new LinkedHashMap<>(); + column.put("COLUMN_NAME", name); + column.put("TYPE_NAME", type); + column.put("NULLABLE", nullable); + for (int index = 0; index < shape.length; index += 2) { + column.put(String.valueOf(shape[index]), shape[index + 1]); + } + return column; + } + + private static final class MetadataDatabase implements TeaQLDatabase { + private final Map> columns = new LinkedHashMap<>(); + private final List executedSql = new ArrayList<>(); + + private MetadataDatabase(List> columns) { + for (Map column : columns) { + this.columns.put(String.valueOf(column.get("COLUMN_NAME")), column); + } + } + + private void replaceColumn(Map column) { + columns.put(String.valueOf(column.get("COLUMN_NAME")), column); + } + + @Override + public List> query(String sql, Object[] args) { + return List.of(); + } + + @Override + public int executeUpdate(String sql, Object[] args) { + return 0; + } + + @Override + public int[] batchUpdate(String sql, List batchArgs) { + return new int[0]; + } + + @Override + public void execute(String sql) { + executedSql.add(sql); + } + + @Override + public void executeInTransaction(Runnable action) { + action.run(); + } + + @Override + public List> getTableColumns(String tableName) { + if ("teaql_id_space".equalsIgnoreCase(tableName)) { + return List.of(Map.of("column_name", "type_name")); + } + return new ArrayList<>(columns.values()); + } + } +} diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/SqliteIdSpaceConcurrencyTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/SqliteIdSpaceConcurrencyTest.java new file mode 100644 index 00000000..7bb7e121 --- /dev/null +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/SqliteIdSpaceConcurrencyTest.java @@ -0,0 +1,153 @@ +package io.teaql.core.sql.portable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +/** Exercises the portable optimistic allocator through independent SQLite connections. */ +public class SqliteIdSpaceConcurrencyTest { + + @Test + public void independentInstancesAllocateUniqueIdsAndRespectBootstrapFloor() throws Exception { + Path databaseFile = Files.createTempFile("teaql-id-space-", ".db"); + try { + String url = "jdbc:sqlite:" + databaseFile; + SqliteDatabase database = new SqliteDatabase(url); + IdSpaceIdGenerator initializer = new IdSpaceIdGenerator(database); + initializer.ensureIdSpaceTable(); + initializer.ensureFloor("SchoolType", 1002L); + assertEquals( + 1003L, + new IdSpaceIdGenerator(new SqliteDatabase(url)).nextId("SchoolType")); + + int workers = 4; + int allocationsPerWorker = 20; + CountDownLatch start = new CountDownLatch(1); + Set ids = ConcurrentHashMap.newKeySet(); + var pool = Executors.newFixedThreadPool(workers); + try { + List> futures = new ArrayList<>(); + for (int worker = 0; worker < workers; worker++) { + futures.add(pool.submit(new Callable<>() { + @Override + public Void call() throws Exception { + IdSpaceIdGenerator generator = new IdSpaceIdGenerator( + new SqliteDatabase(url)); + start.await(); + for (int index = 0; index < allocationsPerWorker; index++) { + assertTrue("duplicate ID allocated", ids.add(generator.nextId("Order"))); + } + return null; + } + })); + } + start.countDown(); + for (Future future : futures) { + future.get(30, TimeUnit.SECONDS); + } + } finally { + pool.shutdownNow(); + } + + assertEquals(workers * allocationsPerWorker, ids.size()); + for (long id = 1; id <= workers * allocationsPerWorker; id++) { + assertTrue("missing ID " + id, ids.contains(id)); + } + assertEquals( + 81L, + new IdSpaceIdGenerator(new SqliteDatabase(url)).nextId("Order")); + } finally { + Files.deleteIfExists(databaseFile); + } + } + + private static final class SqliteDatabase implements TeaQLDatabase { + private final String url; + + private SqliteDatabase(String url) { + this.url = url; + } + + private Connection connect() throws Exception { + Connection connection = DriverManager.getConnection(url); + try (Statement statement = connection.createStatement()) { + statement.execute("PRAGMA busy_timeout = 5000"); + } + return connection; + } + + @Override + public List> query(String sql, Object[] args) { + try (Connection connection = connect(); + PreparedStatement statement = connection.prepareStatement(sql)) { + bind(statement, args); + try (ResultSet result = statement.executeQuery()) { + return result.next() + ? List.of(Map.of("current_level", result.getLong("current_level"))) + : List.of(); + } + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + + @Override + public int executeUpdate(String sql, Object[] args) { + try (Connection connection = connect(); + PreparedStatement statement = connection.prepareStatement(sql)) { + bind(statement, args); + return statement.executeUpdate(); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + + @Override + public void execute(String sql) { + try (Connection connection = connect(); Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + + private static void bind(PreparedStatement statement, Object[] args) throws Exception { + for (int index = 0; index < args.length; index++) { + statement.setObject(index + 1, args[index]); + } + } + + @Override + public int[] batchUpdate(String sql, List batchArgs) { + throw new UnsupportedOperationException("not used by ID allocation"); + } + + @Override + public void executeInTransaction(Runnable action) { + throw new UnsupportedOperationException("not used by ID allocation"); + } + + @Override + public List> getTableColumns(String tableName) { + throw new UnsupportedOperationException("not used by ID allocation"); + } + } +} diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteDataServiceExecutor.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteDataServiceExecutor.java index ac4a8d9d..6b1b3b2b 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteDataServiceExecutor.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteDataServiceExecutor.java @@ -72,6 +72,14 @@ public List> getTableColumns(String tableName) { "PRAGMA table_info(" + tableName + ")", new Object[0]); for (Map col : columns) { col.put("column_name", col.get("name")); + col.put("data_type", col.get("type")); + Object notNull = col.get("notnull"); + Object primaryKey = col.get("pk"); + boolean required = (notNull instanceof Number notNullValue + && notNullValue.intValue() != 0) + || (primaryKey instanceof Number primaryKeyValue + && primaryKeyValue.intValue() != 0); + col.put("nullable", required ? 0 : 1); } return columns; }