diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d4a292f1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +test-vectors/*.tsv whitespace=-blank-at-eol diff --git a/examples/security-foundations/pom.xml b/examples/security-foundations/pom.xml index da1cfae2..84045a01 100644 --- a/examples/security-foundations/pom.xml +++ b/examples/security-foundations/pom.xml @@ -12,6 +12,11 @@ teaql-security-foundations-example TeaQL security foundations focused example + + io.teaql + teaql-sqlite + ${project.version} + io.teaql teaql-runtime diff --git a/examples/security-foundations/src/test/java/io/teaql/examples/security/SecurityFoundationsExampleTest.java b/examples/security-foundations/src/test/java/io/teaql/examples/security/SecurityFoundationsExampleTest.java index 49e5628a..a4c465d2 100644 --- a/examples/security-foundations/src/test/java/io/teaql/examples/security/SecurityFoundationsExampleTest.java +++ b/examples/security-foundations/src/test/java/io/teaql/examples/security/SecurityFoundationsExampleTest.java @@ -20,6 +20,10 @@ import org.junit.Test; public class SecurityFoundationsExampleTest { + @Test + public void sqliteMaskingLifecycle() throws Exception { + SqlMaskingLifecycleExample.verify(); + } private static final class OrderItem extends BaseEntity { OrderItem(long id, long version) { __internalSet("id", id); diff --git a/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlMaskingLifecycleExample.java b/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlMaskingLifecycleExample.java new file mode 100644 index 00000000..ef7c3004 --- /dev/null +++ b/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlMaskingLifecycleExample.java @@ -0,0 +1,149 @@ +package io.teaql.examples.security; + +import io.teaql.core.*; +import io.teaql.core.criteria.Operator; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.*; +import java.nio.file.Files; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; +import org.sqlite.SQLiteDataSource; +import static org.junit.Assert.*; + +/** Runtime-owned fixture, not a hand-edited generated library. */ +final class SqlMaskingLifecycleExample { + static void verify() throws Exception { + var metadata = new SimpleEntityMetaFactory(); + var descriptor = new SQLEntityDescriptor(); + descriptor.setType("MaskFixture"); descriptor.setTargetType(MaskFixture.class); + descriptor.setEntitySupplier(MaskFixture::new); descriptor.setDataService("sqlite"); + descriptor.addSimpleProperty("id", Long.class); + descriptor.addSimpleProperty("version", Long.class); + descriptor.addSimpleProperty("name", String.class); + descriptor.setAuditMaskFields(List.of("name")); + descriptor.with("table_name", "mask_fixture_data"); metadata.register(descriptor); + var ds = new SQLiteDataSource(); + ds.setUrl("jdbc:sqlite:" + Files.createTempFile("teaql-java-mask-example-", ".db")); + var provider = new SqliteDataServiceExecutor("sqlite", new JdbcSqlExecutor(ds), ds); + var logs = new ArrayList(); + var sequence = new AtomicLong(10); + var context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", provider).idGenerationService((caller, entity) -> sequence.getAndIncrement()) + .logSink((caller, entry) -> logs.add(entry)).build()); + + assertThrows(RuntimeException.class, () -> query().executeForStream(context)); + assertEquals(1, logs.size()); + assertEquals("failure", logs.get(0).getExecutionOutcome()); + assertNull(logs.get(0).getResultCount()); + context.ensureSchema(); + for (int i = 0; i < 3; i++) { + logs.clear(); + new MaskFixture().updateName("Riverside").auditAs("seed masking example Riverside").save(context); + var readback = logs.stream().filter(e -> e.getOperation() == DataServiceOperation.QUERY + && String.valueOf(e.getParameterizedQuery()).contains("SELECT *")).findFirst().orElseThrow(); + assertEquals("seed masking example [REDACTED]", readback.getAuditReason()); + assertFalse(readback.getDebugQuery().contains("Riverside")); + assertEquals(Integer.valueOf(1), readback.getResultCount()); + } + MaskFixture changed; + try (var rows = query().executeForStream(context)) { changed = rows.findFirst().orElseThrow(); } + for (String name : List.of("Lakeside", "Riverside")) { + logs.clear(); + changed.updateName(name).auditAs("update masked fixture " + name).save(context); + var readback = logs.stream().filter(e -> e.getOperation() == DataServiceOperation.QUERY + && String.valueOf(e.getParameterizedQuery()).contains("SELECT *")).findFirst().orElseThrow(); + assertEquals("update masked fixture [REDACTED]", readback.getAuditReason()); + assertNull(readback.getIntentRedactions()); + } + // Test-only fault injection, not application persistence SQL. + try (var conn = ds.getConnection(); var statement = conn.createStatement()) { + statement.execute("CREATE TRIGGER disappear_mask_fixture AFTER INSERT ON mask_fixture_data WHEN NEW.id >= 13 BEGIN DELETE FROM mask_fixture_data WHERE id = NEW.id; END"); + } + try { + logs.clear(); + assertThrows(RuntimeException.class, () -> new MaskFixture().updateName("READBACK-CANARY") + .auditAs("verify missing READBACK-CANARY snapshot").save(context)); + var readback = logs.stream().filter(e -> e.getOperation() == DataServiceOperation.QUERY + && String.valueOf(e.getParameterizedQuery()).contains("SELECT *")).findFirst().orElseThrow(); + assertEquals("success", readback.getExecutionOutcome()); + assertEquals(Integer.valueOf(0), readback.getResultCount()); + assertEquals("verify missing [REDACTED] snapshot", readback.getAuditReason()); + assertTrue(logs.stream().anyMatch(e -> e.getOperation() == DataServiceOperation.MUTATION + && "success".equals(e.getExecutionOutcome()) && Long.valueOf(1).equals(e.getAffectedRows()))); + } finally { + try (var conn = ds.getConnection(); var statement = conn.createStatement()) { + statement.execute("DROP TRIGGER disappear_mask_fixture"); + } + } + System.out.println("PASS Java readback inherits sensitive intent for create/update and empty SQLite snapshot"); + logs.clear(); + var duplicateContext = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", provider).idGenerationService((caller, entity) -> 10L) + .logSink((caller, entry) -> logs.add(entry)).build()); + assertThrows(RuntimeException.class, () -> new MaskFixture().updateName("MASKED-DUPLICATE-CANARY") + .auditAs("verify failed duplicate insert").save(duplicateContext)); + assertTrue(logs.stream().anyMatch(entry -> entry.getOperation() == DataServiceOperation.MUTATION)); + assertTrue(logs.stream().noneMatch(entry -> String.valueOf(entry.getDebugQuery()).contains("MASKED-DUPLICATE-CANARY"))); + logs.clear(); + try (var rows = query().executeForStream(context)) { + assertEquals(1, rows.limit(1).count()); + } + assertEquals(1, logs.size()); + assertEquals("cancelled", logs.get(0).getExecutionOutcome()); + assertEquals(Integer.valueOf(1), logs.get(0).getResultCount()); + assertEquals("verify cursor diagnostics", logs.get(0).getPurpose()); + assertTrue(logs.get(0).getDebugQuery().contains("Ri*****de")); + assertFalse(logs.get(0).getDebugQuery().contains("Riverside")); + logs.clear(); + try (var rows = query().executeForStream(context)) { + assertEquals(List.of("Riverside", "Riverside", "Riverside"), rows.map(MaskFixture::getName).toList()); + } + assertEquals(1, logs.size()); + assertEquals("success", logs.get(0).getExecutionOutcome()); + assertEquals(Integer.valueOf(3), logs.get(0).getResultCount()); + // Re-emitting a stored safe record must never upgrade it to debug plaintext. + var retained = logs.get(0); + var replay = LogPrivacy.sql(retained, true); + assertEquals(retained.getDebugQuery(), replay.getDebugQuery()); + var output = Files.createTempFile("teaql-java-masked-replay-", ".log"); + try (var out = new java.io.PrintStream(Files.newOutputStream(output))) { + new DefaultTextRuntimeLogSink(out).writeExecutionLog(context, replay); + } + String text = Files.readString(output); + assertTrue(text.contains("Ri*****de")); + assertFalse(text.contains("Riverside")); + assertFalse(text.contains("DEBUG PLAINTEXT")); + assertTrue(text.contains("verify cursor diagnostics")); + System.out.println("PASS Java retained SQL projection: no plaintext upgrade, masked file output, intent preserved"); + System.out.println("PASS Java SQLite masking lifecycle: failed open/duplicate insert, early close, completion, intent and original data"); + } + + private static ExecutableRequest query() { + var request = new FixtureRequest(); + request.setSize(3); + request.appendSearchCriteria(request.createBasicSearchCriteria("name", Operator.EQUAL, "Riverside")); + return request.comment("read masked fixture").purpose("verify cursor diagnostics"); + } + + public static final class FixtureRequest extends BaseRequest { + FixtureRequest() { super(MaskFixture.class); } + @Override public String getTypeName() { return "MaskFixture"; } + public FixtureRequest comment(String text) { internalComment(text); return this; } + } + + public static final class MaskFixture extends BaseEntity { + private String name; + @Override public String typeName() { return "MaskFixture"; } + public String getName() { return name; } + public MaskFixture updateName(String value) { handleUpdate("name", name, value); name = value; return this; } + @Override public void __internalSet(String property, Object value) { + if (property.equals("name")) name = (String) value; else super.__internalSet(property, value); + } + @Override public Object __internalGet(String property) { + return property.equals("name") ? name : super.__internalGet(property); + } + } +} diff --git a/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlRelationMaskingTest.java b/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlRelationMaskingTest.java new file mode 100644 index 00000000..d240fdde --- /dev/null +++ b/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlRelationMaskingTest.java @@ -0,0 +1,194 @@ +package io.teaql.examples.security; + +import io.teaql.core.*; +import io.teaql.core.criteria.Operator; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.*; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.*; +import java.nio.file.Files; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.sqlite.SQLiteDataSource; +import static org.junit.Assert.*; + +/** Real SQLite graph loading; handwritten runtime fixture, not generated source. */ +@RunWith(Parameterized.class) +public class SqlRelationMaskingTest { + @Parameterized.Parameters(name = "{0}-failure={1}-transaction={2}-sensitiveSink={3}") + public static Collection cases() { + var cases = new ArrayList(); + for (String shape : List.of("probe", "window", "forward", "nested", "idset", "aggregate", "facet")) + for (boolean fail : List.of(false, true)) + for (boolean tx : List.of(false, true)) + for (boolean sensitive : List.of(false, true)) cases.add(new Object[]{shape, fail, tx, sensitive}); + return cases; + } + private final String shape; + private final boolean fail; + private final boolean transaction; + private final boolean sensitiveSink; + public SqlRelationMaskingTest(String shape, boolean fail, boolean transaction, boolean sensitiveSink) { + this.shape = shape; this.fail = fail; this.transaction = transaction; + this.sensitiveSink = sensitiveSink; + } + public static abstract class Row extends BaseEntity { + private final Map fields = new HashMap<>(); + @Override public void __internalSet(String field, Object value) { + if (field.equals("id") || field.equals("version")) super.__internalSet(field, value); + else { fields.put(field, value); markPropertyLoaded(field); } + } + @Override public Object __internalGet(String field) { + return field.equals("id") || field.equals("version") ? super.__internalGet(field) : fields.get(field); + } + } + public static class Parent extends Row { @Override public String typeName() { return "MaskParent"; } } + public static class Child extends Row { @Override public String typeName() { return "MaskChild"; } } + public static class Leaf extends Row { @Override public String typeName() { return "MaskLeaf"; } } + public static class Request extends BaseRequest { + private final String type; + Request(Class cls, String type) { + super(cls); this.type = type; setSize(10); + for (String field : List.of("id", "version", "name", "password")) selectProperty(field); + if (type.equals("MaskChild")) selectProperty("parent"); + if (type.equals("MaskLeaf")) selectProperty("child"); + } + @Override public String getTypeName() { return type; } + Request equal(String field, Object value) { + appendSearchCriteria(createBasicSearchCriteria(field, Operator.EQUAL, value)); return this; + } + Request comment(String value) { internalComment(value); return this; } + } + private static SQLEntityDescriptor descriptor(String type, Class cls, + java.util.function.Supplier factory, String table) { + var d = new SQLEntityDescriptor(); d.setType(type); d.setTargetType(cls); + d.setEntitySupplier(factory); d.setDataService("sqlite"); d.with("table_name", table); + d.addSimpleProperty("id", Long.class); d.addSimpleProperty("version", Long.class); + d.addSimpleProperty("name", String.class); d.addSimpleProperty("password", String.class); + d.setAuditMaskFields(List.of("name")); return d; + } + @Test public void inheritedIntentIsSafeAndGraphAndSqlFactsRemainIntact() throws Exception { + var metadata = new SimpleEntityMetaFactory(); + var parent = descriptor("MaskParent", Parent.class, Parent::new, "mask_parent_data"); + var child = descriptor("MaskChild", Child.class, Child::new, "mask_child_data"); + metadata.register(parent); + var relation = (GenericSQLRelation) child.addObjectProperty(metadata, "parent", "MaskParent", + "children", Parent.class, GenericSQLRelation::new); + relation.setTableName("mask_child_data"); relation.setColumnName("parent"); relation.setColumnType("BIGINT"); + metadata.register(child); + var leaf = descriptor("MaskLeaf", Leaf.class, Leaf::new, "mask_leaf_data"); + var leafRelation = (GenericSQLRelation) leaf.addObjectProperty(metadata, "child", "MaskChild", + "leaves", Child.class, GenericSQLRelation::new); + leafRelation.setTableName("mask_leaf_data"); leafRelation.setColumnName("child"); leafRelation.setColumnType("BIGINT"); + metadata.register(leaf); + var ds = new SQLiteDataSource(); + ds.setUrl("jdbc:sqlite:" + Files.createTempFile("teaql-java-relation-mask-", ".db")); + var provider = new SqliteDataServiceExecutor("sqlite", new JdbcSqlExecutor(ds), ds); + var logs = new ArrayList(); + var ids = new AtomicLong(10); + var context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).dataService("sqlite", provider) + .idGenerationService((c,e) -> ids.getAndIncrement()).logSink(new RuntimeLogSink() { + @Override public void writeExecutionLog(UserContext caller, ExecutionMetadata entry) { logs.add(entry); } + @Override public boolean requiresSensitiveSqlData() { return sensitiveSink; } + }).build()); + context.ensureSchema(); + var p = new Parent(); p.updateProperty("name", "Riverside"); p.updateProperty("password", "RELATION-PASSWORD-CANARY"); + p.auditAs("seed relation fixture").save(context); + var c = new Child(); c.updateProperty("name", "Riverside"); c.updateProperty("password", "RELATION-PASSWORD-CANARY"); + c.updateProperty("parent", p); c.auditAs("seed relation child").save(context); + var l = new Leaf(); l.updateProperty("name", "leaf payload"); l.updateProperty("child", c); + l.auditAs("seed nested leaf").save(context); + boolean forward = shape.equals("forward") || shape.equals("facet"); + var root = forward ? new Request<>(Child.class, "MaskChild") : new Request<>(Parent.class, "MaskParent"); + root.equal("name", "Riverside").equal("password", "RELATION-PASSWORD-CANARY"); + var nested = forward ? new Request<>(Parent.class, "MaskParent") : new Request<>(Child.class, "MaskChild"); + nested.topNProbeParentThreshold(shape.equals("window") ? 0 : 32); + if (shape.equals("nested")) nested.enhanceRelation("leaves", new Request<>(Leaf.class, "MaskLeaf")); + if (shape.equals("facet")) root.addFacet("parentFacet", "parent", nested, true); + else root.enhanceRelation(forward ? "parent" : "children", nested); + if (shape.equals("aggregate")) { + root.enhanceRelations().clear(); + nested.count("count"); nested.setPartitionProperty("parent"); + root.getDynamicAggregateAttributes().add(new SimpleAggregation("childCount", nested, true)); + } + if (shape.equals("idset")) { + root.optimizePaginationWithIdSet(); + root.comment("warm retained IDs").purpose("prime current request cache").executeForList(context); + } + if (fail) { + // Fault injection only: seed data above uses the mutation API. + try (var conn = ds.getConnection(); var statement = conn.createStatement()) { + statement.execute("ALTER TABLE " + (forward ? "mask_parent_data" : "mask_child_data") + " RENAME TO missing_relation_data"); + } + } + logs.clear(); + Runnable execute = () -> { + var rows = root.comment("load Riverside RELATION-PASSWORD-CANARY graph") + .purpose("inspect Riverside RELATION-PASSWORD-CANARY relations").executeForList(context); + assertEquals(1, rows.size()); + assertEquals("Riverside", rows.get(0).getProperty("name")); + if (shape.equals("aggregate")) assertEquals(1, ((Number) rows.get(0).getProperty("childCount")).intValue()); + else if (shape.equals("facet")) { + SmartList parents = rows.getFacet("parentFacet"); + assertEquals(1, parents.size()); + assertEquals("Riverside", parents.get(0).getProperty("name")); + } + else if (forward) { + Entity loadedParent = rows.get(0).getProperty("parent"); + assertEquals("Riverside", loadedParent.getProperty("name")); + } + else { + SmartList children = rows.get(0).getProperty("children"); + assertEquals(1, children.size()); + if (shape.equals("nested")) { + SmartList leaves = children.get(0).getProperty("leaves"); + assertEquals(1, leaves.size()); + assertEquals("leaf payload", leaves.get(0).getProperty("name")); + } + } + }; + Runnable run = transaction ? () -> provider.executeInTransaction(context, () -> { execute.run(); return null; }) : execute; + if (fail) assertThrows(RuntimeException.class, run::run); else run.run(); + // A facet first logs its grouped query, then loads the related rows. The injected + // failure is in that final related-row query, so all three attempts are visible. + assertEquals(shape.equals("facet") || (shape.equals("nested") && !fail) ? 3 : 2, logs.size()); + boolean debug = sensitiveSink && LogPrivacy.plaintextEnabled(); + String name = debug ? "Riverside" : "[REDACTED]"; + for (var entry : logs) { + assertEquals("load " + name + " [REDACTED] graph", entry.getComment()); + assertEquals("inspect " + name + " [REDACTED] relations", entry.getPurpose()); + assertNull(entry.getIntentRedactions()); + assertFalse(entry.getDebugQuery().contains("RELATION-PASSWORD-CANARY")); + if (debug) assertTrue(entry.getDebugQuery().contains("EXPLICIT OPT-IN")); + } + assertEquals(fail ? "failure" : "success", logs.get(logs.size() - 1).getExecutionOutcome()); + if (fail) assertNull(logs.get(logs.size() - 1).getResultCount()); + var output = Files.createTempFile("teaql-java-relation-mask-", ".log"); + try (var out = new java.io.PrintStream(Files.newOutputStream(output))) { + var sink = new DefaultTextRuntimeLogSink(out); + for (var entry : logs) sink.writeExecutionLog(context, entry); + } + var text = Files.readString(output); + assertFalse(text.contains("Riverside")); assertFalse(text.contains("RELATION-PASSWORD-CANARY")); + assertTrue(text.contains("SELECT")); assertTrue(text.contains("inspect")); + if (debug) { + var debugFile = Files.createTempFile("teaql-java-debug-relation-", ".log"); + try (var out = new java.io.PrintStream(Files.newOutputStream(debugFile))) { + var sink = new SensitiveDiagnosticTextRuntimeLogSink(out); + for (var entry : logs) sink.writeExecutionLog(context, entry); + } + String debugText = Files.readString(debugFile); + assertTrue(debugText.contains("Riverside")); assertTrue(debugText.contains("EXPLICIT OPT-IN")); + assertFalse(debugText.contains("RELATION-PASSWORD-CANARY")); + } + logs.clear(); + // Reuse both context and repository; previous query provenance must not bleed into a new request. + var independent = forward ? new Request<>(Child.class, "MaskChild") : new Request<>(Parent.class, "MaskParent"); + independent.comment("independent Riverside").purpose("no inherited bindings").executeForList(context); + assertEquals("independent Riverside", logs.get(0).getComment()); + } +} diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index 2c8d4e63..06cca2f6 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -16,7 +16,7 @@ mvn -q -DskipTests install mvn -q -pl examples/business-id-runtime \ -Dtest=BusinessIdRuntimeExampleTest test mvn -q -pl examples/security-foundations \ - -Dtest=SecurityFoundationsExampleTest test + -Dtest=SecurityFoundationsExampleTest,SqlRelationMaskingTest test mvn -q -pl examples/round-trip-reference-runtime \ -Dtest=RoundTripReferenceRuntimeExampleTest test mvn -q -f examples/conformance/lib/pom.xml install -DskipTests diff --git a/teaql-core/src/main/java/io/teaql/core/ExecutionMetadata.java b/teaql-core/src/main/java/io/teaql/core/ExecutionMetadata.java index 367b781f..66fb49ad 100644 --- a/teaql-core/src/main/java/io/teaql/core/ExecutionMetadata.java +++ b/teaql-core/src/main/java/io/teaql/core/ExecutionMetadata.java @@ -20,6 +20,40 @@ public final class ExecutionMetadata { private String comment; private String purpose; private String auditReason; + private List parameterLogPolicies = List.of(); + private List parameterMasked = List.of(); + private boolean generatedSql; + private String logMode; + private String sqlOmissionReason; + private String executionOutcome; + @com.fasterxml.jackson.annotation.JsonIgnore + private transient SqlIntentRedactions intentRedactions; + + @FrameworkInternal("SQL diagnostic provenance; excluded from serialized metadata") + @com.fasterxml.jackson.annotation.JsonIgnore + public SqlIntentRedactions getIntentRedactions() { return intentRedactions; } + @FrameworkInternal("SQL diagnostic provenance only") + public void setIntentRedactions(SqlIntentRedactions value) { intentRedactions = value; } + + /** Statement/cursor outcome, not transaction commit or business acceptance. Null means unknown. */ + public String getExecutionOutcome() { return executionOutcome; } + public void setExecutionOutcome(String outcome) { executionOutcome = outcome; } + + public List getParameterLogPolicies() { return parameterLogPolicies; } + public void setParameterLogPolicies(List policies) { + parameterLogPolicies = policies == null ? List.of() : List.copyOf(policies); + } + public List getParameterMasked() { return parameterMasked; } + public void setParameterMasked(List masked) { + parameterMasked = masked == null ? List.of() : List.copyOf(masked); + } + /** Set only by a compiler that can attest that inline text contains no user values. */ + public boolean isGeneratedSql() { return generatedSql; } + public void setGeneratedSql(boolean generated) { generatedSql = generated; } + public String getLogMode() { return logMode; } + public void setLogMode(String mode) { logMode = mode; } + public String getSqlOmissionReason() { return sqlOmissionReason; } + public void setSqlOmissionReason(String reason) { sqlOmissionReason = reason; } public String getBackend() { return backend; } public void setBackend(String backend) { this.backend = backend; } @@ -46,7 +80,7 @@ public final class ExecutionMetadata { public String getParameterizedQuery() { return parameterizedQuery; } public void setParameterizedQuery(String parameterizedQuery) { this.parameterizedQuery = parameterizedQuery; } - /** Structured bind values for trusted runtime sinks. Application-safe projections should expose only the count. */ + /** Execution bindings before projection; only independently projected values may reach log sinks. */ public List getParameters() { return parameters; } public void setParameters(List parameters) { this.parameters = parameters == null diff --git a/teaql-core/src/main/java/io/teaql/core/SqlIntentRedactions.java b/teaql-core/src/main/java/io/teaql/core/SqlIntentRedactions.java new file mode 100644 index 00000000..3feeec4b --- /dev/null +++ b/teaql-core/src/main/java/io/teaql/core/SqlIntentRedactions.java @@ -0,0 +1,66 @@ +package io.teaql.core; + +import com.fasterxml.jackson.annotation.JsonIgnoreType; +import io.teaql.core.utils.SensitiveLogNames; +import java.util.*; + +/** Invocation-local provider/runtime provenance; never retained by a projected log. */ +@JsonIgnoreType +public final class SqlIntentRedactions { + private record Secret(String value, boolean forced) {} + private final List secrets = new ArrayList<>(); + + @FrameworkInternal("Independent nested-query provenance snapshot") + public SqlIntentRedactions copy() { + var result = new SqlIntentRedactions(); + result.secrets.addAll(secrets); + return result; + } + + @FrameworkInternal("SQL diagnostic provenance only; not an application policy") + public void capture(List policies, Object[] values) { + boolean invalid = policies.size() != values.length; + for (int i = 0; i < values.length; i++) { + var policy = invalid ? SqlParameterLogPolicy.UNKNOWN : policies.get(i); + boolean forced = policy == SqlParameterLogPolicy.UNKNOWN || policy == SqlParameterLogPolicy.CREDENTIAL + || hasCredentials(values[i]); + if (forced || policy == SqlParameterLogPolicy.MASKED) collect(values[i], forced); + } + } + + /** Keep a mutation's plain ID out of diagnostic prose without changing its SQL binding policy. */ + @FrameworkInternal("Invocation-local SQL intent provenance only") + public void captureTargetId(Object id) { + collect(id, true); + } + + private static boolean hasCredentials(Object value) { + if (value instanceof Map map) return map.entrySet().stream() + .anyMatch(e -> SensitiveLogNames.credential(String.valueOf(e.getKey())) || hasCredentials(e.getValue())); + if (value instanceof Iterable items) for (Object item : items) if (hasCredentials(item)) return true; + if (value != null && value.getClass().isArray()) + for (int i = 0; i < java.lang.reflect.Array.getLength(value); i++) + if (hasCredentials(java.lang.reflect.Array.get(value, i))) return true; + return false; + } + + private void collect(Object value, boolean forced) { + if (value == null) return; + if (value instanceof Map map) { map.values().forEach(v -> collect(v, forced)); return; } + if (value instanceof Iterable items) { items.forEach(v -> collect(v, forced)); return; } + if (value.getClass().isArray()) { + for (int i = 0; i < java.lang.reflect.Array.getLength(value); i++) + collect(java.lang.reflect.Array.get(value, i), forced); + return; + } + String text = String.valueOf(value); + if (!text.isEmpty()) secrets.add(new Secret(text, forced)); + } + + @FrameworkInternal("Runtime log projection only") + public void appendTo(List output, boolean allowPlaintext) { + for (var secret : secrets) if (secret.forced() || !allowPlaintext) output.add(secret.value()); + } + + @Override public String toString() { return "SqlIntentRedactions[count=" + secrets.size() + "]"; } +} diff --git a/teaql-core/src/main/java/io/teaql/core/SqlParameterLogPolicy.java b/teaql-core/src/main/java/io/teaql/core/SqlParameterLogPolicy.java new file mode 100644 index 00000000..3a6da031 --- /dev/null +++ b/teaql-core/src/main/java/io/teaql/core/SqlParameterLogPolicy.java @@ -0,0 +1,6 @@ +package io.teaql.core; + +/** Trusted compiler metadata, never a query caller's opt-out from field masking. */ +public enum SqlParameterLogPolicy { + UNKNOWN, PLAIN, MASKED, CREDENTIAL +} diff --git a/teaql-core/src/main/java/io/teaql/core/meta/EntityDescriptor.java b/teaql-core/src/main/java/io/teaql/core/meta/EntityDescriptor.java index db14ee45..2d4d1598 100644 --- a/teaql-core/src/main/java/io/teaql/core/meta/EntityDescriptor.java +++ b/teaql-core/src/main/java/io/teaql/core/meta/EntityDescriptor.java @@ -59,6 +59,8 @@ public class EntityDescriptor { private List auditMaskFields = new ArrayList<>(); + private boolean auditMaskFieldsDeclared; + private Integer auditValueMaxLength; public List getAuditMaskFields() { @@ -67,6 +69,11 @@ public List getAuditMaskFields() { public void setAuditMaskFields(List auditMaskFields) { this.auditMaskFields = auditMaskFields == null ? new ArrayList<>() : new ArrayList<>(auditMaskFields); + this.auditMaskFieldsDeclared = auditMaskFields != null; + } + + public boolean isAuditMaskFieldsDeclared() { + return auditMaskFieldsDeclared; } public EntityDescriptor auditMaskFields(List auditMaskFields) { diff --git a/teaql-core/src/test/java/io/teaql/core/SqlIntentRedactionsTest.java b/teaql-core/src/test/java/io/teaql/core/SqlIntentRedactionsTest.java new file mode 100644 index 00000000..b244c491 --- /dev/null +++ b/teaql-core/src/test/java/io/teaql/core/SqlIntentRedactionsTest.java @@ -0,0 +1,32 @@ +package io.teaql.core; + +import java.util.List; +import org.junit.Test; +import static org.junit.Assert.*; + +public class SqlIntentRedactionsTest { + @Test public void snapshotDoesNotAccumulateDescendantOrSiblingValues() { + var root = new SqlIntentRedactions(); + root.capture(List.of(SqlParameterLogPolicy.MASKED), new Object[]{"Riverside"}); + var child = root.copy(); + child.capture(List.of(SqlParameterLogPolicy.CREDENTIAL), new Object[]{"CHILD-PASSWORD"}); + var sibling = root.copy(); + var rootValues = new java.util.ArrayList(); + var childValues = new java.util.ArrayList(); + var siblingValues = new java.util.ArrayList(); + root.appendTo(rootValues, false); child.appendTo(childValues, true); sibling.appendTo(siblingValues, false); + assertEquals(List.of("Riverside"), rootValues); + assertEquals(List.of("CHILD-PASSWORD"), childValues); + assertEquals(rootValues, siblingValues); + } + @Test public void provenanceIsNotSerialized() throws Exception { + var source = new SqlIntentRedactions(); + source.capture(List.of(SqlParameterLogPolicy.MASKED), new Object[]{"Riverside"}); + var metadata = new ExecutionMetadata(); + metadata.setIntentRedactions(source); + String json = new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(metadata); + assertFalse(json.contains("intentRedactions")); + assertFalse(json.contains("Riverside")); + assertFalse(source.toString().contains("Riverside")); + } +} diff --git a/teaql-core/src/test/java/io/teaql/core/meta/EntityDescriptorTest.java b/teaql-core/src/test/java/io/teaql/core/meta/EntityDescriptorTest.java index 6ba9b1a6..c4ec39bd 100644 --- a/teaql-core/src/test/java/io/teaql/core/meta/EntityDescriptorTest.java +++ b/teaql-core/src/test/java/io/teaql/core/meta/EntityDescriptorTest.java @@ -9,6 +9,17 @@ public class EntityDescriptorTest { + @Test + public void auditMaskMetadataDistinguishesMissingFromExplicitEmpty() { + EntityDescriptor descriptor = new EntityDescriptor(); + assertFalse(descriptor.isAuditMaskFieldsDeclared()); + descriptor.setAuditMaskFields(List.of()); + assertTrue(descriptor.isAuditMaskFieldsDeclared()); + assertTrue(descriptor.getAuditMaskFields().isEmpty()); + descriptor.setAuditMaskFields(null); + assertFalse(descriptor.isAuditMaskFieldsDeclared()); + } + @Test public void testEntityDescriptorDefaultsAndLookups() { diff --git a/teaql-data-service-sql/pom.xml b/teaql-data-service-sql/pom.xml index be5e6073..2b237526 100644 --- a/teaql-data-service-sql/pom.xml +++ b/teaql-data-service-sql/pom.xml @@ -42,7 +42,7 @@ org.apache.maven.plugins maven-surefire-plugin - --add-reads io.teaql.dataservice.sql=io.teaql.runtime + --add-reads io.teaql.dataservice.sql=io.teaql.runtime --add-opens io.teaql.sql.portable/io.teaql.core.sql.portable=io.teaql.dataservice.sql diff --git a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java index 55cc41d5..ba3a0bec 100644 --- a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java +++ b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java @@ -53,7 +53,21 @@ public QueryResult query(UserContext context, QueryRequest request) { @Override public java.util.stream.Stream queryForStream(UserContext context, io.teaql.core.SearchRequest request) { - return getPortableService(context).queryForStream(context, request); + // Unlike list execution, this path does not enter TeaQLRuntime.executeForList's trace scope. + int pushed = 0; + try { + context.pushTrace(io.teaql.core.TraceKind.OPERATION, request.getTypeName(), "query"); pushed++; + context.pushTrace(io.teaql.core.TraceKind.REQUEST, request.getTypeName(), request.getTypeName()); pushed++; + if (request.comment() != null) { + context.pushTrace(io.teaql.core.TraceKind.COMMENT, request.getTypeName(), request.comment()); pushed++; + } + if (request.purpose() != null) { + context.pushTrace(io.teaql.core.TraceKind.PURPOSE, request.getTypeName(), request.purpose()); pushed++; + } + return getPortableService(context).queryForStream(context, request); + } finally { + for (int i = 0; i < pushed; i++) context.popTrace(); + } } @Override @@ -140,7 +154,25 @@ public java.util.List> query(String sql, Object[] } @Override public java.util.stream.Stream> queryForStream(io.teaql.core.UserContext context, String sql, Object[] args) { - return executionAdapter.queryForStream(sql, args); + return queryForStream(context, sql, args, io.teaql.core.sql.portable.SqlLogBindings.UNKNOWN); + } + @Override + public java.util.stream.Stream> queryForStream(io.teaql.core.UserContext context, + String sql, Object[] args, io.teaql.core.sql.portable.SqlLogBindings bindings) { + boolean logging = context.isQueryExecutionLoggingEnabled(); + long start = logging ? System.nanoTime() : 0L; + io.teaql.core.ExecutionMetadata meta = null; + if (logging) { + meta = statementMetadata(context, sql, args, bindings, io.teaql.core.DataServiceOperation.QUERY); + // Snapshot before request trace scopes are popped; lazy consumption may happen later. + var trace = context.getTraceChain(); + meta.setTraceChain(trace == null || trace.isEmpty() + ? java.util.List.of(new io.teaql.core.TraceNode(io.teaql.core.TraceKind.OPERATION, "stream", "query")) + : java.util.List.copyOf(trace)); + } + var stream = diagnosed(context, sql, args, bindings, io.teaql.core.DataServiceOperation.QUERY, + logging, start, () -> executionAdapter.queryForStream(sql, args)); + return SqlDiagnosticStream.wrap(context, stream, meta, start); } @Override public int executeUpdate(String sql, Object[] args) { @@ -165,9 +197,17 @@ public java.util.List> getTableColumns(String tabl @Override public java.util.List> query(io.teaql.core.UserContext context, String sql, Object[] args) { + return query(context, sql, args, io.teaql.core.sql.portable.SqlLogBindings.UNKNOWN); + } + + @Override + public java.util.List> query(io.teaql.core.UserContext context, String sql, Object[] args, + io.teaql.core.sql.portable.SqlLogBindings bindings) { boolean logging = context.isQueryExecutionLoggingEnabled(); long start = logging ? System.nanoTime() : 0L; - java.util.List> res = executionAdapter.queryForList(sql, args); + java.util.List> res = diagnosed(context, sql, args, bindings, + io.teaql.core.DataServiceOperation.QUERY, logging, start, + () -> executionAdapter.queryForList(sql, args)); if (!logging) return res; long elapsed = (System.nanoTime() - start) / 1000; io.teaql.core.ExecutionMetadata meta = new io.teaql.core.ExecutionMetadata(); @@ -176,11 +216,10 @@ public java.util.List> query(io.teaql.core.UserCon meta.setElapsedUs(elapsed); meta.setResultCount(res.size()); meta.setResultSummary("Fetched " + res.size() + " rows"); + meta.setExecutionOutcome("success"); meta.setParameterizedQuery(sql); - if (context.requiresSensitiveSqlLogData()) { - meta.setParameters(parameters(args)); - meta.setDebugQuery(debugSql(sql, args, debugDatabaseKind)); - } + meta.setParameters(parameters(args)); + bindings.applyTo(meta); context.recordExecutionMetadata(meta); return res; } @@ -189,9 +228,18 @@ public java.util.List> query(io.teaql.core.UserCon public java.util.List query( io.teaql.core.UserContext context, String sql, Object[] args, io.teaql.core.CompiledRowMapper rowMapper) { + return query(context, sql, args, rowMapper, io.teaql.core.sql.portable.SqlLogBindings.UNKNOWN); + } + + @Override + public java.util.List query( + io.teaql.core.UserContext context, String sql, Object[] args, + io.teaql.core.CompiledRowMapper rowMapper, io.teaql.core.sql.portable.SqlLogBindings bindings) { boolean logging = context.isQueryExecutionLoggingEnabled(); long start = logging ? System.nanoTime() : 0L; - java.util.List res = executionAdapter.query(sql, args, rowMapper); + java.util.List res = diagnosed(context, sql, args, bindings, + io.teaql.core.DataServiceOperation.QUERY, logging, start, + () -> executionAdapter.query(sql, args, rowMapper)); if (!logging) return res; long elapsed = (System.nanoTime() - start) / 1000; io.teaql.core.ExecutionMetadata meta = new io.teaql.core.ExecutionMetadata(); @@ -200,20 +248,27 @@ public java.util.List query( meta.setElapsedUs(elapsed); meta.setResultCount(res.size()); meta.setResultSummary("Fetched " + res.size() + " typed rows"); + meta.setExecutionOutcome("success"); meta.setParameterizedQuery(sql); - if (context.requiresSensitiveSqlLogData()) { - meta.setParameters(parameters(args)); - meta.setDebugQuery(debugSql(sql, args, debugDatabaseKind)); - } + meta.setParameters(parameters(args)); + bindings.applyTo(meta); context.recordExecutionMetadata(meta); return res; } @Override public int executeUpdate(io.teaql.core.UserContext context, String sql, Object[] args) { + return executeUpdate(context, sql, args, io.teaql.core.sql.portable.SqlLogBindings.UNKNOWN); + } + + @Override + public int executeUpdate(io.teaql.core.UserContext context, String sql, Object[] args, + io.teaql.core.sql.portable.SqlLogBindings bindings) { boolean logging = context.isMutationExecutionLoggingEnabled(); long start = logging ? System.nanoTime() : 0L; - int res = executionAdapter.update(sql, args); + int res = diagnosed(context, sql, args, bindings, + io.teaql.core.DataServiceOperation.MUTATION, logging, start, + () -> executionAdapter.update(sql, args)); if (!logging) return res; long elapsed = (System.nanoTime() - start) / 1000; io.teaql.core.ExecutionMetadata meta = new io.teaql.core.ExecutionMetadata(); @@ -222,42 +277,33 @@ public int executeUpdate(io.teaql.core.UserContext context, String sql, Object[] meta.setElapsedUs(elapsed); meta.setAffectedRows((long) res); meta.setResultSummary("Affected " + res + " rows"); + meta.setExecutionOutcome("success"); meta.setParameterizedQuery(sql); - if (context.requiresSensitiveSqlLogData()) { - meta.setParameters(parameters(args)); - meta.setDebugQuery(debugSql(sql, args, debugDatabaseKind)); - } + meta.setParameters(parameters(args)); + bindings.applyTo(meta); context.recordExecutionMetadata(meta); return res; } @Override public int[] batchUpdate(io.teaql.core.UserContext context, String sql, java.util.List batchArgs) { + return batchUpdate(context, sql, batchArgs, io.teaql.core.sql.portable.SqlLogBindings.UNKNOWN); + } + + @Override + public int[] batchUpdate(io.teaql.core.UserContext context, String sql, java.util.List batchArgs, + io.teaql.core.sql.portable.SqlLogBindings bindings) { boolean logging = context.isMutationExecutionLoggingEnabled(); long start = logging ? System.nanoTime() : 0L; - int[] res = executionAdapter.batchUpdate(sql, batchArgs); - if (!logging) return res; - long elapsed = (System.nanoTime() - start) / 1000; - int total = 0; if (res != null) { for(int i: res) total += i; } - io.teaql.core.ExecutionMetadata meta = new io.teaql.core.ExecutionMetadata(); - meta.setBackend(debugDatabaseKind.toLowerCase(java.util.Locale.ROOT)); - meta.setOperation(io.teaql.core.DataServiceOperation.MUTATION); - meta.setElapsedUs(elapsed); - meta.setAffectedRows((long) total); - meta.setResultSummary("Batch affected " + total + " rows"); - meta.setParameterizedQuery(sql); - if (context.requiresSensitiveSqlLogData()) { - String loggedSql = sql; - if (batchArgs != null && !batchArgs.isEmpty()) { - loggedSql = debugSql(sql, batchArgs.get(0), debugDatabaseKind); - if (batchArgs.size() > 1) { - loggedSql += " /* + " + (batchArgs.size() - 1) + " more batches */"; - } - } - meta.setParameters(batchParameters(batchArgs)); - meta.setDebugQuery(loggedSql); + int[] res; + try { + res = executionAdapter.batchUpdate(sql, batchArgs); + } catch (RuntimeException failure) { + if (logging) recordBatch(context, sql, batchArgs, bindings, start, batchCounts(failure), failure); + throw failure; } - context.recordExecutionMetadata(meta); + if (logging) recordBatch(context, sql, batchArgs, bindings, start, + res == null ? null : java.util.Arrays.stream(res).asLongStream().toArray(), null); return res; } @@ -265,7 +311,9 @@ public int[] batchUpdate(io.teaql.core.UserContext context, String sql, java.uti public void execute(io.teaql.core.UserContext context, String sql) { boolean logging = context.isMutationExecutionLoggingEnabled(); long start = logging ? System.nanoTime() : 0L; - executionAdapter.execute(sql); + diagnosed(context, sql, null, io.teaql.core.sql.portable.SqlLogBindings.UNKNOWN, + io.teaql.core.DataServiceOperation.SCHEMA, logging, start, + () -> { executionAdapter.execute(sql); return null; }); if (!logging) return; long elapsed = (System.nanoTime() - start) / 1000; io.teaql.core.ExecutionMetadata meta = new io.teaql.core.ExecutionMetadata(); @@ -273,6 +321,7 @@ public void execute(io.teaql.core.UserContext context, String sql) { meta.setOperation(io.teaql.core.DataServiceOperation.SCHEMA); meta.setElapsedUs(elapsed); meta.setResultSummary("Executed"); + meta.setExecutionOutcome("success"); meta.setParameterizedQuery(sql); if (context.requiresSensitiveSqlLogData()) { meta.setDebugQuery(sql); @@ -287,6 +336,79 @@ public void execute(io.teaql.core.UserContext context, String sql) { return portableService; } + private io.teaql.core.ExecutionMetadata statementMetadata(UserContext context, String sql, Object[] args, + io.teaql.core.sql.portable.SqlLogBindings bindings, io.teaql.core.DataServiceOperation operation) { + var meta = new io.teaql.core.ExecutionMetadata(); + meta.setBackend(debugDatabaseKind.toLowerCase(java.util.Locale.ROOT)); + meta.setOperation(operation); + meta.setParameterizedQuery(sql); + meta.setParameters(parameters(args)); + bindings.applyTo(meta); + return meta; + } + + private static long[] batchCounts(Throwable failure) { + var seen = java.util.Collections.newSetFromMap(new java.util.IdentityHashMap()); + while (failure != null && seen.add(failure)) { + if (failure instanceof java.sql.BatchUpdateException batchFailure) return batchFailure.getLargeUpdateCounts(); + failure = failure.getCause(); + } + return null; + } + + private void recordBatch(UserContext context, String sql, java.util.List batchArgs, + io.teaql.core.sql.portable.SqlLogBindings bindings, long start, long[] counts, RuntimeException failure) { + long elapsed = (System.nanoTime() - start) / 1000; + int size = batchArgs == null ? 0 : batchArgs.size(); + for (int row = 0; row < Math.max(size, failure == null ? 0 : 1); row++) { + var meta = statementMetadata(context, sql, row < size ? batchArgs.get(row) : null, + bindings, io.teaql.core.DataServiceOperation.MUTATION); + meta.setElapsedUs(row == 0 ? elapsed : 0); + long count = counts != null && row < counts.length ? counts[row] : Long.MIN_VALUE; + meta.setExecutionOutcome(count >= 0 || count == java.sql.Statement.SUCCESS_NO_INFO ? "success" + : count == java.sql.Statement.EXECUTE_FAILED ? "failure" : "unknown"); + if (count >= 0) meta.setAffectedRows(count); + meta.setResultSummary("Batch " + (failure == null ? "returned" : failure instanceof java.util.concurrent.CancellationException + ? "cancelled" : "failure") + "; member " + (row + 1) + " of " + size + + "; statement outcome " + meta.getExecutionOutcome()); + try { + context.recordExecutionMetadata(meta); + } catch (RuntimeException sinkFailure) { + if (failure == null) throw sinkFailure; + } + } + } + + private T diagnosed(UserContext context, String sql, Object[] args, + io.teaql.core.sql.portable.SqlLogBindings bindings, + io.teaql.core.DataServiceOperation operation, boolean logging, long start, + java.util.function.Supplier execute) { + try { + return execute.get(); + } catch (RuntimeException failure) { + if (logging) { + io.teaql.core.ExecutionMetadata meta = new io.teaql.core.ExecutionMetadata(); + meta.setBackend(debugDatabaseKind.toLowerCase(java.util.Locale.ROOT)); + meta.setOperation(operation); + meta.setElapsedUs((System.nanoTime() - start) / 1000); + meta.setExecutionOutcome(failure instanceof java.util.concurrent.CancellationException + ? "cancelled" : "failure"); + // Driver errors may contain raw bind values. Keep the error for the caller, + // never copy its message/cause into diagnostics or invent affected-row counts. + meta.setResultSummary("Statement did not complete; row count unknown"); + meta.setParameterizedQuery(sql); + meta.setParameters(parameters(args)); + bindings.applyTo(meta); + try { + context.recordExecutionMetadata(meta); + } catch (RuntimeException diagnosticFailure) { + // Preserve the original driver error; no unsafe fallback logger. + } + } + throw failure; + } + } + public static String debugSql(String sql, Object[] args) { return debugSql(sql, args, "sqlite"); } @@ -297,92 +419,17 @@ public static String debugSql( } public static String debugSql(String sql, Object[] args, String databaseKind) { - if (sql == null || args == null || args.length == 0) { - return sql; + if (sql == null) return null; + Object[] parameters = args == null ? new Object[0] : args; + try { + return io.teaql.core.utils.SqlLogRenderer.render(sql, parameters.length, + index -> io.teaql.core.utils.SqlLogRenderer.literal(parameters[index], databaseKind), + databaseKind); + } catch (IllegalArgumentException failure) { + return "[SQL OMITTED; NOT REPLAYABLE; unsupported literal or binding mismatch]"; } - StringBuilder sb = new StringBuilder(); - int argIndex = 0; - SqlScanState state = SqlScanState.SQL; - for (int i = 0; i < sql.length(); i++) { - char c = sql.charAt(i); - char next = i + 1 < sql.length() ? sql.charAt(i + 1) : '\0'; - if (state == SqlScanState.SQL && c == '\'') { - sb.append(c); - state = SqlScanState.SINGLE_QUOTE; - continue; - } - if (state == SqlScanState.SQL && c == '"') { - sb.append(c); - state = SqlScanState.DOUBLE_QUOTE; - continue; - } - if (state == SqlScanState.SQL && c == '-' && next == '-') { - sb.append("--"); i++; state = SqlScanState.LINE_COMMENT; continue; - } - if (state == SqlScanState.SQL && c == '/' && next == '*') { - sb.append("/*"); i++; state = SqlScanState.BLOCK_COMMENT; continue; - } - if (state == SqlScanState.SINGLE_QUOTE) { - sb.append(c); - if (c == '\'' && next == '\'') { sb.append(next); i++; } - else if (c == '\'') { state = SqlScanState.SQL; } - continue; - } - if (state == SqlScanState.DOUBLE_QUOTE) { - sb.append(c); - if (c == '"' && next == '"') { sb.append(next); i++; } - else if (c == '"') { state = SqlScanState.SQL; } - continue; - } - if (state == SqlScanState.LINE_COMMENT) { - sb.append(c); - if (c == '\r' || c == '\n') { state = SqlScanState.SQL; } - continue; - } - if (state == SqlScanState.BLOCK_COMMENT) { - sb.append(c); - if (c == '*' && next == '/') { sb.append(next); i++; state = SqlScanState.SQL; } - continue; - } - if (c == '?' && argIndex < args.length) { - Object arg = args[argIndex++]; - if (arg == null) { - sb.append("NULL"); - continue; - } - if (arg instanceof java.time.LocalDate date) { - String literal = "'" + date + "'"; - if (databaseKind.contains("mysql") || databaseKind.contains("mssql")) sb.append("CAST(").append(literal).append(" AS DATE)"); - else if (databaseKind.contains("sqlite")) sb.append(literal); - else sb.append("DATE ").append(literal); - continue; - } - if (arg instanceof java.time.LocalDateTime dateTime) { - String literal = "'" + java.sql.Timestamp.valueOf(dateTime) + "'"; - if (databaseKind.contains("mysql")) sb.append("CAST(").append(literal).append(" AS DATETIME(3))"); - else if (databaseKind.contains("mssql")) sb.append("CAST(").append(literal).append(" AS DATETIME2(3))"); - else if (databaseKind.contains("sqlite")) sb.append(literal); - else sb.append("TIMESTAMP ").append(literal); - continue; - } - if (arg instanceof String || arg instanceof java.util.Date || arg instanceof java.time.temporal.Temporal) { - sb.append("'").append(arg.toString().replace("'", "''")).append("'"); - continue; - } - if (arg instanceof Boolean) { - sb.append(Boolean.TRUE.equals(arg) ? "TRUE" : "FALSE"); - continue; - } - sb.append(arg.toString()); - continue; - } - sb.append(c); - } - return sb.toString(); } - private enum SqlScanState { SQL, SINGLE_QUOTE, DOUBLE_QUOTE, LINE_COMMENT, BLOCK_COMMENT } - private static java.util.List parameters(Object[] args) { if (args == null || args.length == 0) { return java.util.List.of(); diff --git a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDiagnosticStream.java b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDiagnosticStream.java new file mode 100644 index 00000000..ee2c47a9 --- /dev/null +++ b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDiagnosticStream.java @@ -0,0 +1,91 @@ +package io.teaql.dataservice.sql; + +import io.teaql.core.ExecutionMetadata; +import io.teaql.core.UserContext; +import java.util.Spliterator; +import java.util.concurrent.CancellationException; +import java.util.function.Consumer; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** Package-private cursor lifecycle. Does not make Stream safe for concurrent consumers. */ +final class SqlDiagnosticStream implements Spliterator { + private final UserContext context; + private final Stream source; + private final ExecutionMetadata metadata; + private final long start; + private Spliterator rows; + private boolean done; + private long delivered; + + private SqlDiagnosticStream(UserContext context, Stream source, ExecutionMetadata metadata, long start) { + this.context = context; + this.source = source; + this.metadata = metadata; + this.start = start; + } + + static Stream wrap(UserContext context, Stream source, ExecutionMetadata metadata, long start) { + var cursor = new SqlDiagnosticStream<>(context, source, metadata, start); + try { + cursor.rows = source.spliterator(); + } catch (RuntimeException | Error failure) { + cursor.finish(outcome(failure), failure); + throw failure; + } + return StreamSupport.stream(cursor, false).onClose(() -> cursor.finish("cancelled", null)); + } + + @Override public boolean tryAdvance(Consumer action) { + if (done) return false; + try { + boolean more = rows.tryAdvance(row -> { + // Count rows handed to the caller, not prefetched rows or rows successfully processed. + delivered++; + action.accept(row); + }); + if (!more) finish("success", null); + return more; + } catch (RuntimeException | Error failure) { + finish(outcome(failure), failure); + throw failure; + } + } + + private void finish(String outcome, Throwable original) { + if (done) return; + done = true; + Throwable failure = original; + try { + source.close(); + } catch (RuntimeException | Error closeFailure) { + if (failure == null) { failure = closeFailure; outcome = outcome(closeFailure); } + else if (failure != closeFailure) failure.addSuppressed(closeFailure); + } + if (metadata != null) { + metadata.setElapsedUs((System.nanoTime() - start) / 1000); + metadata.setExecutionOutcome(outcome); + metadata.setResultCount(delivered <= Integer.MAX_VALUE ? (int) delivered : null); + metadata.setResultSummary("Cursor " + outcome + "; delivered " + delivered + " rows"); + try { + context.recordExecutionMetadata(metadata); + } catch (RuntimeException | Error sinkFailure) { + if (failure == null) failure = sinkFailure; + } + } + // The original traversal exception is rethrown by the caller of finish. + if (original == null) { + if (failure instanceof RuntimeException runtimeFailure) throw runtimeFailure; + if (failure instanceof Error fatal) throw fatal; + } + } + + private static String outcome(Throwable failure) { + return failure instanceof CancellationException ? "cancelled" : "failure"; + } + + @Override public Spliterator trySplit() { return null; } + @Override public long estimateSize() { return Long.MAX_VALUE; } + // No SIZED: count() must traverse rather than elide resource cleanup and diagnostics. + @Override public int characteristics() { return ORDERED; } +} diff --git a/teaql-data-service-sql/src/main/java/module-info.java b/teaql-data-service-sql/src/main/java/module-info.java index 84e8049f..4259ea80 100644 --- a/teaql-data-service-sql/src/main/java/module-info.java +++ b/teaql-data-service-sql/src/main/java/module-info.java @@ -2,6 +2,7 @@ requires io.teaql.core; requires transitive io.teaql.sql.portable; requires java.sql; + requires io.teaql.utils; exports io.teaql.dataservice.sql; } diff --git a/teaql-data-service-sql/src/test/java/io/teaql/dataservice/sql/SqlFailureMaskingTest.java b/teaql-data-service-sql/src/test/java/io/teaql/dataservice/sql/SqlFailureMaskingTest.java new file mode 100644 index 00000000..c2f7cfdd --- /dev/null +++ b/teaql-data-service-sql/src/test/java/io/teaql/dataservice/sql/SqlFailureMaskingTest.java @@ -0,0 +1,126 @@ +package io.teaql.dataservice.sql; + +import io.teaql.core.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.portable.*; +import io.teaql.runtime.*; +import java.lang.reflect.Proxy; +import java.util.*; +import org.junit.Test; +import static org.junit.Assert.*; + +/** Test the actual adapter installed by SqlDataServiceExecutor, without exposing a workspace API. */ +public class SqlFailureMaskingTest { + private static final String SQL = "SELECT * FROM customer WHERE name = ? AND address = ? AND password = ?"; + private static final Object[] ARGS = {"Riverside", "1 Runtime Road", "PASSWORD-CANARY"}; + private static final SqlLogBindings BINDINGS = new SqlLogBindings(List.of( + SqlParameterLogPolicy.MASKED, SqlParameterLogPolicy.PLAIN, SqlParameterLogPolicy.CREDENTIAL), true); + + private TeaQLDatabase database(RuntimeException error) throws Exception { + SqlExecutionAdapter adapter = (SqlExecutionAdapter) Proxy.newProxyInstance( + SqlExecutionAdapter.class.getClassLoader(), new Class[]{SqlExecutionAdapter.class}, + (proxy, method, args) -> { throw error; }); + var executor = new SqlDataServiceExecutor("sql", adapter); + var factory = SqlDataServiceExecutor.class.getDeclaredMethod("createPortableService", io.teaql.core.meta.EntityMetaFactory.class); + factory.setAccessible(true); + var service = factory.invoke(executor, new SimpleEntityMetaFactory()); + var field = PortableSQLDataService.class.getDeclaredField("database"); + field.setAccessible(true); + return (TeaQLDatabase) field.get(service); + } + + private void failure(String operation, boolean enabled, boolean brokenSink) throws Exception { + failure(operation, enabled, brokenSink, false); + } + + private void failure(String operation, boolean enabled, boolean brokenSink, boolean cancelled) throws Exception { + RuntimeException driverError = cancelled + ? new java.util.concurrent.CancellationException("driver exposes PASSWORD-CANARY Riverside") + : new IllegalStateException("driver exposes PASSWORD-CANARY Riverside"); + var logs = new ArrayList(); + var context = new DefaultUserContext(TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .logSink((caller, metadata) -> { + logs.add(metadata); + if (brokenSink) throw new IllegalArgumentException("sink unavailable"); + }).build()) { + @Override public boolean isQueryExecutionLoggingEnabled() { return enabled; } + @Override public boolean isMutationExecutionLoggingEnabled() { return enabled; } + }; + context.pushTrace(TraceKind.COMMENT, "Customer", "what: run diagnostic fixture"); + context.pushTrace(TraceKind.PURPOSE, "Customer", "why: verify failed SQL"); + context.pushTrace(TraceKind.AUDIT_REASON, "Customer", "test error handling"); + var db = database(driverError); + var actual = assertThrows(RuntimeException.class, () -> { + switch (operation) { + case "query" -> db.query(context, SQL, ARGS, BINDINGS); + case "typed" -> db.query(context, SQL, ARGS, (CompiledRowMapper) null, BINDINGS); + case "update" -> db.executeUpdate(context, SQL, ARGS, BINDINGS); + case "ddl" -> db.execute(context, "CREATE TABLE broken (name TEXT)"); + default -> throw new AssertionError(operation); + } + }); + assertSame(driverError, actual); + assertEquals(enabled ? 1 : 0, logs.size()); + if (enabled) { + var safe = logs.get(0); + assertEquals(cancelled ? "cancelled" : "failure", safe.getExecutionOutcome()); + assertEquals("what: run diagnostic fixture", safe.getComment()); + assertEquals("why: verify failed SQL", safe.getPurpose()); + assertEquals("test error handling", safe.getAuditReason()); + assertTrue(safe.getTraceChain().stream().anyMatch(node -> node.getKind() == TraceKind.SQL)); + assertNull(safe.getResultCount()); + assertNull(safe.getAffectedRows()); + assertFalse(String.valueOf(safe.getResultSummary()).contains("PASSWORD-CANARY")); + assertFalse(safe.getDebugQuery().contains("PASSWORD-CANARY")); + if (!operation.equals("ddl")) { + assertTrue(safe.getDebugQuery(), safe.getDebugQuery().contains("Ri*****de")); + assertTrue(safe.getDebugQuery().contains("1 Runtime Road")); + assertTrue(safe.getDebugQuery().contains("NOT REPLAYABLE")); + assertFalse(safe.getDebugQuery().contains("Riverside")); + } + } + assertArrayEquals(new Object[]{"Riverside", "1 Runtime Road", "PASSWORD-CANARY"}, ARGS); + } + + @Test public void queryFailure() throws Exception { failure("query", true, false); } + @Test public void typedFailure() throws Exception { failure("typed", true, false); } + @Test public void updateFailure() throws Exception { failure("update", true, false); } + @Test public void ddlFailure() throws Exception { failure("ddl", true, false); } + @Test public void ddlDisabled() throws Exception { failure("ddl", false, false); } + @Test public void queryDisabled() throws Exception { failure("query", false, false); } + @Test public void updateDisabled() throws Exception { failure("update", false, false); } + @Test public void queryBrokenSink() throws Exception { failure("query", true, true); } + @Test public void updateBrokenSink() throws Exception { failure("update", true, true); } + @Test public void queryCancelled() throws Exception { failure("query", true, false, true); } + @Test public void updateCancelled() throws Exception { failure("update", true, false, true); } + + @Test public void inheritedReadbackFailure() throws Exception { inheritedReadback(false, true); } + @Test public void inheritedReadbackCancelled() throws Exception { inheritedReadback(true, true); } + @Test public void inheritedReadbackDisabled() throws Exception { inheritedReadback(false, false); } + + private void inheritedReadback(boolean cancelled, boolean enabled) throws Exception { + RuntimeException error = cancelled ? new java.util.concurrent.CancellationException("DRIVER-CANARY") + : new IllegalStateException("DRIVER-CANARY"); + var logs = new ArrayList(); + var context = new DefaultUserContext(TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .logSink((caller, metadata) -> logs.add(metadata)).build()) { + @Override public boolean isQueryExecutionLoggingEnabled() { return enabled; } + }; + context.pushTrace(TraceKind.AUDIT_REASON, "Customer", "read Riverside PASSWORD-CANARY snapshot"); + var provenance = new SqlIntentRedactions(); + provenance.capture(BINDINGS.policies(), ARGS); + var bindings = new SqlLogBindings(List.of(SqlParameterLogPolicy.PLAIN), true, null, provenance); + var db = database(error); + assertSame(error, assertThrows(RuntimeException.class, + () -> db.query(context, "SELECT * FROM customer WHERE id = ?", new Object[]{1L}, bindings))); + assertEquals(enabled ? 1 : 0, logs.size()); + if (enabled) { + var entry = logs.get(0); + assertEquals("read [REDACTED] [REDACTED] snapshot", entry.getAuditReason()); + assertEquals(cancelled ? "cancelled" : "failure", entry.getExecutionOutcome()); + assertNull(entry.getResultCount()); + assertNull(entry.getIntentRedactions()); + assertTrue(entry.getDebugQuery().contains("id = 1")); + } + } +} diff --git a/teaql-data-service-sql/src/test/java/io/teaql/dataservice/sql/SqlStreamBatchMaskingTest.java b/teaql-data-service-sql/src/test/java/io/teaql/dataservice/sql/SqlStreamBatchMaskingTest.java new file mode 100644 index 00000000..c5bba591 --- /dev/null +++ b/teaql-data-service-sql/src/test/java/io/teaql/dataservice/sql/SqlStreamBatchMaskingTest.java @@ -0,0 +1,200 @@ +package io.teaql.dataservice.sql; + +import io.teaql.core.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.portable.*; +import io.teaql.runtime.*; +import java.lang.reflect.*; +import java.sql.BatchUpdateException; +import java.util.*; +import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.junit.Test; +import static org.junit.Assert.*; + +public class SqlStreamBatchMaskingTest { + private static final String SQL = "SELECT * FROM customer WHERE name = ? AND address = ? AND password = ?"; + private static final Object[] ARGS = {"Riverside", "1 Runtime Road", "PASSWORD-CANARY"}; + private static final SqlLogBindings BINDINGS = new SqlLogBindings(List.of( + SqlParameterLogPolicy.MASKED, SqlParameterLogPolicy.PLAIN, SqlParameterLogPolicy.CREDENTIAL), true); + private final List logs = new ArrayList<>(); + private final AtomicInteger closed = new AtomicInteger(); + private final RuntimeException error = new IllegalStateException("PASSWORD-CANARY Riverside"); + private boolean enabled = true; + private boolean brokenSink; + private final DefaultUserContext context = new DefaultUserContext(TeaQLRuntime.builder() + .metadata(new SimpleEntityMetaFactory()).logSink((caller, metadata) -> { + logs.add(metadata); + if (brokenSink) throw new IllegalStateException("sink failed"); + }).build()) { + @Override public boolean isQueryExecutionLoggingEnabled() { return enabled; } + @Override public boolean isMutationExecutionLoggingEnabled() { return enabled; } + }; + + private TeaQLDatabase database(InvocationHandler action) throws Exception { + var adapter = (SqlExecutionAdapter) Proxy.newProxyInstance(SqlExecutionAdapter.class.getClassLoader(), + new Class[]{SqlExecutionAdapter.class}, action); + var executor = new SqlDataServiceExecutor("sql", adapter); + var factory = SqlDataServiceExecutor.class.getDeclaredMethod("createPortableService", io.teaql.core.meta.EntityMetaFactory.class); + factory.setAccessible(true); + var service = factory.invoke(executor, new SimpleEntityMetaFactory()); + var field = PortableSQLDataService.class.getDeclaredField("database"); + field.setAccessible(true); + return (TeaQLDatabase) field.get(service); + } + + private Stream> stream(String mode) throws Exception { + context.pushTrace(TraceKind.COMMENT, "Customer", "read Riverside"); + context.pushTrace(TraceKind.PURPOSE, "Customer", "verify cursor lifecycle"); + var db = database((proxy, method, args) -> { + if (mode.equals("open")) throw error; + Stream input = mode.equals("empty") ? Stream.empty() : Stream.of(1, 2, 3); + return input.map(index -> { + if (index == 2 && mode.equals("read")) throw error; + if (index == 2 && mode.equals("cancel")) throw new CancellationException("PASSWORD-CANARY"); + return Map.of("id", index); + }).onClose(() -> { + closed.incrementAndGet(); + if (mode.equals("close")) throw error; + }); + }); + try { + return db.queryForStream(context, SQL, ARGS, BINDINGS); + } finally { + context.popTrace(); context.popTrace(); + } + } + + private void safe(ExecutionMetadata log) { + assertTrue(log.getDebugQuery(), log.getDebugQuery().contains("Ri*****de")); + assertTrue(log.getDebugQuery().contains("1 Runtime Road")); + assertFalse(log.getDebugQuery().contains("PASSWORD-CANARY")); + assertFalse(log.getDebugQuery().contains("Riverside")); + assertTrue(log.getDebugQuery().contains("NOT REPLAYABLE")); + assertArrayEquals(new Object[]{"Riverside", "1 Runtime Road", "PASSWORD-CANARY"}, ARGS); + } + + private void terminal(String outcome, Integer delivered) { + assertEquals(1, logs.size()); + var log = logs.get(0); + safe(log); + assertEquals(outcome, log.getExecutionOutcome()); + assertEquals(delivered, log.getResultCount()); + assertEquals("verify cursor lifecycle", log.getPurpose()); + assertTrue(log.getComment().startsWith("read ")); + assertFalse(log.getComment().contains("Riverside")); + assertEquals(1, closed.get()); + } + + @Test public void exhaustedCursorClosesAndLogsOnce() throws Exception { + try (var rows = stream("ok")) { + assertEquals(3, rows.toList().size()); + assertEquals(1, closed.get()); + assertEquals(1, logs.size()); + } + terminal("success", 3); + } + @Test public void emptyCursorCompletesWithKnownZeroRows() throws Exception { + try (var rows = stream("empty")) { + assertEquals(0, rows.count()); + assertEquals(1, closed.get()); + } + terminal("success", 0); + } + @Test public void countCannotSkipDeliveryOrDiagnostics() throws Exception { + try (var rows = stream("ok")) { assertEquals(3L, rows.count()); } + terminal("success", 3); + } + @Test public void earlyCloseCountsOnlyDeliveredRows() throws Exception { + try (var rows = stream("ok")) { assertEquals(1, rows.limit(1).toList().size()); } + terminal("cancelled", 1); + } + @Test public void unopenedConsumptionClosesWithZeroRows() throws Exception { + stream("ok").close(); terminal("cancelled", 0); + } + @Test public void readFailureKeepsDeliveredCountAndOriginalError() throws Exception { + try (var rows = stream("read")) { assertSame(error, assertThrows(RuntimeException.class, rows::toList)); } + terminal("failure", 1); + } + @Test public void cancellationKeepsDeliveredCount() throws Exception { + try (var rows = stream("cancel")) { assertThrows(CancellationException.class, rows::toList); } + terminal("cancelled", 1); + } + @Test public void downstreamFailureClosesCursor() throws Exception { + try (var rows = stream("ok")) { + assertSame(error, assertThrows(RuntimeException.class, () -> rows.forEach(row -> { throw error; }))); + } + terminal("failure", 1); + } + @Test public void downstreamErrorAlsoClosesCursor() throws Exception { + var fatal = new AssertionError("consumer failed"); + try (var rows = stream("ok")) { + assertSame(fatal, assertThrows(AssertionError.class, () -> rows.forEach(row -> { throw fatal; }))); + } + terminal("failure", 1); + } + @Test public void closeFailureIsNotReportedAsSuccess() throws Exception { + var rows = stream("close"); + assertSame(error, assertThrows(RuntimeException.class, rows::toList)); + rows.close(); terminal("failure", 3); + } + @Test public void lateConsumptionRetainsOriginalIntent() throws Exception { + var rows = stream("ok"); + context.pushTrace(TraceKind.PURPOSE, "Other", "unrelated later request"); + try (rows) { rows.toList(); } + terminal("success", 3); + } + @Test public void openFailureHasUnknownCount() throws Exception { + assertSame(error, assertThrows(RuntimeException.class, () -> stream("open"))); + assertEquals(1, logs.size()); safe(logs.get(0)); + assertEquals("failure", logs.get(0).getExecutionOutcome()); + assertNull(logs.get(0).getResultCount()); assertEquals(0, closed.get()); + } + @Test public void sinkFailureDoesNotReplaceReadFailure() throws Exception { + brokenSink = true; + try (var rows = stream("read")) { assertSame(error, assertThrows(RuntimeException.class, rows::toList)); } + terminal("failure", 1); + } + @Test public void disabledLogsStillCloseCursor() throws Exception { + enabled = false; + try (var rows = stream("ok")) { rows.limit(1).toList(); } + assertEquals(0, logs.size()); assertEquals(1, closed.get()); + } + + private void batch(int[] result, RuntimeException failure) throws Exception { + var db = database((proxy, method, args) -> { + if (failure != null) throw failure; + return result; + }); + var rows = List.of(ARGS, ARGS.clone(), ARGS.clone()); + if (failure == null) assertSame(result, db.batchUpdate(context, SQL, rows, BINDINGS)); + else assertSame(failure, assertThrows(RuntimeException.class, () -> db.batchUpdate(context, SQL, rows, BINDINGS))); + assertEquals(enabled ? 3 : 0, logs.size()); + for (var log : logs) { safe(log); assertNull(log.getResultCount()); } + } + @Test public void batchSuccessAndSuccessNoInfo() throws Exception { + batch(new int[]{1, 0, -2}, null); + assertEquals(List.of("success", "success", "success"), logs.stream().map(ExecutionMetadata::getExecutionOutcome).toList()); + assertEquals(Long.valueOf(1), logs.get(0).getAffectedRows()); + assertEquals(Long.valueOf(0), logs.get(1).getAffectedRows()); + assertNull(logs.get(2).getAffectedRows()); + } + @Test public void partialBatchDoesNotInventUnexecutedOutcomes() throws Exception { + batch(null, new RuntimeException(new BatchUpdateException("PASSWORD-CANARY", new int[]{1, -3}))); + assertEquals(List.of("success", "failure", "unknown"), logs.stream().map(ExecutionMetadata::getExecutionOutcome).toList()); + assertEquals(Long.valueOf(1), logs.get(0).getAffectedRows()); + assertNull(logs.get(1).getAffectedRows()); assertNull(logs.get(2).getAffectedRows()); + } + @Test public void genericBatchFailureDoesNotInventProgress() throws Exception { + batch(null, error); + assertTrue(logs.stream().allMatch(log -> "unknown".equals(log.getExecutionOutcome()) && log.getAffectedRows() == null)); + assertTrue(logs.stream().allMatch(log -> log.getResultSummary().contains("Batch failure"))); + } + @Test public void brokenSinkDoesNotReplaceBatchFailure() throws Exception { + brokenSink = true; batch(null, error); + } + @Test public void disabledBatchLogsPreserveOriginalError() throws Exception { + enabled = false; batch(null, error); + } +} diff --git a/teaql-dynamic-fields-jdbc/DESIGN.md b/teaql-dynamic-fields-jdbc/DESIGN.md index ceea75b0..92b35955 100644 --- a/teaql-dynamic-fields-jdbc/DESIGN.md +++ b/teaql-dynamic-fields-jdbc/DESIGN.md @@ -215,8 +215,10 @@ private void tryExecute(String ddl) { try { database.execute(ddl); } catch (Exception e) { - // 已存在，忽略 - LOG.fine("Schema element may already exist: " + e.getMessage()); + // best-effort：driver 异常原文可能含 SQL 字面量或连接细节，不写入日志。 + LOG.log(Level.FINE, + "Dynamic field schema statement failed; continuing best-effort setup ({0})", + e.getClass().getSimpleName()); } } ``` diff --git a/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/DynamicFieldsSchema.java b/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/DynamicFieldsSchema.java index 7fe34674..64edacca 100644 --- a/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/DynamicFieldsSchema.java +++ b/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/DynamicFieldsSchema.java @@ -84,7 +84,10 @@ private static void tryExecute(SqlExecutionAdapter executor, String ddl) { try { executor.execute(ddl); } catch (Exception e) { - LOG.log(Level.FINE, "Schema element may already exist: {0}", e.getMessage()); + // Driver messages can contain connection details or SQL literals. + // Keep the existing best-effort behavior without copying them into logs. + LOG.log(Level.FINE, "Dynamic field schema statement failed; continuing best-effort setup ({0})", + e.getClass().getSimpleName()); } } } diff --git a/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/DynamicFieldsSchemaLogTest.java b/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/DynamicFieldsSchemaLogTest.java new file mode 100644 index 00000000..9c52a55e --- /dev/null +++ b/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/DynamicFieldsSchemaLogTest.java @@ -0,0 +1,58 @@ +package io.teaql.data.dynamic.jdbc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import org.junit.Test; + +public class DynamicFieldsSchemaLogTest { + + @Test + public void driverFailureDoesNotWriteExceptionTextEvenAtFineLevel() { + Logger logger = Logger.getLogger(DynamicFieldsSchema.class.getName()); + Level originalLevel = logger.getLevel(); + List records = new ArrayList<>(); + Handler capture = new Handler() { + @Override public void publish(LogRecord record) { records.add(record); } + @Override public void flush() { } + @Override public void close() { } + }; + capture.setLevel(Level.ALL); + logger.addHandler(capture); + logger.setLevel(Level.FINE); + try { + SqlExecutionAdapter executor = (SqlExecutionAdapter) Proxy.newProxyInstance( + SqlExecutionAdapter.class.getClassLoader(), + new Class[] { SqlExecutionAdapter.class }, + (proxy, method, args) -> { + if (method.getName().equals("execute")) { + throw new IllegalStateException("DRIVER-CANARY PASSWORD-CANARY"); + } + throw new UnsupportedOperationException(method.getName()); + }); + DynamicFieldsSchema.ensureSchema(executor); + } finally { + logger.setLevel(originalLevel); + logger.removeHandler(capture); + } + + assertEquals(3, records.stream().filter(record -> record.getLevel() == Level.FINE).count()); + for (LogRecord record : records) { + String rendered = java.text.MessageFormat.format(record.getMessage(), record.getParameters()); + if (record.getLevel() == Level.FINE) { + assertTrue(rendered.contains("IllegalStateException")); + } + assertFalse(rendered.contains("DRIVER-CANARY")); + assertFalse(rendered.contains("PASSWORD-CANARY")); + } + } +} diff --git a/teaql-hana/pom.xml b/teaql-hana/pom.xml index 93fb5bd7..c347fd0e 100644 --- a/teaql-hana/pom.xml +++ b/teaql-hana/pom.xml @@ -19,10 +19,26 @@ io.teaql teaql-utils + + io.teaql + teaql-runtime + test + junit junit test + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-reads io.teaql.hana=io.teaql.runtime + + + + diff --git a/teaql-hana/src/test/java/io/teaql/core/hana/HanaDialectTest.java b/teaql-hana/src/test/java/io/teaql/core/hana/HanaDialectTest.java index 45ecd243..e1a8228b 100644 --- a/teaql-hana/src/test/java/io/teaql/core/hana/HanaDialectTest.java +++ b/teaql-hana/src/test/java/io/teaql/core/hana/HanaDialectTest.java @@ -1,10 +1,15 @@ package io.teaql.core.hana; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import io.teaql.core.BaseEntity; import io.teaql.core.BaseRequest; +import io.teaql.core.ExecutionMetadata; +import io.teaql.core.SqlParameterLogPolicy; +import io.teaql.runtime.LogPrivacy; +import java.util.List; import org.junit.Test; public class HanaDialectTest { @@ -20,4 +25,49 @@ public void usesHanaPaginationWindowAndLargeText() { assertTrue(dialect.getPartitionSQL().contains("row_number() over(partition by")); assertEquals("NCLOB", dialect.mapColumnType("LARGE_TEXT")); } + + @Test + public void hanaSqlDiagnosticsUseSafeProjectionWithoutChangingExecutionParameters() { + ExecutionMetadata source = new ExecutionMetadata(); + source.setBackend("hana"); + source.setGeneratedSql(true); + source.setParameterizedQuery( + "SELECT \"NAME?\" FROM CUSTOMER WHERE NAME = ? AND ADDRESS = ? AND PASSWORD = ?"); + source.setParameters(List.of("Riverside", "1 Runtime Road", "PASSWORD-CANARY")); + source.setParameterLogPolicies(List.of( + SqlParameterLogPolicy.MASKED, + SqlParameterLogPolicy.PLAIN, + SqlParameterLogPolicy.CREDENTIAL)); + + ExecutionMetadata projected = LogPrivacy.sql(source, false); + String diagnostic = projected.getDebugQuery(); + assertTrue(diagnostic, diagnostic.contains("Ri*****de")); + assertTrue(diagnostic, diagnostic.contains("1 Runtime Road")); + assertTrue(diagnostic, diagnostic.contains("/* masked */")); + assertTrue(diagnostic, diagnostic.contains("NOT REPLAYABLE")); + assertTrue(diagnostic, diagnostic.contains("\"NAME?\"")); + assertTrue(diagnostic, diagnostic.contains("[REDACTED]")); + assertFalse(diagnostic, diagnostic.contains("Riverside")); + assertFalse(diagnostic, diagnostic.contains("PASSWORD-CANARY")); + assertEquals(List.of("Riverside", "1 Runtime Road", "PASSWORD-CANARY"), source.getParameters()); + + String explicitDebug = LogPrivacy.sql(source, true).getDebugQuery(); + assertTrue(explicitDebug, explicitDebug.contains("Riverside")); + assertTrue(explicitDebug, explicitDebug.contains("1 Runtime Road")); + assertFalse(explicitDebug, explicitDebug.contains("PASSWORD-CANARY")); + } + + @Test + public void oldHanaDescriptorsWithoutFieldPolicyFailClosed() { + ExecutionMetadata source = new ExecutionMetadata(); + source.setBackend("hana"); + source.setGeneratedSql(true); + source.setParameterizedQuery("SELECT id FROM CUSTOMER WHERE NAME = ?"); + source.setParameters(List.of("LEGACY-HANA-CANARY")); + + ExecutionMetadata projected = LogPrivacy.sql(source, false); + assertTrue(projected.getDebugQuery(), projected.getDebugQuery().contains("[REDACTED]")); + assertFalse(projected.getDebugQuery(), projected.getDebugQuery().contains("LEGACY-HANA-CANARY")); + assertEquals(List.of("LEGACY-HANA-CANARY"), source.getParameters()); + } } diff --git a/teaql-mysql/src/test/java/io/teaql/mysql/MysqlIntegrationTest.java b/teaql-mysql/src/test/java/io/teaql/mysql/MysqlIntegrationTest.java index a404f13d..bc7d899f 100644 --- a/teaql-mysql/src/test/java/io/teaql/mysql/MysqlIntegrationTest.java +++ b/teaql-mysql/src/test/java/io/teaql/mysql/MysqlIntegrationTest.java @@ -18,6 +18,9 @@ import javax.sql.DataSource; import java.io.PrintWriter; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -392,6 +395,53 @@ private static SQLEntityDescriptor relationEntity( return descriptor; } + @Test + public void testLiveSqlMaskingThroughQueryAndMutation() throws Exception { + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + SQLEntityDescriptor descriptor = new SQLEntityDescriptor(); + descriptor.setType("Task"); + descriptor.setTargetType(Task.class); + descriptor.setEntitySupplier(Task::new); + descriptor.setDataService("mysql"); + descriptor.setAuditMaskFields(List.of("title")); + ((io.teaql.core.sql.GenericSQLProperty) descriptor.addSimpleProperty("id", Long.class)).setColumnType("BIGINT"); + ((io.teaql.core.sql.GenericSQLProperty) descriptor.addSimpleProperty("version", Long.class)).setColumnType("BIGINT"); + ((io.teaql.core.sql.GenericSQLProperty) descriptor.addSimpleProperty("title", String.class)).setColumnType("VARCHAR(200)"); + ((io.teaql.core.sql.GenericSQLProperty) descriptor.addSimpleProperty("status", String.class)).setColumnType("VARCHAR(50)"); + metadata.register(descriptor); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (PrintStream output = new PrintStream(bytes, true, StandardCharsets.UTF_8)) { + TeaQLRuntime maskingRuntime = TeaQLRuntime.builder() + .metadata(metadata) + .dataService("mysql", new MysqlDataServiceExecutor("mysql", new JdbcSqlExecutor(dataSource), dataSource)) + .idGenerationService(new IdSpaceIdGenerator(new JdbcTeaQLDatabase(dataSource))) + .logSink(new io.teaql.runtime.DefaultTextRuntimeLogSink(output)) + .build(); + UserContext maskingContext = new DefaultUserContext(maskingRuntime); + maskingContext.ensureSchema(); + String status = "1 Runtime Road " + UUID.randomUUID().toString().substring(0, 8); + Task task = new Task(); + task.updateTitle("Riverside").updateStatus(status); + task.auditAs("what: create masked task").save(maskingContext); + SmartList rows = new TaskRequest().filterByTitle("Riverside") + .filterByStatus(status) + .comment("what: read masked task") + .purpose("why: verify live-provider SQL masking") + .executeForList(maskingContext); + assertEquals(1, rows.size()); + assertEquals("Riverside", rows.get(0).getTitle()); + assertEquals(status, rows.get(0).getStatus()); + } + String logged = bytes.toString(StandardCharsets.UTF_8); + assertTrue(logged, logged.contains("INSERT")); + assertTrue(logged, logged.contains("SELECT")); + assertTrue(logged, logged.contains("Ri*****de")); + assertTrue(logged, logged.contains("1 Runtime Road")); + assertTrue(logged, logged.contains("what: read masked task")); + assertTrue(logged, logged.contains("why: verify live-provider SQL masking")); + assertFalse(logged, logged.contains("Riverside")); + } + @Test public void testMysqlCrud() { // 1. Create and Save Tasks diff --git a/teaql-mysql/src/test/java/io/teaql/mysql/MysqlTemporalSqlTest.java b/teaql-mysql/src/test/java/io/teaql/mysql/MysqlTemporalSqlTest.java index 8db49d01..a9c0169b 100644 --- a/teaql-mysql/src/test/java/io/teaql/mysql/MysqlTemporalSqlTest.java +++ b/teaql-mysql/src/test/java/io/teaql/mysql/MysqlTemporalSqlTest.java @@ -12,8 +12,13 @@ public class MysqlTemporalSqlTest { @Test public void preparedAndDiagnosticSqlAreEquivalent() throws Exception { String url = System.getenv("TEAQL_TEST_MYSQL_URL"); - Assume.assumeTrue(url != null && !url.isBlank()); - try (Connection connection = DriverManager.getConnection(url)) { + String user = System.getenv("TEAQL_TEST_MYSQL_USER"); + String password = System.getenv("TEAQL_TEST_MYSQL_PASSWORD"); + boolean configured = url != null && !url.isBlank() && user != null && password != null; + if (!configured && Boolean.parseBoolean(System.getenv("TEAQL_REQUIRE_LIVE_DB"))) + fail("MySQL temporal live gate requires URL, USER, and PASSWORD environment variables"); + Assume.assumeTrue(configured); + try (Connection connection = DriverManager.getConnection(url, user, password)) { run(connection, "DROP TABLE IF EXISTS teaql_temporal_java_fixture"); run(connection, "CREATE TABLE teaql_temporal_java_fixture(id INTEGER, d DATE, local_time DATETIME(3))"); String sql = "INSERT INTO teaql_temporal_java_fixture VALUES (?, ?, ?) /* ignored ? */"; diff --git a/teaql-opentelemetry/src/test/java/io/teaql/opentelemetry/OpenTelemetryRuntimeTelemetryTest.java b/teaql-opentelemetry/src/test/java/io/teaql/opentelemetry/OpenTelemetryRuntimeTelemetryTest.java index 76006313..1814a92e 100644 --- a/teaql-opentelemetry/src/test/java/io/teaql/opentelemetry/OpenTelemetryRuntimeTelemetryTest.java +++ b/teaql-opentelemetry/src/test/java/io/teaql/opentelemetry/OpenTelemetryRuntimeTelemetryTest.java @@ -101,6 +101,46 @@ public void exportsSafeSpanAndMetricsThroughOfficialSdk() { loggerProvider.close(); } + @Test + public void failureTelemetryDoesNotExportDriverErrorMessages() { + InMemorySpanExporter spans = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spans)).build(); + SdkMeterProvider meterProvider = SdkMeterProvider.builder().build(); + InMemoryLogRecordExporter logs = InMemoryLogRecordExporter.create(); + SdkLoggerProvider loggerProvider = SdkLoggerProvider.builder() + .addLogRecordProcessor(SimpleLogRecordProcessor.create(logs)).build(); + OpenTelemetryRuntimeTelemetry telemetry = new OpenTelemetryRuntimeTelemetry( + tracerProvider.get("io.teaql.runtime"), + meterProvider.get("io.teaql.runtime"), + loggerProvider.get("io.teaql.runtime")); + + RuntimeTelemetry.Scope scope = RuntimeTelemetry.startSafely(telemetry, + new RuntimeTelemetry.Operation("provider", "sqlite.query", Map.of())); + scope.failure(new IllegalStateException( + "SQL failed for password=OTEL-FAILURE-CANARY")); + + assertEquals(1, spans.getFinishedSpanItems().size()); + assertEquals(1, logs.getFinishedLogRecordItems().size()); + var span = spans.getFinishedSpanItems().get(0); + var log = logs.getFinishedLogRecordItems().get(0); + assertEquals("IllegalStateException", span.getAttributes().get( + AttributeKey.stringKey("teaql.error.type"))); + assertEquals("internal", span.getAttributes().get( + AttributeKey.stringKey("teaql.error.category"))); + assertEquals("failure", log.getAttributes().get( + AttributeKey.stringKey("teaql.operation.outcome"))); + assertFalse(span.getAttributes().toString().contains("OTEL-FAILURE-CANARY")); + assertFalse(span.getStatus().toString().contains("OTEL-FAILURE-CANARY")); + assertFalse(span.getEvents().toString().contains("OTEL-FAILURE-CANARY")); + assertFalse(log.getAttributes().toString().contains("OTEL-FAILURE-CANARY")); + assertFalse(log.getBodyValue().asString().contains("OTEL-FAILURE-CANARY")); + + tracerProvider.close(); + meterProvider.close(); + loggerProvider.close(); + } + @Test public void extractsCaseInsensitiveW3cCarrierAsServerParent() { diff --git a/teaql-postgres/src/test/java/io/teaql/postgres/PostgresIntegrationTest.java b/teaql-postgres/src/test/java/io/teaql/postgres/PostgresIntegrationTest.java index bab08058..db82ad80 100644 --- a/teaql-postgres/src/test/java/io/teaql/postgres/PostgresIntegrationTest.java +++ b/teaql-postgres/src/test/java/io/teaql/postgres/PostgresIntegrationTest.java @@ -18,6 +18,9 @@ import javax.sql.DataSource; import java.io.PrintWriter; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -25,6 +28,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Collections; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -350,6 +354,53 @@ private static SQLEntityDescriptor relationEntity( return descriptor; } + @Test + public void testLiveSqlMaskingThroughQueryAndMutation() throws Exception { + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + SQLEntityDescriptor descriptor = new SQLEntityDescriptor(); + descriptor.setType("Task"); + descriptor.setTargetType(Task.class); + descriptor.setEntitySupplier(Task::new); + descriptor.setDataService("postgres"); + descriptor.setAuditMaskFields(List.of("title")); + ((io.teaql.core.sql.GenericSQLProperty) descriptor.addSimpleProperty("id", Long.class)).setColumnType("BIGINT"); + ((io.teaql.core.sql.GenericSQLProperty) descriptor.addSimpleProperty("version", Long.class)).setColumnType("BIGINT"); + ((io.teaql.core.sql.GenericSQLProperty) descriptor.addSimpleProperty("title", String.class)).setColumnType("VARCHAR(200)"); + ((io.teaql.core.sql.GenericSQLProperty) descriptor.addSimpleProperty("status", String.class)).setColumnType("VARCHAR(50)"); + metadata.register(descriptor); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (PrintStream output = new PrintStream(bytes, true, StandardCharsets.UTF_8)) { + TeaQLRuntime maskingRuntime = TeaQLRuntime.builder() + .metadata(metadata) + .dataService("postgres", new PostgresDataServiceExecutor("postgres", new JdbcSqlExecutor(dataSource))) + .idGenerationService(new IdSpaceIdGenerator(new JdbcTeaQLDatabase(dataSource))) + .logSink(new io.teaql.runtime.DefaultTextRuntimeLogSink(output)) + .build(); + UserContext maskingContext = new DefaultUserContext(maskingRuntime); + maskingContext.ensureSchema(); + String status = "1 Runtime Road " + UUID.randomUUID().toString().substring(0, 8); + Task task = new Task(); + task.updateTitle("Riverside").updateStatus(status); + task.auditAs("what: create masked task").save(maskingContext); + SmartList rows = new TaskRequest().filterByTitle("Riverside") + .filterByStatus(status) + .comment("what: read masked task") + .purpose("why: verify live-provider SQL masking") + .executeForList(maskingContext); + assertEquals(1, rows.size()); + assertEquals("Riverside", rows.get(0).getTitle()); + assertEquals(status, rows.get(0).getStatus()); + } + String logged = bytes.toString(StandardCharsets.UTF_8); + assertTrue(logged, logged.contains("INSERT")); + assertTrue(logged, logged.contains("SELECT")); + assertTrue(logged, logged.contains("Ri*****de")); + assertTrue(logged, logged.contains("1 Runtime Road")); + assertTrue(logged, logged.contains("what: read masked task")); + assertTrue(logged, logged.contains("why: verify live-provider SQL masking")); + assertFalse(logged, logged.contains("Riverside")); + } + @Test public void testPostgresCrud() { // 1. Create and Save Tasks diff --git a/teaql-postgres/src/test/java/io/teaql/postgres/PostgresTemporalSqlTest.java b/teaql-postgres/src/test/java/io/teaql/postgres/PostgresTemporalSqlTest.java index 448be6dc..bad5cf69 100644 --- a/teaql-postgres/src/test/java/io/teaql/postgres/PostgresTemporalSqlTest.java +++ b/teaql-postgres/src/test/java/io/teaql/postgres/PostgresTemporalSqlTest.java @@ -12,8 +12,13 @@ public class PostgresTemporalSqlTest { @Test public void preparedAndDiagnosticSqlAreEquivalent() throws Exception { String url = System.getenv("TEAQL_TEST_POSTGRES_URL"); - Assume.assumeTrue(url != null && !url.isBlank()); - try (Connection connection = DriverManager.getConnection(url)) { + String user = System.getenv("TEAQL_TEST_POSTGRES_USER"); + String password = System.getenv("TEAQL_TEST_POSTGRES_PASSWORD"); + boolean configured = url != null && !url.isBlank() && user != null && password != null; + if (!configured && Boolean.parseBoolean(System.getenv("TEAQL_REQUIRE_LIVE_DB"))) + fail("PostgreSQL temporal live gate requires URL, USER, and PASSWORD environment variables"); + Assume.assumeTrue(configured); + try (Connection connection = DriverManager.getConnection(url, user, password)) { run(connection, "DROP TABLE IF EXISTS teaql_temporal_java_fixture"); run(connection, "CREATE TABLE teaql_temporal_java_fixture(id INTEGER, d DATE, local_time TIMESTAMP(3))"); String sql = "INSERT INTO teaql_temporal_java_fixture VALUES (?, ?, ?) /* ignored ? */"; diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java index 7000a96f..97b3ac3b 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java @@ -54,16 +54,17 @@ public Stream queryForStream(String sql, Map params, SqlR @Override public Stream> queryForStream(String sql, Object[] params) { + StreamResources resources = new StreamResources(); try { - Connection connection = openConnection(); - PreparedStatement ps = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + Connection connection = resources.connection = openConnection(); + PreparedStatement ps = resources.statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); ps.setFetchSize(200); if (params != null) { for (int i = 0; i < params.length; i++) { bind(ps, i + 1, params[i]); } } - ResultSet rs = ps.executeQuery(); + ResultSet rs = resources.resultSet = ps.executeQuery(); String[] columnLabels = columnLabels(rs); java.util.Iterator> iterator = new java.util.Iterator<>() { private boolean ready; @@ -76,11 +77,11 @@ public boolean hasNext() { hasNext = rs.next(); ready = true; } catch (SQLException e) { - closeResources(rs, ps, connection); + resources.close(); throw new RuntimeException(e); } } - if (!hasNext) closeResources(rs, ps, connection); + if (!hasNext) resources.close(); return hasNext; } @@ -95,22 +96,36 @@ public Map next() { } return row; } catch (SQLException e) { - closeResources(rs, ps, connection); + resources.close(); throw new RuntimeException(e); } } }; return java.util.stream.StreamSupport.stream(java.util.Spliterators.spliteratorUnknownSize(iterator, java.util.Spliterator.ORDERED), false) - .onClose(() -> closeResources(rs, ps, connection)); + .onClose(resources::close); } catch (SQLException e) { + resources.close(); throw new RuntimeException("JDBC streaming query failed", e); + } catch (RuntimeException | Error failure) { + resources.close(); + throw failure; } } - private static void closeResources(ResultSet resultSet, PreparedStatement statement, Connection connection) { - try { resultSet.close(); } catch (SQLException ignored) { } - try { statement.close(); } catch (SQLException ignored) { } - try { connection.close(); } catch (SQLException ignored) { } + private static final class StreamResources { + private Connection connection; + private PreparedStatement statement; + private ResultSet resultSet; + private boolean closed; + + private void close() { + if (closed) return; + closed = true; + // Retain the adapter's existing best-effort cleanup policy, including partially acquired resources. + try { if (resultSet != null) resultSet.close(); } catch (SQLException ignored) { } + try { if (statement != null) statement.close(); } catch (SQLException ignored) { } + try { if (connection != null) connection.close(); } catch (SQLException ignored) { } + } } @Override diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java index a02d6663..cab8d8a2 100644 --- a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java @@ -179,6 +179,38 @@ public void testQueryForStreamReadsLazilyAndSupportsEarlyClose() { assertEquals(5, total.intValue()); } + @Test + public void streamingPrepareFailureClosesAcquiredConnection() throws Exception { + failedStreamClosesConnection("SELECT * FROM table_does_not_exist"); + } + + @Test + public void streamingExecuteFailureClosesAcquiredConnection() throws Exception { + failedStreamClosesConnection("SELECT id FROM test_user WHERE id = ?"); + } + + private void failedStreamClosesConnection(String sql) throws Exception { + var acquired = new ArrayList(); + var tracking = (DataSource) java.lang.reflect.Proxy.newProxyInstance( + DataSource.class.getClassLoader(), new Class[]{DataSource.class}, (proxy, method, args) -> { + try { + var value = method.invoke(dataSource, args); + if (value instanceof Connection connection) acquired.add(connection); + return value; + } catch (java.lang.reflect.InvocationTargetException failure) { + throw failure.getCause(); + } + }); + try { + org.junit.Assert.assertThrows(RuntimeException.class, + () -> new JdbcSqlExecutor(tracking).queryForStream(sql, new Object[0])); + assertEquals(1, acquired.size()); + assertTrue("failed stream leaked its connection", acquired.get(0).isClosed()); + } finally { + for (var connection : acquired) connection.close(); + } + } + private static class SimpleDataSource implements DataSource { private final String url; private final String user; diff --git a/teaql-query-json/src/main/java/io/teaql/query/json/DynamicSearchHelper.java b/teaql-query-json/src/main/java/io/teaql/query/json/DynamicSearchHelper.java index 20c7e894..26052790 100644 --- a/teaql-query-json/src/main/java/io/teaql/query/json/DynamicSearchHelper.java +++ b/teaql-query-json/src/main/java/io/teaql/query/json/DynamicSearchHelper.java @@ -115,12 +115,11 @@ protected void warnUnknownField(BaseRequest request, String clause, String field warnings.add(warning); LOGGER.log( Level.WARNING, - "Ignored unknown dynamic search field: code={0}, requestType={1}, clause={2}, fieldPath={3}", + "Ignored unknown dynamic search field: code={0}, requestType={1}, clause={2}, fieldPath=", new Object[] { warning.getCode(), warning.getRequestType(), - warning.getClause(), - warning.getFieldPath() + warning.getClause() }); } diff --git a/teaql-query-json/src/main/java/io/teaql/query/json/LocalDynamicSearch.java b/teaql-query-json/src/main/java/io/teaql/query/json/LocalDynamicSearch.java index 9c533a5c..d5e90c12 100644 --- a/teaql-query-json/src/main/java/io/teaql/query/json/LocalDynamicSearch.java +++ b/teaql-query-json/src/main/java/io/teaql/query/json/LocalDynamicSearch.java @@ -145,7 +145,7 @@ private static void emit(List warnings, Consumer warn) { for (Warning warning : warnings) { if (warn != null) warn.accept(warning); else LOG.warning(() -> warning.code() + " entity=" + warning.entity() - + " clause=" + warning.clause() + " fieldPath=" + warning.fieldPath()); + + " clause=" + warning.clause() + " fieldPath="); } } diff --git a/teaql-query-json/src/test/java/io/teaql/query/json/DynamicSearchHelperTest.java b/teaql-query-json/src/test/java/io/teaql/query/json/DynamicSearchHelperTest.java index a1fae953..bb961fa6 100644 --- a/teaql-query-json/src/test/java/io/teaql/query/json/DynamicSearchHelperTest.java +++ b/teaql-query-json/src/test/java/io/teaql/query/json/DynamicSearchHelperTest.java @@ -9,10 +9,42 @@ import io.teaql.core.BaseRequest; import io.teaql.core.Entity; import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; import org.junit.Test; public class DynamicSearchHelperTest { + @Test + public void legacyWarningDefaultLogOmitsUntrustedPathButRetainsStructuredWarning() { + String fieldPath = "CLIENT_SECRET_FIELD_PATH_91"; + Logger logger = Logger.getLogger(DynamicSearchHelper.class.getName()); + AtomicInteger count = new AtomicInteger(); + Handler handler = new Handler() { + @Override public void publish(LogRecord record) { + String message = java.text.MessageFormat.format(record.getMessage(), record.getParameters()); + assertTrue(message.contains("DYNAMIC_SEARCH_UNKNOWN_FIELD")); + assertTrue(message.contains("fieldPath=")); + assertTrue(!message.contains(fieldPath)); + assertTrue(!message.contains("SECRET_VALUE_99")); + count.incrementAndGet(); + } + @Override public void flush() {} + @Override public void close() {} + }; + logger.addHandler(handler); + try { + StubRequest request = new StubRequest("Order"); + new DynamicSearchHelper().mergeClauses(request, + DynamicSearchHelper.jsonFromString( + "{\"" + fieldPath + "\":\"SECRET_VALUE_99\"}")); + assertWarning(request, "FILTER", fieldPath); + assertEquals(1, count.get()); + } finally { logger.removeHandler(handler); } + } + @Test public void invalidPayloadPreservesExistingQueryAndWarnings() { StubRequest request = new StubRequest("Order"); diff --git a/teaql-query-json/src/test/java/io/teaql/query/json/LocalDynamicSearchTest.java b/teaql-query-json/src/test/java/io/teaql/query/json/LocalDynamicSearchTest.java index 4b753161..b8207f0a 100644 --- a/teaql-query-json/src/test/java/io/teaql/query/json/LocalDynamicSearchTest.java +++ b/teaql-query-json/src/test/java/io/teaql/query/json/LocalDynamicSearchTest.java @@ -96,12 +96,14 @@ private static final class SchoolRequest extends BaseRequest { } @Test public void warningLoggingIsEnabledWithoutCallback() { + String fieldPath = "CLIENT_SECRET_FIELD_PATH_91"; Logger logger = Logger.getLogger(LocalDynamicSearch.class.getName()); AtomicInteger count = new AtomicInteger(); Handler handler = new Handler() { public void publish(LogRecord record) { assertFalse(record.getMessage().contains("SECRET")); assertTrue(record.getMessage().contains("DYNAMIC_SEARCH_UNKNOWN_FIELD")); + assertTrue(record.getMessage().contains("fieldPath=")); count.incrementAndGet(); } public void flush() {} @@ -109,7 +111,9 @@ public void close() {} }; logger.addHandler(handler); try { - LocalDynamicSearch.normalize("{\"filter\":{\"removed\":\"SECRET\"}}", "School", MODELS, null); + var result = LocalDynamicSearch.normalize( + "{\"filter\":{\"" + fieldPath + "\":\"SECRET\"}}", "School", MODELS, null); + assertEquals(fieldPath, result.warnings().get(0).fieldPath()); assertEquals(1, count.get()); } finally { logger.removeHandler(handler); } } diff --git a/teaql-runtime-log/src/main/java/io/teaql/runtime/log/HumanReaderFormatter.java b/teaql-runtime-log/src/main/java/io/teaql/runtime/log/HumanReaderFormatter.java index f780143b..3f597241 100644 --- a/teaql-runtime-log/src/main/java/io/teaql/runtime/log/HumanReaderFormatter.java +++ b/teaql-runtime-log/src/main/java/io/teaql/runtime/log/HumanReaderFormatter.java @@ -28,17 +28,11 @@ public String formatExecutionLog(io.teaql.core.ExecutionMetadata metadata) { String traceStr = formatTraceChain(metadata.getTraceChain()); String traceDisplay = traceStr.isEmpty() ? "" : " - [" + traceStr + "]"; - String parameterized = metadata.getParameterizedQuery() == null ? "" : metadata.getParameterizedQuery().replace('\n', ' '); - String execution = String.format("[%s]-[%5dµs]-[DEBUG]-ExecutionLog%s - [%s] comment=%s purpose=%s auditReason=%s\n Parameterized SQL: %s", + return String.format("[%s]-[%5dµs]-[SQL]-ExecutionLog%s - [%s] outcome=%s comment=%s purpose=%s auditReason=%s\n SQL: %s", ts, metadata.getElapsedUs(), traceDisplay, metadata.getResultSummary(), + metadata.getExecutionOutcome() == null ? "unknown" : metadata.getExecutionOutcome(), metadata.getComment(), metadata.getPurpose(), metadata.getAuditReason(), - parameterized); - if (metadata.getDebugQuery() == null) { - return execution; - } - String debug = metadata.getDebugQuery().replace('\n', ' '); - return execution + String.format(" params=%s\n Debug SQL: %s", - metadata.getParameters(), debug); + metadata.getDebugQuery() == null ? "" : metadata.getDebugQuery()); } @Override diff --git a/teaql-runtime-log/src/main/java/io/teaql/runtime/log/JsonReaderFormatter.java b/teaql-runtime-log/src/main/java/io/teaql/runtime/log/JsonReaderFormatter.java index e437b081..d7ef7a54 100644 --- a/teaql-runtime-log/src/main/java/io/teaql/runtime/log/JsonReaderFormatter.java +++ b/teaql-runtime-log/src/main/java/io/teaql/runtime/log/JsonReaderFormatter.java @@ -18,30 +18,29 @@ private String formatTraceChain(List traceChain) { private String escapeJson(String text) { if (text == null) return ""; - return text.replace("\"", "\\\"").replace("\n", "\\n"); + StringBuilder result = new StringBuilder(); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '\\' || c == '"') result.append('\\').append(c); + else if (c < 32) result.append(String.format("\\u%04x", (int) c)); + else result.append(c); + } + return result.toString(); } @Override public String formatExecutionLog(io.teaql.core.ExecutionMetadata metadata) { metadata = io.teaql.runtime.LogPrivacy.sql(metadata, io.teaql.runtime.LogPrivacy.plaintextEnabled()); - String execution = String.format("{\"type\":\"EXEC_LOG\",\"tracePath\":%s,\"backend\":\"%s\",\"operation\":\"%s\",\"comment\":\"%s\",\"purpose\":\"%s\",\"auditReason\":\"%s\",\"elapsedUs\":%d,\"resultCount\":%s,\"affectedRows\":%s,\"summary\":\"%s\",\"parameterizedSQL\":\"%s\"", + return String.format("{\"type\":\"EXEC_LOG\",\"tracePath\":%s,\"backend\":\"%s\",\"operation\":\"%s\",\"comment\":\"%s\",\"purpose\":\"%s\",\"auditReason\":\"%s\",\"elapsedUs\":%d,\"resultCount\":%s,\"affectedRows\":%s,\"summary\":\"%s\",\"sql\":\"%s\",\"logMode\":\"%s\",\"maskedParameters\":%s,\"sqlOmissionReason\":\"%s\",\"executionOutcome\":\"%s\"}", formatTraceChain(metadata.getTraceChain()), - escapeJson(metadata.getBackend()), - metadata.getOperation(), - escapeJson(metadata.getComment()), - escapeJson(metadata.getPurpose()), - escapeJson(metadata.getAuditReason()), - metadata.getElapsedUs(), - metadata.getResultCount(), - metadata.getAffectedRows(), - escapeJson(metadata.getResultSummary()), - escapeJson(metadata.getParameterizedQuery())); - if (metadata.getDebugQuery() == null) { - return execution + "}"; - } - return execution + String.format(",\"parameters\":\"%s\",\"debugSQL\":\"%s\"}", - escapeJson(String.valueOf(metadata.getParameters())), - escapeJson(metadata.getDebugQuery())); + escapeJson(metadata.getBackend()), metadata.getOperation(), + escapeJson(metadata.getComment()), escapeJson(metadata.getPurpose()), + escapeJson(metadata.getAuditReason()), metadata.getElapsedUs(), + metadata.getResultCount(), metadata.getAffectedRows(), + escapeJson(metadata.getResultSummary()), escapeJson(metadata.getDebugQuery()), + escapeJson(metadata.getLogMode()), metadata.getParameterMasked(), + escapeJson(metadata.getSqlOmissionReason()), + escapeJson(metadata.getExecutionOutcome() == null ? "unknown" : metadata.getExecutionOutcome())); } @Override diff --git a/teaql-runtime-log/src/test/java/io/teaql/runtime/log/ExecutionLogPrivacyTest.java b/teaql-runtime-log/src/test/java/io/teaql/runtime/log/ExecutionLogPrivacyTest.java index 6322de76..48c8dee4 100644 --- a/teaql-runtime-log/src/test/java/io/teaql/runtime/log/ExecutionLogPrivacyTest.java +++ b/teaql-runtime-log/src/test/java/io/teaql/runtime/log/ExecutionLogPrivacyTest.java @@ -13,7 +13,7 @@ public class ExecutionLogPrivacyTest { private static final String SECRET = "customer-secret"; @Test - public void safeFormattersOmitValueBearingFields() { + public void safeFormattersExpandMaskedValuesWithoutSeparateParameterArrays() { ExecutionMetadata metadata = metadata(); // A custom provider may populate parameters without rendered SQL. // The ordinary formatters must still keep those values private. @@ -22,11 +22,15 @@ public void safeFormattersOmitValueBearingFields() { String human = new HumanReaderFormatter().formatExecutionLog(metadata); String json = new JsonReaderFormatter().formatExecutionLog(metadata); - assertTrue(human.contains("WHERE name = ?")); + assertTrue(human.contains("WHERE name = '[REDACTED]' /* masked */")); + assertFalse(human.contains("WHERE name = ?")); assertFalse(human.contains("params=")); assertFalse(human.contains("Debug SQL:")); assertFalse(human.contains(SECRET)); - assertTrue(json.contains("\"parameterizedSQL\"")); + assertTrue(json.contains("\"sql\"")); + assertTrue(json.contains("WHERE name = '[REDACTED]' /* masked */")); + assertTrue(json.contains("\"maskedParameters\":[true]")); + assertFalse(json.contains("\"parameterizedSQL\"")); assertFalse(json.contains("\"parameters\"")); assertFalse(json.contains("\"debugSQL\"")); assertFalse(json.contains(SECRET)); @@ -62,4 +66,38 @@ private ExecutionMetadata metadata() { metadata.setParameterizedQuery("SELECT id FROM customer_data WHERE name = ?"); return metadata; } + + @Test + public void failureOutcomeAndUnknownCountsSurviveBothFormatters() { + var metadata = metadata(); + metadata.setParameters(List.of(SECRET)); + metadata.setExecutionOutcome("failure"); + metadata.setResultSummary("Statement did not complete; row count unknown"); + String human = new HumanReaderFormatter().formatExecutionLog(metadata); + String json = new JsonReaderFormatter().formatExecutionLog(metadata); + assertTrue(human.contains("outcome=failure")); + assertTrue(json.contains("\"executionOutcome\":\"failure\"")); + assertTrue(json.contains("\"resultCount\":null,\"affectedRows\":null")); + assertFalse(human.contains(SECRET)); + assertFalse(json.contains(SECRET)); + } + + @Test + public void copiedDebugIntentIsHiddenByBothFormatters() { + var copy = new ExecutionMetadata(); + copy.setBackend("sqlite"); copy.setGeneratedSql(true); + copy.setParameterizedQuery("SELECT name FROM customer_data WHERE id=? LIMIT 10000"); + copy.setParameters(List.of(1L)); + copy.setParameterLogPolicies(List.of(io.teaql.core.SqlParameterLogPolicy.PLAIN)); + copy.setLogMode("DEBUG PLAINTEXT"); + copy.setComment("what: reload " + SECRET); copy.setPurpose("why: check " + SECRET); + copy.setAuditReason("persist " + SECRET); + copy.setTraceChain(List.of(new io.teaql.core.TraceNode(SECRET))); + for (String text : List.of(new HumanReaderFormatter().formatExecutionLog(copy), + new JsonReaderFormatter().formatExecutionLog(copy))) { + assertFalse(text, text.contains(SECRET)); + assertTrue(text, text.contains("LIMIT 10000")); + assertTrue(text, text.contains("[REDACTED]")); + } + } } diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/DefaultTextRuntimeLogSink.java b/teaql-runtime/src/main/java/io/teaql/runtime/DefaultTextRuntimeLogSink.java index 33c3fbfa..53210b01 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/DefaultTextRuntimeLogSink.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/DefaultTextRuntimeLogSink.java @@ -7,7 +7,7 @@ import java.util.List; import java.util.stream.IntStream; -/** Default operator log. It deliberately excludes bind values and rendered Debug SQL. */ +/** Default operator log, showing safe expanded SQL rather than a separate bind array. */ public class DefaultTextRuntimeLogSink implements RuntimeLogSink { protected final PrintStream output; @@ -28,13 +28,14 @@ public boolean requiresSensitiveSqlData() { public void writeExecutionLog(UserContext context, ExecutionMetadata metadata) { metadata = LogPrivacy.sql(metadata, false); output.printf( - "[TeaQL SQL][%s][%dus] %s comment=%s purpose=%s auditReason=%s tracePath=%s%n" - + "Parameterized SQL: %s%n", + "[TeaQL SQL][%s][%dus] %s outcome=%s comment=%s purpose=%s auditReason=%s tracePath=%s%n" + + "SQL: %s%n", metadata.getOperation() == null ? "unknown" : metadata.getOperation().name().toLowerCase(), metadata.getElapsedUs(), resultSummary(metadata), + metadata.getExecutionOutcome() == null ? "unknown" : metadata.getExecutionOutcome(), nullToEmpty(metadata.getComment()), nullToEmpty(metadata.getPurpose()), nullToEmpty(metadata.getAuditReason()), formatTrace(metadata.getTraceChain()), - nullToEmpty(metadata.getParameterizedQuery())); + nullToEmpty(metadata.getDebugQuery())); } protected static String formatTrace(List nodes) { diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/LogPrivacy.java b/teaql-runtime/src/main/java/io/teaql/runtime/LogPrivacy.java index c1fe3751..66a2c95c 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/LogPrivacy.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/LogPrivacy.java @@ -2,6 +2,8 @@ import io.teaql.core.ExecutionMetadata; import io.teaql.core.TraceNode; +import io.teaql.core.SqlParameterLogPolicy; +import io.teaql.core.utils.SqlLogRenderer; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -11,6 +13,7 @@ public final class LogPrivacy { public static final String ACKNOWLEDGEMENT = "I_UNDERSTAND_SENSITIVE_DATA_MAY_BE_WRITTEN_TO_DISK"; public static final String REDACTED = "[REDACTED]"; private static final AtomicBoolean WARNED = new AtomicBoolean(); + private static final String DEBUG_LABEL = "-- TeaQL DEBUG PLAINTEXT; EXPLICIT OPT-IN\n"; private LogPrivacy() {} public static boolean plaintextEnabled() { @@ -21,16 +24,16 @@ public static boolean plaintextEnabled() { } public static boolean credential(String name) { - String key = Objects.toString(name, "").replaceAll("[^a-zA-Z0-9]", "").toLowerCase(Locale.ROOT); - return List.of("password", "passwd", "passphrase", "privatekey", "secret", "accesstoken", - "refreshtoken", "idtoken", "apikey", "authorization", "credential", "sessiontoken", "magiclinktoken") - .stream().anyMatch(key::contains); + return io.teaql.core.utils.SensitiveLogNames.credential(name); } public static boolean hasCredentials(Object value) { if (value instanceof Map map) return map.entrySet().stream() .anyMatch(e -> credential(String.valueOf(e.getKey())) || hasCredentials(e.getValue())); if (value instanceof Iterable items) for (Object item : items) if (hasCredentials(item)) return true; + if (value != null && value.getClass().isArray()) + for (int i = 0; i < java.lang.reflect.Array.getLength(value); i++) + if (hasCredentials(java.lang.reflect.Array.get(value, i))) return true; return false; } @@ -38,6 +41,11 @@ private static void collect(Object value, List result) { if (value == null) return; if (value instanceof Map map) { map.values().forEach(item -> collect(item, result)); return; } if (value instanceof Iterable items) { items.forEach(item -> collect(item, result)); return; } + if (value.getClass().isArray()) { + for (int i = 0; i < java.lang.reflect.Array.getLength(value); i++) + collect(java.lang.reflect.Array.get(value, i), result); + return; + } String text = String.valueOf(value); if (!text.isEmpty()) result.add(text); } @@ -58,26 +66,129 @@ public static List trace(List nodes, Collection values) } public static ExecutionMetadata sql(ExecutionMetadata source, boolean allow) { - boolean reveal = allow && !credential(source.getParameterizedQuery()) - && !credential(source.getDebugQuery()) && !hasCredentials(source.getParameters()); - List secrets = reveal ? List.of() : source.getParameters(); + String mode = source.getLogMode(); + boolean debugSource = (mode != null && mode.startsWith("DEBUG")) + || (source.getDebugQuery() != null && source.getDebugQuery().startsWith(DEBUG_LABEL)); + if (Set.of("PLAIN", "MASKED", "OMITTED").contains(mode == null ? "" : mode)) allow = false; + boolean orphanedDebug = false; + if (!allow && debugSource) { + ExecutionMetadata remembered = SqlLogProjectionCache.safeCopy(source); + if (remembered != null) return remembered; + orphanedDebug = true; + } + List policies = source.getParameterLogPolicies(); + boolean invalidPolicies = !policies.isEmpty() && policies.size() != source.getParameterCount(); + boolean invalidFlags = !source.getParameterMasked().isEmpty() + && source.getParameterMasked().size() != source.getParameterCount(); + boolean legacyCredential = (policies.isEmpty() || !source.isGeneratedSql()) && (credential(source.getParameterizedQuery()) + || credential(source.getDebugQuery())); + List values = new ArrayList<>(); + List masked = new ArrayList<>(); + List secrets = new ArrayList<>(); + for (int i = 0; i < source.getParameterCount(); i++) { + Object raw = source.getParameters().get(i); + var policy = invalidPolicies || invalidFlags || policies.isEmpty() ? SqlParameterLogPolicy.UNKNOWN : policies.get(i); + boolean forced = legacyCredential || policy == SqlParameterLogPolicy.CREDENTIAL || hasCredentials(raw); + boolean alreadyMasked = source.getParameterMasked().size() == source.getParameterCount() + && source.getParameterMasked().get(i); + boolean hide = forced || invalidPolicies || alreadyMasked || policy == SqlParameterLogPolicy.UNKNOWN + || (!allow && policy != SqlParameterLogPolicy.PLAIN); + masked.add(hide); + if (hide) { + secrets.add(raw); + values.add(raw == null ? null : !forced && policy == SqlParameterLogPolicy.MASKED + && (raw instanceof CharSequence || raw instanceof Number || raw instanceof Boolean + || raw instanceof java.time.temporal.Temporal) + ? TeaQLRuntime.maskAuditValue(raw.toString()) : REDACTED); + } else values.add(copyValue(raw)); + } + if (source.getIntentRedactions() != null) source.getIntentRedactions().appendTo(secrets, allow); ExecutionMetadata safe = new ExecutionMetadata(); safe.setBackend(source.getBackend()); safe.setOperation(source.getOperation()); + safe.setExecutionOutcome(source.getExecutionOutcome()); safe.setStartedAt(source.getStartedAt()); safe.setEndedAt(source.getEndedAt()); safe.setElapsedUs(source.getElapsedUs()); safe.setAffectedRows(source.getAffectedRows()); - safe.setResultCount(source.getResultCount()); safe.setResultSummary(scrub(source.getResultSummary(), secrets)); + safe.setResultCount(source.getResultCount()); + safe.setResultSummary(source.getResultCount() != null ? source.getResultCount() + " rows returned" + : source.getAffectedRows() != null ? source.getAffectedRows() + " rows affected" + : scrub(source.getResultSummary(), secrets)); safe.setBackendRequestId(scrub(source.getBackendRequestId(), secrets)); safe.setComment(scrub(source.getComment(), secrets)); safe.setPurpose(scrub(source.getPurpose(), secrets)); safe.setAuditReason(scrub(source.getAuditReason(), secrets)); safe.setTraceChain(trace(source.getTraceChain(), secrets)); + if (orphanedDebug) { + safe.setComment(hideIntent(source.getComment())); safe.setPurpose(hideIntent(source.getPurpose())); + safe.setAuditReason(hideIntent(source.getAuditReason())); + safe.setBackendRequestId(hideIntent(source.getBackendRequestId())); + if (source.getResultCount() == null && source.getAffectedRows() == null) + safe.setResultSummary(hideIntent(source.getResultSummary())); + safe.setTraceChain(source.getTraceChain() == null ? List.of() : source.getTraceChain().stream() + .map(node -> new TraceNode(node.getKind(), hideIntent(node.getName()), hideIntent(node.getComment()))).toList()); + } String sql = source.getParameterizedQuery(); - if (!reveal && sql != null && (sql.matches("(?s).*[0-9'\"`$].*") || sql.contains("--") || sql.contains("/*"))) - sql = "[REDACTED SQL; NOT REPLAYABLE]"; safe.setParameterizedQuery(sql); - safe.setParameters(reveal ? source.getParameters() : Collections.nCopies(source.getParameterCount(), null)); - safe.setDebugQuery(reveal ? source.getDebugQuery() : null); + safe.setParameters(values); safe.setParameterMasked(masked); + safe.setParameterLogPolicies(policies); safe.setGeneratedSql(source.isGeneratedSql()); + boolean anyMasked = masked.contains(true); + safe.setLogMode(allow ? anyMasked ? "DEBUG PARTIALLY MASKED" : "DEBUG PLAINTEXT" : anyMasked ? "MASKED" : "PLAIN"); + boolean unsafeInline = !source.isGeneratedSql() && sql != null + && (sql.matches("(?s).*[0-9'\"`$].*") || sql.contains("--") || sql.contains("/*")); + String omission = sql == null ? "missing_sql" : invalidPolicies ? "policy_count_mismatch" + : invalidFlags ? "mask_count_mismatch" + : unsafeInline && (!allow || credential(sql)) ? "untrusted_inline_sql" : null; + if (source.getSqlOmissionReason() != null) { + omission = Set.of("missing_sql", "policy_count_mismatch", "mask_count_mismatch", "untrusted_inline_sql", + "unsupported_literal_or_binding_mismatch").contains(source.getSqlOmissionReason()) + ? source.getSqlOmissionReason() : "unavailable_sql"; + } + String rendered = null; + if (omission == null) { + try { + rendered = SqlLogRenderer.render(sql, values.size(), i -> + SqlLogRenderer.literal(values.get(i), source.getBackend()) + + (masked.get(i) ? " /* masked */" : ""), source.getBackend()); + } catch (IllegalArgumentException failure) { + omission = "unsupported_literal_or_binding_mismatch"; + } + } + if (omission != null) { + safe.setLogMode("OMITTED"); + safe.setSqlOmissionReason(omission); + safe.setParameterizedQuery("[REDACTED SQL; NOT REPLAYABLE]"); + safe.setDebugQuery("-- TeaQL SQL OMITTED; NOT REPLAYABLE; reason=" + omission); + } else { + String header = allow ? DEBUG_LABEL : "-- TeaQL " + safe.getLogMode() + "\n"; + if (anyMasked) header += (allow ? "-- DEBUG PARTIALLY MASKED; " : "-- ") + "NOT REPLAYABLE; masked_parameters=" + + masked.stream().filter(Boolean::booleanValue).count() + "\n"; + safe.setDebugQuery(header + rendered); + } + if (allow && safe.getLogMode().startsWith("DEBUG")) + SqlLogProjectionCache.remember(safe, sql(source, false)); return safe; } + private static String hideIntent(String value) { return value == null || value.isEmpty() ? value : REDACTED; } + + static Object copyValue(Object value) { + if (value instanceof Map map) { + Map copy = new LinkedHashMap<>(); + map.forEach((key, item) -> copy.put(key, copyValue(item))); + return Collections.unmodifiableMap(copy); + } + if (value instanceof Iterable items) { + List copy = new ArrayList<>(); + items.forEach(item -> copy.add(copyValue(item))); + return Collections.unmodifiableList(copy); + } + if (value != null && value.getClass().isArray()) { + int count = java.lang.reflect.Array.getLength(value); + Object copy = java.lang.reflect.Array.newInstance(value.getClass().getComponentType(), count); + for (int i = 0; i < count; i++) java.lang.reflect.Array.set(copy, i, copyValue(java.lang.reflect.Array.get(value, i))); + return copy; + } + if (value instanceof java.util.Date date) return date.clone(); + return value; + } + public static RawAuditEvent audit(RawAuditEvent source, boolean allow) { List secrets = new ArrayList<>(); List changes = source.changes().stream().map(change -> { @@ -87,8 +198,10 @@ public static RawAuditEvent audit(RawAuditEvent source, boolean allow) { return new AuditFieldChange(change.field(), change.oldValue() == null ? null : REDACTED, change.newValue() == null ? null : REDACTED); }).toList(); + List intentValues = new ArrayList<>(secrets); + if (source.entityId() != null) intentValues.add(source.entityId()); return new RawAuditEvent(source.kind(), source.entityType(), source.entityId(), changes, - trace(source.traceChain(), secrets), scrub(source.actor(), secrets), source.category(), - scrub(source.reason(), secrets), source.resultingVersion(), source.occurredAt()); + trace(source.traceChain(), intentValues), scrub(source.actor(), intentValues), source.category(), + scrub(source.reason(), intentValues), source.resultingVersion(), source.occurredAt()); } } diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/SensitiveDiagnosticTextRuntimeLogSink.java b/teaql-runtime/src/main/java/io/teaql/runtime/SensitiveDiagnosticTextRuntimeLogSink.java index c0c712c8..51eabb48 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/SensitiveDiagnosticTextRuntimeLogSink.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/SensitiveDiagnosticTextRuntimeLogSink.java @@ -29,13 +29,13 @@ public boolean requiresSensitiveSqlData() { public void writeExecutionLog(UserContext context, ExecutionMetadata metadata) { metadata = LogPrivacy.sql(metadata, LogPrivacy.plaintextEnabled()); output.printf( - "[TeaQL SQL][%s][%dus] %s comment=%s purpose=%s auditReason=%s tracePath=%s%n" - + "Parameterized SQL: %s params=%s%nDebug SQL: %s%n", + "[TeaQL SQL][%s][%dus] %s outcome=%s comment=%s purpose=%s auditReason=%s tracePath=%s%n" + + "SQL: %s%n", metadata.getOperation() == null ? "unknown" : metadata.getOperation().name().toLowerCase(), metadata.getElapsedUs(), resultSummary(metadata), + metadata.getExecutionOutcome() == null ? "unknown" : metadata.getExecutionOutcome(), nullToEmpty(metadata.getComment()), nullToEmpty(metadata.getPurpose()), nullToEmpty(metadata.getAuditReason()), formatTrace(metadata.getTraceChain()), - nullToEmpty(metadata.getParameterizedQuery()), metadata.getParameters(), nullToEmpty(metadata.getDebugQuery())); } } diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/SqlLogProjectionCache.java b/teaql-runtime/src/main/java/io/teaql/runtime/SqlLogProjectionCache.java new file mode 100644 index 00000000..08ed28c7 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/SqlLogProjectionCache.java @@ -0,0 +1,79 @@ +package io.teaql.runtime; + +import io.teaql.core.ExecutionMetadata; +import io.teaql.core.utils.SqlLogRenderer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collections; +import java.util.HexFormat; +import java.util.Map; +import java.util.WeakHashMap; + +/** Private debug downgrade state: a fingerprint and already-safe values, never raw provenance. */ +final class SqlLogProjectionCache { + private record Entry(String fingerprint, ExecutionMetadata safe) {} + private static final Map ENTRIES = Collections.synchronizedMap(new WeakHashMap<>()); + + static void remember(ExecutionMetadata debug, ExecutionMetadata safe) { + String fingerprint = fingerprint(debug); + if (fingerprint != null) ENTRIES.put(debug, new Entry(fingerprint, copy(safe))); + } + + static ExecutionMetadata safeCopy(ExecutionMetadata debug) { + Entry entry = ENTRIES.get(debug); + return entry != null && entry.fingerprint().equals(fingerprint(debug)) ? copy(entry.safe()) : null; + } + + private static String fingerprint(ExecutionMetadata m) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + add(digest, m.getBackend()); add(digest, m.getOperation()); + add(digest, m.getStartedAt()); add(digest, m.getEndedAt()); add(digest, m.getElapsedUs()); + add(digest, m.getAffectedRows()); add(digest, m.getResultCount()); add(digest, m.getResultSummary()); + add(digest, m.getBackendRequestId()); add(digest, m.getParameterizedQuery()); add(digest, m.getDebugQuery()); + add(digest, m.getComment()); add(digest, m.getPurpose()); add(digest, m.getAuditReason()); + add(digest, m.getParameterLogPolicies()); add(digest, m.getParameterMasked()); + add(digest, m.isGeneratedSql()); add(digest, m.getLogMode()); + add(digest, m.getSqlOmissionReason()); add(digest, m.getExecutionOutcome()); + add(digest, m.getParameterCount()); + for (Object value : m.getParameters()) add(digest, SqlLogRenderer.literal(value, m.getBackend())); + add(digest, m.getTraceChain() == null ? -1 : m.getTraceChain().size()); + if (m.getTraceChain() != null) for (var node : m.getTraceChain()) { + add(digest, node.getKind()); add(digest, node.getName()); add(digest, node.getComment()); + } + return HexFormat.of().formatHex(digest.digest()); + } catch (IllegalArgumentException unsupported) { + // Unsupported values cannot establish identity; downgrade conservatively. + return null; + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 unavailable", impossible); + } + } + + private static void add(MessageDigest digest, Object value) { + if (value == null) { digest.update((byte) 0); return; } + digest.update((byte) 1); + byte[] bytes = value.toString().getBytes(StandardCharsets.UTF_8); + int size = bytes.length; + digest.update((byte) (size >>> 24)); digest.update((byte) (size >>> 16)); + digest.update((byte) (size >>> 8)); digest.update((byte) size); + digest.update(bytes); + } + + private static ExecutionMetadata copy(ExecutionMetadata m) { + var c = new ExecutionMetadata(); + c.setBackend(m.getBackend()); c.setOperation(m.getOperation()); + c.setStartedAt(m.getStartedAt()); c.setEndedAt(m.getEndedAt()); c.setElapsedUs(m.getElapsedUs()); + c.setAffectedRows(m.getAffectedRows()); c.setResultCount(m.getResultCount()); + c.setResultSummary(m.getResultSummary()); c.setBackendRequestId(m.getBackendRequestId()); + c.setParameterizedQuery(m.getParameterizedQuery()); c.setDebugQuery(m.getDebugQuery()); + c.setParameters(m.getParameters().stream().map(LogPrivacy::copyValue).toList()); + c.setParameterLogPolicies(m.getParameterLogPolicies()); c.setParameterMasked(m.getParameterMasked()); + c.setGeneratedSql(m.isGeneratedSql()); c.setLogMode(m.getLogMode()); + c.setSqlOmissionReason(m.getSqlOmissionReason()); c.setExecutionOutcome(m.getExecutionOutcome()); + c.setComment(m.getComment()); c.setPurpose(m.getPurpose()); c.setAuditReason(m.getAuditReason()); + c.setTraceChain(m.getTraceChain() == null ? null : java.util.List.copyOf(m.getTraceChain())); + return c; + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 96e3ab9d..91a16c0f 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -127,7 +127,13 @@ GeneratedSchemaBootstrap getGeneratedSchemaBootstrap() { public void recordExecutionMetadata(UserContext context, ExecutionMetadata metadata) { if (logSink != null) { - logSink.writeExecutionLog(context, LogPrivacy.sql(metadata, requiresSensitiveSqlLogData())); + try { + logSink.writeExecutionLog(context, + LogPrivacy.sql(metadata, requiresSensitiveSqlLogData())); + } catch (RuntimeException ignored) { + // Diagnostics are fail-open. Never print the exception: a custom + // sink failure may itself contain the unmasked SQL or values. + } } } @@ -904,7 +910,10 @@ private SafeAuditEvent buildSafeAuditEvent(RawAuditEvent event) { boolean masked = raw != null && ((!allowPlaintext && maskFields.contains(change.field())) || LogPrivacy.credential(change.field()) || LogPrivacy.hasCredentials(change.oldValue()) || LogPrivacy.hasCredentials(change.newValue())); - String safe = masked ? LogPrivacy.REDACTED : LogPrivacy.scrub(raw, sensitiveValues); + boolean credential = LogPrivacy.credential(change.field()) + || LogPrivacy.hasCredentials(change.oldValue()) || LogPrivacy.hasCredentials(change.newValue()); + String safe = masked ? (credential ? LogPrivacy.REDACTED : maskAuditValue(raw)) + : LogPrivacy.scrub(raw, sensitiveValues); int rawLength = raw == null ? 0 : raw.length(); boolean truncated = safe != null && maxLength != null && safe.length() > maxLength; if (truncated) safe = limitAuditValue(safe, maxLength); @@ -913,17 +922,19 @@ private SafeAuditEvent buildSafeAuditEvent(RawAuditEvent event) { raw == null ? null : rawLength, safe == null ? null : safe.length())); } + List intentValues = new ArrayList<>(sensitiveValues); + if (event.entityId() != null) intentValues.add(event.entityId()); return new SafeAuditEvent( - event.kind(), event.entityType(), event.entityId(), fields, LogPrivacy.trace(event.traceChain(), sensitiveValues)); + event.kind(), event.entityType(), event.entityId(), fields, LogPrivacy.trace(event.traceChain(), intentValues)); } static String maskAuditValue(String value) { if (value == null || value.isEmpty()) return value; - if (value.chars().allMatch(Character::isDigit)) return "*".repeat(value.length()); - if (value.length() < 8) return "*".repeat(value.length()); - return value.substring(0, 2) - + "*".repeat(value.length() - 4) - + value.substring(value.length() - 2); + int length = value.codePointCount(0, value.length()); + if (length < 8 || value.codePoints().allMatch(c -> c >= '0' && c <= '9')) return "*".repeat(length); + return value.substring(0, value.offsetByCodePoints(0, 2)) + + "*".repeat(length - 4) + + value.substring(value.offsetByCodePoints(0, length - 2)); } static String limitAuditValue(String value, int maxLength) { diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/LogPrivacyTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/LogPrivacyTest.java index 6890332b..d443534c 100644 --- a/teaql-runtime/src/test/java/io/teaql/runtime/LogPrivacyTest.java +++ b/teaql-runtime/src/test/java/io/teaql/runtime/LogPrivacyTest.java @@ -32,7 +32,9 @@ public class LogPrivacyTest { public static void main(String[] args) { var sink = new SensitiveDiagnosticTextRuntimeLogSink(System.out); - sink.writeExecutionLog(null, entry("name", "PRIVATE-CUSTOMER-CANARY")); + var business = entry("name", "PRIVATE-CUSTOMER-CANARY"); + business.setParameterLogPolicies(List.of(io.teaql.core.SqlParameterLogPolicy.MASKED)); + sink.writeExecutionLog(null, business); sink.writeExecutionLog(null, entry("password", "PASSWORD-CANARY")); } @@ -41,7 +43,8 @@ private static String childClasspath() throws Exception { paths.add(System.getProperty("surefire.test.class.path", System.getProperty("java.class.path"))); // Surefire places named runtime modules on a module path, not necessarily // in java.class.path. Locate the actually loaded local classes explicitly. - for (Class type : List.of(LogPrivacyTest.class, LogPrivacy.class, ExecutionMetadata.class)) + for (Class type : List.of(LogPrivacyTest.class, LogPrivacy.class, ExecutionMetadata.class, + io.teaql.core.utils.SqlLogRenderer.class)) paths.add(java.nio.file.Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toString()); return String.join(java.io.File.pathSeparator, paths); } @@ -59,17 +62,33 @@ private static String childClasspath() throws Exception { assertEquals(newValue, source.changes().get(0).newValue()); } + @Test public void allMutationKindsScrubTargetIdFromFreeTextOnly() { + for (MutationAuditKind kind : MutationAuditKind.values()) { + var source = new RawAuditEvent(kind, "Order", 1001L, List.of(), + List.of(new TraceNode("change order 1001")), "operator", "mutation", + "change order 1001", 2L, null); + var safe = LogPrivacy.audit(source, false); + assertEquals(kind.name(), 1001L, safe.entityId()); + assertEquals(kind.name(), "change order [REDACTED]", safe.traceChain().get(0).getComment()); + assertEquals(kind.name(), "change order [REDACTED]", safe.reason()); + assertEquals(kind.name(), "change order 1001", source.reason()); + } + } + @Test public void projectionIsIndependentAndCredentialsStayHidden() { ExecutionMetadata source = entry("name", "PRIVATE-CUSTOMER-CANARY"); ExecutionMetadata safe = LogPrivacy.sql(source, false); - assertNull(safe.getParameters().get(0)); + assertEquals(LogPrivacy.REDACTED, safe.getParameters().get(0)); assertFalse(safe.getComment().contains("PRIVATE-CUSTOMER-CANARY")); assertFalse(safe.getTraceChain().toString().contains("PRIVATE-CUSTOMER-CANARY")); assertEquals("PRIVATE-CUSTOMER-CANARY", source.getParameters().get(0)); + assertEquals(List.of(LogPrivacy.REDACTED), LogPrivacy.sql(source, true).getParameters()); + source.setParameterLogPolicies(List.of(io.teaql.core.SqlParameterLogPolicy.MASKED)); assertEquals(source.getParameters(), LogPrivacy.sql(source, true).getParameters()); ExecutionMetadata credential = LogPrivacy.sql(entry("password", "PASSWORD-CANARY"), true); - assertNull(credential.getParameters().get(0)); - assertNull(credential.getDebugQuery()); + assertEquals(LogPrivacy.REDACTED, credential.getParameters().get(0)); + assertTrue(credential.getDebugQuery().contains("/* masked */")); + assertFalse(credential.getDebugQuery().contains("PASSWORD-CANARY")); } @Test public void directFileSinkCannotLeakCredentials() throws Exception { @@ -86,6 +105,25 @@ private static String childClasspath() throws Exception { assertEquals("[REDACTED SQL; NOT REPLAYABLE]", LogPrivacy.sql(source, false).getParameterizedQuery()); } + @Test public void mutatedDebugRecordIsSafeInDefaultFileSink() throws Exception { + var source = entry("name", "PRIVATE-CUSTOMER-CANARY"); + source.setParameterLogPolicies(List.of(io.teaql.core.SqlParameterLogPolicy.MASKED)); + source.setGeneratedSql(true); + var debug = LogPrivacy.sql(source, true); + debug.setParameterizedQuery("SELECT id FROM customer WHERE id=? LIMIT 10000"); + debug.setParameters(List.of(1L)); + debug.setParameterLogPolicies(List.of(io.teaql.core.SqlParameterLogPolicy.PLAIN)); + var file = Files.createTempFile("teaql-debug-downgrade-", ".log"); + try (var out = new PrintStream(Files.newOutputStream(file))) { + new DefaultTextRuntimeLogSink(out).writeExecutionLog(null, debug); + } + var text = Files.readString(file); + assertFalse(text, text.contains("PRIVATE-CUSTOMER-CANARY")); + assertTrue(text, text.contains("LIMIT 10000")); + assertTrue(text, text.contains("[REDACTED]")); + assertTrue(debug.getComment().contains("PRIVATE-CUSTOMER-CANARY")); + } + private static ExecutionMetadata entry(String field, String value) { var metadata = new ExecutionMetadata(); metadata.setParameterizedQuery("select * from customer where " + field + " = ?"); diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/MaskingContractTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/MaskingContractTest.java new file mode 100644 index 00000000..f34c2c66 --- /dev/null +++ b/teaql-runtime/src/test/java/io/teaql/runtime/MaskingContractTest.java @@ -0,0 +1,100 @@ +package io.teaql.runtime; + +import io.teaql.core.ExecutionMetadata; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.List; +import org.junit.Test; +import org.junit.Rule; +import org.junit.rules.ErrorCollector; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** Acceptance contract, not a snapshot of the current implementation. */ +public class MaskingContractTest { + @Rule public ErrorCollector errors = new ErrorCollector(); + private static final String[][] CASES = { + {"", ""}, {"Ada", "***"}, {"12345678", "********"}, + {"ABCDEFGH", "AB****GH"}, {"Riverside", "Ri*****de"}, {"O'Reilly", "O'****ly"} + }; + + @Test public void legacyMaskAlgorithmMustRemainAvailable() { + for (var c : CASES) assertEquals(c[0], c[1], TeaQLRuntime.maskAuditValue(c[0])); + } + + @Test public void sharedUnicodeGoldenVectors() throws Exception { + var fixture = java.nio.file.Path.of("../test-vectors/masking-v1.tsv"); + for (String line : java.nio.file.Files.readAllLines(fixture).subList(1, 11)) { + var values = line.split("\t", -1); + errors.checkThat(values[0], TeaQLRuntime.maskAuditValue(values[1]), is(values[2])); + } + } + + @Test public void defaultSqlMustRemainExpandedWithUnknownParametersMasked() { + for (var c : CASES) { + var source = entry(c[0]); + var bytes = new ByteArrayOutputStream(); + new DefaultTextRuntimeLogSink(new PrintStream(bytes)).writeExecutionLog(null, source); + String log = bytes.toString(java.nio.charset.StandardCharsets.UTF_8); + assertEquals(c[0], source.getParameters().get(0)); + // No field policy is attached: unknown provenance must not expose a prefix/suffix. + errors.checkThat(log, log.contains("name = '"), is(true)); + errors.checkThat(log, log.toLowerCase().contains("masked"), is(true)); + errors.checkThat(log, log.contains("name = ?"), is(false)); + errors.checkThat(log, log.contains("[REDACTED SQL"), is(false)); + if (!c[0].isEmpty()) errors.checkThat(log, log.contains("'" + c[0].replace("'", "''") + "'"), is(false)); + if (c[0].length() >= 8 && !c[1].startsWith("*")) errors.checkThat(log, log.contains(c[1].replace("'", "''")), is(false)); + } + } + + @Test public void debugSinkMustLabelEveryPlaintextRecord() throws Exception { + var process = new ProcessBuilder( + java.nio.file.Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", childClasspath(), MaskingContractTest.class.getName()); + process.environment().put(LogPrivacy.ENVIRONMENT, LogPrivacy.ACKNOWLEDGEMENT); + var output = java.nio.file.Files.createTempFile("mask-contract-", ".log"); + try { + var child = process.redirectErrorStream(true).redirectOutput(output.toFile()).start(); + if (!child.waitFor(30, java.util.concurrent.TimeUnit.SECONDS)) { + child.destroyForcibly(); fail("log child timed out"); + } + String text = java.nio.file.Files.readString(output); + assertEquals(text, 0, child.exitValue()); + String[] records = text.split("RECORD_BOUNDARY", -1); + assertEquals(text, 3, records.length); + for (int i = 1; i < records.length; i++) { + errors.checkThat(records[i], records[i].contains("'Riverside'"), is(true)); + errors.checkThat(records[i], records[i].toUpperCase().contains("DEBUG"), is(true)); + errors.checkThat(records[i], records[i].toUpperCase().contains("PLAINTEXT"), is(true)); + } + } finally { java.nio.file.Files.deleteIfExists(output); } + } + + public static void main(String[] args) { + var sink = new SensitiveDiagnosticTextRuntimeLogSink(System.out); + for (int i = 0; i < 2; i++) { + System.out.println("RECORD_BOUNDARY"); + var business = entry("Riverside"); + business.setParameterLogPolicies(List.of(io.teaql.core.SqlParameterLogPolicy.MASKED)); + sink.writeExecutionLog(null, business); + } + } + + private static String childClasspath() throws Exception { + var paths = new java.util.ArrayList(); + paths.add(System.getProperty("surefire.test.class.path", System.getProperty("java.class.path"))); + for (Class type : List.of(MaskingContractTest.class, LogPrivacy.class, ExecutionMetadata.class, + io.teaql.core.utils.SqlLogRenderer.class)) + paths.add(java.nio.file.Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toString()); + return String.join(java.io.File.pathSeparator, paths); + } + + private static ExecutionMetadata entry(String value) { + var m = new ExecutionMetadata(); + m.setParameterizedQuery("UPDATE customer SET name = ?"); + m.setParameters(List.of(value)); + m.setDebugQuery("UPDATE customer SET name = '" + value.replace("'", "''") + "'"); + m.setComment("what: edit customer"); m.setPurpose("why: verify mask contract"); + return m; + } +} diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/SqlInheritedIntentTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/SqlInheritedIntentTest.java new file mode 100644 index 00000000..1118ab3d --- /dev/null +++ b/teaql-runtime/src/test/java/io/teaql/runtime/SqlInheritedIntentTest.java @@ -0,0 +1,85 @@ +package io.teaql.runtime; + +import io.teaql.core.*; +import java.util.*; +import org.junit.Test; +import static org.junit.Assert.*; + +public class SqlInheritedIntentTest { + private ExecutionMetadata readback() { + var source = new SqlIntentRedactions(); + source.capture(List.of(SqlParameterLogPolicy.MASKED, SqlParameterLogPolicy.PLAIN, + SqlParameterLogPolicy.CREDENTIAL, SqlParameterLogPolicy.UNKNOWN), + new Object[]{"Riverside", "PublicAddress", "PASSWORD-CANARY", "UNKNOWN-CANARY"}); + var m = new ExecutionMetadata(); + m.setBackend("sqlite"); m.setGeneratedSql(true); + m.setParameterizedQuery("SELECT * FROM customer WHERE id = ? LIMIT 10000"); + m.setParameters(List.of(1L)); m.setParameterLogPolicies(List.of(SqlParameterLogPolicy.PLAIN)); + m.setIntentRedactions(source); + m.setComment("what: Riverside PublicAddress PASSWORD-CANARY UNKNOWN-CANARY"); + m.setPurpose(m.getComment()); m.setAuditReason(m.getComment()); + m.setTraceChain(List.of(new TraceNode(TraceKind.AUDIT_REASON, "Customer", m.getComment()))); + m.setResultCount(1); + return m; + } + + @Test public void safeAndDebugViewsClearRawProvenance() { + var m = readback(); + for (boolean debug : new boolean[]{false, true}) { + var safe = LogPrivacy.sql(m, debug); + assertNull(safe.getIntentRedactions()); + assertEquals(debug, safe.getAuditReason().contains("Riverside")); + assertTrue(safe.getAuditReason().contains("PublicAddress")); + assertFalse(safe.getTraceChain().toString().contains("CANARY")); + assertTrue(safe.getDebugQuery().contains("id = 1 LIMIT 10000")); + assertEquals(Integer.valueOf(1), safe.getResultCount()); + } + assertTrue(m.getAuditReason().contains("Riverside")); + assertFalse(m.getIntentRedactions().toString().contains("Riverside")); + } + + @Test public void retainedDebugDowngradesWithoutLosingSafeIntent() { + var debug = LogPrivacy.sql(readback(), true); + var safe = LogPrivacy.sql(debug, false); + assertEquals("what: [REDACTED] PublicAddress [REDACTED] [REDACTED]", safe.getAuditReason()); + assertNull(safe.getIntentRedactions()); + assertEquals(safe.getAuditReason(), LogPrivacy.sql(LogPrivacy.sql(debug, true), false).getAuditReason()); + } + + @Test public void nestedCredentialsAndMismatchedPoliciesStayHiddenInDebug() { + var m = readback(); + var intent = new SqlIntentRedactions(); + intent.capture(List.of(SqlParameterLogPolicy.PLAIN), new Object[]{Map.of("api_key", "NESTED-CANARY")}); + intent.capture(List.of(), new Object[]{"MISMATCH-CANARY"}); + intent.capture(List.of(SqlParameterLogPolicy.MASKED), new Object[]{new java.math.BigDecimal("12.3400")}); + m.setIntentRedactions(intent); + m.setAuditReason("NESTED-CANARY MISMATCH-CANARY 12.3400"); + assertEquals("[REDACTED] [REDACTED] [REDACTED]", LogPrivacy.sql(m, false).getAuditReason()); + assertEquals("[REDACTED] [REDACTED] 12.3400", LogPrivacy.sql(m, true).getAuditReason()); + } + + @Test public void mutationTargetIdMasksIntentButKeepsPlainBindingAndRowCount() { + var source = new SqlIntentRedactions(); + source.captureTargetId(1L); + var m = new ExecutionMetadata(); + m.setBackend("sqlite"); m.setGeneratedSql(true); + m.setParameterizedQuery("UPDATE customer SET version = version + 1 WHERE id = ?"); + m.setParameters(List.of(1L)); m.setParameterLogPolicies(List.of(SqlParameterLogPolicy.PLAIN)); + m.setIntentRedactions(source); + m.setAuditReason("what: update customer 1"); + m.setComment("what: update customer 1"); + m.setTraceChain(List.of(new TraceNode(TraceKind.AUDIT_REASON, "Customer", "what: update customer 1"))); + m.setAffectedRows(1L); + for (boolean debug : new boolean[]{false, true}) { + var safe = LogPrivacy.sql(m, debug); + assertEquals("what: update customer [REDACTED]", safe.getAuditReason()); + assertEquals("what: update customer [REDACTED]", safe.getComment()); + assertEquals("what: update customer [REDACTED]", safe.getTraceChain().get(0).getComment()); + assertEquals("1 rows affected", safe.getResultSummary()); + assertEquals(List.of(1L), safe.getParameters()); + assertTrue(safe.getDebugQuery().contains("id = 1")); + assertNull(safe.getIntentRedactions()); + } + assertEquals("what: update customer 1", m.getAuditReason()); + } +} diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/SqlMaskingPolicyTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/SqlMaskingPolicyTest.java new file mode 100644 index 00000000..5506b50a --- /dev/null +++ b/teaql-runtime/src/test/java/io/teaql/runtime/SqlMaskingPolicyTest.java @@ -0,0 +1,237 @@ +package io.teaql.runtime; + +import io.teaql.core.ExecutionMetadata; +import io.teaql.core.SqlParameterLogPolicy; +import io.teaql.core.TraceNode; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.junit.Test; +import static org.junit.Assert.*; + +public class SqlMaskingPolicyTest { + private ExecutionMetadata mixed() { + var m = new ExecutionMetadata(); + m.setBackend("sqlite"); + m.setGeneratedSql(true); + m.setParameterizedQuery("UPDATE \"customer\" SET name = ?, active = ?, password = ? WHERE id = ?"); + m.setParameters(List.of("Riverside", true, "PASSWORD-CANARY", 1L)); + m.setParameterLogPolicies(List.of(SqlParameterLogPolicy.MASKED, SqlParameterLogPolicy.PLAIN, + SqlParameterLogPolicy.CREDENTIAL, SqlParameterLogPolicy.PLAIN)); + m.setComment("what: replace PASSWORD-CANARY"); + m.setPurpose("why: edit customer"); + m.setTraceChain(List.of(new TraceNode("PASSWORD-CANARY"))); + m.setAffectedRows(1L); + return m; + } + + @Test public void mixedPoliciesPreserveOrdinaryValuesAndIntentButNotCredentials() { + var original = mixed(); + var safe = LogPrivacy.sql(original, false); + assertTrue(safe.getDebugQuery().contains("name = 'Ri*****de' /* masked */")); + assertTrue(safe.getDebugQuery().contains("active = TRUE")); + assertTrue(safe.getDebugQuery().contains("password = '[REDACTED]' /* masked */ WHERE id = 1")); + assertFalse(safe.getDebugQuery().contains("Riverside")); + assertFalse(safe.getDebugQuery().contains("PASSWORD-CANARY")); + assertFalse(safe.getComment().contains("PASSWORD-CANARY")); + assertFalse(safe.getTraceChain().toString().contains("PASSWORD-CANARY")); + assertEquals("why: edit customer", safe.getPurpose()); + assertEquals("1 rows affected", safe.getResultSummary()); + assertEquals(List.of(true, false, true, false), safe.getParameterMasked()); + assertEquals(List.of("Riverside", true, "PASSWORD-CANARY", 1L), original.getParameters()); + assertNull(original.getDebugQuery()); + } + + @Test public void repeatedProjectionDoesNotUpgradeMaskedValuesOrDuplicateHeaders() { + var once = LogPrivacy.sql(mixed(), false); + var twice = LogPrivacy.sql(once, false); + assertEquals(once.getDebugQuery(), twice.getDebugQuery()); + assertEquals(once.getParameters(), twice.getParameters()); + var upgrade = LogPrivacy.sql(twice, true); + assertFalse(upgrade.getDebugQuery().contains("Riverside")); + assertTrue(upgrade.getDebugQuery().contains("NOT REPLAYABLE")); + } + + @Test public void debugIsPerRecordAndStillProtectsCredentials() { + for (int i = 0; i < 2; i++) { + var safe = LogPrivacy.sql(mixed(), true); + assertTrue(safe.getDebugQuery().contains("DEBUG PLAINTEXT; EXPLICIT OPT-IN")); + assertTrue(safe.getDebugQuery().contains("NOT REPLAYABLE")); + assertTrue(safe.getDebugQuery().contains("name = 'Riverside'")); + assertFalse(safe.getDebugQuery().contains("PASSWORD-CANARY")); + assertEquals("DEBUG PARTIALLY MASKED", safe.getLogMode()); + } + } + + @Test public void policyAndBindMismatchesHaveSafeReasons() { + var m = mixed(); + m.setParameterLogPolicies(List.of(SqlParameterLogPolicy.PLAIN)); + var safe = LogPrivacy.sql(m, true); + assertEquals("policy_count_mismatch", safe.getSqlOmissionReason()); + assertFalse(safe.getParameters().toString().contains("PASSWORD-CANARY")); + m = mixed(); m.setParameterizedQuery("SELECT ?"); + safe = LogPrivacy.sql(m, false); + assertEquals("unsupported_literal_or_binding_mismatch", safe.getSqlOmissionReason()); + assertFalse(safe.getDebugQuery().contains("PASSWORD-CANARY")); + } + + @Test public void unknownNestedCredentialAndNullValuesRemainSafe() { + var m = new ExecutionMetadata(); + m.setParameterizedQuery("SELECT ?, ?, ?"); + m.setParameters(Arrays.asList(null, new Object[]{Map.of("access_token", "NESTED-TOKEN-CANARY")}, "Riverside")); + var safe = LogPrivacy.sql(m, false); + assertTrue(safe.getDebugQuery().contains("NULL /* masked */")); + assertFalse(safe.getDebugQuery().contains("Ri*****de")); + assertFalse(safe.getDebugQuery().contains("NESTED-TOKEN-CANARY")); + var debug = LogPrivacy.sql(m, true); + assertFalse(debug.getDebugQuery().contains("Riverside")); + assertFalse(debug.getDebugQuery().contains("NESTED-TOKEN-CANARY")); + } + + @Test public void binaryLogCopiesCannotMutateDriverParameters() { + var m = new ExecutionMetadata(); + byte[] original = {1, 2}; + m.setParameterizedQuery("SELECT ?"); + m.setParameters(List.of(original)); + m.setParameterLogPolicies(List.of(SqlParameterLogPolicy.PLAIN)); + var safe = LogPrivacy.sql(m, false); + assertTrue(safe.getDebugQuery().contains("X'0102'")); + ((byte[]) safe.getParameters().get(0))[0] = 42; + assertArrayEquals(new byte[]{1, 2}, original); + } + + @Test public void unknownBindingsStayPrivateWithDebugOptIn() { + for (var policies : List.of(List.of(), List.of(SqlParameterLogPolicy.UNKNOWN))) { + var raw = new ExecutionMetadata(); + raw.setParameterizedQuery("SELECT ?"); + raw.setParameters(List.of("UNKNOWN-BINDING-CANARY")); + raw.setParameterLogPolicies(policies); + raw.setComment("what: locate UNKNOWN-BINDING-CANARY"); + var safe = LogPrivacy.sql(raw, true); + assertFalse(safe.getDebugQuery().contains("UNKNOWN-BINDING-CANARY")); + assertFalse(safe.getComment().contains("UNKNOWN-BINDING-CANARY")); + assertEquals(List.of("[REDACTED]"), safe.getParameters()); + assertEquals(List.of(true), safe.getParameterMasked()); + assertTrue(safe.getDebugQuery().contains("SELECT")); + assertTrue(safe.getDebugQuery().contains("NOT REPLAYABLE")); + assertEquals(List.of("UNKNOWN-BINDING-CANARY"), raw.getParameters()); + } + } + + @Test public void debugOnlyExposesExplicitBusinessPolicies() { + var raw = new ExecutionMetadata(); + raw.setParameterizedQuery("SELECT ?, ?, ?, ?"); + raw.setParameters(List.of("Ordinary", "Riverside", "UNKNOWN-BINDING-CANARY", "CREDENTIAL-CANARY")); + raw.setParameterLogPolicies(List.of(SqlParameterLogPolicy.PLAIN, SqlParameterLogPolicy.MASKED, + SqlParameterLogPolicy.UNKNOWN, SqlParameterLogPolicy.CREDENTIAL)); + var safe = LogPrivacy.sql(raw, true); + assertEquals(List.of("Ordinary", "Riverside", "[REDACTED]", "[REDACTED]"), safe.getParameters()); + assertEquals(List.of(false, false, true, true), safe.getParameterMasked()); + } + + @Test public void untrustedInlineCredentialCannotEscapeUnderDebug() { + var m = mixed(); m.setGeneratedSql(false); + m.setParameterizedQuery("SELECT 'INLINE-SECRET-CANARY' AS password, ?, ?, ?, ?"); + var safe = LogPrivacy.sql(m, true); + assertEquals("untrusted_inline_sql", safe.getSqlOmissionReason()); + assertFalse(safe.getDebugQuery().contains("INLINE-SECRET-CANARY")); + assertFalse(safe.getParameterizedQuery().contains("INLINE-SECRET-CANARY")); + var twice = LogPrivacy.sql(safe, false); + assertEquals(safe.getSqlOmissionReason(), twice.getSqlOmissionReason()); + assertEquals(safe.getDebugQuery(), twice.getDebugQuery()); + assertEquals("OMITTED", twice.getLogMode()); + } + + @Test public void safeProjectionCannotBeRelabeledAsDebugPlaintext() { + var safe = LogPrivacy.sql(mixed(), false); + assertEquals(safe.getDebugQuery(), LogPrivacy.sql(safe, true).getDebugQuery()); + assertEquals(safe.getLogMode(), LogPrivacy.sql(safe, true).getLogMode()); + } + + @Test public void invalidMaskFlagsFailClosedInsteadOfTrustingPlainPolicy() { + var raw = mixed(); + raw.setParameterLogPolicies(List.of(SqlParameterLogPolicy.PLAIN, SqlParameterLogPolicy.PLAIN, + SqlParameterLogPolicy.PLAIN, SqlParameterLogPolicy.PLAIN)); + raw.setParameterMasked(List.of(true)); + var safe = LogPrivacy.sql(raw, true); + assertEquals("mask_count_mismatch", safe.getSqlOmissionReason()); + assertFalse(safe.getParameters().toString().contains("CANARY")); + assertFalse(safe.getParameters().toString().contains("Riverside")); + assertEquals(safe.getDebugQuery(), LogPrivacy.sql(safe, false).getDebugQuery()); + } + + @Test public void debugRevocationPreservesOriginalSafeIntent() { + var raw = mixed(); + raw.setAuditReason("what: persist Riverside PASSWORD-CANARY"); + var debug = LogPrivacy.sql(raw, true); + var safe = LogPrivacy.sql(debug, false); + assertEquals("what: persist [REDACTED] [REDACTED]", safe.getAuditReason()); + assertEquals("why: edit customer", safe.getPurpose()); + // A caller may mutate a returned record; this must not corrupt cached safety. + safe.setAuditReason("Riverside"); + assertEquals("what: persist [REDACTED] [REDACTED]", LogPrivacy.sql(debug, false).getAuditReason()); + } + + @Test public void modifiedDebugRecordCannotLeakInheritedIntent() { + var raw = mixed(); + raw.setAuditReason("what: persist Riverside"); + var debug = LogPrivacy.sql(raw, true); + // A derived ID-only request no longer carries the original secret bindings. + debug.setParameterizedQuery("SELECT name FROM customer WHERE id=? LIMIT 10000"); + debug.setParameters(List.of(1L)); + debug.setParameterLogPolicies(List.of(SqlParameterLogPolicy.PLAIN)); + debug.setParameterMasked(List.of(false)); + var safe = LogPrivacy.sql(debug, false); + assertFalse(safe.getAuditReason().contains("Riverside")); + assertEquals("[REDACTED]", safe.getAuditReason()); + assertEquals(debug.getParameterizedQuery(), safe.getParameterizedQuery()); + assertTrue(safe.getDebugQuery().contains("LIMIT 10000")); + assertEquals(List.of(1L), safe.getParameters()); + } + + @Test public void copiedDebugRecordWithoutPrivateProvenanceHidesIntent() { + var copy = new ExecutionMetadata(); + copy.setBackend("sqlite"); copy.setGeneratedSql(true); + copy.setLogMode("DEBUG PLAINTEXT"); + copy.setParameterizedQuery("SELECT name FROM customer WHERE id=? LIMIT 10000"); + copy.setParameters(List.of(1L)); + copy.setParameterLogPolicies(List.of(SqlParameterLogPolicy.PLAIN)); + copy.setComment("what: reload Riverside"); copy.setPurpose("why: confirm Riverside"); + copy.setAuditReason("persist Riverside"); copy.setBackendRequestId("Riverside"); + copy.setTraceChain(List.of(new TraceNode("Riverside"))); + var safe = LogPrivacy.sql(copy, false); + for (String text : List.of(safe.getComment(), safe.getPurpose(), safe.getAuditReason(), + safe.getBackendRequestId(), safe.getTraceChain().toString())) assertFalse(text, text.contains("Riverside")); + assertTrue(safe.getDebugQuery().contains("LIMIT 10000")); + copy.setLogMode(null); + copy.setDebugQuery("-- TeaQL DEBUG PLAINTEXT; EXPLICIT OPT-IN\nSELECT name FROM customer WHERE id=1 LIMIT 10000"); + assertEquals("[REDACTED]", LogPrivacy.sql(copy, false).getAuditReason()); + } + + @Test public void debugFallbackCopiesMutableBinaryAndNullableBindings() { + var raw = mixed(); + raw.setParameterizedQuery("SELECT ?, ?, ?"); + raw.setParameters(Arrays.asList("Riverside", new byte[]{1, 2}, null)); + raw.setParameterLogPolicies(List.of(SqlParameterLogPolicy.MASKED, SqlParameterLogPolicy.PLAIN, SqlParameterLogPolicy.PLAIN)); + raw.setAuditReason("Riverside"); + var debug = LogPrivacy.sql(raw, true); + var safe = LogPrivacy.sql(debug, false); + ((byte[]) safe.getParameters().get(1))[0] = 42; + assertArrayEquals(new byte[]{1, 2}, (byte[]) LogPrivacy.sql(debug, false).getParameters().get(1)); + assertNull(safe.getParameters().get(2)); + ((byte[]) debug.getParameters().get(1))[0] = 99; + assertEquals("[REDACTED]", LogPrivacy.sql(debug, false).getAuditReason()); + assertArrayEquals(new byte[]{1, 2}, (byte[]) raw.getParameters().get(1)); + } + + @Test public void numericAndTableNameParametersDoNotChangeSqlStructure() { + for (Object value : List.of(1L, "customer")) { + var raw = mixed(); + raw.setParameterizedQuery("SELECT name FROM customer WHERE name=? LIMIT 10000"); + raw.setParameters(List.of(value)); raw.setParameterLogPolicies(List.of(SqlParameterLogPolicy.MASKED)); + var safe = LogPrivacy.sql(raw, false); + assertEquals(raw.getParameterizedQuery(), LogPrivacy.sql(safe, false).getParameterizedQuery()); + assertEquals(safe.getDebugQuery(), LogPrivacy.sql(safe, false).getDebugQuery()); + } + } +} diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/TeaQLRuntimeTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/TeaQLRuntimeTest.java index f2d02269..1233071f 100644 --- a/teaql-runtime/src/test/java/io/teaql/runtime/TeaQLRuntimeTest.java +++ b/teaql-runtime/src/test/java/io/teaql/runtime/TeaQLRuntimeTest.java @@ -13,10 +13,32 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicInteger; import java.util.Map; public class TeaQLRuntimeTest { + @Test + public void brokenSqlDiagnosticSinkDoesNotFailTheBusinessOperation() { + AtomicInteger attempts = new AtomicInteger(); + RuntimeLogSink broken = (context, projected) -> { + attempts.incrementAndGet(); + Assert.assertFalse(projected.getDebugQuery().contains("PASSWORD-CANARY")); + throw new IllegalStateException("LOG-SINK-FAILURE"); + }; + TeaQLRuntime runtime = TeaQLRuntime.builder() + .metadata(new DummyMetaFactory()).logSink(broken).build(); + DefaultUserContext context = new DefaultUserContext(runtime); + ExecutionMetadata metadata = new ExecutionMetadata(); + metadata.setOperation(DataServiceOperation.QUERY); + metadata.setParameterizedQuery("SELECT id FROM customer WHERE password = ?"); + metadata.setParameters(List.of("PASSWORD-CANARY")); + + context.recordExecutionMetadata(metadata); + + Assert.assertEquals(1, attempts.get()); + } + @Test public void executionLoggingDefaultsOnAndQueryMutationCanBeDisabledIndependently() { TeaQLRuntime defaultRuntime = TeaQLRuntime.builder() @@ -195,7 +217,8 @@ public void executionMetadataRetainsStructuredIntentAndThreeLevelTrace() { Assert.assertEquals(TraceKind.PROVIDER, recorded.getTraceChain().get(5).getKind()); Assert.assertEquals(TraceKind.SQL, recorded.getTraceChain().get(6).getKind()); Assert.assertEquals("SELECT name FROM school_data WHERE id = ?", recorded.getParameterizedQuery()); - Assert.assertNull(recorded.getDebugQuery()); + Assert.assertTrue(recorded.getDebugQuery().contains("WHERE id = '[REDACTED]' /* masked */")); + Assert.assertFalse(recorded.getDebugQuery().contains("WHERE id = 7")); } public static class DummyMetaFactory implements EntityMetaFactory { @@ -819,7 +842,7 @@ public EntityDescriptor resolveEntityDescriptor(String type) { DummyEntity entity = new DummyEntity(); entity.updateProperty("name", "private-value"); - entity.auditAs("create audited entity").save(context); + entity.auditAs("create audited entity 700").save(context); Assert.assertEquals(1, standardSink.auditEvents.size()); RawAuditEvent raw = standardSink.auditEvents.get(0); @@ -830,13 +853,18 @@ public EntityDescriptor resolveEntityDescriptor(String type) { .findFirst().orElseThrow().newValue()); Assert.assertTrue(raw.traceChain().stream().anyMatch(node -> node.getKind() == TraceKind.AUDIT_REASON - && "create audited entity".equals(node.getComment()))); + && "create audited entity [REDACTED]".equals(node.getComment()))); Assert.assertEquals(1, appEvents.size()); + Assert.assertEquals(Long.valueOf(700L), appEvents.get(0).entityId()); + Assert.assertTrue(appEvents.get(0).traceChain().stream().anyMatch(node -> + node.getKind() == TraceKind.AUDIT_REASON + && "create audited entity [REDACTED]".equals(node.getComment()))); SafeAuditField safeName = appEvents.get(0).fields().stream() .filter(field -> "name".equals(field.name())) .findFirst().orElseThrow(); Assert.assertTrue(safeName.masked()); Assert.assertNotEquals("private-value", safeName.value()); + Assert.assertEquals("pr*********ue", safeName.value()); } } diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/TextRuntimeLogSinkTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/TextRuntimeLogSinkTest.java index 183452db..f43aa994 100644 --- a/teaql-runtime/src/test/java/io/teaql/runtime/TextRuntimeLogSinkTest.java +++ b/teaql-runtime/src/test/java/io/teaql/runtime/TextRuntimeLogSinkTest.java @@ -15,12 +15,14 @@ public class TextRuntimeLogSinkTest { private static final String SECRET = "customer-secret"; @Test - public void defaultSinkKeepsIntentAndParameterizedSqlButOmitsValues() { + public void defaultSinkKeepsIntentAndExpandedSqlButOmitsSensitiveValues() { String text = render(new DefaultTextRuntimeLogSink(output())); assertTrue(text.contains("comment=what: load customer")); assertTrue(text.contains("purpose=why: render profile")); - assertTrue(text.contains("WHERE name = ?")); + assertTrue(text.contains("WHERE name = '[REDACTED]' /* masked */")); + assertTrue(text.contains("NOT REPLAYABLE")); + assertFalse(text.contains("WHERE name = ?")); assertFalse(text.contains("params=")); assertFalse(text.contains("Debug SQL:")); assertFalse(text.contains(SECRET)); diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/SQLColumnResolver.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/SQLColumnResolver.java index 0c4d943f..86bba550 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/SQLColumnResolver.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/SQLColumnResolver.java @@ -10,6 +10,10 @@ public interface SQLColumnResolver { + default io.teaql.core.SqlParameterLogPolicy parameterLogPolicy(String property) { + return io.teaql.core.SqlParameterLogPolicy.UNKNOWN; + } + default SQLColumn getPropertyColumn(String idTable, String property) { return CollUtil.getFirst(getPropertyColumns(idTable, property)); } diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/SqlParameters.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/SqlParameters.java new file mode 100644 index 00000000..a0920ecd --- /dev/null +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/SqlParameters.java @@ -0,0 +1,34 @@ +package io.teaql.core.sql; + +import io.teaql.core.SqlParameterLogPolicy; +import java.util.HashMap; +import java.util.Map; + +/** Per-compilation bindings; never stored on UserContext or a shared dialect instance. */ +public final class SqlParameters extends HashMap { + private final Map policies = new HashMap<>(); + private SqlParameterLogPolicy currentPolicy = SqlParameterLogPolicy.UNKNOWN; + private boolean generated = true; + + public SqlParameterLogPolicy policy(String name) { + return policies.getOrDefault(name, SqlParameterLogPolicy.UNKNOWN); + } + public SqlParameterLogPolicy currentPolicy() { return currentPolicy; } + public void currentPolicy(SqlParameterLogPolicy policy) { currentPolicy = policy; } + public boolean generated() { return generated; } + public void untrusted() { generated = false; } + + @Override public Object put(String name, Object value) { + policies.remove(name); + return super.put(name, value); + } + @Override public void clear() { policies.clear(); super.clear(); } + @Override public Object remove(Object key) { policies.remove(key); return super.remove(key); } + @Override public void putAll(Map values) { values.forEach(this::put); } + + public static void bind(Map parameters, String name, Object value, + SqlParameterLogPolicy policy) { + parameters.put(name, value); + if (parameters instanceof SqlParameters tracked) tracked.policies.put(name, policy); + } +} diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/ExpressionHelper.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/ExpressionHelper.java index 0543e5f6..625fc19a 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/ExpressionHelper.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/ExpressionHelper.java @@ -9,6 +9,12 @@ public class ExpressionHelper { + private static final java.util.Set> BUILTIN = java.util.Set.of( + ANDExpressionParser.class, AggrExpressionParser.class, BetweenParser.class, + FunctionApplyParser.class, NOTExpressionParser.class, NamedExpressionParser.class, + ORExpressionParser.class, OneOperatorExpressionParser.class, OrderByExpressionParser.class, + OrderBysParser.class, ParameterParser.class, PropertyParser.class, SubQueryParser.class, + TwoOperatorExpressionParser.class, TypeCriteriaParser.class, VersionSearchCriteriaParser.class); public static String toSql( UserContext userContext, @@ -41,6 +47,7 @@ private static String toSqlInternal( return null; } if (expression instanceof SQLExpressionParser) { + if (parameters instanceof io.teaql.core.sql.SqlParameters tracked) tracked.untrusted(); return ((SQLExpressionParser) expression) .toSql(userContext, expression, idTable, parameters, columnResolver); } @@ -58,6 +65,27 @@ private static String toSqlInternal( if (parser == null) { throw new TeaQLRuntimeException("no parse for expression type:" + expression.getClass()); } - return parser.toSql(userContext, expression, idTable, parameters, columnResolver); + if (!(parameters instanceof io.teaql.core.sql.SqlParameters tracked)) + return parser.toSql(userContext, expression, idTable, parameters, columnResolver); + if (!BUILTIN.contains(parser.getClass())) tracked.untrusted(); + var previous = tracked.currentPolicy(); + try { + if (expression instanceof io.teaql.core.criteria.TwoOperatorCriteria + || expression instanceof io.teaql.core.criteria.Between) { + var properties = expression.properties(userContext); + var policy = io.teaql.core.SqlParameterLogPolicy.PLAIN; + if (properties == null || properties.isEmpty()) policy = io.teaql.core.SqlParameterLogPolicy.UNKNOWN; + else for (String property : properties) { + var candidate = columnResolver.parameterLogPolicy(property); + if (rank(candidate) > rank(policy)) policy = candidate; + } + tracked.currentPolicy(policy); + } + return parser.toSql(userContext, expression, idTable, parameters, columnResolver); + } finally { tracked.currentPolicy(previous); } + } + + private static int rank(io.teaql.core.SqlParameterLogPolicy policy) { + return switch (policy) { case PLAIN -> 0; case UNKNOWN -> 1; case MASKED -> 2; case CREDENTIAL -> 3; }; } } diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/ParameterParser.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/ParameterParser.java index da7d5d8b..bb5b034b 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/ParameterParser.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/ParameterParser.java @@ -30,7 +30,11 @@ public String toSql( if (operator != null) { value = fixValue(operator, parameter.getValue()); } - parameters.put(key, value); + var policy = parameters instanceof io.teaql.core.sql.SqlParameters tracked + ? tracked.currentPolicy() : io.teaql.core.SqlParameterLogPolicy.UNKNOWN; + if (io.teaql.core.utils.SensitiveLogNames.credential(parameter.getName())) + policy = io.teaql.core.SqlParameterLogPolicy.CREDENTIAL; + io.teaql.core.sql.SqlParameters.bind(parameters, key, value, policy); return StrUtil.format(":{}", key); } diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java index 0095025b..39bf50db 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java @@ -77,17 +77,18 @@ public QueryResult query(UserContext context, QueryRequest request) { SearchRequest searchRequest = ((DefaultQueryRequest) request).getSearchRequest(); String typeName = searchRequest.getTypeName(); PortableSQLRepository repository = getRepository(typeName); + SqlIntentRedactions intent = SqlDiagnosticRequest.source(context, searchRequest); if (searchRequest.hasSimpleAgg()) { AggregationResult aggregation = - repository.doAggregateInternal(context, (SearchRequest) searchRequest); + repository.doAggregateInternal(context, (SearchRequest) searchRequest, intent); return new DefaultQueryResult(new SmartList<>(), aggregation); } - SmartList result = repository.loadInternal(context, (SearchRequest) searchRequest); + SmartList result = repository.loadInternal(context, (SearchRequest) searchRequest, intent); if (searchRequest.enhanceRelations() != null && !searchRequest.enhanceRelations().isEmpty()) { - enhanceRelations(context, (SmartList) result, searchRequest); + enhanceRelations(context, (SmartList) result, searchRequest, intent); } - attachDynamicAggregations(context, (SmartList) result, searchRequest); + attachDynamicAggregations(context, (SmartList) result, searchRequest, intent); return new DefaultQueryResult((SmartList) result); } @@ -103,7 +104,7 @@ public java.util.stream.Stream queryForStream(UserContext private void attachDynamicAggregations( UserContext userContext, SmartList results, - SearchRequest parentRequest) { + SearchRequest parentRequest, SqlIntentRedactions intent) { List attributes = parentRequest.getDynamicAggregateAttributes(); if (results == null || results.isEmpty() || attributes == null || attributes.isEmpty()) { return; @@ -132,7 +133,7 @@ private void attachDynamicAggregations( PortableSQLRepository aggregateRepository = getRepository(aggregateRequest.getTypeName()); AggregationResult aggregation = - aggregateRepository.doAggregateInternal(userContext, request); + aggregateRepository.doAggregateInternal(userContext, request, intent == null ? null : intent.copy()); if (attribute.isSingleNumber()) { for (Entity parent : parentsById.values()) { parent.addDynamicProperty(attribute.getName(), 0); @@ -172,7 +173,7 @@ private Entity parentByAggregationKey(Map parentsById, Object pare } private void enhanceRelations( - UserContext userContext, SmartList dataSet, SearchRequest request) { + UserContext userContext, SmartList dataSet, SearchRequest request, SqlIntentRedactions intent) { if (dataSet == null || dataSet.isEmpty()) { return; } @@ -188,10 +189,10 @@ private void enhanceRelations( if (!(property instanceof Relation)) return; if (shouldHandle(entityDescriptor, (Relation) property)) { - enhanceParent(userContext, dataSet, (Relation) property, r); + enhanceParent(userContext, dataSet, (Relation) property, r, intent); return; } - collectChildren(userContext, dataSet, (Relation) property, r); + collectChildren(userContext, dataSet, (Relation) property, r, intent); }); } @@ -223,7 +224,7 @@ private void enhanceParent( UserContext userContext, SmartList results, Relation relation, - SearchRequest parentRequest) { + SearchRequest parentRequest, SqlIntentRedactions intent) { List parents = results.stream() .map(e -> e.getProperty(relation.getName())) @@ -233,8 +234,12 @@ private void enhanceParent( .toList(); if (io.teaql.core.utils.ObjectUtil.isEmpty(parents)) return; - io.teaql.core.internal.TempRequest parentTemp = new io.teaql.core.internal.TempRequest(parentRequest); + io.teaql.core.internal.TempRequest parentTemp = new SqlDiagnosticRequest(parentRequest, intent); parentTemp.appendSearchCriteria(parentTemp.createBasicSearchCriteria(BaseEntity.ID_PROPERTY, io.teaql.core.criteria.Operator.IN, parents)); + // This is a framework-owned lookup over the already materialized child page. + // A caller may project the parent without specifying a separate page size, but + // the distinct referenced IDs give this internal query an exact upper bound. + if (parentTemp.getSlice() == null) parentTemp.setSize(parents.size()); SmartList parentItems = userContext.internalExecuteForList(parentTemp); @@ -254,8 +259,8 @@ private void collectChildren( UserContext userContext, SmartList dataSet, Relation relation, - SearchRequest childRequest) { - io.teaql.core.internal.TempRequest childTempRequest = new io.teaql.core.internal.TempRequest(childRequest); + SearchRequest childRequest, SqlIntentRedactions intent) { + io.teaql.core.internal.TempRequest childTempRequest = new SqlDiagnosticRequest(childRequest, intent); PropertyDescriptor reverseProperty = relation.getReverseProperty(); childTempRequest.selectProperty(reverseProperty.getName()); Slice slice = childTempRequest.getSlice(); @@ -270,7 +275,7 @@ private void collectChildren( "probe", dataSet.size()); for (Entity parent : dataSet) { io.teaql.core.internal.TempRequest probeRequest = - new io.teaql.core.internal.TempRequest(childRequest); + new SqlDiagnosticRequest(childRequest, intent); probeRequest.selectProperty(reverseProperty.getName()); probeRequest.setPartitionProperty(null); ensureStableEntityIdOrder(probeRequest); @@ -361,6 +366,9 @@ public MutationResult mutate(UserContext context, MutationRequest request) { Entity entity = mutation.getEntity(); String typeName = entity.typeName(); PortableSQLRepository repository = getRepository(typeName); + // Local to this mutation, never stored on context or a shared repository. + var readbackIntent = context.isQueryExecutionLoggingEnabled() || context.isMutationExecutionLoggingEnabled() + ? new io.teaql.core.SqlIntentRedactions() : null; if (mutation.getAction() == DefaultMutationRequest.Action.SAVE) { if (entity.getId() == null) { @@ -369,19 +377,19 @@ public MutationResult mutate(UserContext context, MutationRequest request) { } if (entity.newItem()) { ((BaseEntity) entity).__internalSet("version", 1L); - repository.createInternal(context, Collections.singletonList(entity)); + repository.createInternal(context, Collections.singletonList(entity), readbackIntent); } else if (entity.updateItem()) { - repository.updateInternal(context, Collections.singletonList(entity)); + repository.updateInternal(context, Collections.singletonList(entity), readbackIntent); ((BaseEntity) entity).__internalSet("version", entity.getVersion() + 1); } else if (entity.recoverItem()) { - repository.recoverInternal(context, Collections.singletonList(entity)); + repository.recoverInternal(context, Collections.singletonList(entity), readbackIntent); ((BaseEntity) entity).__internalSet("version", -entity.getVersion() + 1); } if (entity instanceof BaseEntity) { ((BaseEntity) entity).gotoNextStatus(EntityAction.PERSIST); } } else if (mutation.getAction() == DefaultMutationRequest.Action.DELETE) { - repository.deleteInternal(context, Collections.singletonList(entity)); + repository.deleteInternal(context, Collections.singletonList(entity), readbackIntent); ((BaseEntity) entity).__internalSet("version", -(entity.getVersion() + 1)); if (entity instanceof BaseEntity) { ((BaseEntity) entity).gotoNextStatus(EntityAction.PERSIST); @@ -392,7 +400,7 @@ public MutationResult mutate(UserContext context, MutationRequest request) { if (entity.getId() != null && (mutation.getAction() == DefaultMutationRequest.Action.SAVE || mutation.getAction() == DefaultMutationRequest.Action.DELETE)) { - persisted = repository.loadPersistedById(context, entity.getId()); + persisted = repository.loadPersistedById(context, entity.getId(), readbackIntent); } return new io.teaql.core.DefaultMutationResult(persisted); } diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index 88bced3e..cac8c1cb 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -239,16 +239,19 @@ public String buildDataSQL(UserContext userContext, SearchRequest request, Map rowMapper) {} private record ColumnBinding( @@ -259,23 +262,44 @@ private record ColumnBinding( private record QueryShape(String key, Object[] arguments) {} + private PositionalSQL withQueryIntent(PositionalSQL sql, io.teaql.core.SqlIntentRedactions intent) { + if (intent == null) return sql; + intent.capture(sql.logBindings.policies(), sql.args); + return new PositionalSQL(sql.sql, sql.args, new SqlLogBindings(sql.logBindings.policies(), + sql.logBindings.generated(), sql.logBindings.diagnosticSql(), intent.copy())); + } + + private SqlLogBindings withMutationIntent(SqlLogBindings bindings, io.teaql.core.SqlIntentRedactions intent) { + if (intent == null) return bindings; + return new SqlLogBindings(bindings.policies(), bindings.generated(), bindings.diagnosticSql(), intent.copy()); + } + private PositionalSQL toPositional(String namedSql, Map params) { List args = new ArrayList<>(); + List policies = new ArrayList<>(); Matcher m = NAMED_PARAM.matcher(namedSql); StringBuffer sb = new StringBuffer(); while (m.find()) { String paramName = m.group(1); Object value = params.get(paramName); + var policy = params instanceof io.teaql.core.sql.SqlParameters tracked + ? tracked.policy(paramName) : io.teaql.core.SqlParameterLogPolicy.UNKNOWN; + if (params instanceof io.teaql.core.sql.SqlParameters tracked && !tracked.generated() + && policy != io.teaql.core.SqlParameterLogPolicy.CREDENTIAL) + policy = io.teaql.core.SqlParameterLogPolicy.UNKNOWN; Collection expandedValues = expandedParameterValues(value); if (expandedValues != null) { + policies.addAll(java.util.Collections.nCopies(Math.max(1, expandedValues.size()), policy)); appendExpandedParameter(expandedValues, args, m, sb); continue; } args.add(value); + policies.add(policy); m.appendReplacement(sb, "?"); } m.appendTail(sb); - return new PositionalSQL(sb.toString(), args.toArray()); + return new PositionalSQL(sb.toString(), args.toArray(), new SqlLogBindings(policies, + params instanceof io.teaql.core.sql.SqlParameters tracked && tracked.generated())); } private Collection expandedParameterValues(Object value) { @@ -525,7 +549,8 @@ private record IdSetExecution( SearchRequest request, long[] pageIds, boolean optimized) {} @SuppressWarnings("unchecked") - private IdSetExecution prepareIdSetPage(UserContext context, SearchRequest original) { + private IdSetExecution prepareIdSetPage(UserContext context, SearchRequest original, + io.teaql.core.SqlIntentRedactions intent) { IdSetPaginationOptions options = original.idSetPaginationOptions(); if (options == null) return idSetFallback(context, original, "ID_SET_DISABLED", null, "UNKNOWN"); Slice slice = original.getSlice(); @@ -557,12 +582,14 @@ private IdSetExecution prepareIdSetPage(UserContext context, SearchRequest idRequest.getProjections().clear(); idRequest.selectProperty("id"); idRequest.offset(0, options.maxIds() + 1); - Map idParams = new HashMap<>(); + Map idParams = new io.teaql.core.sql.SqlParameters(); String idSql = buildDataSQL(context, idRequest, idParams); if (ObjectUtil.isEmpty(idSql)) { return idSetFallback(context, original, "ID_SET_FALLBACK_UNSUPPORTED_SHAPE", null, "UNKNOWN"); } String key = idSetQueryKey(context, working, options, idSql, idParams); + // Capture current bindings even when retained IDs avoid executing the discovery query. + PositionalSQL idStatement = withQueryIntent(toPositional(idSql, idParams), intent); IdSetStore store = idSetStore(context); RetainedIdSet retained; try { @@ -581,8 +608,8 @@ private IdSetExecution prepareIdSetPage(UserContext context, SearchRequest retained = store.get(key).orElse(null); if (retained != null) hit = true; else { - PositionalSQL positional = toPositional(idSql, idParams); - List> rows = database.query(context, positional.sql, positional.args); + PositionalSQL positional = idStatement; + List> rows = database.query(context, positional.sql, positional.args, positional.logBindings); if (rows.size() > options.maxIds()) { return idSetFallback(context, original, "ID_SET_FALLBACK_LIMIT_EXCEEDED", @@ -662,7 +689,18 @@ private String idSetQueryKey(UserContext context, SearchRequest request, } public SmartList loadInternal(UserContext userContext, SearchRequest request) { - IdSetExecution idSetExecution = prepareIdSetPage(userContext, request); + return loadWithIntent(userContext, request, SqlDiagnosticRequest.source(userContext, request)); + } + + SmartList loadInternal(UserContext userContext, SearchRequest request, + io.teaql.core.SqlIntentRedactions intent) { + // Preserve virtual dispatch through the existing provider customization hook. + return loadInternal(userContext, intent == null ? request : SqlDiagnosticRequest.forExecution(request, intent)); + } + + private SmartList loadWithIntent(UserContext userContext, SearchRequest request, + io.teaql.core.SqlIntentRedactions intent) { + IdSetExecution idSetExecution = prepareIdSetPage(userContext, request, intent); if (idSetExecution.optimized() && idSetExecution.pageIds().length == 0) { return SmartList.empty(request.returnType()); } @@ -673,12 +711,12 @@ public SmartList loadInternal(UserContext userContext, SearchRequest reque SearchRequest executedRequest; PositionalSQL psql; if (plan != null && plan.parameterCount() == shape.arguments().length) { - psql = new PositionalSQL(plan.sql(), shape.arguments()); + psql = new PositionalSQL(plan.sql(), shape.arguments(), plan.logBindings()); pageExecution = fallback(userContext, request, null, "DISABLED"); executedRequest = request; } else { - Map params = new HashMap<>(); + Map params = new io.teaql.core.sql.SqlParameters(); String sql = buildDataSQL(userContext, requestedPage, params); if (ObjectUtil.isEmpty(sql)) { return new SmartList<>(); @@ -688,7 +726,7 @@ public SmartList loadInternal(UserContext userContext, SearchRequest reque : prepareContinuousPage(userContext, requestedPage, sql, params); executedRequest = pageExecution.request(); if (pageExecution.request() != requestedPage) { - params = new HashMap<>(); + params = new io.teaql.core.sql.SqlParameters(); sql = buildDataSQL(userContext, executedRequest, params); } psql = toPositional(sql, params); @@ -699,11 +737,14 @@ public SmartList loadInternal(UserContext userContext, SearchRequest reque CompiledQueryPlan candidate = new CompiledQueryPlan( psql.sql, psql.args.length, + psql.logBindings, compileRowMapper(executedRequest)); CompiledQueryPlan existing = compiledQueryPlans.putIfAbsent(shape.key(), candidate); plan = existing == null ? candidate : existing; } } + // Attach only after inserting the reusable plan: no original values enter the plan cache. + psql = withQueryIntent(psql, intent); SmartList smartList; Object mapperExtension = request.getExtension(COMPILED_ROW_MAPPER); io.teaql.core.CompiledRowMapper selectedMapper = @@ -715,7 +756,7 @@ public SmartList loadInternal(UserContext userContext, SearchRequest reque @SuppressWarnings("unchecked") io.teaql.core.CompiledRowMapper mapper = (io.teaql.core.CompiledRowMapper) rawMapper; - List entities = database.query(userContext, psql.sql, psql.args, mapper); + List entities = database.query(userContext, psql.sql, psql.args, mapper, psql.logBindings); if (entities.isEmpty() && ObjectUtil.isEmpty(request.getFacetRequests())) { registerContinuousPage(userContext, request, pageExecution, List.of()); return SmartList.empty(request.returnType()); @@ -723,7 +764,7 @@ public SmartList loadInternal(UserContext userContext, SearchRequest reque smartList = SmartList.takeOwnership(entities); } else { - List> rows = database.query(userContext, psql.sql, psql.args); + List> rows = database.query(userContext, psql.sql, psql.args, psql.logBindings); if (rows.isEmpty() && ObjectUtil.isEmpty(request.getFacetRequests())) { registerContinuousPage(userContext, request, pageExecution, List.of()); return SmartList.empty(request.returnType()); @@ -751,12 +792,13 @@ public SmartList loadInternal(UserContext userContext, SearchRequest reque tr.groupBy(facetRequest.getRelationName()); tr.count("count"); - Map facetParams = new HashMap<>(); + Map facetParams = new io.teaql.core.sql.SqlParameters(); java.util.List facetTables = compiler.collectAggregationTables(this.sqlMetadata, this, userContext, tr); String facetSql = compiler.buildAggregationSQL(this.sqlMetadata, this, userContext, tr, facetParams, facetTables); if (!io.teaql.core.utils.ObjectUtil.isEmpty(facetSql)) { - PositionalSQL psqlFacet = toPositional(facetSql, facetParams); - List> facetRows = database.query(userContext, psqlFacet.sql, psqlFacet.args); + var facetIntent = intent == null ? null : intent.copy(); + PositionalSQL psqlFacet = withQueryIntent(toPositional(facetSql, facetParams), facetIntent); + List> facetRows = database.query(userContext, psqlFacet.sql, psqlFacet.args, psqlFacet.logBindings); SmartList facetEntities = new SmartList<>(); io.teaql.core.SearchRequest relationReq = facetRequest.getRequest(); @@ -779,7 +821,7 @@ public SmartList loadInternal(UserContext userContext, SearchRequest reque if (facetRequest.isMergeCriteria()) { fetchRelReq.appendSearchCriteria(request.getSearchCriteria()); } - SmartList loadedRels = relationRepo.loadInternal(userContext, fetchRelReq); + SmartList loadedRels = relationRepo.loadInternal(userContext, fetchRelReq, facetIntent); java.util.List countAliases = relationReq.getAggregations().getAggregates() .stream().map(io.teaql.core.SimpleNamedExpression::name).toList(); if (countAliases.isEmpty()) countAliases = java.util.List.of("count"); @@ -927,6 +969,7 @@ private QueryShape simpleQueryShape(UserContext context, SearchRequest reques private boolean appendExpressionShape( Expression expression, StringBuilder key, List arguments, boolean inlineParameter) { + if (expression instanceof SQLExpressionParser) return false; if (expression == null) { key.append("null"); return true; @@ -949,6 +992,9 @@ private boolean appendExpressionShape( key.append("C:0"); return true; } + key.append("N:").append(parameter.getName()).append(':'); + if (parameter.getOperator() != null) + value = new io.teaql.core.sql.expression.ParameterParser().fixValue(parameter.getOperator(), value); Collection expanded = expandedParameterValues(value); if (expanded == null) { key.append("?:1"); @@ -985,11 +1031,17 @@ private boolean isActiveVersionPredicate(SearchCriteria criteria) { @SuppressWarnings("unchecked") public T loadPersistedById(UserContext userContext, Long id) { + return loadPersistedById(userContext, id, null); + } + + @SuppressWarnings("unchecked") + T loadPersistedById(UserContext userContext, Long id, io.teaql.core.SqlIntentRedactions intent) { String primaryTable = thisPrimaryTableName != null ? thisPrimaryTableName : tableName(entityDescriptor.getType()); String sql = "SELECT * FROM " + escapeIdentifier(primaryTable) + " WHERE " + escapeIdentifier("id") + " = ?"; - List> rows = database.query(userContext, sql, new Object[] {id}); + List> rows = database.query(userContext, sql, new Object[] {id}, + new SqlLogBindings(List.of(parameterLogPolicy("id")), true, null, intent)); if (rows.size() != 1) { throw new TeaQLRuntimeException( "Persisted " + entityDescriptor.getType() + "(" + id + ") could not be read back"); @@ -1023,11 +1075,11 @@ public T loadPersistedById(UserContext userContext, Long id) { } public Stream streamInternal(UserContext userContext, SearchRequest request) { - Map params = new HashMap<>(); + Map params = new io.teaql.core.sql.SqlParameters(); String sql = buildDataSQL(userContext, request, params); if (ObjectUtil.isEmpty(sql)) return Stream.empty(); PositionalSQL psql = toPositional(sql, params); - return database.queryForStream(userContext, psql.sql, psql.args) + return database.queryForStream(userContext, psql.sql, psql.args, psql.logBindings) .map(row -> mapRowToEntity(userContext, request, row)); } @@ -1161,7 +1213,12 @@ private EntityDescriptor resolveDescriptor(Class entityType) { throw new IllegalStateException("No entity descriptor registered for " + entityType.getName()); } - public void createInternal(UserContext userContext, Collection createItems) { + public void createInternal(UserContext userContext, Collection createItems) { + createInternal(userContext, createItems, null); + } + + void createInternal(UserContext userContext, Collection createItems, io.teaql.core.SqlIntentRedactions intent) { + if (intent != null) createItems.forEach(item -> intent.captureTargetId(item.getId())); List sqlEntities = CollectionUtil.map(createItems, i -> convertToSQLEntityForInsert(userContext, i), true); if (ObjectUtil.isEmpty(sqlEntities)) return; @@ -1192,11 +1249,18 @@ public void createInternal(UserContext userContext, Collection createItems) { List columns = tableColumns.get(k); io.teaql.core.sql.SqlAstCompiler compiler = new io.teaql.core.sql.SqlAstCompiler(); String sql = compiler.buildInsertSQL(this, k, columns, sqlEntity.getTraceChain()); - database.batchUpdate(userContext, sql, v); + var bindings = logBindings(k, columns, sql, sqlEntity.getTraceChain()); + if (intent != null) for (Object[] args : v) intent.capture(bindings.policies(), args); + database.batchUpdate(userContext, sql, v, withMutationIntent(bindings, intent)); }); } public void updateInternal(UserContext userContext, Collection updateItems) { + updateInternal(userContext, updateItems, null); + } + + void updateInternal(UserContext userContext, Collection updateItems, io.teaql.core.SqlIntentRedactions intent) { + if (intent != null) updateItems.forEach(item -> intent.captureTargetId(item.getId())); if (ObjectUtil.isEmpty(updateItems)) return; List sqlEntities = CollectionUtil.map(updateItems, i -> convertToSQLEntityForUpdate(userContext, i), true); @@ -1215,41 +1279,50 @@ public void updateInternal(UserContext userContext, Collection updateItems) { boolean primaryTable = this.primaryTableNames.contains(k); if (versionTable) { - updateVersionTable(userContext, sqlEntity, versionTableUpdated, k, columns, l); + updateVersionTable(userContext, sqlEntity, versionTableUpdated, k, columns, l, intent); return; } if (primaryTable) { - updatePrimaryTable(userContext, sqlEntity, k, columns, l); + updatePrimaryTable(userContext, sqlEntity, k, columns, l, intent); return; } String updateSql = dialect.buildSubsidiaryInsertSql(k, columns); - database.executeUpdate(userContext, updateSql, l.toArray()); + var bindings = logBindings(k, columns); + if (intent != null) intent.capture(bindings.policies(), l.toArray()); + database.executeUpdate(userContext, updateSql, l.toArray(), withMutationIntent(bindings, intent)); }); if (!versionTableUpdated.get()) { - updateVersionTableVersion(userContext, sqlEntity); + updateVersionTableVersion(userContext, sqlEntity, intent); } } } - private void updateVersionTableVersion(UserContext userContext, SQLEntity sqlEntity) { + private void updateVersionTableVersion(UserContext userContext, SQLEntity sqlEntity, io.teaql.core.SqlIntentRedactions intent) { io.teaql.core.sql.SqlAstCompiler compiler = new io.teaql.core.sql.SqlAstCompiler(); String updateSql = compiler.buildUpdateVersionTableVersionSQL(this, this.versionTableName); Object[] parameters = {sqlEntity.getVersion() + 1, sqlEntity.getId(), sqlEntity.getVersion()}; - int update = database.executeUpdate(userContext, updateSql, parameters); + var bindings = logBindings(this.versionTableName, List.of("version", "id", "version")); + if (intent != null) intent.capture(bindings.policies(), parameters); + int update = database.executeUpdate(userContext, updateSql, parameters, withMutationIntent(bindings, intent)); if (update != 1) throw new ConcurrentModifyException(); } - private void updatePrimaryTable(UserContext userContext, SQLEntity sqlEntity, String k, List columns, List l) { + private void updatePrimaryTable(UserContext userContext, SQLEntity sqlEntity, String k, List columns, List l, + io.teaql.core.SqlIntentRedactions intent) { l.add(sqlEntity.getId()); io.teaql.core.sql.SqlAstCompiler compiler = new io.teaql.core.sql.SqlAstCompiler(); String updateSql = compiler.buildUpdatePrimarySQL(this, k, columns, sqlEntity.getTraceChain()); - int update = database.executeUpdate(userContext, updateSql, l.toArray()); + List bindings = new ArrayList<>(columns); bindings.add("id"); + var policies = logBindings(k, bindings, updateSql, sqlEntity.getTraceChain()); + if (intent != null) intent.capture(policies.policies(), l.toArray()); + int update = database.executeUpdate(userContext, updateSql, l.toArray(), withMutationIntent(policies, intent)); if (update != 1) throw new TeaQLRuntimeException("primary table update failed"); } private void updateVersionTable(UserContext userContext, SQLEntity sqlEntity, - AtomicBoolean versionTableUpdated, String k, List columns, List l) { + AtomicBoolean versionTableUpdated, String k, List columns, List l, + io.teaql.core.SqlIntentRedactions intent) { versionTableUpdated.set(true); columns.add("version"); l.add(sqlEntity.getVersion() + 1); @@ -1257,11 +1330,19 @@ private void updateVersionTable(UserContext userContext, SQLEntity sqlEntity, l.add(sqlEntity.getVersion()); io.teaql.core.sql.SqlAstCompiler compiler = new io.teaql.core.sql.SqlAstCompiler(); String updateSql = compiler.buildUpdateVersionSQL(this, k, columns, sqlEntity.getTraceChain()); - int update = database.executeUpdate(userContext, updateSql, l.toArray()); + List bindings = new ArrayList<>(columns); bindings.add("id"); bindings.add("version"); + var policies = logBindings(k, bindings, updateSql, sqlEntity.getTraceChain()); + if (intent != null) intent.capture(policies.policies(), l.toArray()); + int update = database.executeUpdate(userContext, updateSql, l.toArray(), withMutationIntent(policies, intent)); if (update != 1) throw new ConcurrentModifyException(); } public void deleteInternal(UserContext userContext, Collection entities) { + deleteInternal(userContext, entities, null); + } + + void deleteInternal(UserContext userContext, Collection entities, io.teaql.core.SqlIntentRedactions intent) { + if (intent != null) entities.forEach(item -> intent.captureTargetId(item.getId())); if (ObjectUtil.isEmpty(entities)) return; io.teaql.core.sql.SqlAstCompiler compiler = new io.teaql.core.sql.SqlAstCompiler(); String updateSql = compiler.buildDeleteSQL(this, this.versionTableName); @@ -1269,13 +1350,20 @@ public void deleteInternal(UserContext userContext, Collection entities) { .filter(e -> e.getVersion() > 0) .map(e -> new Object[]{-(e.getVersion() + 1), e.getId(), e.getVersion()}) .collect(Collectors.toList()); - int[] rets = database.batchUpdate(userContext, updateSql, args); + var bindings = logBindings(this.versionTableName, List.of("version", "id", "version")); + if (intent != null) for (Object[] row : args) intent.capture(bindings.policies(), row); + int[] rets = database.batchUpdate(userContext, updateSql, args, withMutationIntent(bindings, intent)); for (int ret : rets) { if (ret != 1) throw new ConcurrentModifyException(); } } public void recoverInternal(UserContext userContext, Collection entities) { + recoverInternal(userContext, entities, null); + } + + void recoverInternal(UserContext userContext, Collection entities, io.teaql.core.SqlIntentRedactions intent) { + if (intent != null) entities.forEach(item -> intent.captureTargetId(item.getId())); if (ObjectUtil.isEmpty(entities)) return; io.teaql.core.sql.SqlAstCompiler compiler = new io.teaql.core.sql.SqlAstCompiler(); String updateSql = compiler.buildDeleteSQL(this, this.versionTableName); @@ -1283,7 +1371,9 @@ public void recoverInternal(UserContext userContext, Collection entities) { .filter(e -> e.getVersion() < 0) .map(e -> new Object[]{(-e.getVersion() + 1), e.getId(), e.getVersion()}) .collect(Collectors.toList()); - int[] rets = database.batchUpdate(userContext, updateSql, args); + var bindings = logBindings(this.versionTableName, List.of("version", "id", "version")); + if (intent != null) for (Object[] row : args) intent.capture(bindings.policies(), row); + int[] rets = database.batchUpdate(userContext, updateSql, args, withMutationIntent(bindings, intent)); for (int ret : rets) { if (ret != 1) throw new ConcurrentModifyException(); } @@ -1922,6 +2012,55 @@ public String prepareLimit(SearchRequest request) { return prepareLimit(request, new java.util.HashMap<>()); } + @Override + public io.teaql.core.SqlParameterLogPolicy parameterLogPolicy(String name) { + if (io.teaql.core.utils.SensitiveLogNames.credential(name)) + return io.teaql.core.SqlParameterLogPolicy.CREDENTIAL; + for (PropertyDescriptor property : allProperties) { + if (!property.getName().equals(name)) continue; + if (entityDescriptor.getAuditMaskFields().contains(name) + || property.getOwner() != null && property.getOwner().getAuditMaskFields().contains(name) + || "masked".equalsIgnoreCase(property.getAdditionalInfo().get("logPolicy"))) + return io.teaql.core.SqlParameterLogPolicy.MASKED; + if ("credential".equalsIgnoreCase(property.getAdditionalInfo().get("logPolicy"))) + return io.teaql.core.SqlParameterLogPolicy.CREDENTIAL; + if ("plain".equalsIgnoreCase(property.getAdditionalInfo().get("logPolicy"))) + return io.teaql.core.SqlParameterLogPolicy.PLAIN; + EntityDescriptor owner = property.getOwner(); + return (owner == null ? entityDescriptor : owner).isAuditMaskFieldsDeclared() + ? io.teaql.core.SqlParameterLogPolicy.PLAIN + : io.teaql.core.SqlParameterLogPolicy.UNKNOWN; + } + return io.teaql.core.SqlParameterLogPolicy.UNKNOWN; + } + + private SqlLogBindings logBindings(String table, List columns) { + List policies = new ArrayList<>(); + for (String column : columns) { + var policy = io.teaql.core.SqlParameterLogPolicy.UNKNOWN; + for (PropertyDescriptor property : allProperties) { + if (!shouldHandle(property)) continue; + for (SQLColumn mapping : getSqlColumns(property)) { + if (table.equals(mapping.getTableName()) && column.equals(mapping.getColumnName())) + policy = parameterLogPolicy(property.getName()); + } + } + policies.add(policy); + } + return new SqlLogBindings(policies, true); + } + + private SqlLogBindings logBindings(String table, List columns, String sql, String trace) { + SqlLogBindings bindings = logBindings(table, columns); + if (trace == null || trace.isEmpty()) return bindings; + String suffix = " /* [" + trace + "] */"; + // Execution retains its original trace comment. Logs use compiler-known SQL structure + // and the separately projected context trace, never a raw business annotation inline. + return sql.endsWith(suffix) + ? new SqlLogBindings(bindings.policies(), true, sql.substring(0, sql.length() - suffix.length())) + : new SqlLogBindings(bindings.policies(), false); + } + @Override public String prepareLimit(SearchRequest request, java.util.Map parameters) { Slice slice = request.getSlice(); @@ -1929,7 +2068,7 @@ public String prepareLimit(SearchRequest request, java.util.Map String limitKey = "limit0"; while (parameters.containsKey(limitKey)) limitKey += "_1"; - parameters.put(limitKey, slice.getSize()); + io.teaql.core.sql.SqlParameters.bind(parameters, limitKey, slice.getSize(), io.teaql.core.SqlParameterLogPolicy.PLAIN); String offsetPlaceholder; if (slice.getOffset() == 0) { @@ -1939,7 +2078,7 @@ public String prepareLimit(SearchRequest request, java.util.Map else { String offsetKey = "offset0"; while (parameters.containsKey(offsetKey)) offsetKey += "_1"; - parameters.put(offsetKey, slice.getOffset()); + io.teaql.core.sql.SqlParameters.bind(parameters, offsetKey, slice.getOffset(), io.teaql.core.SqlParameterLogPolicy.PLAIN); offsetPlaceholder = ":" + offsetKey; } @@ -1964,11 +2103,21 @@ public String getPartitionSQL() { // ========================================== protected AggregationResult doAggregateInternal(UserContext userContext, SearchRequest request) { + return aggregateWithIntent(userContext, request, SqlDiagnosticRequest.source(userContext, request)); + } + + AggregationResult doAggregateInternal(UserContext userContext, SearchRequest request, + io.teaql.core.SqlIntentRedactions intent) { + return doAggregateInternal(userContext, intent == null ? request : SqlDiagnosticRequest.forExecution(request, intent)); + } + + private AggregationResult aggregateWithIntent(UserContext userContext, SearchRequest request, + io.teaql.core.SqlIntentRedactions intent) { if (!request.hasSimpleAgg()) return null; io.teaql.core.sql.SqlAstCompiler compiler = new io.teaql.core.sql.SqlAstCompiler(); List tables = compiler.collectAggregationTables(sqlMetadata, this, userContext, request); - Map parameters = new HashMap<>(); + Map parameters = new io.teaql.core.sql.SqlParameters(); Object preConfig = userContext.getObj(MULTI_TABLE); userContext.putAttribute(MULTI_TABLE, tables.size() > 1); @@ -1976,8 +2125,8 @@ protected AggregationResult doAggregateInternal(UserContext userContext, SearchR String sql = compiler.buildAggregationSQL(sqlMetadata, this, userContext, request, parameters, tables); if (sql == null) return null; - PositionalSQL psql = toPositional(sql, parameters); - List> rows = database.query(userContext, psql.sql, psql.args); + PositionalSQL psql = withQueryIntent(toPositional(sql, parameters), intent); + List> rows = database.query(userContext, psql.sql, psql.args, psql.logBindings); AggregationResult result = new AggregationResult(); result.setName(request.getAggregations().getName()); @@ -2025,9 +2174,9 @@ protected boolean ensureTableEnabled(UserContext context) { } static void logInfo(String message) { - // Legacy schema/bootstrap SQL may embed literal seed values. This layer - // has no field provenance and must not bypass the runtime log policy. - System.out.println("[SQL-PORTABLE] schema/bootstrap statement; SQL payload omitted"); + // The runtime database adapter owns schema diagnostics and applies the + // mutation-family logging switch plus safe projection. This legacy hook + // must not write directly to stdout, even when its payload is omitted. } protected int toIntOrZero(Object cnt) { diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/SqlDiagnosticRequest.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/SqlDiagnosticRequest.java new file mode 100644 index 00000000..cf28fba0 --- /dev/null +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/SqlDiagnosticRequest.java @@ -0,0 +1,44 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.SearchRequest; +import io.teaql.core.SqlIntentRedactions; +import io.teaql.core.UserContext; +import io.teaql.core.internal.TempRequest; + +/** Invocation-local provenance, never an extension/wire field or context/repository state. */ +final class SqlDiagnosticRequest extends TempRequest { + private final transient SqlIntentRedactions source; + private final transient boolean executionScope; + private final transient SearchRequest original; + + SqlDiagnosticRequest(SearchRequest request, SqlIntentRedactions source) { + this(request, source, false); + } + + private SqlDiagnosticRequest(SearchRequest request, SqlIntentRedactions source, boolean executionScope) { + super(request); + this.original = request; + // TempRequest's relation-oriented copy omits these root-query semantics. + this.comment = request.comment(); + this.purpose = request.purpose(); + this.searchForText = request.getSearchForText(); + this.dynamicFieldSelection = request.getDynamicFieldSelection(); + this.hardLimit = request.hardLimit(); + this.source = source == null || executionScope ? source : source.copy(); + this.executionScope = executionScope; + } + + static SqlDiagnosticRequest forExecution(SearchRequest request, SqlIntentRedactions source) { + return new SqlDiagnosticRequest(request, source, true); + } + + @Override public io.teaql.core.Entity internalNewEntity() { return original.internalNewEntity(); } + @Override public boolean tryUseSubQuery() { return original.tryUseSubQuery(); } + + static SqlIntentRedactions source(UserContext context, SearchRequest request) { + if (!context.isQueryExecutionLoggingEnabled()) return null; + if (request instanceof SqlDiagnosticRequest scoped && scoped.source != null) + return scoped.executionScope ? scoped.source : scoped.source.copy(); + return new SqlIntentRedactions(); + } +} diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/SqlLogBindings.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/SqlLogBindings.java new file mode 100644 index 00000000..c5ceb7a2 --- /dev/null +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/SqlLogBindings.java @@ -0,0 +1,24 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.ExecutionMetadata; +import io.teaql.core.SqlParameterLogPolicy; +import java.util.List; + +/** Immutable compiler-owned parameter provenance, separate from execution values. */ +public record SqlLogBindings(List policies, boolean generated, String diagnosticSql, + io.teaql.core.SqlIntentRedactions intentRedactions) { + public static final SqlLogBindings UNKNOWN = new SqlLogBindings(List.of(), false); + + public SqlLogBindings { policies = List.copyOf(policies); } + public SqlLogBindings(List policies, boolean generated) { this(policies, generated, null); } + public SqlLogBindings(List policies, boolean generated, String diagnosticSql) { + this(policies, generated, diagnosticSql, null); + } + + public void applyTo(ExecutionMetadata metadata) { + metadata.setParameterLogPolicies(policies); + metadata.setGeneratedSql(generated); + metadata.setIntentRedactions(intentRedactions); + if (diagnosticSql != null) metadata.setParameterizedQuery(diagnosticSql); + } +} diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java index 3e6aeda9..33f36914 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java @@ -12,6 +12,27 @@ */ public interface TeaQLDatabase { + default List> query(io.teaql.core.UserContext context, String sql, + Object[] args, SqlLogBindings bindings) { + return query(context, sql, args); + } + default List query(io.teaql.core.UserContext context, + String sql, Object[] args, io.teaql.core.CompiledRowMapper rowMapper, SqlLogBindings bindings) { + return query(context, sql, args, rowMapper); + } + default Stream> queryForStream(io.teaql.core.UserContext context, + String sql, Object[] args, SqlLogBindings bindings) { + return queryForStream(context, sql, args); + } + default int executeUpdate(io.teaql.core.UserContext context, String sql, Object[] args, + SqlLogBindings bindings) { + return executeUpdate(context, sql, args); + } + default int[] batchUpdate(io.teaql.core.UserContext context, String sql, List batchArgs, + SqlLogBindings bindings) { + return batchUpdate(context, sql, batchArgs); + } + default boolean supportsCompiledRowMapping() { return false; } diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableLogPrivacyTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableLogPrivacyTest.java index f32cab13..52376d98 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableLogPrivacyTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableLogPrivacyTest.java @@ -6,7 +6,7 @@ import org.junit.Test; public class PortableLogPrivacyTest { - @Test public void legacyBootstrapLoggerNeverPrintsLiteralSeedValues() { + @Test public void legacyBootstrapLoggerDoesNotBypassRuntimeLogSwitch() { var original = System.out; var bytes = new ByteArrayOutputStream(); try (var output = new PrintStream(bytes)) { @@ -16,8 +16,6 @@ public class PortableLogPrivacyTest { System.setOut(original); } String text = bytes.toString(java.nio.charset.StandardCharsets.UTF_8); - assertTrue(text.contains("SQL-PORTABLE")); - assertFalse(text.contains("PRIVATE-SEED-CANARY")); - assertFalse(text.contains("INSERT")); + assertEquals("Schema diagnostics belong to the governed runtime adapter", "", text); } } diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java index 7c368665..4bbba68f 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java @@ -242,6 +242,36 @@ public void relationHydrationUsesInvokingRuntimeWhenEntityNamesMatch() throws Ex } } + @Test + public void forwardRelationWithoutItsOwnSliceUsesReferencedIdBound() throws Exception { + RelationRuntime fixture = relationRuntime("parent", 31L); + TopNChildRequest child = new TopNChildRequest(); + child.selectProperty("id"); + child.selectProperty("version"); + child.selectProperty("name"); + child.selectProperty("parent"); + child.top(10); + TopNParentRequest parent = new TopNParentRequest(); + parent.selectProperty("id"); + parent.selectProperty("version"); + parent.selectProperty("name"); + parent.unlimited(); // Generated forward helpers use this relation-local form. + assertNull("the relation request itself is deliberately unbounded", parent.getSlice()); + child.enhanceRelation("parent", parent); + + SmartList rows = child + .comment("load a bounded child page and its referenced parent") + .purpose("verify internal forward lookup has a derived bound") + .executeForList(fixture.context()); + + assertEquals(1, rows.size()); + TopNParent loadedParent = rows.get(0).getProperty("parent"); + assertNotNull(loadedParent); + assertEquals(1L, loadedParent.getId().longValue()); + assertEquals("parent", loadedParent.getProperty("name")); + assertNull("the caller's nested request must remain unchanged", parent.getSlice()); + } + private static RelationRuntime relationRuntime(String foreignKeyColumn, long childId) throws Exception { SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); @@ -1016,7 +1046,6 @@ public SQLiteTeaQLDatabase() throws Exception { @Override public List> query(String sql, Object[] args) { - System.out.println("[SQL-QUERY] " + sql + " | args: " + Arrays.toString(args)); queryTrace.add(sql + " | args: " + Arrays.toString(args)); List> results = new ArrayList<>(); try (PreparedStatement stmt = connection.prepareStatement(sql)) { @@ -1064,7 +1093,6 @@ public List executeTrace() { @Override public int executeUpdate(String sql, Object[] args) { - System.out.println("[SQL-UPDATE] " + sql + " | args: " + Arrays.toString(args)); try (PreparedStatement stmt = connection.prepareStatement(sql)) { for (int i = 0; i < args.length; i++) { bindSqlite(stmt, i + 1, args[i]); @@ -1077,7 +1105,6 @@ public int executeUpdate(String sql, Object[] args) { @Override public int[] batchUpdate(String sql, List batchArgs) { - System.out.println("[SQL-BATCH] " + sql + " | batch count: " + batchArgs.size()); try (PreparedStatement stmt = connection.prepareStatement(sql)) { for (Object[] args : batchArgs) { for (int i = 0; i < args.length; i++) { diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/SqlParameterPropagationTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/SqlParameterPropagationTest.java new file mode 100644 index 00000000..8aee3759 --- /dev/null +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/SqlParameterPropagationTest.java @@ -0,0 +1,271 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.*; +import io.teaql.core.criteria.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.*; +import io.teaql.core.sql.expression.*; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.lang.reflect.Proxy; +import java.util.*; +import java.util.concurrent.*; +import org.junit.Test; +import static org.junit.Assert.*; + +public class SqlParameterPropagationTest { + public static class Customer extends BaseEntity { @Override public String typeName() { return "Customer"; } } + public static class Request extends BaseRequest { + public Request() { super(Customer.class); } + @Override public String getTypeName() { return "Customer"; } + } + private record Captured(String sql, Object[] args, SqlLogBindings bindings) {} + private static class Fixture { + final List captured = new ArrayList<>(); + final SQLEntityDescriptor descriptor = new SQLEntityDescriptor(); + final PortableSQLRepository repository; + final TeaQLRuntime runtime; + final java.util.concurrent.atomic.AtomicInteger compilations = new java.util.concurrent.atomic.AtomicInteger(); + final java.util.concurrent.atomic.AtomicInteger loads = new java.util.concurrent.atomic.AtomicInteger(); + final java.util.concurrent.atomic.AtomicInteger aggregates = new java.util.concurrent.atomic.AtomicInteger(); + Fixture() { this(true); } + Fixture(boolean declared) { + descriptor.setType("Customer"); descriptor.setTargetType(Customer.class); + descriptor.setEntitySupplier(Customer::new); descriptor.setDataService("test"); + descriptor.with("table_name", "customer_data"); + for (String field : List.of("id", "version", "title", "status", "password")) { + var property = (GenericSQLProperty) descriptor.addSimpleProperty(field, + field.equals("id") || field.equals("version") ? Long.class : String.class); + property.setColumnType(field.equals("id") || field.equals("version") ? "BIGINT" : "VARCHAR(200)"); + } + if (declared) descriptor.setAuditMaskFields(List.of("title")); + var factory = new SimpleEntityMetaFactory(); factory.register(descriptor); + runtime = TeaQLRuntime.builder().metadata(factory).build(); + TeaQLDatabase db = (TeaQLDatabase) Proxy.newProxyInstance(TeaQLDatabase.class.getClassLoader(), + new Class[]{TeaQLDatabase.class}, (proxy, method, args) -> { + if (method.getName().equals("supportsCompiledRowMapping")) return false; + if (method.getName().equals("query") && args.length == 4 && args[3] instanceof SqlLogBindings bindings) { + captured.add(new Captured((String) args[1], (Object[]) args[2], bindings)); + return List.of(); + } + if (method.getName().equals("batchUpdate") && args.length == 4 && args[3] instanceof SqlLogBindings bindings) { + var rows = (List) args[2]; + for (var row : rows) captured.add(new Captured((String) args[1], row, bindings)); + int[] counts = new int[rows.size()]; Arrays.fill(counts, 1); return counts; + } + throw new AssertionError("unexpected database call " + method); + }); + repository = new PortableSQLRepository<>(descriptor, db, null, factory) { + @Override public SmartList loadInternal(UserContext context, SearchRequest request) { + loads.incrementAndGet(); return super.loadInternal(context, request); + } + @Override protected AggregationResult doAggregateInternal(UserContext context, SearchRequest request) { + aggregates.incrementAndGet(); return super.doAggregateInternal(context, request); + } + @Override public String buildDataSQL(UserContext context, SearchRequest request, Map params) { + compilations.incrementAndGet(); + return super.buildDataSQL(context, request, params); + } + }; + } + DefaultUserContext context() { return new DefaultUserContext(runtime); } + Request request(String field, Operator op, Object... values) { + Request request = new Request(); request.selectSelf(); request.offset(0, 10); + request.appendSearchCriteria(request.createBasicSearchCriteria(field, op, values)); + return request; + } + } + + @Test public void legacyMissingMaskMetadataFailsClosedUntilExplicitlyDeclared() { + var f = new Fixture(false); + assertEquals(SqlParameterLogPolicy.UNKNOWN, f.repository.parameterLogPolicy("status")); + assertEquals(SqlParameterLogPolicy.CREDENTIAL, f.repository.parameterLogPolicy("password")); + f.repository.loadInternal(f.context(), f.request("status", Operator.EQUAL, "PRIVATE-CANARY")); + var captured = f.captured.get(0); + assertEquals(SqlParameterLogPolicy.UNKNOWN, + captured.bindings.policies().get(Arrays.asList(captured.args).indexOf("PRIVATE-CANARY"))); + f.descriptor.setAuditMaskFields(List.of()); + assertEquals(SqlParameterLogPolicy.PLAIN, f.repository.parameterLogPolicy("status")); + } + + @Test public void cachedAndUncachedQueriesCarryFieldPoliciesAndDerivedLikeValues() { + var f = new Fixture(); + for (String value : List.of("Riverside", "Lakeside")) { + Request request = f.request("title", Operator.CONTAIN, value); + request.appendSearchCriteria(request.createBasicSearchCriteria("status", Operator.EQUAL, "ACTIVE")); + f.repository.loadInternal(f.context(), request); + } + assertEquals(2, f.captured.size()); + assertEquals("the second query must hit the compiled plan", 1, f.compilations.get()); + for (int i = 0; i < 2; i++) { + var c = f.captured.get(i); + assertTrue(c.bindings.generated()); + assertEquals(c.args.length, c.bindings.policies().size()); + int title = Arrays.asList(c.args).indexOf(i == 0 ? "%Riverside%" : "%Lakeside%"); + assertTrue(Arrays.toString(c.args), title >= 0); + assertEquals(SqlParameterLogPolicy.MASKED, c.bindings.policies().get(title)); + assertEquals(SqlParameterLogPolicy.PLAIN, c.bindings.policies().get(Arrays.asList(c.args).indexOf("ACTIVE"))); + assertEquals(SqlParameterLogPolicy.PLAIN, c.bindings.policies().get(c.args.length - 1)); + } + } + + @Test public void cachedPlansNeverRetainQueryIntentAndScopesKeepCurrentBindingsOnly() { + var f = new Fixture(); + for (String value : List.of("Riverside", "Lakeside")) { + var source = new SqlIntentRedactions(); + source.capture(List.of(SqlParameterLogPolicy.CREDENTIAL), new Object[]{"ancestor-" + value}); + var scoped = new SqlDiagnosticRequest(f.request("title", Operator.EQUAL, value), source); + f.repository.loadInternal(f.context(), scoped); + var captured = f.captured.get(f.captured.size() - 1); + var values = new ArrayList(); + captured.bindings.intentRedactions().appendTo(values, false); + assertEquals(List.of("ancestor-" + value, value), values); + var ancestor = new ArrayList(); source.appendTo(ancestor, false); + assertEquals(List.of("ancestor-" + value), ancestor); + assertFalse(scoped.getExtensions().toString().contains("ancestor-")); + } + assertEquals("same query shape compiles only once", 1, f.compilations.get()); + } + + @Test public void scopedExecutionPreservesRepositoryOverridesAndSharesOnlyInvocationOutput() { + var f = new Fixture(); + var source = new SqlIntentRedactions(); + f.repository.loadInternal(f.context(), f.request("title", Operator.EQUAL, "Riverside"), source); + assertEquals(1, f.loads.get()); + var values = new ArrayList(); source.appendTo(values, false); + assertEquals(List.of("Riverside"), values); + var aggregate = f.request("title", Operator.EQUAL, "Lakeside"); aggregate.count("count"); + var aggregateSource = new SqlIntentRedactions(); + f.repository.doAggregateInternal(f.context(), aggregate, aggregateSource); + assertEquals(1, f.aggregates.get()); + values.clear(); aggregateSource.appendTo(values, false); + assertEquals(List.of("Lakeside"), values); + } + + @Test public void diagnosticWrapperPreservesQuerySemanticsBeyondSqlCriteria() { + var selection = new io.teaql.data.dynamic.DynamicFieldSelection().selectString("preferred_channel"); + var original = new Request() { + @Override public String getSearchForText() { return "Riverside"; } + @Override public String comment() { return "query comment"; } + @Override public String purpose() { return "query purpose"; } + @Override public int hardLimit() { return 27; } + @Override public boolean tryUseSubQuery() { return false; } + @Override public Customer internalNewEntity() { return new Customer(); } + @Override public io.teaql.data.dynamic.DynamicFieldSelection getDynamicFieldSelection() { return selection; } + }; + for (var request : List.of(new SqlDiagnosticRequest(original, new SqlIntentRedactions()), + SqlDiagnosticRequest.forExecution(original, new SqlIntentRedactions()))) { + assertEquals(original.getSearchForText(), request.getSearchForText()); + assertEquals(original.comment(), request.comment()); + assertEquals(original.purpose(), request.purpose()); + assertEquals(original.hardLimit(), request.hardLimit()); + assertFalse(request.tryUseSubQuery()); + assertTrue(request.internalNewEntity() instanceof Customer); + assertSame(selection, request.getDynamicFieldSelection()); + } + } + + @Test public void expandedCollectionsAndCredentialPredicatesKeepPolicies() { + var f = new Fixture(); + Request request = f.request("title", Operator.IN, "Riverside", "Lakeside"); + request.appendSearchCriteria(request.createBasicSearchCriteria("password", Operator.EQUAL, "PASSWORD-CANARY")); + f.repository.loadInternal(f.context(), request); + var c = f.captured.get(0); + for (String value : List.of("Riverside", "Lakeside")) + assertEquals(SqlParameterLogPolicy.MASKED, c.bindings.policies().get(Arrays.asList(c.args).indexOf(value))); + assertEquals(SqlParameterLogPolicy.CREDENTIAL, + c.bindings.policies().get(Arrays.asList(c.args).indexOf("PASSWORD-CANARY"))); + } + + @Test public void parameterNameCannotPretendToBeAnOrdinaryField() { + var f = new Fixture(); var params = new SqlParameters(); + Request request = new Request(); request.selectSelf(); request.offset(0, 1); + request.appendSearchCriteria(new EQ(new PropertyReference("title"), new Parameter("status", "Riverside", Operator.EQUAL))); + f.repository.buildDataSQL(f.context(), request, params); + assertEquals(SqlParameterLogPolicy.MASKED, params.policy("status")); + } + + private static class Raw extends Parameter implements SQLExpressionParser { + Raw() { super("raw", "RAW-CANARY", Operator.EQUAL); } + @Override public String toSql(UserContext context, Raw expression, String table, + Map params, SQLColumnResolver resolver) { return "'RAW-CANARY'"; } + } + + @Test public void customSqlCannotClaimGeneratedSqlProvenance() { + var f = new Fixture(); var parameters = new SqlParameters(); + var request = new Request(); request.selectSelf(); request.offset(0, 1); + request.appendSearchCriteria(new EQ(new PropertyReference("status"), new Raw())); + String sql = f.repository.buildDataSQL(f.context(), request, parameters); + assertTrue(sql.contains("RAW-CANARY")); + assertFalse(parameters.generated()); + } + + @Test public void compilationStateIsIndependentAcrossConcurrentCalls() throws Exception { + var f = new Fixture(); var pool = Executors.newFixedThreadPool(4); + try { + List> calls = new ArrayList<>(); + for (int i = 0; i < 64; i++) { + String field = i % 2 == 0 ? "title" : "status"; + calls.add(() -> { + var parameters = new SqlParameters(); + f.repository.buildDataSQL(f.context(), f.request(field, Operator.EQUAL, "SAME-VALUE"), parameters); + assertEquals(field.equals("title") ? SqlParameterLogPolicy.MASKED : SqlParameterLogPolicy.PLAIN, + parameters.policy(field)); + assertEquals(SqlParameterLogPolicy.UNKNOWN, parameters.currentPolicy()); + return null; + }); + } + for (Future result : pool.invokeAll(calls)) result.get(); + } finally { pool.shutdownNow(); } + } + + @Test public void rawMapReplacementCannotRetainOldOrdinaryClassification() { + var parameters = new SqlParameters(); + SqlParameters.bind(parameters, "status", "ACTIVE", SqlParameterLogPolicy.PLAIN); + parameters.put("status", "REPLACEMENT-CANARY"); + assertEquals(SqlParameterLogPolicy.UNKNOWN, parameters.policy("status")); + SqlParameters.bind(parameters, "status", "ACTIVE", SqlParameterLogPolicy.PLAIN); + parameters.putAll(Map.of("status", "SECOND-CANARY")); + assertEquals(SqlParameterLogPolicy.UNKNOWN, parameters.policy("status")); + } + + @Test public void deleteAndRecoverCaptureVersionsWithoutContaminatingLaterReads() { + var f = new Fixture(); + f.descriptor.setAuditMaskFields(List.of("title", "version")); + for (long version : new long[]{3, -3}) { + var entity = new Customer(); + entity.__internalSet("id", 77L); entity.__internalSet("version", version); + var intent = new SqlIntentRedactions(); + if (version > 0) f.repository.deleteInternal(f.context(), List.of(entity), intent); + else f.repository.recoverInternal(f.context(), List.of(entity), intent); + var secrets = new ArrayList(); + intent.appendTo(secrets, false); + assertTrue("ordinary mutation target ID must redact SQL intent", secrets.contains("77")); + assertTrue(secrets.contains(Long.toString(version))); + assertTrue(secrets.contains(Long.toString(version > 0 ? -4 : 4))); + var write = f.captured.get(f.captured.size() - 1); + assertNotNull("write SQL must carry invocation-local provenance", write.bindings.intentRedactions()); + var writeSecrets = new ArrayList(); + write.bindings.intentRedactions().appendTo(writeSecrets, false); + assertTrue(writeSecrets.contains("77")); + var metadata = new ExecutionMetadata(); + write.bindings.applyTo(metadata); + metadata.setBackend("sqlite"); + metadata.setParameterizedQuery(write.sql); + metadata.setParameters(Arrays.asList(write.args)); + metadata.setAuditReason("what: mutate Customer 77"); + metadata.setAffectedRows(1L); + for (boolean debug : new boolean[]{false, true}) { + var projected = io.teaql.runtime.LogPrivacy.sql(metadata, debug); + assertEquals("what: mutate Customer [REDACTED]", projected.getAuditReason()); + assertEquals("1 rows affected", projected.getResultSummary()); + assertTrue(projected.getDebugQuery().contains("77")); + assertNull(projected.getIntentRedactions()); + } + secrets.clear(); intent.appendTo(secrets, true); + assertEquals(List.of("77"), secrets); + assertThrows(TeaQLRuntimeException.class, () -> f.repository.loadPersistedById(f.context(), 77L)); + assertNull(f.captured.get(f.captured.size() - 1).bindings.intentRedactions()); + } + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java index 0a3c62b5..88805dd5 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java @@ -37,15 +37,49 @@ public class SqliteIntegrationTest { @Test - public void ordinarySqlLogsSkipSensitivePayloadConstruction() { + public void realSqliteFailureRetainsSafeIntentAndRecoversAfterSchemaInitialization() throws Exception { + var logs = new ArrayList(); + var ds = new SimpleDataSource("jdbc:sqlite:" + java.nio.file.Files.createTempFile("teaql-failed-query-", ".db"), "", ""); + var provider = new SqliteDataServiceExecutor("sqlite", new JdbcSqlExecutor(ds), ds); + var local = new DefaultUserContext(TeaQLRuntime.builder().metadata(runtime.getMetadata()) + .dataService("sqlite", provider).idGenerationService(runtime.getIdGenerationService()) + .logSink((caller, metadata) -> logs.add(metadata)).build()); + assertThrows(RuntimeException.class, () -> new TaskRequest().filterByTitle("PRIVATE-QUERY-CANARY") + .comment("read absent table").purpose("verify driver failure evidence").executeForList(local)); + var failures = logs.stream().filter(log -> "failure".equals(log.getExecutionOutcome())).toList(); + assertEquals(1, failures.size()); + var failure = failures.get(0); + assertNull(failure.getResultCount()); + assertNull(failure.getAffectedRows()); + assertEquals("read absent table", failure.getComment()); + assertEquals("verify driver failure evidence", failure.getPurpose()); + assertTrue(failure.getDebugQuery().contains("NOT REPLAYABLE")); + assertFalse(failure.getDebugQuery().contains("PRIVATE-QUERY-CANARY")); + local.ensureSchema(); + var task = new Task(); + task.updateTitle("PRIVATE-QUERY-CANARY"); + task.updateStatus("RECOVERED"); + task.auditAs("verify same runtime can recover").save(local); + var rows = new TaskRequest().filterByTitle("PRIVATE-QUERY-CANARY") + .comment("read initialized table").purpose("verify original data retained").executeForList(local); + assertEquals(1, rows.size()); + assertEquals("PRIVATE-QUERY-CANARY", rows.get(0).getTitle()); + assertTrue(logs.stream().anyMatch(log -> "success".equals(log.getExecutionOutcome()))); + assertTrue(logs.stream().noneMatch(log -> String.valueOf(log.getDebugQuery()).contains("PRIVATE-QUERY-CANARY"))); + } + + @Test + public void ordinarySqlLogsExpandSafeBindingsWithoutEarlyPlaintextSql() { List safeLogs = new ArrayList<>(); RuntimeLogSink safeSink = (context, metadata) -> safeLogs.add(metadata); executeLoggedQueryAndMutation(safeSink); assertTrue(safeLogs.stream().anyMatch(log -> log.getOperation() == DataServiceOperation.QUERY)); assertTrue(safeLogs.stream().anyMatch(log -> log.getOperation() == DataServiceOperation.MUTATION)); assertTrue(safeLogs.stream().allMatch(log -> log.getParameterizedQuery() != null)); - assertTrue(safeLogs.stream().allMatch(log -> log.getParameters().isEmpty())); - assertTrue(safeLogs.stream().allMatch(log -> log.getDebugQuery() == null)); + assertTrue(safeLogs.stream().allMatch(log -> !log.getParameters().isEmpty())); + assertTrue(safeLogs.stream().allMatch(log -> log.getDebugQuery() != null && log.getSqlOmissionReason() == null)); + assertTrue(safeLogs.stream().allMatch(log -> !log.getDebugQuery().contains("diagnostic-payload-check"))); + assertTrue(safeLogs.stream().anyMatch(log -> log.getDebugQuery().contains("'LOG-CHECK'"))); List diagnosticLogs = new ArrayList<>(); executeLoggedQueryAndMutation(new RuntimeLogSink() { @@ -62,8 +96,8 @@ public boolean requiresSensitiveSqlData() { // Requesting sensitive data is not authorization: the exact environment // acknowledgement is also required (covered by LogPrivacyTest). assertFalse(diagnosticLogs.isEmpty()); - assertTrue(diagnosticLogs.stream().allMatch(log -> log.getParameters().isEmpty())); - assertTrue(diagnosticLogs.stream().allMatch(log -> log.getDebugQuery() == null)); + assertTrue(diagnosticLogs.stream().allMatch(log -> log.getDebugQuery() != null)); + assertTrue(diagnosticLogs.stream().allMatch(log -> !log.getDebugQuery().contains("diagnostic-payload-check"))); } @Test @@ -263,7 +297,7 @@ public Connection getConnection(String username, String password) throws SQLExce @BeforeClass public static void setup() throws Exception { // Use embedded sqlite - String url = "jdbc:sqlite:teaql_test.db"; + String url = "jdbc:sqlite:" + java.nio.file.Files.createTempFile("teaql-sqlite-regression-", ".db"); String user = ""; String password = ""; @@ -274,6 +308,7 @@ public static void setup() throws Exception { taskDescriptor.setTargetType(Task.class); taskDescriptor.setEntitySupplier(Task::new); taskDescriptor.setDataService("sqlite"); + taskDescriptor.setAuditMaskFields(List.of("title")); io.teaql.core.sql.GenericSQLProperty idProp = (io.teaql.core.sql.GenericSQLProperty) taskDescriptor.addSimpleProperty("id", Long.class); idProp.setColumnType("BIGINT"); diff --git a/teaql-utils/src/main/java/io/teaql/core/utils/SensitiveLogNames.java b/teaql-utils/src/main/java/io/teaql/core/utils/SensitiveLogNames.java new file mode 100644 index 00000000..7c6899bf --- /dev/null +++ b/teaql-utils/src/main/java/io/teaql/core/utils/SensitiveLogNames.java @@ -0,0 +1,17 @@ +package io.teaql.core.utils; + +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +/** Shared conservative credential classification for compilers and log projections. */ +public final class SensitiveLogNames { + private SensitiveLogNames() {} + + public static boolean credential(String name) { + String key = Objects.toString(name, "").replaceAll("[^a-zA-Z0-9]", "").toLowerCase(Locale.ROOT); + return List.of("password", "passwd", "passphrase", "privatekey", "secret", "accesstoken", + "refreshtoken", "idtoken", "apikey", "authorization", "credential", "sessiontoken", "magiclinktoken") + .stream().anyMatch(key::contains); + } +} diff --git a/teaql-utils/src/main/java/io/teaql/core/utils/SqlLogRenderer.java b/teaql-utils/src/main/java/io/teaql/core/utils/SqlLogRenderer.java new file mode 100644 index 00000000..4feb7daa --- /dev/null +++ b/teaql-utils/src/main/java/io/teaql/core/utils/SqlLogRenderer.java @@ -0,0 +1,142 @@ +package io.teaql.core.utils; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Locale; +import java.util.Objects; +import java.util.function.IntFunction; + +/** + * JDBC diagnostic SQL rendering, never SQL execution. The callback receives a zero-based + * binding position and must return an already safe SQL literal, including any mask annotation. + * This class has no access to raw execution parameters when rendering a safe projection. + */ +public final class SqlLogRenderer { + private SqlLogRenderer() {} + + public static String render(String sql, int parameterCount, IntFunction literal, + String databaseKind) { + Objects.requireNonNull(sql, "sql"); + Objects.requireNonNull(literal, "literal"); + if (parameterCount < 0) throw invalid(); + String kind = kind(databaseKind); + StringBuilder output = new StringBuilder(); + int index = 0; + for (int i = 0; i < sql.length();) { + char c = sql.charAt(i); + char next = i + 1 < sql.length() ? sql.charAt(i + 1) : '\0'; + if (c == '\'' || c == '"' || c == '`' || (c == '[' && kind.contains("mssql"))) { + char end = c == '[' ? ']' : c; + int start = i++; + boolean closed = false; + while (i < sql.length()) { + char ch = sql.charAt(i++); + if (ch == '\\' && kind.contains("mysql")) { + if (i >= sql.length()) throw invalid(); + i++; + } else if (ch == end) { + if (i < sql.length() && sql.charAt(i) == end) i++; + else { closed = true; break; } + } + } + if (!closed) throw invalid(); + output.append(sql, start, i); + } else if (c == '-' && next == '-') { + int start = i; + while (i < sql.length() && sql.charAt(i) != '\n' && sql.charAt(i) != '\r') i++; + output.append(sql, start, i); + } else if (c == '/' && next == '*') { + int start = i; + i += 2; + int depth = 1; + // MySQL executable comments are not inert diagnostic text. + if (i < sql.length() && (sql.charAt(i) == '!' || sql.charAt(i) == '+')) throw invalid(); + while (i < sql.length() && depth > 0) { + if (i + 1 < sql.length() && sql.startsWith("/*", i)) { depth++; i += 2; } + else if (i + 1 < sql.length() && sql.startsWith("*/", i)) { depth--; i += 2; } + else i++; + } + if (depth != 0) throw invalid(); + output.append(sql, start, i); + } else if (c == '$' && kind.contains("postgres")) { + int end = sql.indexOf('$', i + 1); + String tag = end < 0 ? "!" : sql.substring(i + 1, end); + if (tag.isEmpty() || tag.matches("[A-Za-z_][A-Za-z0-9_]*")) { + String delimiter = sql.substring(i, end + 1); + int close = sql.indexOf(delimiter, end + 1); + if (close < 0) throw invalid(); + output.append(sql, i, close + delimiter.length()); + i = close + delimiter.length(); + } else { output.append(c); i++; } + } else if (c == '?' && next == '?' && kind.contains("postgres")) { + // PgJDBC escapes a literal question-mark operator as ??. + output.append('?'); i += 2; + } else if (c == '?') { + if (index >= parameterCount) throw invalid(); + String value = literal.apply(index++); + if (value == null) throw invalid(); + output.append(value); i++; + } else { + // E'...' escape strings and MySQL # comments need a dedicated parser; + // reject instead of silently treating an inner ? as a bind position. + if (kind.contains("postgres") && (c == 'E' || c == 'e') && next == '\'') throw invalid(); + if (kind.contains("mysql") && c == '#') throw invalid(); + output.append(c); i++; + } + } + if (index != parameterCount) throw invalid(); + return output.toString(); + } + + /** A typed literal for a value already authorized for this log destination. */ + public static String literal(Object value, String databaseKind) { + String kind = kind(databaseKind); + if (value == null) return "NULL"; + if (value instanceof LocalDate date) { + String quoted = quote(date.toString(), kind); + if (kind.contains("mysql") || kind.contains("mssql")) return "CAST(" + quoted + " AS DATE)"; + return kind.contains("sqlite") ? quoted : "DATE " + quoted; + } + if (value instanceof LocalDateTime time) { + String quoted = quote(java.sql.Timestamp.valueOf(time).toString(), kind); + if (kind.contains("mysql")) return "CAST(" + quoted + " AS DATETIME(3))"; + if (kind.contains("mssql")) return "CAST(" + quoted + " AS DATETIME2(3))"; + return kind.contains("sqlite") ? quoted : "TIMESTAMP " + quoted; + } + if (value instanceof String || value instanceof Character || value instanceof java.util.Date + || value instanceof java.time.temporal.Temporal || value instanceof java.util.UUID) + return quote(value.toString(), kind); + if (value instanceof Boolean b) return b ? "TRUE" : "FALSE"; + if (value instanceof Byte || value instanceof Short || value instanceof Integer + || value instanceof Long || value instanceof BigInteger || value instanceof BigDecimal) + return value.toString(); + if (value instanceof Float f && Float.isFinite(f)) return f.toString(); + if (value instanceof Double d && Double.isFinite(d)) return d.toString(); + if (value instanceof byte[] bytes) { + String hex = java.util.HexFormat.of().formatHex(bytes); + if (kind.contains("postgres")) return "decode('" + hex + "','hex')"; + if (kind.contains("mssql")) return "0x" + hex; + if (kind.contains("oracle")) return "HEXTORAW('" + hex + "')"; + return "X'" + hex + "'"; + } + // Do not invoke arbitrary toString(): it can leak nested secrets or produce invalid SQL. + throw new IllegalArgumentException("SQL log literal type is unsupported"); + } + + private static String quote(String value, String kind) { + if (value.indexOf('\0') >= 0) throw new IllegalArgumentException("SQL log literal contains NUL"); + if (kind.contains("mysql")) value = value.replace("\\", "\\\\"); + return "'" + value.replace("'", "''") + "'"; + } + + private static String kind(String value) { + return Objects.toString(value, "sqlite").toLowerCase(Locale.ROOT); + } + + private static IllegalArgumentException invalid() { + // Never include SQL or raw parameter values in failure diagnostics. + return new IllegalArgumentException("SQL log binding mismatch or unsupported SQL syntax"); + } +} diff --git a/teaql-utils/src/main/java/module-info.java b/teaql-utils/src/main/java/module-info.java index 2db3255e..cf091fde 100644 --- a/teaql-utils/src/main/java/module-info.java +++ b/teaql-utils/src/main/java/module-info.java @@ -1,4 +1,5 @@ module io.teaql.utils { + requires java.sql; requires org.apache.commons.lang3; requires org.apache.commons.collections4; diff --git a/teaql-utils/src/test/java/io/teaql/core/utils/SqlLogRendererTest.java b/teaql-utils/src/test/java/io/teaql/core/utils/SqlLogRendererTest.java new file mode 100644 index 00000000..76c5e958 --- /dev/null +++ b/teaql-utils/src/test/java/io/teaql/core/utils/SqlLogRendererTest.java @@ -0,0 +1,74 @@ +package io.teaql.core.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class SqlLogRendererTest { + @Test void rendersOnlySafeCallbackValuesAndIgnoresQuotedPlaceholders() { + var calls = new AtomicInteger(); + String sql = "-- ?\nUPDATE \"customer?\" SET name = ?, active = ? WHERE note = '?' /* ? */"; + String rendered = SqlLogRenderer.render(sql, 2, i -> { + calls.incrementAndGet(); + return i == 0 ? "'Ri*****de' /* masked */" : "TRUE"; + }, "sqlite"); + assertEquals(2, calls.get()); + assertEquals("-- ?\nUPDATE \"customer?\" SET name = 'Ri*****de' /* masked */, active = TRUE WHERE note = '?' /* ? */", rendered); + assertFalse(rendered.contains("Riverside")); + } + + @Test void strictBindingCountsFailClosedWithoutValuesInErrors() { + for (String sql : List.of("SELECT ?, ?", "SELECT 1", "SELECT 'secret-canary ?", + "SELECT ? /* unterminated", "SELECT /*! ? */ ?")) { + var error = assertThrows(IllegalArgumentException.class, + () -> SqlLogRenderer.render(sql, 1, i -> "'secret-canary'", "sqlite")); + assertFalse(error.getMessage().contains("secret-canary")); + assertFalse(error.getMessage().contains(sql)); + } + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.render("?", 1, i -> null, "sqlite")); + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.render("", -1, i -> "NULL", "sqlite")); + } + + @Test void respectsDialectQuotedRegionsAndEscapedOperators() { + assertEquals("SELECT `a``?`, 'x\\\'?y', 10", SqlLogRenderer.render( + "SELECT `a``?`, 'x\\\'?y', ?", 1, i -> "10", "mysql")); + assertEquals("SELECT [a]]?], 10", SqlLogRenderer.render("SELECT [a]]?], ?", 1, i -> "10", "mssql")); + assertEquals("SELECT $$?$$, $body$?$body$, data ? 'a', 10 /* outer /* ? */ ? */", + SqlLogRenderer.render("SELECT $$?$$, $body$?$body$, data ?? 'a', ? /* outer /* ? */ ? */", + 1, i -> "10", "postgresql")); + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.render("SELECT $body$?", 0, i -> "NULL", "postgresql")); + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.render("SELECT E'\\\'?', ?", 1, i -> "NULL", "postgresql")); + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.render("SELECT ? # ?", 1, i -> "NULL", "mysql")); + } + + @Test void retainsTypedTemporalLiterals() { + var date = LocalDate.of(2024, 2, 29); + var time = LocalDateTime.of(2026, 8, 19, 3, 30, 0, 123_000_000); + assertEquals("'2024-02-29'", SqlLogRenderer.literal(date, "sqlite")); + assertEquals("DATE '2024-02-29'", SqlLogRenderer.literal(date, "postgresql")); + assertEquals("CAST('2024-02-29' AS DATE)", SqlLogRenderer.literal(date, "mysql")); + assertEquals("TIMESTAMP '2026-08-19 03:30:00.123'", SqlLogRenderer.literal(time, "postgresql")); + assertEquals("CAST('2026-08-19 03:30:00.123' AS DATETIME(3))", SqlLogRenderer.literal(time, "mysql")); + assertEquals("CAST('2026-08-19 03:30:00.123' AS DATETIME2(3))", SqlLogRenderer.literal(time, "mssql")); + } + + @Test void safeTypedLiteralsDoNotInvokeUnknownObjectToString() { + assertEquals("NULL", SqlLogRenderer.literal(null, "sqlite")); + assertEquals("TRUE", SqlLogRenderer.literal(true, "sqlite")); + assertEquals("128.000001", SqlLogRenderer.literal(new BigDecimal("128.000001"), "sqlite")); + assertEquals("'O''Reilly'", SqlLogRenderer.literal("O'Reilly", "sqlite")); + assertEquals("'a\\\\b'", SqlLogRenderer.literal("a\\b", "mysql")); + assertEquals("X'00ff'", SqlLogRenderer.literal(new byte[]{0, (byte) 255}, "sqlite")); + assertEquals("decode('00ff','hex')", SqlLogRenderer.literal(new byte[]{0, (byte) 255}, "postgresql")); + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.literal(Double.NaN, "sqlite")); + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.literal(Double.POSITIVE_INFINITY, "sqlite")); + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.literal("canary\0value", "sqlite")); + Object secret = new Object() { @Override public String toString() { fail("must not stringify unknown values"); return "secret"; } }; + assertThrows(IllegalArgumentException.class, () -> SqlLogRenderer.literal(secret, "sqlite")); + } +} diff --git a/test-vectors/masking-v1.tsv b/test-vectors/masking-v1.tsv new file mode 100644 index 00000000..476f5a87 --- /dev/null +++ b/test-vectors/masking-v1.tsv @@ -0,0 +1,11 @@ +id input expected +empty +short Ada *** +digits 12345678 ******** +boundary ABCDEFGH AB****GH +long Riverside Ri*****de +quote O'Reilly O'****ly +emoji 😀😀1234😀😀 😀😀****😀😀 +non_ascii_digits １２３４５６７８ １２****７８ +combining éabcdef é****ef +cjk 甲乙丙丁戊己庚辛 甲乙****庚辛