From dc45aceb8d39c3d7dc4debe0571978faf3f46dbc Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Fri, 20 Nov 2026 13:35:31 -0500 Subject: [PATCH] fix(bigquery-jdbc): gate Storage Write API on direct VALUES inserts and align columns Stack: PR 4 of 5 (base: `fix/swa-proto-3-type-parity`) ### Summary - Fix silent SQL expression/UDF bypass (`b/587228731`) and `STRING -> BYTES` transform crashes when `EnableWriteAPI=true` is used with SQL expressions, `CAST`s, or UDFs inside `INSERT ... VALUES (...)`. - Add a zero-regex whitelist parser (`parseDirectInsertValues`) so only single-row `INSERT [INTO] [()] VALUES (?, ...) [;]` statements with bare `?` placeholders use the Storage Write API; all other `INSERT` forms fall back automatically to BigQuery SQL DML. - Align reordered column lists (`resolveInsertSchema`), reject partial/duplicate/pseudo-column lists so DDL `DEFAULT` values execute on BigQuery's SQL engine, honor `DefaultDataset` vs. `catalog` project matching (`resolveTargetProject`), skip positional dry runs in Legacy SQL mode, retry unbound dry runs at most once when tables are created after `prepareStatement()`, and fall back to SQL inside active transactions or `_SESSION` temp tables. ### Testing - `make lint`, `mvn -B -ntp dependency:analyze-only -DfailOnWarning=true`, `mvn -q -Denforcer.skip=true -Dclirr.skip=true test-compile`, and `make unittest` (parameterized SQL whitelist/rejection tests, `MASK_UDF(?)` `b/587228731` & `UDF(CAST(? AS STRING), ...)` SQL fallback tests, reordered column `ProtoRows` round-trips, project/session/transaction gating, and persistent dry-run error non-retry tests; ITs require GCP credentials). --- .../jdbc/BigQueryParameterHandler.java | 2 +- .../jdbc/BigQueryPreparedStatement.java | 459 ++++++++++++++++-- .../BigQueryPreparedStatementSettersTest.java | 6 +- ...BigQueryPreparedStatementWriteApiTest.java | 298 ++++++++++++ 4 files changed, 728 insertions(+), 37 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java index 97c0fed9b5f3..de528a1dd3fd 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java @@ -116,7 +116,7 @@ static Tuple> bindingForJavaTime(Object value) { this(parameterCount, new ArrayList<>(parameterCount), enableTimestampPicos); } - BigQueryParameterHandler( + private BigQueryParameterHandler( int parameterCount, ArrayList parametersList, boolean enableTimestampPicos) { diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java index 354c08cc28a8..7a7a929261c8 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java @@ -19,6 +19,7 @@ import com.google.api.gax.retrying.RetrySettings; import com.google.api.services.bigquery.model.QueryParameter; import com.google.cloud.Tuple; +import com.google.cloud.bigquery.DatasetId; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.JobStatistics.QueryStatistics; @@ -66,9 +67,15 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Queue; +import java.util.Set; class BigQueryPreparedStatement extends BigQueryStatement implements PreparedStatement { private static final BigQueryJdbcCustomLogger LOG = @@ -77,9 +84,11 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta // parameterHandler is inherited from BigQueryStatement protected final int parameterCount; protected final String currentQuery; + private final DirectInsertShape directInsertShape; + private boolean insertMetadataEvaluated = false; private Queue> batchParameters = new LinkedList<>(); - // Both hold statistics.getSchema(), which reports different things per statement type: the rows - // a SELECT returns, versus the columns an INSERT writes. + // resultSchema feeds getMetaData() for SELECT statements; insertSchema holds the target columns + // in placeholder order for single-table direct-VALUES INSERT statements on the Storage Write API. Schema resultSchema = null; // Feeds getMetaData(). Schema insertSchema = null; // Feeds the Storage Write API. private TableName insertTableName = null; @@ -90,6 +99,7 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta throw new BigQueryJdbcException("SQL query cannot be null."); } this.currentQuery = query; + this.directInsertShape = parseDirectInsertValues(query); QueryStatistics queryStatistics = describeQueryQuietly(); this.parameterCount = resolveParameterCount(query, queryStatistics); this.parameterHandler = @@ -116,7 +126,7 @@ private int resolveParameterCount(String query, QueryStatistics queryStatistics) * of the query text, so a statement stays usable against a service that refuses the dry run. */ private QueryStatistics describeQueryQuietly() { - if (this.currentQuery.indexOf(POSITIONAL_PARAMETER_CHAR) == -1) { + if (this.currentQuery.indexOf(POSITIONAL_PARAMETER_CHAR) == -1 || getUseLegacySql()) { return null; } try { @@ -160,27 +170,390 @@ private static StandardSQLTypeName toStandardSqlType(String typeName) { } } - // Remembers the table and schema of single-table INSERT targets, for the Storage Write API. - // Failure is delegated to executeBatch instead of the constructor. + // Remembers the table and target column schema of direct single-table INSERT ... VALUES (?, ...) + // statements for the Storage Write API. Statements with SQL expressions, partial column lists, or + // unsupported column types leave insertSchema null so executeBatch uses standard SQL execution. private void captureInsertMetadata(QueryStatistics statistics) { - Schema schema = statistics.getSchema(); - if (!QueryStatistics.StatementType.INSERT.equals(statistics.getStatementType()) - || schema == null - || schema.getFields() == null + if (statistics == null) { + return; + } + this.insertMetadataEvaluated = true; + if (this.directInsertShape == null + || !QueryStatistics.StatementType.INSERT.equals(statistics.getStatementType()) + || statistics.getSchema() == null || statistics.getReferencedTables() == null || statistics.getReferencedTables().stream().distinct().count() != 1) { return; } - for (Field field : schema.getFields()) { + TableId tableId = statistics.getReferencedTables().get(0); + String project = resolveTargetProject(tableId); + if (project == null || tableId.getDataset() == null || tableId.getTable() == null) { + return; + } + Schema resolvedSchema = + resolveInsertSchema(statistics.getSchema(), this.directInsertShape, this.parameterCount); + if (resolvedSchema == null) { + LOG.fine( + "INSERT target schema is incompatible with direct Storage Write API streaming;" + + " falling back to standard SQL batch execution."); + return; + } + this.insertSchema = resolvedSchema; + this.insertTableName = TableName.of(project, tableId.getDataset(), tableId.getTable()); + LOG.finer("insertTableName: %s, insertSchema: %s", this.insertTableName, this.insertSchema); + } + + private String resolveTargetProject(TableId tableId) { + if (tableId.getProject() != null) { + return tableId.getProject(); + } + DatasetId defaultDataset = this.querySettings.getDefaultDataset(); + String catalog = this.connection != null ? this.connection.getCatalog() : null; + if (defaultDataset != null + && defaultDataset.getProject() != null + && (catalog == null + || (tableId.getDataset() != null + && tableId.getDataset().equals(defaultDataset.getDataset())))) { + return defaultDataset.getProject(); + } + return catalog; + } + + static Schema resolveInsertSchema( + Schema tableSchema, DirectInsertShape shape, int parameterCount) { + if (tableSchema == null || shape == null) { + return null; + } + FieldList tableFields = tableSchema.getFields(); + if (tableFields == null + || shape.placeholderCount == 0 + || shape.placeholderCount != parameterCount + || tableFields.size() != shape.placeholderCount) { + return null; + } + for (Field field : tableFields) { if (!BigQueryTypeRegistry.isSupportedWriteApiField(field)) { - return; + return null; } } - this.insertSchema = schema; - TableId tableId = statistics.getReferencedTables().get(0); - this.insertTableName = - TableName.of(tableId.getProject(), tableId.getDataset(), tableId.getTable()); - LOG.finer("insertTableName: %s, insertSchema: %s", this.insertTableName, this.insertSchema); + if (shape.columnNames == null) { + return tableSchema; + } + if (shape.columnNames.size() != tableFields.size()) { + return null; + } + Map fieldsByLowerName = new HashMap<>(tableFields.size() * 2); + for (Field field : tableFields) { + fieldsByLowerName.put(field.getName().toLowerCase(Locale.ROOT), field); + } + List orderedFields = new ArrayList<>(shape.columnNames.size()); + Set seenColumns = new HashSet<>(shape.columnNames.size() * 2); + for (String columnName : shape.columnNames) { + String normalized = columnName.toLowerCase(Locale.ROOT); + if (!seenColumns.add(normalized)) { + return null; + } + Field matched = fieldsByLowerName.get(normalized); + if (matched == null) { + return null; + } + orderedFields.add(matched); + } + return Schema.of(orderedFields); + } + + static final class DirectInsertShape { + final int placeholderCount; + final List columnNames; + + DirectInsertShape(int placeholderCount, List columnNames) { + this.placeholderCount = placeholderCount; + this.columnNames = + columnNames == null ? null : Collections.unmodifiableList(new ArrayList<>(columnNames)); + } + } + + /** + * Recognizes single-row {@code INSERT [INTO]
[(, ...)] VALUES (?, ...) [;]} + * statements whose {@code VALUES} tuple consists strictly of bare positional {@code ?} + * placeholders. Returns {@code null} for any SQL expression, function/UDF call, cast, literal, + * {@code DEFAULT}, {@code SELECT}, multi-row tuple list, or {@code _SESSION}-qualified temporary + * table so the statement executes on BigQuery's SQL engine instead of the Storage Write API. + */ + static DirectInsertShape parseDirectInsertValues(String sql) { + if (sql == null) { + return null; + } + int pos = skipTrivia(sql, 0); + if (pos < 0) { + return null; + } + pos = matchKeyword(sql, pos, "INSERT"); + if (pos < 0) { + return null; + } + pos = skipTrivia(sql, pos); + if (pos < 0) { + return null; + } + int afterInto = matchKeyword(sql, pos, "INTO"); + if (afterInto >= 0) { + pos = skipTrivia(sql, afterInto); + if (pos < 0) { + return null; + } + } + pos = consumeTablePath(sql, pos); + if (pos < 0 || pos >= sql.length()) { + return null; + } + + List columnNames = null; + if (sql.charAt(pos) == '(') { + columnNames = new ArrayList<>(); + pos++; + while (true) { + pos = skipTrivia(sql, pos); + if (pos < 0 || pos >= sql.length()) { + return null; + } + Tuple column = readColumnIdentifier(sql, pos); + if (column == null) { + return null; + } + columnNames.add(column.x()); + pos = skipTrivia(sql, column.y()); + if (pos < 0 || pos >= sql.length()) { + return null; + } + char sep = sql.charAt(pos); + if (sep == ',') { + pos++; + } else if (sep == ')') { + pos++; + break; + } else { + return null; + } + } + } + + pos = skipTrivia(sql, pos); + if (pos < 0) { + return null; + } + pos = matchKeyword(sql, pos, "VALUES"); + if (pos < 0) { + return null; + } + pos = skipTrivia(sql, pos); + if (pos < 0 || pos >= sql.length() || sql.charAt(pos) != '(') { + return null; + } + pos++; + + int placeholderCount = 0; + while (true) { + pos = skipTrivia(sql, pos); + if (pos < 0 || pos >= sql.length() || sql.charAt(pos) != POSITIONAL_PARAMETER_CHAR) { + return null; + } + placeholderCount++; + pos = skipTrivia(sql, pos + 1); + if (pos < 0 || pos >= sql.length()) { + return null; + } + char sep = sql.charAt(pos); + if (sep == ',') { + pos++; + } else if (sep == ')') { + pos++; + break; + } else { + return null; + } + } + + pos = skipTrivia(sql, pos); + if (pos < 0) { + return null; + } + if (pos < sql.length() && sql.charAt(pos) == ';') { + pos = skipTrivia(sql, pos + 1); + if (pos < 0) { + return null; + } + } + if (pos != sql.length()) { + return null; + } + return new DirectInsertShape(placeholderCount, columnNames); + } + + private static int consumeTablePath(String sql, int pos) { + Tuple firstSegment = readTableSegment(sql, pos); + if (firstSegment == null) { + return -1; + } + String firstName = firstSegment.x(); + pos = firstSegment.y(); + int segmentCount = 1; + while (true) { + int next = skipTrivia(sql, pos); + if (next < 0) { + return -1; + } + if (next < sql.length() && (sql.charAt(next) == '.' || sql.charAt(next) == ':')) { + pos = skipTrivia(sql, next + 1); + if (pos < 0) { + return -1; + } + Tuple segment = readTableSegment(sql, pos); + if (segment == null) { + return -1; + } + pos = segment.y(); + segmentCount++; + } else { + pos = next; + break; + } + } + if (segmentCount == 2 && "_session".equalsIgnoreCase(firstName)) { + return -1; + } + if (segmentCount == 1 + && firstName.length() > 9 + && firstName.regionMatches(true, 0, "_session.", 0, 9)) { + return -1; + } + return pos; + } + + private static Tuple readTableSegment(String sql, int pos) { + if (pos >= sql.length()) { + return null; + } + char first = sql.charAt(pos); + if (first == '`') { + return readBacktickIdentifier(sql, pos); + } + if (!isUnquotedTableStart(first)) { + return null; + } + int i = pos + 1; + while (i < sql.length() && isUnquotedTablePart(sql.charAt(i))) { + i++; + } + return Tuple.of(sql.substring(pos, i), i); + } + + private static Tuple readColumnIdentifier(String sql, int pos) { + if (pos >= sql.length()) { + return null; + } + char first = sql.charAt(pos); + if (first == '`') { + return readBacktickIdentifier(sql, pos); + } + if (!isIdentifierStart(first)) { + return null; + } + int i = pos + 1; + while (i < sql.length() && isIdentifierPart(sql.charAt(i))) { + i++; + } + return Tuple.of(sql.substring(pos, i), i); + } + + private static Tuple readBacktickIdentifier(String sql, int pos) { + StringBuilder sb = new StringBuilder(); + int i = pos + 1; + while (i < sql.length()) { + char c = sql.charAt(i); + if (c == '\\') { + if (i + 1 >= sql.length()) { + return null; + } + char escaped = sql.charAt(i + 1); + if (escaped != '\\' && escaped != '`') { + return null; + } + sb.append(escaped); + i += 2; + } else if (c == '`') { + if (sb.length() == 0) { + return null; + } + return Tuple.of(sb.toString(), i + 1); + } else { + sb.append(c); + i++; + } + } + return null; + } + + private static int matchKeyword(String sql, int pos, String keyword) { + int end = pos + keyword.length(); + if (end > sql.length() || !sql.regionMatches(true, pos, keyword, 0, keyword.length())) { + return -1; + } + if (end < sql.length() && isIdentifierPart(sql.charAt(end))) { + return -1; + } + return end; + } + + private static int skipTrivia(String sql, int pos) { + int len = sql.length(); + while (pos < len) { + char ch = sql.charAt(pos); + if (Character.isWhitespace(ch)) { + pos++; + } else if (ch == '#') { + pos = skipToEndOfLine(sql, pos + 1); + } else if (ch == '-' && pos + 1 < len && sql.charAt(pos + 1) == '-') { + pos = skipToEndOfLine(sql, pos + 2); + } else if (ch == '/' && pos + 1 < len && sql.charAt(pos + 1) == '*') { + int closeIdx = sql.indexOf("*/", pos + 2); + if (closeIdx < 0) { + return -1; + } + pos = closeIdx + 2; + } else { + break; + } + } + return pos; + } + + private static int skipToEndOfLine(String sql, int pos) { + int len = sql.length(); + while (pos < len) { + char ch = sql.charAt(pos); + if (ch == '\n' || ch == '\r') { + return pos + 1; + } + pos++; + } + return len; + } + + private static boolean isIdentifierStart(char ch) { + return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch == '_'; + } + + private static boolean isIdentifierPart(char ch) { + return isIdentifierStart(ch) || (ch >= '0' && ch <= '9'); + } + + private static boolean isUnquotedTableStart(char ch) { + return isIdentifierPart(ch); + } + + private static boolean isUnquotedTablePart(char ch) { + return isIdentifierPart(ch) || ch == '-'; } // This populates the schema of the ResultSet that is being returned by the query. @@ -448,12 +821,19 @@ public int[] executeBatch() throws SQLException { private int[] executeBatchInternal(int[] result) throws SQLException { // The constructor's dry run may have happened before the target table existed. - if (this.insertSchema == null + if (!this.insertMetadataEvaluated + && this.directInsertShape != null && this.querySettings.isUseWriteAPI() + && !getUseLegacySql() + && (this.connection == null || !this.connection.isTransactionStarted()) && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount()) { + this.insertMetadataEvaluated = true; try { - captureInsertMetadata( - getQueryStatistics(getWriteBatchJobConfiguration(this.batchParameters.peek()))); + QueryStatistics statistics = describePositionalParameterQuery(this.currentQuery); + if (statistics != null) { + applyInferredParameterTypes(statistics); + captureInsertMetadata(statistics); + } } catch (SQLException | RuntimeException ex) { LOG.warning(ex, "Could not describe INSERT target; using the standard batch path."); } @@ -502,6 +882,12 @@ private int[] executeBatchInternal(int[] result) throws SQLException { } } else { + if (this.querySettings.isUseWriteAPI() + && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount()) { + LOG.fine( + "Batch query is not eligible for Storage Write API; falling back to standard SQL" + + " batch execution."); + } try { LOG.info("Using individual INSERT query runs."); int count = this.batchParameters.size(); @@ -602,18 +988,6 @@ private long bulkInsertWithWriteAPI(BigQueryWriteClient bigQueryWriteClient) } } - QueryJobConfiguration getWriteBatchJobConfiguration( - ArrayList currentParameterList) throws SQLException { - LOG.finer("++enter++"); - BigQueryParameterHandler batchHandler = - new BigQueryParameterHandler( - this.parameterCount, currentParameterList, this.isEnableTimestampPicos()); - QueryJobConfiguration.Builder jobConfiguration = getJobConfig(this.currentQuery); - jobConfiguration.setParameterMode("POSITIONAL"); - jobConfiguration = batchHandler.configureParameters(jobConfiguration); - return jobConfiguration.build(); - } - QueryJobConfiguration getStandardBatchJobConfiguration(String query) throws SQLException { LOG.finer("++enter++"); QueryJobConfiguration.Builder jobConfiguration = getJobConfig(query); @@ -640,13 +1014,34 @@ QueryJobConfiguration getStandardBatchJobConfiguration(String query) throws SQLE Boolean useWriteAPI() { LOG.finer("++enter++"); return this.querySettings.isUseWriteAPI() - // the constructor dry run already confirmed a single-table INSERT && this.insertSchema != null && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount() + && isWriteApiAllowedBySessionAndTransaction() && BigQueryParameterHandler.areBatchParametersCompatibleWithWriteApi( this.insertSchema.getFields(), this.batchParameters); } + private boolean isWriteApiAllowedBySessionAndTransaction() { + if (this.connection == null) { + return true; + } + if (this.connection.isTransactionStarted()) { + return false; + } + if (this.insertTableName != null && this.insertTableName.getDataset().startsWith("_")) { + BigQueryConnection.SessionState snapshot = this.connection.getSessionStateSnapshot(); + boolean sessionActive = + this.connection.isSessionEnabled() || (snapshot != null && snapshot.sessionInfo != null); + DatasetId defaultDataset = this.querySettings.getDefaultDataset(); + if (sessionActive + && (defaultDataset == null + || !this.insertTableName.getDataset().equals(defaultDataset.getDataset()))) { + return false; + } + } + return true; + } + @Override public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java index c5ff936d2bf8..a73ff55bb502 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java @@ -59,7 +59,6 @@ import java.time.LocalTime; import java.time.OffsetDateTime; import java.time.ZonedDateTime; -import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.TimeZone; @@ -354,9 +353,8 @@ public void testBatchJobConfigurationWithSetObjectNull() throws Exception { preparedStatement.setBoolean(5, true); preparedStatement.addBatch(); - ArrayList batchParams = - preparedStatement.parameterHandler.parametersList; - QueryJobConfiguration config = preparedStatement.getWriteBatchJobConfiguration(batchParams); + QueryJobConfiguration config = + preparedStatement.getStandardBatchJobConfiguration("INSERT INTO t VALUES (?, ?, ?, ?, ?);"); assertNotNull(config); assertEquals(5, config.getPositionalParameters().size()); assertNull(config.getPositionalParameters().get(0).getValue()); diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java index 284f8bef923c..271223652214 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -27,6 +28,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -45,6 +47,8 @@ import com.google.api.services.bigquery.model.QueryParameterType; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.ConnectionProperty; +import com.google.cloud.bigquery.DatasetId; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobInfo; @@ -55,6 +59,7 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; @@ -109,8 +114,13 @@ import java.util.Locale; import java.util.TimeZone; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; @@ -293,6 +303,7 @@ public void testExecuteBatchKeepsWriteClientOpenAcrossBatchesAndWrapsErrors() th } private static final class InProcessWriteServer implements AutoCloseable { + final List createParents = new CopyOnWriteArrayList<>(); final List appendRequests = new CopyOnWriteArrayList<>(); final Descriptor descriptor; final Server server; @@ -312,6 +323,7 @@ private static final class InProcessWriteServer implements AutoCloseable { @Override public void createWriteStream( CreateWriteStreamRequest req, StreamObserver obs) { + createParents.add(req.getParent()); obs.onNext( WriteStream.newBuilder() .setName(req.getParent() + "/streams/_p1") @@ -677,4 +689,290 @@ public void testUnsupportedColumnsAndIncompatibleValuesFallBackToSqlBatch() thro } verify(connection, never()).getBigQueryWriteClient(); } + + private static Stream validDirectInsertQueries() { + return Stream.of( + Arguments.of("INSERT INTO TAB VALUES (?)", 1, null), + Arguments.of( + "insert tab (TARG_FIELD) values (?);", 1, Collections.singletonList("TARG_FIELD")), + Arguments.of( + "/* header */ INSERT INTO `my-proj.my_ds.TAB` (`col_b`, col_a) -- comment\n" + + "VALUES ( ? , ? ) ; # trailing", + 2, + Arrays.asList("col_b", "col_a")), + Arguments.of( + "INSERT INTO proj:123_ds.tab (c1) -- mac line ending\rVALUES (?)", + 1, + Collections.singletonList("c1")), + Arguments.of( + "INSERT INTO `my\\`tab` (`col\\\\1`, `col\\`2`) VALUES (?, ?)", + 2, + Arrays.asList("col\\1", "col`2"))); + } + + @ParameterizedTest + @MethodSource("validDirectInsertQueries") + public void testParseDirectInsertValuesAcceptsValidForms( + String sql, int expectedPlaceholders, List expectedColumns) { + BigQueryPreparedStatement.DirectInsertShape shape = + BigQueryPreparedStatement.parseDirectInsertValues(sql); + assertNotNull(shape); + assertEquals(expectedPlaceholders, shape.placeholderCount); + assertEquals(expectedColumns, shape.columnNames); + } + + @ParameterizedTest + @ValueSource( + strings = { + "", + " ", + "SELECT ?", + "UPDATE TAB SET a = ? WHERE b = ?", + "MERGE INTO TAB USING (SELECT ? AS v) ON FALSE WHEN NOT MATCHED THEN INSERT (a) VALUES (v)", + "INSERT INTO TAB (TARG_FIELD) VALUES (UDF(CAST(? AS STRING), 'XXXXXX'))", + "INSERT INTO TAB (STR_FIELD) VALUES (MASK_UDF(?))", + "INSERT INTO TAB (a) VALUES ((?))", + "INSERT INTO TAB (a, b) VALUES (?, 'constant')", + "INSERT INTO TAB (a, b) VALUES (?, DEFAULT)", + "INSERT INTO TAB (a, b) VALUES (?, NULL)", + "INSERT INTO TAB (a) VALUES (?), (?)", + "INSERT INTO TAB (a) SELECT a FROM TAB WHERE a = ?", + "INSERT INTO _SESSION.temp_tab (a) VALUES (?)", + "INSERT INTO `_SESSION.temp_tab` (a) VALUES (?)", + "INSERT INTO `\\u005fSESSION.temp_tab` (a) VALUES (?)", + "INSERT INTO TAB () VALUES (?)", + "INSERT INTO TAB (s.a) VALUES (?)", + "INSERT INTO TAB (1col) VALUES (?)", + "INSERT INTO TAB (`bad\\nescape`) VALUES (?)", + "INSERT INTO `` VALUES (?)", + "INSERT INTO TAB (a) VALUES (?); INSERT INTO TAB (a) VALUES (?)", + "/* unclosed comment INSERT INTO TAB VALUES (?)" + }) + public void testParseDirectInsertValuesRejectsIneligibleSql(String sql) { + assertNull(BigQueryPreparedStatement.parseDirectInsertValues(null)); + assertNull(BigQueryPreparedStatement.parseDirectInsertValues(sql)); + } + + private static Stream expressionFallbackQueries() { + return Stream.of( + Arguments.of( + "INSERT INTO TAB (STR_FIELD) VALUES (MASK_UDF(?));", StandardSQLTypeName.STRING), + Arguments.of( + "INSERT INTO TAB (TARG_FIELD) VALUES (UDF(CAST(? AS STRING), 'XXXXXX'));", + StandardSQLTypeName.BYTES)); + } + + @ParameterizedTest + @MethodSource("expressionFallbackQueries") + public void testExpressionInValuesFallsBackToStandardSqlBatch( + String sql, StandardSQLTypeName targetColType) throws Exception { + doReturn( + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + Schema.of(Field.of("COL", targetColType)), + Collections.singletonList(queryParam("STRING")))) + .when(bigQuery) + .create(any(JobInfo.class)); + + TableResult dmlResult = mock(TableResult.class); + doReturn(StatementType.INSERT).when(dmlResult).getStatementType(); + doReturn(1L).when(dmlResult).getNumDmlAffectedRows(); + doReturn(dmlResult).when(bigQuery).queryWithTimeout(any(), any(), any()); + + BigQueryPreparedStatement ps = new BigQueryPreparedStatement(connection, sql); + assertNull(ps.insertSchema); + for (int i = 0; i < 3; i++) { + ps.setString(1, "val-" + i); + ps.addBatch(); + } + assertFalse(ps.useWriteAPI()); + assertEquals(3, ps.executeBatch().length); + verify(connection, never()).getBigQueryWriteClient(); + verify(bigQuery, times(1)).create(any(JobInfo.class)); + + ArgumentCaptor jobCaptor = + ArgumentCaptor.forClass(QueryJobConfiguration.class); + verify(bigQuery).queryWithTimeout(jobCaptor.capture(), any(), any()); + assertEquals(3, jobCaptor.getValue().getPositionalParameters().size()); + assertEquals("val-0", jobCaptor.getValue().getPositionalParameters().get(0).getValue()); + } + + @Test + public void testResolveInsertSchemaAndReorderedColumnProtoRowsExecution() throws Exception { + Schema twoCol = + Schema.of( + Field.of("id", StandardSQLTypeName.INT64), + Field.of("name", StandardSQLTypeName.STRING)); + BigQueryPreparedStatement.DirectInsertShape reordered = + BigQueryPreparedStatement.parseDirectInsertValues( + "INSERT INTO TAB (NAME, id) VALUES (?, ?)"); + Schema resolved = BigQueryPreparedStatement.resolveInsertSchema(twoCol, reordered, 2); + assertEquals("name", resolved.getFields().get(0).getName()); + assertEquals("id", resolved.getFields().get(1).getName()); + + // Subset, duplicate, and unknown/pseudo columns return null. + for (String badSql : + new String[] { + "INSERT INTO TAB (id) VALUES (?)", + "INSERT INTO TAB (id, ID) VALUES (?, ?)", + "INSERT INTO TAB (id, _PARTITIONTIME) VALUES (?, ?)" + }) { + BigQueryPreparedStatement.DirectInsertShape s = + BigQueryPreparedStatement.parseDirectInsertValues(badSql); + assertNull(BigQueryPreparedStatement.resolveInsertSchema(twoCol, s, s.placeholderCount)); + } + + // End-to-end reordered executeBatch maps placeholder 1 -> "name" and placeholder 2 -> "id". + TableSchema storageTwoCol = + TableSchema.newBuilder() + .addFields(tfs("id", TableFieldSchema.Type.INT64)) + .addFields(tfs("name", TableFieldSchema.Type.STRING)) + .build(); + try (InProcessWriteServer srv = new InProcessWriteServer(storageTwoCol)) { + doReturn(srv.client).when(connection).getBigQueryWriteClient(); + doReturn( + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + twoCol, + Arrays.asList(queryParam("STRING"), queryParam("INT64")))) + .when(bigQuery) + .create(any(JobInfo.class)); + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB (name, id) VALUES (?, ?)"); + for (int i = 0; i < 3; i++) { + ps.setString(1, "u-" + i); + ps.setInt(2, 10 + i); + ps.addBatch(); + } + assertArrayEquals(new int[] {1, 1, 1}, ps.executeBatch()); + DynamicMessage first = srv.decodedRows().get(0); + assertEquals("u-0", first.getField(srv.descriptor.findFieldByName("name"))); + assertEquals(10L, first.getField(srv.descriptor.findFieldByName("id"))); + } + } + + @Test + public void testTargetProjectTransactionsSessionsLegacySqlAndRetryDryRun() throws Exception { + // 1. Legacy SQL skips positional dry run in constructor. + doReturn("BIG_QUERY").when(connection).getQueryDialect(); + assertEquals( + 1, new BigQueryPreparedStatement(connection, "SELECT ? FROM [proj:ds.tab]").parameterCount); + verify(bigQuery, never()).create(any(JobInfo.class)); + doReturn("SQL").when(connection).getQueryDialect(); + + // 2. Active transaction & active session anonymous "_" dataset gating. + Schema oneStr = Schema.of(Field.of("c1", StandardSQLTypeName.STRING)); + doReturn( + mockInsertDryRunJob( + TableId.of("proj", "_anon_session_ds", "TAB"), + oneStr, + Collections.singletonList(queryParam("STRING")))) + .when(bigQuery) + .create(any(JobInfo.class)); + BigQueryPreparedStatement psGate = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?)"); + for (int i = 0; i < 3; i++) { + psGate.setString(1, "v" + i); + psGate.addBatch(); + } + assertTrue(psGate.useWriteAPI()); + doReturn(true).when(connection).isTransactionStarted(); + assertFalse(psGate.useWriteAPI()); + doReturn(false).when(connection).isTransactionStarted(); + + doReturn( + new BigQueryConnection.SessionState( + ConnectionProperty.of("session_id", "s1"), Collections.emptyList(), true)) + .when(connection) + .getSessionStateSnapshot(); + assertFalse(psGate.useWriteAPI()); + doReturn(DatasetId.of("proj", "_anon_session_ds")).when(connection).getDefaultDataset(); + BigQueryPreparedStatement psExplicitHidden = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?)"); + for (int i = 0; i < 3; i++) { + psExplicitHidden.setString(1, "v" + i); + psExplicitHidden.addBatch(); + } + assertTrue(psExplicitHidden.useWriteAPI()); + doReturn(BigQueryConnection.SessionState.empty()).when(connection).getSessionStateSnapshot(); + + // 3. Target project resolution & retry dry run after 404 (and non-repetition on null/403). + doReturn("catalog-proj").when(connection).getCatalog(); + doReturn(DatasetId.of("default-proj", "default_ds")).when(connection).getDefaultDataset(); + TableSchema oneStrStorage = + TableSchema.newBuilder().addFields(tfs("c1", TableFieldSchema.Type.STRING)).build(); + try (InProcessWriteServer srv = new InProcessWriteServer(oneStrStorage)) { + doReturn(srv.client).when(connection).getBigQueryWriteClient(); + for (String ds : new String[] {"other_ds", "default_ds"}) { + srv.appendRequests.clear(); + doReturn( + mockInsertDryRunJob( + TableId.of(ds, "TAB"), oneStr, Collections.singletonList(queryParam("STRING")))) + .when(bigQuery) + .create(any(JobInfo.class)); + BigQueryPreparedStatement psProj = + new BigQueryPreparedStatement(connection, "INSERT INTO " + ds + ".TAB VALUES (?)"); + for (int i = 0; i < 3; i++) { + psProj.setString(1, "v" + i); + psProj.addBatch(); + } + assertArrayEquals(new int[] {1, 1, 1}, psProj.executeBatch()); + } + assertEquals( + Arrays.asList( + "projects/catalog-proj/datasets/other_ds/tables/TAB", + "projects/default-proj/datasets/default_ds/tables/TAB"), + srv.createParents); + + // Retry dry run after 404 uses unbound query and unlocks Write API execution. + srv.appendRequests.clear(); + clearInvocations(bigQuery); + when(bigQuery.create(any(JobInfo.class))) + .thenThrow(new BigQueryException(404, "Not found")) + .thenReturn( + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + oneStr, + Collections.singletonList(queryParam("STRING")))); + BigQueryPreparedStatement psRetry = + new BigQueryPreparedStatement(connection, "INSERT INTO ds.TAB (c1) VALUES (?)"); + for (int i = 0; i < 3; i++) { + psRetry.setString(1, "r" + i); + psRetry.addBatch(); + } + assertArrayEquals(new int[] {1, 1, 1}, psRetry.executeBatch()); + assertNotNull(psRetry.insertSchema); + ArgumentCaptor jobCaptor = ArgumentCaptor.forClass(JobInfo.class); + verify(bigQuery, times(2)).create(jobCaptor.capture()); + QueryJobConfiguration retryCfg = jobCaptor.getAllValues().get(1).getConfiguration(); + assertTrue( + retryCfg.getPositionalParameters() == null + || retryCfg.getPositionalParameters().isEmpty()); + } + + TableResult dmlResult = mock(TableResult.class); + doReturn(StatementType.INSERT).when(dmlResult).getStatementType(); + doReturn(1L).when(dmlResult).getNumDmlAffectedRows(); + doReturn(dmlResult).when(bigQuery).queryWithTimeout(any(), any(), any()); + for (boolean throwOnRetry : new boolean[] {false, true}) { + clearInvocations(bigQuery); + if (throwOnRetry) { + when(bigQuery.create(any(JobInfo.class))).thenThrow(new BigQueryException(403, "Denied")); + } else { + when(bigQuery.create(any(JobInfo.class))) + .thenThrow(new BigQueryException(404, "Not found")) + .thenReturn(mock(Job.class)); + } + BigQueryPreparedStatement psOnce = + new BigQueryPreparedStatement(connection, "INSERT INTO ds.TAB (c1) VALUES (?)"); + for (int b = 0; b < 2; b++) { + for (int i = 0; i < 3; i++) { + psOnce.setString(1, "v" + i); + psOnce.addBatch(); + } + psOnce.executeBatch(); + } + verify(bigQuery, times(2)).create(any(JobInfo.class)); + } + } }