From d8c218ab56cb979fc4b069516cef5bddfcd8116c Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Tue, 8 Sep 2026 16:00:36 +0000 Subject: [PATCH 1/6] Allow configuring schema alignment behavior for DSv2 writes --- .../catalog/SchemaAlignmentConfig.java | 52 ++++ .../spark/sql/connector/catalog/Table.java | 10 + .../sql/catalyst/analysis/Analyzer.scala | 15 +- .../catalyst/analysis/AssignmentUtils.scala | 44 ++- .../ResolveRowLevelCommandAssignments.scala | 38 ++- .../analysis/TableOutputResolver.scala | 120 ++++--- .../write/RowLevelOperationTable.scala | 3 +- .../connector/catalog/InMemoryBaseTable.scala | 3 +- .../InMemoryRowLevelOperationTable.scala | 21 +- .../sql/connector/catalog/InMemoryTable.scala | 5 +- .../spark/sql/connector/catalog/txns.scala | 3 +- .../InsertSchemaEvolutionSuite.scala | 192 ++++++++++++ .../catalog/SchemaAlignmentConfigSuite.scala | 294 ++++++++++++++++++ .../command/AlignAssignmentsSuiteBase.scala | 10 +- .../command/PlanResolutionSuite.scala | 9 +- 15 files changed, 734 insertions(+), 85 deletions(-) create mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/connector/InsertSchemaEvolutionSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java new file mode 100644 index 000000000000..2351f9a6229c --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog; + +import org.apache.spark.annotation.Evolving; + +/** + * Schema-alignment configuration for writes to a {@link Table}. This allows connectors to + * configure casting behavior and handling of schema mismatches during writes. + * + * @since 4.3.0 + */ +@Evolving +public interface SchemaAlignmentConfig { + + /** The strict data source v2 configuration, returned by {@link Table} by default. */ + SchemaAlignmentConfig DEFAULT = new SchemaAlignmentConfig() {}; + + /** + * Whether {@code spark.sql.storeAssignmentPolicy=LEGACY} is allowed for writes and row-level + * operations targeting this table. Data source v2 rejects LEGACY by default; a table can decide + * to opt-out from this restriction. + */ + default boolean allowLegacyStoreAssignmentPolicy() { + return false; + } + + /** + * Whether the {@code ANSI} store-assignment cast check is deferred from analysis to runtime under + * {@code spark.sql.storeAssignmentPolicy=ANSI}. When {@code true}, the analyzer skips the + * store-assignment compatibility check and inserts an ANSI cast, so malformed values or + * overflows surface at execution time. + */ + default boolean deferCastValidationToRuntime() { + return false; + } +} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java index ec27bcf6c82e..1c740afbf80a 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java @@ -112,4 +112,14 @@ default Map properties() { * the version that corresponds to the current state of this table instance. */ default String version() { return null; } + + /** + * Returns the schema-alignment configuration for writes to this table. This allows connectors to + * configure casting behavior and handling of schema mismatches during writes. + * It is recommended to use the DEFAULT configuration to provide a unified behavior across data + * sources, but some connectors may require deviating from the default behavior. + */ + default SchemaAlignmentConfig schemaAlignmentConfig() { + return SchemaAlignmentConfig.DEFAULT; + } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 35b9052686dc..6322fccd7494 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -3945,7 +3945,10 @@ class Analyzer( case v2Write: V2WriteCommand if v2Write.table.resolved && v2Write.query.resolved && !v2Write.outputResolved && v2Write.pendingSchemaChanges.isEmpty => - validateStoreAssignmentPolicy() + val schemaAlignment = v2Write.table.collectFirst { + case r: DataSourceV2Relation => r.table.schemaAlignmentConfig() + }.getOrElse(SchemaAlignmentConfig.DEFAULT) + validateStoreAssignmentPolicy(schemaAlignment) TableOutputResolver.suitableForByNameCheck(v2Write.isByName, expected = v2Write.table.output, queryOutput = v2Write.query.output) // With schema evolution + coercion flag, missing top-level columns AND missing nested @@ -3965,7 +3968,8 @@ class Analyzer( val (projection, autoFilledGenCols) = TableOutputResolver.resolveOutputColumnsWithGeneratedInfo( v2Write.table.name, expected, v2Write.query, v2Write.isByName, conf, - defaultValueFillMode) + defaultValueFillMode, + deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) if (projection != v2Write.query) { val cleanedTable = v2Write.table match { case r: DataSourceV2Relation => @@ -3981,9 +3985,10 @@ class Analyzer( } } - private def validateStoreAssignmentPolicy(): Unit = { - // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2. - if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY) { + private def validateStoreAssignmentPolicy(schemaAlignment: SchemaAlignmentConfig): Unit = { + // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2 by default. + if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY && + !schemaAlignment.allowLegacyStoreAssignmentPolicy()) { throw QueryCompilationErrors.legacyStoreAssignmentPolicyError() } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala index df4b0646ed42..42065caebb68 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getDefaultValueExprOrNullLit import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ +import org.apache.spark.sql.connector.catalog.SchemaAlignmentConfig import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StructType} @@ -63,7 +64,8 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { attrs: Seq[Attribute], assignments: Seq[Assignment], fromStar: Boolean, - coerceNestedTypes: Boolean): Seq[Assignment] = { + coerceNestedTypes: Boolean, + schemaAlignment: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT): Seq[Assignment] = { val errors = new mutable.ArrayBuffer[String]() @@ -75,7 +77,8 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { addError = err => errors += err, colPath = Seq(attr.name), coerceNestedTypes, - fromStar) + fromStar, + schemaAlignment = schemaAlignment) } if (errors.nonEmpty) { @@ -103,7 +106,8 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { def alignInsertAssignments( attrs: Seq[Attribute], assignments: Seq[Assignment], - coerceNestedTypes: Boolean = false): Seq[Assignment] = { + coerceNestedTypes: Boolean = false, + schemaAlignment: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT): Seq[Assignment] = { val errors = new mutable.ArrayBuffer[String]() @@ -137,7 +141,8 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { val value = matchingAssignments.head.value val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate( - "", value, actualAttr, conf, err => errors += err, colPath, coerceMode) + "", value, actualAttr, conf, err => errors += err, colPath, coerceMode, + deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) } Assignment(attr, resolvedValue) } @@ -160,7 +165,8 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { addError: String => Unit, colPath: Seq[String], coerceNestedTypes: Boolean = false, - updateStar: Boolean = false): Expression = { + updateStar: Boolean = false, + schemaAlignment: SchemaAlignmentConfig): Expression = { val (exactAssignments, otherAssignments) = assignments.partition { assignment => assignment.key.semanticEquals(colExpr) @@ -188,21 +194,24 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { case _: StructType => // Expand assignments to leaf fields (fixNullExpansion is applied inside) applyNestedFieldAssignments(col, colExpr, value, addError, colPath, - coerceNestedTypes) + coerceNestedTypes, schemaAlignment = schemaAlignment) case _ => // For non-struct types, resolve directly val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate("", value, col, conf, addError, colPath, - coerceMode) + coerceMode, + deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) } } else { val value = exactAssignments.head.value val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate("", value, col, conf, addError, - colPath, coerceMode) + colPath, coerceMode, + deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) } } else { - applyFieldAssignments(col, colExpr, fieldAssignments, addError, colPath, coerceNestedTypes) + applyFieldAssignments(col, colExpr, fieldAssignments, addError, colPath, coerceNestedTypes, + schemaAlignment = schemaAlignment) } } @@ -212,7 +221,8 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { assignments: Seq[Assignment], addError: String => Unit, colPath: Seq[String], - coerceNestedTypes: Boolean): Expression = { + coerceNestedTypes: Boolean, + schemaAlignment: SchemaAlignmentConfig): Expression = { col.dataType match { case structType: StructType => @@ -222,7 +232,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { } val updatedFieldExprs = fieldAttrs.zip(fieldExprs).map { case (fieldAttr, fieldExpr) => applyAssignments(fieldAttr, fieldExpr, assignments, addError, colPath :+ fieldAttr.name, - coerceNestedTypes) + coerceNestedTypes, schemaAlignment = schemaAlignment) } toNamedStruct(structType, updatedFieldExprs) @@ -240,7 +250,8 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { value: Expression, addError: String => Unit, colPath: Seq[String], - coerceNestedTypes: Boolean): Expression = { + coerceNestedTypes: Boolean, + schemaAlignment: SchemaAlignmentConfig): Expression = { col.dataType match { case structType: StructType => @@ -273,12 +284,15 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { case _: StructType => // Field is a struct, recurse applyNestedFieldAssignments(fieldAttr, targetFieldExpr, - sourceFieldValue, addError, fieldPath, coerceNestedTypes) + sourceFieldValue, addError, fieldPath, coerceNestedTypes, + schemaAlignment = schemaAlignment) case _ => // Field is not a struct, resolve with TableOutputResolver val coerceMode = if (coerceNestedTypes) RECURSE else NONE - TableOutputResolver.resolveUpdate("", sourceFieldValue, fieldAttr, conf, addError, - fieldPath, coerceMode) + TableOutputResolver.resolveUpdate( + "", sourceFieldValue, fieldAttr, conf, addError, + fieldPath, coerceMode, + deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) } } val namedStruct = toNamedStruct(structType, updatedFieldExprs) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala index 76035ea819ff..ca04b577c582 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Assignment, DeleteAction, In import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.COMMAND import org.apache.spark.sql.catalyst.util.CharVarcharUtils +import org.apache.spark.sql.connector.catalog.SchemaAlignmentConfig import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy @@ -40,32 +41,37 @@ object ResolveRowLevelCommandAssignments extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning( _.containsPattern(COMMAND), ruleId) { case u: UpdateTable if !u.skipSchemaResolution && u.resolved && u.rewritable && !u.aligned => - validateStoreAssignmentPolicy() + val schemaAlignment = schemaAlignmentConfig(u.table) + validateStoreAssignmentPolicy(schemaAlignment) val newTable = cleanAttrMetadata(u.table) val newAssignments = AssignmentUtils.alignUpdateAssignments(u.table.output, u.assignments, - fromStar = false, coerceNestedTypes = false) + fromStar = false, coerceNestedTypes = false, schemaAlignment = schemaAlignment) u.copy(table = newTable, assignments = newAssignments) case u: UpdateTable if !u.skipSchemaResolution && u.resolved && !u.aligned => resolveAssignments(u) case m: MergeIntoTable if m.rewritable && shouldAlignAssignments(m) && containsFinalSchema(m) => - validateStoreAssignmentPolicy() + val schemaAlignment = schemaAlignmentConfig(m.targetTable) + validateStoreAssignmentPolicy(schemaAlignment) val coerceNestedTypes = conf.coerceMergeNestedTypes && m.withSchemaEvolution m.copy( targetTable = cleanAttrMetadata(m.targetTable), matchedActions = alignActions( m.targetTable.output, m.matchedActions, - coerceNestedTypes), + coerceNestedTypes, + schemaAlignment), notMatchedActions = alignActions( m.targetTable.output, m.notMatchedActions, - coerceNestedTypes), + coerceNestedTypes, + schemaAlignment), notMatchedBySourceActions = alignActions( m.targetTable.output, m.notMatchedBySourceActions, - coerceNestedTypes)) + coerceNestedTypes, + schemaAlignment)) case m: MergeIntoTable if shouldAlignAssignments(m) && containsFinalSchema(m) => resolveAssignments(m) @@ -79,9 +85,16 @@ object ResolveRowLevelCommandAssignments extends Rule[LogicalPlan] { !m.schemaEvolutionEnabled || (m.schemaEvolutionReady && m.pendingSchemaChanges.isEmpty) } - private def validateStoreAssignmentPolicy(): Unit = { - // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2 - if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY) { + private def schemaAlignmentConfig(target: LogicalPlan): SchemaAlignmentConfig = + target.collectFirst { + case relation: DataSourceV2Relation => relation.table.schemaAlignmentConfig() + }.getOrElse(SchemaAlignmentConfig.DEFAULT) + + private def validateStoreAssignmentPolicy(schemaAlignment: SchemaAlignmentConfig): Unit = { + // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2, unless the + // target table opts into it via its SchemaAlignmentConfig. + if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY && + !schemaAlignment.allowLegacyStoreAssignmentPolicy()) { throw QueryCompilationErrors.legacyStoreAssignmentPolicyError() } } @@ -127,16 +140,17 @@ object ResolveRowLevelCommandAssignments extends Rule[LogicalPlan] { private def alignActions( attrs: Seq[Attribute], actions: Seq[MergeAction], - coerceNestedTypes: Boolean): Seq[MergeAction] = { + coerceNestedTypes: Boolean, + schemaAlignment: SchemaAlignmentConfig): Seq[MergeAction] = { actions.map { case u @ UpdateAction(_, assignments, fromStar) => u.copy(assignments = AssignmentUtils.alignUpdateAssignments(attrs, assignments, - fromStar, coerceNestedTypes)) + fromStar, coerceNestedTypes, schemaAlignment = schemaAlignment)) case d: DeleteAction => d case i @ InsertAction(_, assignments) => i.copy(assignments = AssignmentUtils.alignInsertAssignments(attrs, assignments, - coerceNestedTypes)) + coerceNestedTypes, schemaAlignment = schemaAlignment)) case other => throw new AnalysisException( errorClass = "_LEGACY_ERROR_TEMP_3052", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala index 933513654dc1..5c1958b1b221 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala @@ -94,9 +94,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { query: LogicalPlan, byName: Boolean, conf: SQLConf, - defaultValueFillMode: DefaultValueFillMode.Value = NONE): LogicalPlan = { + defaultValueFillMode: DefaultValueFillMode.Value = NONE, + deferCastValidationToRuntime: Boolean = false): LogicalPlan = { resolveOutputColumnsInternal( - tableName, expected, query, byName, conf, defaultValueFillMode)._1 + tableName, expected, query, byName, conf, defaultValueFillMode, + deferCastValidationToRuntime)._1 } /** @@ -111,10 +113,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { query: LogicalPlan, byName: Boolean, conf: SQLConf, - defaultValueFillMode: DefaultValueFillMode.Value = NONE + defaultValueFillMode: DefaultValueFillMode.Value = NONE, + deferCastValidationToRuntime: Boolean = false ): (LogicalPlan, Set[String]) = { resolveOutputColumnsInternal( - tableName, expected, query, byName, conf, defaultValueFillMode) + tableName, expected, query, byName, conf, defaultValueFillMode, deferCastValidationToRuntime) } private def resolveOutputColumnsInternal( @@ -123,7 +126,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { query: LogicalPlan, byName: Boolean, conf: SQLConf, - defaultValueFillMode: DefaultValueFillMode.Value + defaultValueFillMode: DefaultValueFillMode.Value, + deferCastValidationToRuntime: Boolean ): (LogicalPlan, Set[String]) = { if (expected.size < query.output.size) { @@ -149,7 +153,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { errors += _, Nil, defaultValueFillMode, - enforceFullOutput = true) + enforceFullOutput = true, + deferCastValidationToRuntime = deferCastValidationToRuntime) } else { if (expected.size > query.output.size && !fillDefaultValue) { throw QueryCompilationErrors.cannotWriteNotEnoughColumnsToTableError( @@ -157,7 +162,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { } resolveColumnsByPosition( tableName, query.output, expected, conf, errors += _, - fillDefaultValue = fillDefaultValue) + fillDefaultValue = fillDefaultValue, + deferCastValidationToRuntime = deferCastValidationToRuntime) } if (errors.nonEmpty) { @@ -180,14 +186,16 @@ object TableOutputResolver extends SQLConfHelper with Logging { conf: SQLConf, addError: String => Unit, colPath: Seq[String], - defaultValueFillMode: DefaultValueFillMode.Value): Expression = { + defaultValueFillMode: DefaultValueFillMode.Value, + deferCastValidationToRuntime: Boolean = false): Expression = { val fillChildDefaultValue = defaultValueFillMode == RECURSE (value.dataType, col.dataType) match { // no need to reorder inner fields or cast if types are already compatible case (valueType, colType) if DataType.equalsIgnoreCompatibleNullability(valueType, colType) => val canWriteExpr = canWrite( - tableName, valueType, colType, byName = true, conf, addError, colPath) + tableName, valueType, colType, byName = true, conf, addError, colPath, + deferCastValidationToRuntime = deferCastValidationToRuntime) if (canWriteExpr) { val nullsHandled = checkNullability(value, col, conf, colPath) applyColumnMetadata(nullsHandled, col) @@ -197,20 +205,23 @@ object TableOutputResolver extends SQLConfHelper with Logging { case (valueType: StructType, colType: StructType) => val resolvedValue = resolveStructType( tableName, value, valueType, col, colType, - byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false) + byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, + deferCastValidationToRuntime = deferCastValidationToRuntime) resolvedValue.getOrElse(value) case (valueType: ArrayType, colType: ArrayType) => val resolvedValue = resolveArrayType( tableName, value, valueType, col, colType, - byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false) + byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, + deferCastValidationToRuntime = deferCastValidationToRuntime) resolvedValue.getOrElse(value) case (valueType: MapType, colType: MapType) => val resolvedValue = resolveMapType( tableName, value, valueType, col, colType, - byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false) + byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, + deferCastValidationToRuntime = deferCastValidationToRuntime) resolvedValue.getOrElse(value) case _ => - checkUpdate(tableName, value, col, conf, addError, colPath) + checkUpdate(tableName, value, col, conf, addError, colPath, deferCastValidationToRuntime) } } @@ -220,7 +231,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { attr: Attribute, conf: SQLConf, addError: String => Unit, - colPath: Seq[String]): Expression = { + colPath: Seq[String], + deferCastValidationToRuntime: Boolean): Expression = { val attrTypeHasCharVarchar = CharVarcharUtils.hasCharVarchar(attr.dataType) val attrTypeWithoutCharVarchar = if (attrTypeHasCharVarchar) { @@ -231,7 +243,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { val canWriteValue = canWrite( tableName, value.dataType, attrTypeWithoutCharVarchar, - byName = true, conf, addError, colPath) + byName = true, conf, addError, colPath, + deferCastValidationToRuntime = deferCastValidationToRuntime) if (canWriteValue) { val nullCheckedValue = checkNullability(value, attr, conf, colPath) @@ -354,8 +367,14 @@ object TableOutputResolver extends SQLConfHelper with Logging { byName: Boolean, conf: SQLConf, addError: String => Unit, - colPath: Seq[String]): Boolean = { + colPath: Seq[String], + deferCastValidationToRuntime: Boolean): Boolean = { conf.storeAssignmentPolicy match { + case StoreAssignmentPolicy.ANSI if deferCastValidationToRuntime => + // The target validates casts at runtime, so skip the ANSI store-assignment analysis check + // and let the inserted cast surface overflows / malformed values at execution time. Casts + // between structurally incompatible types are still rejected when the cast is resolved. + true case StoreAssignmentPolicy.STRICT | StoreAssignmentPolicy.ANSI => DataTypeUtils.canWrite( tableName, valueType, expectedType, byName, conf.resolver, colPath.quoted, @@ -373,7 +392,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { addError: String => Unit, colPath: Seq[String] = Nil, defaultValueFillMode: DefaultValueFillMode.Value, - enforceFullOutput: Boolean = false): (Seq[NamedExpression], Set[String]) = { + enforceFullOutput: Boolean = false, + deferCastValidationToRuntime: Boolean): (Seq[NamedExpression], Set[String]) = { // Names of generated columns that were auto-filled (not provided by the user). Only populated // for top-level columns, since generated columns cannot be nested. val autoFilledGenCols = mutable.Set.empty[String] @@ -423,18 +443,22 @@ object TableOutputResolver extends SQLConfHelper with Logging { case (matchedType: StructType, expectedType: StructType) => resolveStructType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, - byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput) + byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, + deferCastValidationToRuntime = deferCastValidationToRuntime) case (matchedType: ArrayType, expectedType: ArrayType) => resolveArrayType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, - byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput) + byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, + deferCastValidationToRuntime = deferCastValidationToRuntime) case (matchedType: MapType, expectedType: MapType) => resolveMapType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, - byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput) + byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, + deferCastValidationToRuntime = deferCastValidationToRuntime) case _ => checkField( - tableName, actualExpectedCol, matchedCol, byName = true, conf, addError, newColPath) + tableName, actualExpectedCol, matchedCol, byName = true, conf, addError, newColPath, + deferCastValidationToRuntime = deferCastValidationToRuntime) } } } @@ -488,7 +512,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { conf: SQLConf, addError: String => Unit, colPath: Seq[String] = Nil, - fillDefaultValue: Boolean = false): (Seq[NamedExpression], Set[String]) = { + fillDefaultValue: Boolean = false, + deferCastValidationToRuntime: Boolean): (Seq[NamedExpression], Set[String]) = { // Names of generated columns that were auto-filled (not provided by the user). Only populated // for top-level columns, since generated columns cannot be nested. val autoFilledGenCols = mutable.Set.empty[String] @@ -528,17 +553,21 @@ object TableOutputResolver extends SQLConfHelper with Logging { case (inputType: StructType, expectedType: StructType) => resolveStructType( tableName, inputCol, inputType, expectedCol, expectedType, - byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true) + byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, + deferCastValidationToRuntime = deferCastValidationToRuntime) case (inputType: ArrayType, expectedType: ArrayType) => resolveArrayType( tableName, inputCol, inputType, expectedCol, expectedType, - byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true) + byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, + deferCastValidationToRuntime = deferCastValidationToRuntime) case (inputType: MapType, expectedType: MapType) => resolveMapType( tableName, inputCol, inputType, expectedCol, expectedType, - byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true) + byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, + deferCastValidationToRuntime = deferCastValidationToRuntime) case _ => - checkField(tableName, expectedCol, inputCol, byName = false, conf, addError, newColPath) + checkField(tableName, expectedCol, inputCol, byName = false, conf, addError, newColPath, + deferCastValidationToRuntime = deferCastValidationToRuntime) } } @@ -622,7 +651,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { addError: String => Unit, colPath: Seq[String], fillDefaultValue: Boolean, - enforceFullOutput: Boolean): Option[NamedExpression] = { + enforceFullOutput: Boolean, + deferCastValidationToRuntime: Boolean): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val fields = inputType.zipWithIndex.map { case (f, i) => Alias(GetStructField(nullCheckedInput, i, Some(f.name)), f.name)() @@ -631,10 +661,12 @@ object TableOutputResolver extends SQLConfHelper with Logging { // Generated columns cannot be nested, so the auto-filled set is always empty here. val (resolved, _) = if (byName) { reorderColumnsByName(tableName, fields, toAttributes(expectedType), conf, addError, colPath, - defaultValueMode, enforceFullOutput) + defaultValueMode, enforceFullOutput, + deferCastValidationToRuntime = deferCastValidationToRuntime) } else { resolveColumnsByPosition( - tableName, fields, toAttributes(expectedType), conf, addError, colPath, fillDefaultValue) + tableName, fields, toAttributes(expectedType), conf, addError, colPath, fillDefaultValue, + deferCastValidationToRuntime = deferCastValidationToRuntime) } if (resolved.length == expectedType.length) { val struct = CreateStruct(resolved) @@ -665,7 +697,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { addError: String => Unit, colPath: Seq[String], fillDefaultValue: Boolean, - enforceFullOutput: Boolean): Option[NamedExpression] = { + enforceFullOutput: Boolean, + deferCastValidationToRuntime: Boolean): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val param = NamedLambdaVariable("element", inputType.elementType, inputType.containsNull) val fakeAttr = @@ -674,10 +707,12 @@ object TableOutputResolver extends SQLConfHelper with Logging { val (res, _) = if (byName) { val defaultValueMode = if (fillDefaultValue) RECURSE else NONE reorderColumnsByName(tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, - defaultValueMode, enforceFullOutput) + defaultValueMode, enforceFullOutput, + deferCastValidationToRuntime = deferCastValidationToRuntime) } else { resolveColumnsByPosition( - tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, fillDefaultValue) + tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, fillDefaultValue, + deferCastValidationToRuntime = deferCastValidationToRuntime) } if (res.length == 1) { val castedArray = @@ -708,7 +743,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { addError: String => Unit, colPath: Seq[String], fillDefaultValue: Boolean, - enforceFullOutput: Boolean): Option[NamedExpression] = { + enforceFullOutput: Boolean, + deferCastValidationToRuntime: Boolean): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val keyParam = NamedLambdaVariable("key", inputType.keyType, nullable = false) @@ -717,10 +753,12 @@ object TableOutputResolver extends SQLConfHelper with Logging { // Generated columns cannot be nested, so the auto-filled set is always empty here. val (resKey, _) = if (byName) { reorderColumnsByName(tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, - defaultValueFillMode, enforceFullOutput) + defaultValueFillMode, enforceFullOutput, + deferCastValidationToRuntime = deferCastValidationToRuntime) } else { resolveColumnsByPosition( - tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, fillDefaultValue) + tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, fillDefaultValue, + deferCastValidationToRuntime = deferCastValidationToRuntime) } val valueParam = @@ -730,10 +768,12 @@ object TableOutputResolver extends SQLConfHelper with Logging { // Generated columns cannot be nested, so the auto-filled set is always empty here. val (resValue, _) = if (byName) { reorderColumnsByName(tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, - defaultValueFillMode, enforceFullOutput) + defaultValueFillMode, enforceFullOutput, + deferCastValidationToRuntime = deferCastValidationToRuntime) } else { resolveColumnsByPosition( - tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, fillDefaultValue) + tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, fillDefaultValue, + deferCastValidationToRuntime = deferCastValidationToRuntime) } if (resKey.length == 1 && resValue.length == 1) { @@ -833,7 +873,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { byName: Boolean, conf: SQLConf, addError: String => Unit, - colPath: Seq[String]): Option[NamedExpression] = { + colPath: Seq[String], + deferCastValidationToRuntime: Boolean): Option[NamedExpression] = { val attrTypeHasCharVarchar = CharVarcharUtils.hasCharVarchar(tableAttr.dataType) val attrTypeWithoutCharVarchar = if (attrTypeHasCharVarchar) { @@ -844,7 +885,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { val canWriteExpr = canWrite( tableName, queryExpr.dataType, attrTypeWithoutCharVarchar, - byName, conf, addError, colPath) + byName, conf, addError, colPath, + deferCastValidationToRuntime = deferCastValidationToRuntime) if (canWriteExpr) { val prepared = diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala index 50179824e255..04169c0e927d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.connector.write import java.util -import org.apache.spark.sql.connector.catalog.{Column, SupportsRead, SupportsRowLevelOperations, SupportsWrite, Table, TableCapability} +import org.apache.spark.sql.connector.catalog.{Column, SchemaAlignmentConfig, SupportsRead, SupportsRowLevelOperations, SupportsWrite, Table, TableCapability} import org.apache.spark.sql.connector.catalog.constraints.Constraint import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -40,6 +40,7 @@ private[sql] case class RowLevelOperationTable( override def columns: Array[Column] = table.columns() override def capabilities: util.Set[TableCapability] = table.capabilities override def constraints(): Array[Constraint] = table.constraints() + override def schemaAlignmentConfig(): SchemaAlignmentConfig = table.schemaAlignmentConfig() override def toString: String = table.toString override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala index 39634eb4c834..056cc35e835f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala @@ -64,7 +64,8 @@ abstract class InMemoryBaseTable( val numPartitions: Option[Int] = None, val advisoryPartitionSize: Option[Long] = None, val isDistributionStrictlyRequired: Boolean = true, - val numRowsPerSplit: Int = Int.MaxValue) + val numRowsPerSplit: Int = Int.MaxValue, + override val schemaAlignmentConfig: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT) extends Table with SupportsRead with SupportsWrite with SupportsMetadataColumns with SupportsSchemaEvolution { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala index 7556bc96912f..b79bd5fbb3d4 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala @@ -47,14 +47,16 @@ class InMemoryRowLevelOperationTable private ( partitioning: Array[Transform], properties: util.Map[String, String], constraints: Array[Constraint], - tableId: String) + tableId: String, + schemaAlignmentConfig: SchemaAlignmentConfig) extends InMemoryTable( name, columns, partitioning, properties, constraints, - id = tableId) + id = tableId, + schemaAlignmentConfig = schemaAlignmentConfig) with SupportsRowLevelOperations { def this( @@ -63,14 +65,16 @@ class InMemoryRowLevelOperationTable private ( partitioning: Array[Transform], properties: util.Map[String, String], constraints: Array[Constraint] = Array.empty, - tableId: String = java.util.UUID.randomUUID().toString) = { + tableId: String = java.util.UUID.randomUUID().toString, + schemaAlignmentConfig: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT) = { this( name = name, columns = CatalogV2Util.structTypeToV2Columns(schema), partitioning = partitioning, properties = properties, constraints = constraints, - tableId = tableId) + tableId = tableId, + schemaAlignmentConfig = schemaAlignmentConfig) } private final val PARTITION_COLUMN_REF = FieldReference(PartitionKeyColumn.name) @@ -106,7 +110,8 @@ class InMemoryRowLevelOperationTable private ( partitioning = partitioning, properties = properties, constraints = constraints, - tableId = id) + tableId = id, + schemaAlignmentConfig = schemaAlignmentConfig) dataMap.synchronized { dataMap.foreach { case (key, splits) => val copiedSplits = splits.map { bufferedRows => @@ -373,9 +378,11 @@ object InMemoryRowLevelOperationTable { partitioning: Array[Transform], properties: util.Map[String, String], constraints: Array[Constraint] = Array.empty, - tableId: String = java.util.UUID.randomUUID().toString): InMemoryRowLevelOperationTable = { + tableId: String = java.util.UUID.randomUUID().toString, + schemaAlignmentConfig: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT) + : InMemoryRowLevelOperationTable = { new InMemoryRowLevelOperationTable( - name, columns, partitioning, properties, constraints, tableId) + name, columns, partitioning, properties, constraints, tableId, schemaAlignmentConfig) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala index c783bfbece14..f4a55cd402fb 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala @@ -47,10 +47,11 @@ class InMemoryTable( advisoryPartitionSize: Option[Long] = None, isDistributionStrictlyRequired: Boolean = true, override val numRowsPerSplit: Int = Int.MaxValue, - override val id: String = UUID.randomUUID().toString) + override val id: String = UUID.randomUUID().toString, + override val schemaAlignmentConfig: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT) extends InMemoryBaseTable(name, columns, partitioning, properties, constraints, distribution, ordering, numPartitions, advisoryPartitionSize, isDistributionStrictlyRequired, - numRowsPerSplit) with SupportsDelete { + numRowsPerSplit, schemaAlignmentConfig) with SupportsDelete { def this( name: String, diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/txns.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/txns.scala index f48f99994a8c..0a18f32e7029 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/txns.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/txns.scala @@ -110,7 +110,8 @@ class TxnTable( schema, delegate.partitioning, delegate.properties, - delegate.constraints) { + delegate.constraints, + schemaAlignmentConfig = delegate.schemaAlignmentConfig) { // Expose the same id as the delegate so that identity checks during transaction re-resolution // don't false-positive on the TxnTable wrapper having a different UUID. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertSchemaEvolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertSchemaEvolutionSuite.scala new file mode 100644 index 000000000000..f10cf3e31734 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertSchemaEvolutionSuite.scala @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector + +import org.scalatest.BeforeAndAfter + +import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.connector.catalog.InMemoryCatalog +import org.apache.spark.sql.internal.SQLConf.{PARTITION_OVERWRITE_MODE, PartitionOverwriteMode} +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types._ + +class InsertSchemaEvolutionSuite + extends QueryTest with SharedSparkSession with BeforeAndAfter { + + private val catalogName = "testcat" + private val namespace = "ns" + private val tableIdent = s"$catalogName.$namespace.test_table" + + before { + spark.conf.set(s"spark.sql.catalog.$catalogName", classOf[InMemoryCatalog].getName) + } + + after { + spark.sessionState.catalogManager.reset() + spark.sessionState.conf.unsetConf(s"spark.sql.catalog.$catalogName") + } + + test("INSERT BY NAME with extra source column adds column to table") { + withTable(tableIdent) { + sql(s"CREATE TABLE $tableIdent (id INT, data STRING)") + sql( + s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME + |SELECT * FROM VALUES (1, 'a', CAST(10.0 AS DOUBLE)), + | (2, 'b', CAST(20.0 AS DOUBLE)) AS t(id, data, amount) + |""".stripMargin) + + val result = spark.table(tableIdent) + checkAnswer(result, Seq(Row(1, "a", 10.0d), Row(2, "b", 20.0d))) + assert(result.schema == StructType(Seq( + StructField("id", IntegerType), + StructField("data", StringType), + StructField("amount", DoubleType)))) + } + } + + test("INSERT BY NAME with type widening updates column type") { + withTable(tableIdent) { + sql(s"CREATE TABLE $tableIdent (id INT, value INT)") + sql( + s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME + |SELECT * FROM VALUES (1, CAST(100 AS LONG)), + | (2, CAST(200 AS LONG)) AS t(id, value) + |""".stripMargin) + + val result = spark.table(tableIdent) + checkAnswer(result, Seq(Row(1, 100L), Row(2, 200L))) + assert(result.schema == StructType(Seq( + StructField("id", IntegerType), + StructField("value", LongType)))) + } + } + + test("INSERT BY NAME with nested struct evolution") { + withTable(tableIdent) { + sql(s"CREATE TABLE $tableIdent (id INT, info STRUCT)") + sql( + s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME + |SELECT id, named_struct('name', name, 'age', age) AS info + |FROM VALUES (1, 'Alice', 30), (2, 'Bob', 25) AS t(id, name, age) + |""".stripMargin) + + val result = spark.table(tableIdent) + checkAnswer(result, Seq(Row(1, Row("Alice", 30)), Row(2, Row("Bob", 25)))) + val expectedInfoType = StructType(Seq( + StructField("name", StringType), + StructField("age", IntegerType))) + assert(result.schema == StructType(Seq( + StructField("id", IntegerType), + StructField("info", expectedInfoType)))) + } + } + + test("INSERT BY NAME with matching schema - no evolution needed") { + withTable(tableIdent) { + sql(s"CREATE TABLE $tableIdent (id INT, data STRING)") + sql( + s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME + |SELECT * FROM VALUES (1, 'a'), (2, 'b') AS t(id, data) + |""".stripMargin) + + val result = spark.table(tableIdent) + checkAnswer(result, Seq(Row(1, "a"), Row(2, "b"))) + assert(result.schema == StructType(Seq( + StructField("id", IntegerType), + StructField("data", StringType)))) + } + } + + test("INSERT BY POSITION with schema evolution adds extra columns") { + withTable(tableIdent) { + sql(s"CREATE TABLE $tableIdent (id INT, data STRING)") + sql( + s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent + |SELECT * FROM VALUES (1, 'a', CAST(10.0 AS DOUBLE)), + | (2, 'b', CAST(20.0 AS DOUBLE)) AS t(id, data, amount) + |""".stripMargin) + + val result = spark.table(tableIdent) + checkAnswer(result, Seq(Row(1, "a", 10.0d), Row(2, "b", 20.0d))) + assert(result.schema == StructType(Seq( + StructField("id", IntegerType), + StructField("data", StringType), + StructField("amount", DoubleType)))) + } + } + + test("table without AUTOMATIC_SCHEMA_EVOLUTION - no evolution") { + withTable(tableIdent) { + sql( + s"""CREATE TABLE $tableIdent (id INT, data STRING) + |TBLPROPERTIES ('auto-schema-evolution' = 'false')""".stripMargin) + + intercept[Exception] { + sql( + s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME + |SELECT * FROM VALUES (1, 'a', CAST(10.0 AS DOUBLE)), + | (2, 'b', CAST(20.0 AS DOUBLE)) AS t(id, data, amount) + |""".stripMargin) + } + } + } + + test("OVERWRITE BY EXPRESSION with schema evolution") { + withTable(tableIdent) { + sql(s"CREATE TABLE $tableIdent (id INT, data STRING)") + sql(s"INSERT INTO $tableIdent VALUES (1, 'a'), (2, 'b')") + + withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { + sql( + s"""INSERT WITH SCHEMA EVOLUTION OVERWRITE $tableIdent BY NAME + |SELECT * FROM VALUES (3, 'c', CAST(30.0 AS DOUBLE)), + | (4, 'd', CAST(40.0 AS DOUBLE)) AS t(id, data, amount) + |""".stripMargin) + } + + val result = spark.table(tableIdent) + checkAnswer(result, Seq(Row(3, "c", 30.0d), Row(4, "d", 40.0d))) + assert(result.schema == StructType(Seq( + StructField("id", IntegerType), + StructField("data", StringType), + StructField("amount", DoubleType)))) + } + } + + test("OVERWRITE PARTITIONS DYNAMIC with schema evolution") { + withTable(tableIdent) { + sql(s"CREATE TABLE $tableIdent (id INT, data STRING) PARTITIONED BY (id)") + sql(s"INSERT INTO $tableIdent VALUES (1, 'a'), (2, 'b')") + + withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) { + sql( + s"""INSERT WITH SCHEMA EVOLUTION OVERWRITE $tableIdent BY NAME + |SELECT * FROM VALUES (1, 'c', CAST(30.0 AS DOUBLE)) AS t(id, data, amount) + |""".stripMargin) + } + + val result = spark.table(tableIdent) + checkAnswer(result.orderBy("id"), + Seq(Row(1, "c", 30.0d), Row(2, "b", null))) + assert(result.schema == StructType(Seq( + StructField("id", IntegerType), + StructField("data", StringType), + StructField("amount", DoubleType)))) + } + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala new file mode 100644 index 000000000000..f1827d3ca4cf --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog + +import scala.util.{Failure, Success, Try} + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} +import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException +import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} + +/** + * A catalog that creates [[InMemoryRowLevelOperationTable]]s carrying a fixed + * [[SchemaAlignmentConfig]] supplied by the concrete subclass. It returns the live table instance + * on load (rather than a copy) so the config is preserved for the analyzer. + */ +abstract class SchemaAlignmentTestCatalog extends InMemoryRowLevelOperationTableCatalog { + + protected def tableConfig: SchemaAlignmentConfig + + override def loadTable(ident: Identifier): Table = liveTable(ident) + + override def createTable(ident: Identifier, tableInfo: TableInfo): Table = { + if (tables.containsKey(ident)) { + throw new TableAlreadyExistsException(ident.asMultipartIdentifier) + } + val name = s"${this.name}.${ident.quoted}" + val schema = CatalogV2Util.v2ColumnsToStructType(tableInfo.columns) + val table = new InMemoryRowLevelOperationTable( + name, schema, tableInfo.partitions, tableInfo.properties, tableInfo.constraints(), + schemaAlignmentConfig = tableConfig) + tables.put(ident, table) + namespaces.putIfAbsent(ident.namespace.toList, Map()) + table + } +} + +/** A catalog whose tables opt into every [[SchemaAlignmentConfig]] relaxation. */ +class RelaxedSchemaAlignmentCatalog extends SchemaAlignmentTestCatalog { + override protected def tableConfig: SchemaAlignmentConfig = new SchemaAlignmentConfig { + override def allowLegacyStoreAssignmentPolicy(): Boolean = true + override def deferCastValidationToRuntime(): Boolean = true + } +} + +/** A catalog whose tables keep the strict data source v2 defaults. */ +class StrictSchemaAlignmentCatalog extends SchemaAlignmentTestCatalog { + override protected def tableConfig: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT +} + +/** + * End-to-end coverage for [[SchemaAlignmentConfig]]: a table that opts into a relaxation gets the + * more permissive analyzer behavior, while an otherwise identical table using the default (strict) + * config keeps the data source v2 behavior. Exercised on both the INSERT path + * ([[org.apache.spark.sql.catalyst.analysis.Analyzer.ResolveOutputRelation]]) and the row-level + * path ([[org.apache.spark.sql.catalyst.analysis.ResolveRowLevelCommandAssignments]]). + */ +class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { + + private val relaxed = "relaxed" + private val strict = "strict" + + override def sparkConf: SparkConf = + super.sparkConf + .set(s"spark.sql.catalog.$relaxed", classOf[RelaxedSchemaAlignmentCatalog].getName) + .set(s"spark.sql.catalog.$strict", classOf[StrictSchemaAlignmentCatalog].getName) + + private def withLegacyPolicy(f: => Unit): Unit = + withSQLConf( + SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString)(f) + + private def withAnsiPolicy(f: => Unit): Unit = + withSQLConf( + SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.ANSI.toString)(f) + + private def legacyRejected(f: => Unit): Unit = + checkError( + exception = intercept[AnalysisException](f), + condition = "_LEGACY_ERROR_TEMP_1000", + parameters = Map("configKey" -> SQLConf.STORE_ASSIGNMENT_POLICY.key)) + + test("allowLegacyStoreAssignmentPolicy: INSERT under LEGACY policy") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") + sql(s"CREATE TABLE $strict.t (id INT) USING foo") + withLegacyPolicy { + sql(s"INSERT INTO $relaxed.t VALUES (1)") + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1)) + legacyRejected(sql(s"INSERT INTO $strict.t VALUES (1)")) + } + } + } + + test("allowLegacyStoreAssignmentPolicy: UPDATE under LEGACY policy") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (id INT, data STRING) USING foo") + sql(s"CREATE TABLE $strict.t (id INT, data STRING) USING foo") + sql(s"INSERT INTO $relaxed.t VALUES (1, 'a')") + sql(s"INSERT INTO $strict.t VALUES (1, 'a')") + withLegacyPolicy { + sql(s"UPDATE $relaxed.t SET data = 'b' WHERE id = 1") + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1, "b")) + legacyRejected(sql(s"UPDATE $strict.t SET data = 'b' WHERE id = 1")) + } + } + } + + test("deferCastValidationToRuntime: INSERT of an ANSI-incompatible cast") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") + sql(s"CREATE TABLE $strict.t (id INT) USING foo") + withAnsiPolicy { + sql(s"INSERT INTO $relaxed.t VALUES ('1')") + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1)) + checkError( + exception = intercept[AnalysisException] { + sql(s"INSERT INTO $strict.t VALUES ('1')") + }, + condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST", + parameters = Map( + "tableName" -> s"`$strict`.`t`", + "colName" -> "`id`", + "srcType" -> "\"STRING\"", + "targetType" -> "\"INT\"")) + } + } + } + + test("deferCastValidationToRuntime: UPDATE with an ANSI-incompatible cast") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (id INT, data INT) USING foo") + sql(s"CREATE TABLE $strict.t (id INT, data INT) USING foo") + sql(s"INSERT INTO $relaxed.t VALUES (1, 0)") + sql(s"INSERT INTO $strict.t VALUES (1, 0)") + withAnsiPolicy { + sql(s"UPDATE $relaxed.t SET data = '5' WHERE id = 1") + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1, 5)) + checkError( + exception = intercept[AnalysisException] { + sql(s"UPDATE $strict.t SET data = '5' WHERE id = 1") + }, + condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST", + parameters = Map( + "tableName" -> "``", + "colName" -> "`data`", + "srcType" -> "\"STRING\"", + "targetType" -> "\"INT\"")) + } + } + } + + test("allowLegacyStoreAssignmentPolicy: MERGE under LEGACY policy") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (id INT, data STRING) USING foo") + sql(s"CREATE TABLE $strict.t (id INT, data STRING) USING foo") + sql(s"INSERT INTO $relaxed.t VALUES (1, 'a')") + sql(s"INSERT INTO $strict.t VALUES (1, 'a')") + def merge(target: String): String = + s"""MERGE INTO $target t + |USING (SELECT 1 AS id, 'b' AS data) s + |ON t.id = s.id + |WHEN MATCHED THEN UPDATE SET t.data = s.data""".stripMargin + withLegacyPolicy { + sql(merge(s"$relaxed.t")) + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1, "b")) + legacyRejected(sql(merge(s"$strict.t"))) + } + } + } + + test("deferCastValidationToRuntime: MERGE with an ANSI-incompatible cast") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (id INT, data INT) USING foo") + sql(s"CREATE TABLE $strict.t (id INT, data INT) USING foo") + sql(s"INSERT INTO $relaxed.t VALUES (1, 0)") + sql(s"INSERT INTO $strict.t VALUES (1, 0)") + def merge(target: String): String = + s"""MERGE INTO $target t + |USING (SELECT 1 AS id, '5' AS data) s + |ON t.id = s.id + |WHEN MATCHED THEN UPDATE SET t.data = s.data""".stripMargin + withAnsiPolicy { + sql(merge(s"$relaxed.t")) + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1, 5)) + checkError( + exception = intercept[AnalysisException](sql(merge(s"$strict.t"))), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST", + parameters = Map( + "tableName" -> "``", + "colName" -> "`data`", + "srcType" -> "\"STRING\"", + "targetType" -> "\"INT\"")) + } + } + } + + test("deferCastValidationToRuntime: structurally impossible casts are still rejected") { + withTable(s"$relaxed.t") { + sql(s"CREATE TABLE $relaxed.t (d DATE) USING foo") + withAnsiPolicy { + // BOOLEAN cannot be cast to DATE at all, so the write is rejected even though the table + // defers store-assignment cast validation to runtime. + intercept[AnalysisException] { + sql(s"INSERT INTO $relaxed.t VALUES (true)") + } + } + } + } + + private def appendByName( + catalog: String, targetSchema: StructType, source: DataFrame): Try[Seq[Row]] = { + var result: Try[Seq[Row]] = Try(Seq.empty[Row]) + withTable(s"$catalog.t") { + spark.createDataFrame(new java.util.ArrayList[Row](), targetSchema) + .writeTo(s"$catalog.t").create() + result = Try { + source.writeTo(s"$catalog.t").append() + spark.table(s"$catalog.t").collect().toSeq + } + } + result + } + + private def assertRelaxedMatchesStrict(targetSchema: StructType, source: DataFrame): Unit = + withAnsiPolicy { + val fromRelaxed = appendByName(relaxed, targetSchema, source) + val fromStrict = appendByName(strict, targetSchema, source) + (fromRelaxed, fromStrict) match { + case (Success(relaxedRows), Success(strictRows)) => + assert(relaxedRows.map(_.toString).sorted == strictRows.map(_.toString).sorted, + s"relaxed=$relaxedRows strict=$strictRows") + case (Failure(relaxedError: AnalysisException), Failure(strictError: AnalysisException)) => + assert(relaxedError.getCondition == strictError.getCondition, + s"relaxed=${relaxedError.getCondition} strict=${strictError.getCondition}") + case (Failure(_), Failure(_)) => + case _ => + fail(s"relaxed and strict diverged: relaxed=$fromRelaxed strict=$fromStrict") + } + } + + test("deferCastValidationToRuntime: renamed nested struct field is still rejected") { + val target = new StructType() + .add("s", new StructType().add("a", IntegerType).add("b", IntegerType)) + val source = spark.createDataFrame( + java.util.Arrays.asList(Row(Row(1, 2))), + new StructType().add("s", new StructType().add("a", IntegerType).add("c", IntegerType))) + assertRelaxedMatchesStrict(target, source) + } + + test("deferCastValidationToRuntime: nullable array element into non-null element type") { + val target = new StructType().add("a", ArrayType(IntegerType, containsNull = false)) + val source = spark.createDataFrame( + java.util.Arrays.asList(Row(Seq(1, 2))), + new StructType().add("a", ArrayType(IntegerType, containsNull = true))) + assertRelaxedMatchesStrict(target, source) + } + + test("deferCastValidationToRuntime: nullable map value into non-null value type") { + val target = new StructType() + .add("m", MapType(StringType, IntegerType, valueContainsNull = false)) + val source = spark.createDataFrame( + java.util.Arrays.asList(Row(Map("k" -> 1))), + new StructType().add("m", MapType(StringType, IntegerType, valueContainsNull = true))) + assertRelaxedMatchesStrict(target, source) + } + + test("deferCastValidationToRuntime: nullable child into non-null struct field") { + val target = new StructType() + .add("s", new StructType().add("a", IntegerType, nullable = false)) + val source = spark.createDataFrame( + java.util.Arrays.asList(Row(Row(1))), + new StructType().add("s", new StructType().add("a", IntegerType, nullable = true))) + assertRelaxedMatchesStrict(target, source) + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala index bcc4895616bd..a8c7e3b559aa 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, Column, ColumnDefaultValue, Identifier, SupportsRowLevelOperations, TableCapability, TableCatalog, TableContext, TableWritePrivilege} +import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, Column, ColumnDefaultValue, Identifier, SchemaAlignmentConfig, SupportsRowLevelOperations, TableCapability, TableCatalog, TableContext, TableWritePrivilege} import org.apache.spark.sql.connector.expressions.{LiteralValue, Transform} import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.datasources.v2.V2SessionCatalog @@ -48,6 +48,7 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -59,6 +60,7 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("i", "INT") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -73,6 +75,7 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -84,6 +87,7 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -96,6 +100,7 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -121,6 +126,7 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { nullable = false) when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -132,6 +138,7 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) when(t.capabilities()).thenReturn(Collections.singleton(TableCapability.ACCEPT_ANY_SCHEMA)) t } @@ -143,6 +150,7 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { Column.create("b", BooleanType, true, null, null), Column.create("i", IntegerType, true, null, iDefault, null))) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala index f960507b300e..09f2abb00cf8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{AlterColumns, AlterColumnSpe import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.TypeUtils.toSQLId import org.apache.spark.sql.connector.FakeV2Provider -import org.apache.spark.sql.connector.catalog.{CatalogManager, Column, ColumnDefaultValue, Identifier, SupportsDelete, Table, TableCapability, TableCatalog, TableChange, TableContext, TableWritePrivilege, V1Table} +import org.apache.spark.sql.connector.catalog.{CatalogManager, Column, ColumnDefaultValue, Identifier, SchemaAlignmentConfig, SupportsDelete, Table, TableCapability, TableCatalog, TableChange, TableContext, TableWritePrivilege, V1Table} import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME import org.apache.spark.sql.connector.expressions.{LiteralValue, Transform} import org.apache.spark.sql.errors.QueryExecutionErrors @@ -62,6 +62,7 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { when(t.columns()).thenReturn( Array(Column.create("i", IntegerType), Column.create("s", StringType))) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -70,6 +71,7 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { when(t.columns()).thenReturn( Array(Column.create("s", StringType), Column.create("i", IntegerType))) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -78,6 +80,7 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { when(t.columns()).thenReturn( Array(Column.create("i", IntegerType), Column.create("x", StringType, false))) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) when(t.name()).thenReturn("tab2") t } @@ -95,6 +98,7 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { when(t.columns()).thenReturn( Array(Column.create("c1", CharType(5)), Column.create("c2", VarcharType(5)))) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -106,6 +110,7 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { Column.create("i", BooleanType, true, null, default1, null), Column.create("s", IntegerType, true, null, default2, null))) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -117,6 +122,7 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { Column.create("i", StringType), Column.create("e", StringType, true, null, default, null))) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -126,6 +132,7 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { Column.create("s", StringType), Column.create("default", StringType))) when(t.partitioning()).thenReturn(Array.empty[Transform]) + when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } From 22ff5ce38a5627a3a28dbec825e9130f85b2d167 Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Tue, 15 Sep 2026 14:02:45 +0000 Subject: [PATCH 2/6] Address comments --- docs/sql-ref-ansi-compliance.md | 3 + docs/sql-v2-data-sources.md | 1 + .../catalog/SchemaAlignmentConfig.java | 9 +- .../sql/catalyst/analysis/Analyzer.scala | 2 +- .../catalyst/analysis/AssignmentUtils.scala | 9 +- .../analysis/TableOutputResolver.scala | 91 +++++---- .../sql/catalyst/types/DataTypeUtils.scala | 17 +- ...nMemoryRowLevelOperationTableCatalog.scala | 6 +- .../sql/connector/catalog/InMemoryTable.scala | 3 +- .../DataTypeWriteCompatibilitySuite.scala | 28 +++ .../InsertSchemaEvolutionSuite.scala | 192 ------------------ .../catalog/SchemaAlignmentConfigSuite.scala | 125 +++++++++--- 12 files changed, 199 insertions(+), 287 deletions(-) delete mode 100644 sql/core/src/test/scala/org/apache/spark/sql/connector/InsertSchemaEvolutionSuite.scala diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index 2ec0b7872e52..23c1f8afe2c4 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -226,6 +226,9 @@ INSERT INTO test VALUES (2147483648L); org.apache.spark.SparkArithmeticException: [CAST_OVERFLOW_IN_TABLE_INSERT] Fail to insert a value of "BIGINT" type into the "INT" type column `i` due to an overflow. Use `try_cast` on the input value to tolerate overflow and return NULL instead. ``` +By default, invalid source/target combinations are rejected during analysis. +Data sources may instead defer this validation to execution time, so an insertion is rejected only when a value is actually malformed or overflows, not during analysis. + ### Type coercion #### Type Promotion and Precedence When `spark.sql.ansi.enabled` is set to `true`, Spark SQL uses several rules that govern how conflicts between data types are resolved. diff --git a/docs/sql-v2-data-sources.md b/docs/sql-v2-data-sources.md index 4b4d0f35cb4e..33dc5bce4883 100644 --- a/docs/sql-v2-data-sources.md +++ b/docs/sql-v2-data-sources.md @@ -205,6 +205,7 @@ A `Table` provides: | `partitioning()` | Physical partitioning expressed as `Transform` arrays | | `properties()` | A string map of table properties | | `capabilities()` | A set of `TableCapability` values declaring what the table supports | +| `schemaAlignmentConfig()` | A `SchemaAlignmentConfig` controlling how a query is aligned to the table schema during batch / row-level writes | ### Read and Write Mix-ins diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java index 2351f9a6229c..519e44d96204 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java @@ -20,10 +20,11 @@ import org.apache.spark.annotation.Evolving; /** - * Schema-alignment configuration for writes to a {@link Table}. This allows connectors to - * configure casting behavior and handling of schema mismatches during writes. + * Schema-alignment configuration for batch/row-level writes to a {@link Table}. This allows + * connectors to configure casting behavior and handling of schema mismatches during DSv2 writes. + * It is not consulted for streaming writes, which do not go through this alignment path. * - * @since 4.3.0 + * @since 4.4.0 */ @Evolving public interface SchemaAlignmentConfig { @@ -46,7 +47,7 @@ default boolean allowLegacyStoreAssignmentPolicy() { * store-assignment compatibility check and inserts an ANSI cast, so malformed values or * overflows surface at execution time. */ - default boolean deferCastValidationToRuntime() { + default boolean deferAnsiCastValidationToRuntime() { return false; } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 6322fccd7494..12d412b6b6df 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -3969,7 +3969,7 @@ class Analyzer( TableOutputResolver.resolveOutputColumnsWithGeneratedInfo( v2Write.table.name, expected, v2Write.query, v2Write.isByName, conf, defaultValueFillMode, - deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) + deferAnsiCastValidationToRuntime = schemaAlignment.deferAnsiCastValidationToRuntime()) if (projection != v2Write.query) { val cleanedTable = v2Write.table match { case r: DataSourceV2Relation => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala index 42065caebb68..042e0b08504c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala @@ -142,7 +142,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate( "", value, actualAttr, conf, err => errors += err, colPath, coerceMode, - deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) + deferAnsiCastValidationToRuntime = schemaAlignment.deferAnsiCastValidationToRuntime()) } Assignment(attr, resolvedValue) } @@ -200,14 +200,14 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate("", value, col, conf, addError, colPath, coerceMode, - deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) + deferAnsiCastValidationToRuntime = schemaAlignment.deferAnsiCastValidationToRuntime()) } } else { val value = exactAssignments.head.value val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate("", value, col, conf, addError, colPath, coerceMode, - deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) + deferAnsiCastValidationToRuntime = schemaAlignment.deferAnsiCastValidationToRuntime()) } } else { applyFieldAssignments(col, colExpr, fieldAssignments, addError, colPath, coerceNestedTypes, @@ -292,7 +292,8 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { TableOutputResolver.resolveUpdate( "", sourceFieldValue, fieldAttr, conf, addError, fieldPath, coerceMode, - deferCastValidationToRuntime = schemaAlignment.deferCastValidationToRuntime()) + deferAnsiCastValidationToRuntime = + schemaAlignment.deferAnsiCastValidationToRuntime()) } } val namedStruct = toNamedStruct(structType, updatedFieldExprs) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala index 5c1958b1b221..3a19935dfdbb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala @@ -95,10 +95,10 @@ object TableOutputResolver extends SQLConfHelper with Logging { byName: Boolean, conf: SQLConf, defaultValueFillMode: DefaultValueFillMode.Value = NONE, - deferCastValidationToRuntime: Boolean = false): LogicalPlan = { + deferAnsiCastValidationToRuntime: Boolean = false): LogicalPlan = { resolveOutputColumnsInternal( tableName, expected, query, byName, conf, defaultValueFillMode, - deferCastValidationToRuntime)._1 + deferAnsiCastValidationToRuntime)._1 } /** @@ -114,10 +114,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { byName: Boolean, conf: SQLConf, defaultValueFillMode: DefaultValueFillMode.Value = NONE, - deferCastValidationToRuntime: Boolean = false + deferAnsiCastValidationToRuntime: Boolean = false ): (LogicalPlan, Set[String]) = { resolveOutputColumnsInternal( - tableName, expected, query, byName, conf, defaultValueFillMode, deferCastValidationToRuntime) + tableName, expected, query, byName, conf, defaultValueFillMode, + deferAnsiCastValidationToRuntime) } private def resolveOutputColumnsInternal( @@ -127,7 +128,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { byName: Boolean, conf: SQLConf, defaultValueFillMode: DefaultValueFillMode.Value, - deferCastValidationToRuntime: Boolean + deferAnsiCastValidationToRuntime: Boolean ): (LogicalPlan, Set[String]) = { if (expected.size < query.output.size) { @@ -154,7 +155,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { Nil, defaultValueFillMode, enforceFullOutput = true, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } else { if (expected.size > query.output.size && !fillDefaultValue) { throw QueryCompilationErrors.cannotWriteNotEnoughColumnsToTableError( @@ -163,7 +164,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { resolveColumnsByPosition( tableName, query.output, expected, conf, errors += _, fillDefaultValue = fillDefaultValue, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } if (errors.nonEmpty) { @@ -187,7 +188,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { addError: String => Unit, colPath: Seq[String], defaultValueFillMode: DefaultValueFillMode.Value, - deferCastValidationToRuntime: Boolean = false): Expression = { + deferAnsiCastValidationToRuntime: Boolean = false): Expression = { val fillChildDefaultValue = defaultValueFillMode == RECURSE (value.dataType, col.dataType) match { @@ -195,7 +196,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { case (valueType, colType) if DataType.equalsIgnoreCompatibleNullability(valueType, colType) => val canWriteExpr = canWrite( tableName, valueType, colType, byName = true, conf, addError, colPath, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) if (canWriteExpr) { val nullsHandled = checkNullability(value, col, conf, colPath) applyColumnMetadata(nullsHandled, col) @@ -206,22 +207,23 @@ object TableOutputResolver extends SQLConfHelper with Logging { val resolvedValue = resolveStructType( tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) resolvedValue.getOrElse(value) case (valueType: ArrayType, colType: ArrayType) => val resolvedValue = resolveArrayType( tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) resolvedValue.getOrElse(value) case (valueType: MapType, colType: MapType) => val resolvedValue = resolveMapType( tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) resolvedValue.getOrElse(value) case _ => - checkUpdate(tableName, value, col, conf, addError, colPath, deferCastValidationToRuntime) + checkUpdate( + tableName, value, col, conf, addError, colPath, deferAnsiCastValidationToRuntime) } } @@ -232,7 +234,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { conf: SQLConf, addError: String => Unit, colPath: Seq[String], - deferCastValidationToRuntime: Boolean): Expression = { + deferAnsiCastValidationToRuntime: Boolean): Expression = { val attrTypeHasCharVarchar = CharVarcharUtils.hasCharVarchar(attr.dataType) val attrTypeWithoutCharVarchar = if (attrTypeHasCharVarchar) { @@ -244,7 +246,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { val canWriteValue = canWrite( tableName, value.dataType, attrTypeWithoutCharVarchar, byName = true, conf, addError, colPath, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) if (canWriteValue) { val nullCheckedValue = checkNullability(value, attr, conf, colPath) @@ -368,17 +370,16 @@ object TableOutputResolver extends SQLConfHelper with Logging { conf: SQLConf, addError: String => Unit, colPath: Seq[String], - deferCastValidationToRuntime: Boolean): Boolean = { + deferAnsiCastValidationToRuntime: Boolean): Boolean = { conf.storeAssignmentPolicy match { - case StoreAssignmentPolicy.ANSI if deferCastValidationToRuntime => - // The target validates casts at runtime, so skip the ANSI store-assignment analysis check - // and let the inserted cast surface overflows / malformed values at execution time. Casts - // between structurally incompatible types are still rejected when the cast is resolved. - true case StoreAssignmentPolicy.STRICT | StoreAssignmentPolicy.ANSI => + // Always delegate to DataTypeUtils.canWrite so structural checks (struct field names and + // nullability, array/map element nullability, field counts) still run. Only the atomic + // ANSI store-assignment cast check is relaxed, via deferAnsiCastValidationToRuntime. DataTypeUtils.canWrite( tableName, valueType, expectedType, byName, conf.resolver, colPath.quoted, - conf.storeAssignmentPolicy, addError) + conf.storeAssignmentPolicy, addError, + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) case _ => true } @@ -393,7 +394,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { colPath: Seq[String] = Nil, defaultValueFillMode: DefaultValueFillMode.Value, enforceFullOutput: Boolean = false, - deferCastValidationToRuntime: Boolean): (Seq[NamedExpression], Set[String]) = { + deferAnsiCastValidationToRuntime: Boolean): (Seq[NamedExpression], Set[String]) = { // Names of generated columns that were auto-filled (not provided by the user). Only populated // for top-level columns, since generated columns cannot be nested. val autoFilledGenCols = mutable.Set.empty[String] @@ -444,21 +445,21 @@ object TableOutputResolver extends SQLConfHelper with Logging { resolveStructType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) case (matchedType: ArrayType, expectedType: ArrayType) => resolveArrayType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) case (matchedType: MapType, expectedType: MapType) => resolveMapType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) case _ => checkField( tableName, actualExpectedCol, matchedCol, byName = true, conf, addError, newColPath, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } } } @@ -513,7 +514,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { addError: String => Unit, colPath: Seq[String] = Nil, fillDefaultValue: Boolean = false, - deferCastValidationToRuntime: Boolean): (Seq[NamedExpression], Set[String]) = { + deferAnsiCastValidationToRuntime: Boolean): (Seq[NamedExpression], Set[String]) = { // Names of generated columns that were auto-filled (not provided by the user). Only populated // for top-level columns, since generated columns cannot be nested. val autoFilledGenCols = mutable.Set.empty[String] @@ -554,20 +555,20 @@ object TableOutputResolver extends SQLConfHelper with Logging { resolveStructType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) case (inputType: ArrayType, expectedType: ArrayType) => resolveArrayType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) case (inputType: MapType, expectedType: MapType) => resolveMapType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) case _ => checkField(tableName, expectedCol, inputCol, byName = false, conf, addError, newColPath, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } } @@ -652,7 +653,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { colPath: Seq[String], fillDefaultValue: Boolean, enforceFullOutput: Boolean, - deferCastValidationToRuntime: Boolean): Option[NamedExpression] = { + deferAnsiCastValidationToRuntime: Boolean): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val fields = inputType.zipWithIndex.map { case (f, i) => Alias(GetStructField(nullCheckedInput, i, Some(f.name)), f.name)() @@ -662,11 +663,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val (resolved, _) = if (byName) { reorderColumnsByName(tableName, fields, toAttributes(expectedType), conf, addError, colPath, defaultValueMode, enforceFullOutput, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } else { resolveColumnsByPosition( tableName, fields, toAttributes(expectedType), conf, addError, colPath, fillDefaultValue, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } if (resolved.length == expectedType.length) { val struct = CreateStruct(resolved) @@ -698,7 +699,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { colPath: Seq[String], fillDefaultValue: Boolean, enforceFullOutput: Boolean, - deferCastValidationToRuntime: Boolean): Option[NamedExpression] = { + deferAnsiCastValidationToRuntime: Boolean): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val param = NamedLambdaVariable("element", inputType.elementType, inputType.containsNull) val fakeAttr = @@ -708,11 +709,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val defaultValueMode = if (fillDefaultValue) RECURSE else NONE reorderColumnsByName(tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, defaultValueMode, enforceFullOutput, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } else { resolveColumnsByPosition( tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, fillDefaultValue, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } if (res.length == 1) { val castedArray = @@ -744,7 +745,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { colPath: Seq[String], fillDefaultValue: Boolean, enforceFullOutput: Boolean, - deferCastValidationToRuntime: Boolean): Option[NamedExpression] = { + deferAnsiCastValidationToRuntime: Boolean): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val keyParam = NamedLambdaVariable("key", inputType.keyType, nullable = false) @@ -754,11 +755,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val (resKey, _) = if (byName) { reorderColumnsByName(tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, defaultValueFillMode, enforceFullOutput, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } else { resolveColumnsByPosition( tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, fillDefaultValue, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } val valueParam = @@ -769,11 +770,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val (resValue, _) = if (byName) { reorderColumnsByName(tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, defaultValueFillMode, enforceFullOutput, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } else { resolveColumnsByPosition( tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, fillDefaultValue, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) } if (resKey.length == 1 && resValue.length == 1) { @@ -874,7 +875,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { conf: SQLConf, addError: String => Unit, colPath: Seq[String], - deferCastValidationToRuntime: Boolean): Option[NamedExpression] = { + deferAnsiCastValidationToRuntime: Boolean): Option[NamedExpression] = { val attrTypeHasCharVarchar = CharVarcharUtils.hasCharVarchar(tableAttr.dataType) val attrTypeWithoutCharVarchar = if (attrTypeHasCharVarchar) { @@ -886,7 +887,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { val canWriteExpr = canWrite( tableName, queryExpr.dataType, attrTypeWithoutCharVarchar, byName, conf, addError, colPath, - deferCastValidationToRuntime = deferCastValidationToRuntime) + deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) if (canWriteExpr) { val prepared = diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala index f7ab8b06baf5..485cc3b83ce4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala @@ -111,13 +111,14 @@ object DataTypeUtils { resolver: Resolver, context: String, storeAssignmentPolicy: StoreAssignmentPolicy.Value, - addError: String => Unit): Boolean = { + addError: String => Unit, + deferAnsiCastValidationToRuntime: Boolean = false): Boolean = { (write, read) match { case (wArr: ArrayType, rArr: ArrayType) => // run compatibility check first to produce all error messages val typesCompatible = canWrite( tableName, wArr.elementType, rArr.elementType, byName, resolver, context + ".element", - storeAssignmentPolicy, addError) + storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) if (wArr.containsNull && !rArr.containsNull) { throw QueryCompilationErrors.incompatibleDataToTableNullableArrayElementsError( @@ -134,10 +135,10 @@ object DataTypeUtils { // run compatibility check first to produce all error messages val keyCompatible = canWrite( tableName, wMap.keyType, rMap.keyType, byName, resolver, context + ".key", - storeAssignmentPolicy, addError) + storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) val valueCompatible = canWrite( tableName, wMap.valueType, rMap.valueType, byName, resolver, context + ".value", - storeAssignmentPolicy, addError) + storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) if (wMap.valueContainsNull && !rMap.valueContainsNull) { throw QueryCompilationErrors.incompatibleDataToTableNullableMapValuesError( @@ -155,7 +156,7 @@ object DataTypeUtils { val fieldContext = s"$context.${rField.name}" val typesCompatible = canWrite( tableName, wField.dataType, rField.dataType, byName, resolver, fieldContext, - storeAssignmentPolicy, addError) + storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) if (byName && !nameMatch) { throw QueryCompilationErrors.incompatibleDataToTableUnexpectedColumnNameError( @@ -199,7 +200,7 @@ object DataTypeUtils { case (_: NullType, _) if storeAssignmentPolicy == ANSI => true case (w: AtomicType, r: AtomicType) if storeAssignmentPolicy == ANSI => - if (!Cast.canANSIStoreAssign(w, r)) { + if (!Cast.canANSIStoreAssign(w, r) && !deferAnsiCastValidationToRuntime) { throw QueryCompilationErrors.incompatibleDataToTableCannotSafelyCastError( tableName, context, w.catalogString, r.catalogString ) @@ -213,12 +214,12 @@ object DataTypeUtils { // If write-side data type is a user-defined type, check with its underlying data type. case (w, r) if w.isInstanceOf[UserDefinedType[_]] && !r.isInstanceOf[UserDefinedType[_]] => canWrite(tableName, w.asInstanceOf[UserDefinedType[_]].sqlType, r, byName, resolver, - context, storeAssignmentPolicy, addError) + context, storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) // If read-side data type is a user-defined type, check with its underlying data type. case (w, r) if r.isInstanceOf[UserDefinedType[_]] && !w.isInstanceOf[UserDefinedType[_]] => canWrite(tableName, w, r.asInstanceOf[UserDefinedType[_]].sqlType, byName, resolver, - context, storeAssignmentPolicy, addError) + context, storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) case (w, r) => throw QueryCompilationErrors.incompatibleDataToTableCannotSafelyCastError( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala index 95e6659956ca..5c9060fcca6c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala @@ -97,7 +97,8 @@ class InMemoryRowLevelOperationTableCatalog partitioning = partitioning, properties = properties, constraints = constraints, - tableId = table.id) + tableId = table.id, + schemaAlignmentConfig = table.schemaAlignmentConfig) newTable.alterTableWithData(table.data, schema) newTable.setVersionAndValidatedVersionFrom(table) @@ -139,7 +140,8 @@ class PartialSchemaEvolutionCatalog extends InMemoryRowLevelOperationTableCatalo schema = schema, partitioning = table.partitioning, properties = properties, - constraints = table.constraints) + constraints = table.constraints, + schemaAlignmentConfig = table.schemaAlignmentConfig) newTable.alterTableWithData(table.data, table.schema) newTable.setVersionAndValidatedVersionFrom(table) tables.put(ident, newTable) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala index f4a55cd402fb..5565dc190fce 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala @@ -144,7 +144,8 @@ class InMemoryTable( advisoryPartitionSize, isDistributionStrictlyRequired, numRowsPerSplit, - id) + id, + schemaAlignmentConfig = schemaAlignmentConfig) dataMap.synchronized { dataMap.foreach { case (key, splits) => diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala index 131eab34762f..8f0233df5de4 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala @@ -152,6 +152,34 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase ) } + test("deferAnsiCastValidationToRuntime relaxes only the atomic cast, not struct field names") { + // deferAnsiCastValidationToRuntime relaxes the atomic STRING -> INT store-assignment cast on the + // mismatched field, but the field-name check must still reject the write. This guards against + // canWrite becoming too broad under the deferred-cast path (it must keep the structural checks). + val writeStruct = StructType(Seq( + StructField("first", FloatType, nullable = false), + StructField("second", StringType, nullable = false))) + val readStruct = StructType(Seq( + StructField("first", FloatType, nullable = false), + StructField("renamed", IntegerType, nullable = false))) + + val errs = new mutable.ArrayBuffer[String]() + checkError( + exception = intercept[AnalysisException]( + DataTypeUtils.canWrite("", writeStruct, readStruct, byName = true, + analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, + errMsg => errs += errMsg, deferAnsiCastValidationToRuntime = true) + ), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.UNEXPECTED_COLUMN_NAME", + parameters = Map( + "tableName" -> "``", + "colName" -> "`t`", + "order" -> "1", + "expected" -> "`renamed`", + "found" -> "`second`") + ) + } + private val stringPoint2 = StructType(Seq( StructField("x", StringType, nullable = false), StructField("y", StringType, nullable = false))) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertSchemaEvolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertSchemaEvolutionSuite.scala deleted file mode 100644 index f10cf3e31734..000000000000 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertSchemaEvolutionSuite.scala +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.connector - -import org.scalatest.BeforeAndAfter - -import org.apache.spark.sql.{QueryTest, Row} -import org.apache.spark.sql.connector.catalog.InMemoryCatalog -import org.apache.spark.sql.internal.SQLConf.{PARTITION_OVERWRITE_MODE, PartitionOverwriteMode} -import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types._ - -class InsertSchemaEvolutionSuite - extends QueryTest with SharedSparkSession with BeforeAndAfter { - - private val catalogName = "testcat" - private val namespace = "ns" - private val tableIdent = s"$catalogName.$namespace.test_table" - - before { - spark.conf.set(s"spark.sql.catalog.$catalogName", classOf[InMemoryCatalog].getName) - } - - after { - spark.sessionState.catalogManager.reset() - spark.sessionState.conf.unsetConf(s"spark.sql.catalog.$catalogName") - } - - test("INSERT BY NAME with extra source column adds column to table") { - withTable(tableIdent) { - sql(s"CREATE TABLE $tableIdent (id INT, data STRING)") - sql( - s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME - |SELECT * FROM VALUES (1, 'a', CAST(10.0 AS DOUBLE)), - | (2, 'b', CAST(20.0 AS DOUBLE)) AS t(id, data, amount) - |""".stripMargin) - - val result = spark.table(tableIdent) - checkAnswer(result, Seq(Row(1, "a", 10.0d), Row(2, "b", 20.0d))) - assert(result.schema == StructType(Seq( - StructField("id", IntegerType), - StructField("data", StringType), - StructField("amount", DoubleType)))) - } - } - - test("INSERT BY NAME with type widening updates column type") { - withTable(tableIdent) { - sql(s"CREATE TABLE $tableIdent (id INT, value INT)") - sql( - s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME - |SELECT * FROM VALUES (1, CAST(100 AS LONG)), - | (2, CAST(200 AS LONG)) AS t(id, value) - |""".stripMargin) - - val result = spark.table(tableIdent) - checkAnswer(result, Seq(Row(1, 100L), Row(2, 200L))) - assert(result.schema == StructType(Seq( - StructField("id", IntegerType), - StructField("value", LongType)))) - } - } - - test("INSERT BY NAME with nested struct evolution") { - withTable(tableIdent) { - sql(s"CREATE TABLE $tableIdent (id INT, info STRUCT)") - sql( - s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME - |SELECT id, named_struct('name', name, 'age', age) AS info - |FROM VALUES (1, 'Alice', 30), (2, 'Bob', 25) AS t(id, name, age) - |""".stripMargin) - - val result = spark.table(tableIdent) - checkAnswer(result, Seq(Row(1, Row("Alice", 30)), Row(2, Row("Bob", 25)))) - val expectedInfoType = StructType(Seq( - StructField("name", StringType), - StructField("age", IntegerType))) - assert(result.schema == StructType(Seq( - StructField("id", IntegerType), - StructField("info", expectedInfoType)))) - } - } - - test("INSERT BY NAME with matching schema - no evolution needed") { - withTable(tableIdent) { - sql(s"CREATE TABLE $tableIdent (id INT, data STRING)") - sql( - s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME - |SELECT * FROM VALUES (1, 'a'), (2, 'b') AS t(id, data) - |""".stripMargin) - - val result = spark.table(tableIdent) - checkAnswer(result, Seq(Row(1, "a"), Row(2, "b"))) - assert(result.schema == StructType(Seq( - StructField("id", IntegerType), - StructField("data", StringType)))) - } - } - - test("INSERT BY POSITION with schema evolution adds extra columns") { - withTable(tableIdent) { - sql(s"CREATE TABLE $tableIdent (id INT, data STRING)") - sql( - s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent - |SELECT * FROM VALUES (1, 'a', CAST(10.0 AS DOUBLE)), - | (2, 'b', CAST(20.0 AS DOUBLE)) AS t(id, data, amount) - |""".stripMargin) - - val result = spark.table(tableIdent) - checkAnswer(result, Seq(Row(1, "a", 10.0d), Row(2, "b", 20.0d))) - assert(result.schema == StructType(Seq( - StructField("id", IntegerType), - StructField("data", StringType), - StructField("amount", DoubleType)))) - } - } - - test("table without AUTOMATIC_SCHEMA_EVOLUTION - no evolution") { - withTable(tableIdent) { - sql( - s"""CREATE TABLE $tableIdent (id INT, data STRING) - |TBLPROPERTIES ('auto-schema-evolution' = 'false')""".stripMargin) - - intercept[Exception] { - sql( - s"""INSERT WITH SCHEMA EVOLUTION INTO $tableIdent BY NAME - |SELECT * FROM VALUES (1, 'a', CAST(10.0 AS DOUBLE)), - | (2, 'b', CAST(20.0 AS DOUBLE)) AS t(id, data, amount) - |""".stripMargin) - } - } - } - - test("OVERWRITE BY EXPRESSION with schema evolution") { - withTable(tableIdent) { - sql(s"CREATE TABLE $tableIdent (id INT, data STRING)") - sql(s"INSERT INTO $tableIdent VALUES (1, 'a'), (2, 'b')") - - withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { - sql( - s"""INSERT WITH SCHEMA EVOLUTION OVERWRITE $tableIdent BY NAME - |SELECT * FROM VALUES (3, 'c', CAST(30.0 AS DOUBLE)), - | (4, 'd', CAST(40.0 AS DOUBLE)) AS t(id, data, amount) - |""".stripMargin) - } - - val result = spark.table(tableIdent) - checkAnswer(result, Seq(Row(3, "c", 30.0d), Row(4, "d", 40.0d))) - assert(result.schema == StructType(Seq( - StructField("id", IntegerType), - StructField("data", StringType), - StructField("amount", DoubleType)))) - } - } - - test("OVERWRITE PARTITIONS DYNAMIC with schema evolution") { - withTable(tableIdent) { - sql(s"CREATE TABLE $tableIdent (id INT, data STRING) PARTITIONED BY (id)") - sql(s"INSERT INTO $tableIdent VALUES (1, 'a'), (2, 'b')") - - withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) { - sql( - s"""INSERT WITH SCHEMA EVOLUTION OVERWRITE $tableIdent BY NAME - |SELECT * FROM VALUES (1, 'c', CAST(30.0 AS DOUBLE)) AS t(id, data, amount) - |""".stripMargin) - } - - val result = spark.table(tableIdent) - checkAnswer(result.orderBy("id"), - Seq(Row(1, "c", 30.0d), Row(2, "b", null))) - assert(result.schema == StructType(Seq( - StructField("id", IntegerType), - StructField("data", StringType), - StructField("amount", DoubleType)))) - } - } -} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala index f1827d3ca4cf..f18b0b05063d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.connector.catalog import scala.util.{Failure, Success, Try} import org.apache.spark.SparkConf +import org.apache.spark.SparkThrowable import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ @@ -30,15 +31,12 @@ import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, /** * A catalog that creates [[InMemoryRowLevelOperationTable]]s carrying a fixed - * [[SchemaAlignmentConfig]] supplied by the concrete subclass. It returns the live table instance - * on load (rather than a copy) so the config is preserved for the analyzer. + * [[SchemaAlignmentConfig]] supplied by the concrete subclass. */ abstract class SchemaAlignmentTestCatalog extends InMemoryRowLevelOperationTableCatalog { protected def tableConfig: SchemaAlignmentConfig - override def loadTable(ident: Identifier): Table = liveTable(ident) - override def createTable(ident: Identifier, tableInfo: TableInfo): Table = { if (tables.containsKey(ident)) { throw new TableAlreadyExistsException(ident.asMultipartIdentifier) @@ -58,7 +56,7 @@ abstract class SchemaAlignmentTestCatalog extends InMemoryRowLevelOperationTable class RelaxedSchemaAlignmentCatalog extends SchemaAlignmentTestCatalog { override protected def tableConfig: SchemaAlignmentConfig = new SchemaAlignmentConfig { override def allowLegacyStoreAssignmentPolicy(): Boolean = true - override def deferCastValidationToRuntime(): Boolean = true + override def deferAnsiCastValidationToRuntime(): Boolean = true } } @@ -124,7 +122,7 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } - test("deferCastValidationToRuntime: INSERT of an ANSI-incompatible cast") { + test("deferAnsiCastValidationToRuntime: INSERT of an ANSI-incompatible cast") { withTable(s"$relaxed.t", s"$strict.t") { sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") sql(s"CREATE TABLE $strict.t (id INT) USING foo") @@ -145,7 +143,7 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } - test("deferCastValidationToRuntime: UPDATE with an ANSI-incompatible cast") { + test("deferAnsiCastValidationToRuntime: UPDATE with an ANSI-incompatible cast") { withTable(s"$relaxed.t", s"$strict.t") { sql(s"CREATE TABLE $relaxed.t (id INT, data INT) USING foo") sql(s"CREATE TABLE $strict.t (id INT, data INT) USING foo") @@ -187,7 +185,7 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } - test("deferCastValidationToRuntime: MERGE with an ANSI-incompatible cast") { + test("deferAnsiCastValidationToRuntime: MERGE with an ANSI-incompatible cast") { withTable(s"$relaxed.t", s"$strict.t") { sql(s"CREATE TABLE $relaxed.t (id INT, data INT) USING foo") sql(s"CREATE TABLE $strict.t (id INT, data INT) USING foo") @@ -213,7 +211,7 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } - test("deferCastValidationToRuntime: structurally impossible casts are still rejected") { + test("deferAnsiCastValidationToRuntime: structurally impossible casts are still rejected") { withTable(s"$relaxed.t") { sql(s"CREATE TABLE $relaxed.t (d DATE) USING foo") withAnsiPolicy { @@ -226,6 +224,53 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } + test("schema alignment config survives ALTER TABLE ADD COLUMNS") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") + sql(s"CREATE TABLE $strict.t (id INT) USING foo") + sql(s"ALTER TABLE $relaxed.t ADD COLUMNS (data INT)") + sql(s"ALTER TABLE $strict.t ADD COLUMNS (data INT)") + withAnsiPolicy { + sql(s"INSERT INTO $relaxed.t VALUES (1, '5')") + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1, 5)) + checkError( + exception = intercept[AnalysisException](sql(s"INSERT INTO $strict.t VALUES (1, '5')")), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST", + parameters = Map( + "tableName" -> s"`$strict`.`t`", + "colName" -> "`data`", + "srcType" -> "\"STRING\"", + "targetType" -> "\"INT\"")) + } + } + } + + test("schema alignment config survives schema evolution") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") + sql(s"CREATE TABLE $strict.t (id INT) USING foo") + // A schema-evolving write reconstructs the table (adds the `data` column) via the catalog's + // alterTable; the config must survive that reconstruction. + val evolving = spark.createDataFrame( + java.util.Arrays.asList(Row(1, 5)), + new StructType().add("id", IntegerType).add("data", IntegerType)) + evolving.write.mode("append").withSchemaEvolution().insertInto(s"$relaxed.t") + evolving.write.mode("append").withSchemaEvolution().insertInto(s"$strict.t") + withAnsiPolicy { + sql(s"INSERT INTO $relaxed.t VALUES (2, '5')") + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Seq(Row(1, 5), Row(2, 5))) + checkError( + exception = intercept[AnalysisException](sql(s"INSERT INTO $strict.t VALUES (2, '5')")), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST", + parameters = Map( + "tableName" -> s"`$strict`.`t`", + "colName" -> "`data`", + "srcType" -> "\"STRING\"", + "targetType" -> "\"INT\"")) + } + } + } + private def appendByName( catalog: String, targetSchema: StructType, source: DataFrame): Try[Seq[Row]] = { var result: Try[Seq[Row]] = Try(Seq.empty[Row]) @@ -240,55 +285,75 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { result } - private def assertRelaxedMatchesStrict(targetSchema: StructType, source: DataFrame): Unit = + /** + * Append `source` by name into a fresh table with `targetSchema` on both the relaxed and the + * strict catalog, and assert both reject it at analysis with the same error condition. The point + * is that [[SchemaAlignmentConfig.deferAnsiCastValidationToRuntime]] relaxes only the atomic ANSI + * store-assignment cast; it must not relax structural checks, so both catalogs behave the same. + */ + private def assertBothReject( + targetSchema: StructType, source: DataFrame, condition: String): Unit = withAnsiPolicy { - val fromRelaxed = appendByName(relaxed, targetSchema, source) - val fromStrict = appendByName(strict, targetSchema, source) - (fromRelaxed, fromStrict) match { - case (Success(relaxedRows), Success(strictRows)) => - assert(relaxedRows.map(_.toString).sorted == strictRows.map(_.toString).sorted, - s"relaxed=$relaxedRows strict=$strictRows") - case (Failure(relaxedError: AnalysisException), Failure(strictError: AnalysisException)) => - assert(relaxedError.getCondition == strictError.getCondition, - s"relaxed=${relaxedError.getCondition} strict=${strictError.getCondition}") - case (Failure(_), Failure(_)) => - case _ => - fail(s"relaxed and strict diverged: relaxed=$fromRelaxed strict=$fromStrict") + Seq(relaxed, strict).foreach { catalog => + appendByName(catalog, targetSchema, source) match { + case Failure(error: SparkThrowable) => + assert(error.getCondition == condition, + s"$catalog: expected $condition, got ${error.getCondition}") + case other => + fail(s"$catalog: expected rejection with $condition, got $other") + } + } + } + + /** + * Append `source` by name into a fresh table with `targetSchema` on both the relaxed and the + * strict catalog, and assert both accept it and store exactly `source`'s rows (round trip). + */ + private def assertBothStore(targetSchema: StructType, source: DataFrame): Unit = + withAnsiPolicy { + val expected = source.collect().toSeq.map(_.toString).sorted + Seq(relaxed, strict).foreach { catalog => + appendByName(catalog, targetSchema, source) match { + case Success(rows) => + assert(rows.map(_.toString).sorted == expected, s"$catalog: got $rows") + case other => + fail(s"$catalog: expected success storing $expected, got $other") + } } } - test("deferCastValidationToRuntime: renamed nested struct field is still rejected") { + test("deferAnsiCastValidationToRuntime: renamed nested struct field is still rejected") { val target = new StructType() .add("s", new StructType().add("a", IntegerType).add("b", IntegerType)) val source = spark.createDataFrame( java.util.Arrays.asList(Row(Row(1, 2))), new StructType().add("s", new StructType().add("a", IntegerType).add("c", IntegerType))) - assertRelaxedMatchesStrict(target, source) + assertBothReject(target, source, "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA") } - test("deferCastValidationToRuntime: nullable array element into non-null element type") { + test("deferAnsiCastValidationToRuntime: nullable array element into non-null element type") { val target = new StructType().add("a", ArrayType(IntegerType, containsNull = false)) val source = spark.createDataFrame( java.util.Arrays.asList(Row(Seq(1, 2))), new StructType().add("a", ArrayType(IntegerType, containsNull = true))) - assertRelaxedMatchesStrict(target, source) + assertBothStore(target, source) } - test("deferCastValidationToRuntime: nullable map value into non-null value type") { + test("deferAnsiCastValidationToRuntime: nullable map value into non-null value type") { val target = new StructType() .add("m", MapType(StringType, IntegerType, valueContainsNull = false)) val source = spark.createDataFrame( java.util.Arrays.asList(Row(Map("k" -> 1))), new StructType().add("m", MapType(StringType, IntegerType, valueContainsNull = true))) - assertRelaxedMatchesStrict(target, source) + assertBothStore(target, source) } - test("deferCastValidationToRuntime: nullable child into non-null struct field") { + test("deferAnsiCastValidationToRuntime: nullable child into non-null struct field") { val target = new StructType() .add("s", new StructType().add("a", IntegerType, nullable = false)) val source = spark.createDataFrame( java.util.Arrays.asList(Row(Row(1))), new StructType().add("s", new StructType().add("a", IntegerType, nullable = true))) - assertRelaxedMatchesStrict(target, source) + assertBothStore(target, source) } } From 4de013fee87abc4926eee22c6894968bce8f8efd Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Mon, 21 Sep 2026 07:57:22 +0000 Subject: [PATCH 3/6] Remove allowing LEGACY, restructure interface, address comments --- docs/sql-ref-ansi-compliance.md | 3 - docs/sql-v2-data-sources.md | 1 - .../catalog/SchemaAlignmentConfig.java | 53 ------ .../spark/sql/connector/catalog/Table.java | 10 -- .../ConfigurableSchemaAlignment.java | 31 ++++ .../connector/SchemaAlignmentConfig.java | 50 ++++++ .../sql/catalyst/analysis/Analyzer.scala | 24 +-- .../catalyst/analysis/AssignmentUtils.scala | 38 +++-- .../ResolveRowLevelCommandAssignments.scala | 44 ++--- .../analysis/TableOutputResolver.scala | 87 +++++----- .../sql/catalyst/types/DataTypeUtils.scala | 24 ++- .../write/RowLevelOperationTable.scala | 12 +- .../connector/catalog/InMemoryBaseTable.scala | 4 +- .../InMemoryRowLevelOperationTable.scala | 1 + .../sql/connector/catalog/InMemoryTable.scala | 1 + .../catalog/InMemoryTableCatalog.scala | 7 +- ...oryTableCatalystRuntimeFilterCatalog.scala | 5 +- .../InMemoryTableWithV2FilterCatalog.scala | 5 +- .../DataTypeWriteCompatibilitySuite.scala | 94 ++++++++++- .../catalog/SchemaAlignmentConfigSuite.scala | 155 ++++-------------- .../command/AlignAssignmentsSuiteBase.scala | 10 +- .../command/PlanResolutionSuite.scala | 9 +- 22 files changed, 352 insertions(+), 316 deletions(-) delete mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java create mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/internal/connector/ConfigurableSchemaAlignment.java create mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/internal/connector/SchemaAlignmentConfig.java diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index 23c1f8afe2c4..2ec0b7872e52 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -226,9 +226,6 @@ INSERT INTO test VALUES (2147483648L); org.apache.spark.SparkArithmeticException: [CAST_OVERFLOW_IN_TABLE_INSERT] Fail to insert a value of "BIGINT" type into the "INT" type column `i` due to an overflow. Use `try_cast` on the input value to tolerate overflow and return NULL instead. ``` -By default, invalid source/target combinations are rejected during analysis. -Data sources may instead defer this validation to execution time, so an insertion is rejected only when a value is actually malformed or overflows, not during analysis. - ### Type coercion #### Type Promotion and Precedence When `spark.sql.ansi.enabled` is set to `true`, Spark SQL uses several rules that govern how conflicts between data types are resolved. diff --git a/docs/sql-v2-data-sources.md b/docs/sql-v2-data-sources.md index 33dc5bce4883..4b4d0f35cb4e 100644 --- a/docs/sql-v2-data-sources.md +++ b/docs/sql-v2-data-sources.md @@ -205,7 +205,6 @@ A `Table` provides: | `partitioning()` | Physical partitioning expressed as `Transform` arrays | | `properties()` | A string map of table properties | | `capabilities()` | A set of `TableCapability` values declaring what the table supports | -| `schemaAlignmentConfig()` | A `SchemaAlignmentConfig` controlling how a query is aligned to the table schema during batch / row-level writes | ### Read and Write Mix-ins diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java deleted file mode 100644 index 519e44d96204..000000000000 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfig.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.connector.catalog; - -import org.apache.spark.annotation.Evolving; - -/** - * Schema-alignment configuration for batch/row-level writes to a {@link Table}. This allows - * connectors to configure casting behavior and handling of schema mismatches during DSv2 writes. - * It is not consulted for streaming writes, which do not go through this alignment path. - * - * @since 4.4.0 - */ -@Evolving -public interface SchemaAlignmentConfig { - - /** The strict data source v2 configuration, returned by {@link Table} by default. */ - SchemaAlignmentConfig DEFAULT = new SchemaAlignmentConfig() {}; - - /** - * Whether {@code spark.sql.storeAssignmentPolicy=LEGACY} is allowed for writes and row-level - * operations targeting this table. Data source v2 rejects LEGACY by default; a table can decide - * to opt-out from this restriction. - */ - default boolean allowLegacyStoreAssignmentPolicy() { - return false; - } - - /** - * Whether the {@code ANSI} store-assignment cast check is deferred from analysis to runtime under - * {@code spark.sql.storeAssignmentPolicy=ANSI}. When {@code true}, the analyzer skips the - * store-assignment compatibility check and inserts an ANSI cast, so malformed values or - * overflows surface at execution time. - */ - default boolean deferAnsiCastValidationToRuntime() { - return false; - } -} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java index 1c740afbf80a..ec27bcf6c82e 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java @@ -112,14 +112,4 @@ default Map properties() { * the version that corresponds to the current state of this table instance. */ default String version() { return null; } - - /** - * Returns the schema-alignment configuration for writes to this table. This allows connectors to - * configure casting behavior and handling of schema mismatches during writes. - * It is recommended to use the DEFAULT configuration to provide a unified behavior across data - * sources, but some connectors may require deviating from the default behavior. - */ - default SchemaAlignmentConfig schemaAlignmentConfig() { - return SchemaAlignmentConfig.DEFAULT; - } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/internal/connector/ConfigurableSchemaAlignment.java b/sql/catalyst/src/main/java/org/apache/spark/sql/internal/connector/ConfigurableSchemaAlignment.java new file mode 100644 index 000000000000..db91fdb60587 --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/internal/connector/ConfigurableSchemaAlignment.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.internal.connector; + +import org.apache.spark.sql.connector.catalog.Table; + +/** + * Allows connectors to configure schema alignment and casting behavior for DSv2 batch/row-level + * writes to a {@link Table}. It is not consulted for streaming writes, where alignment is + * delegated to the connector. + */ +public interface ConfigurableSchemaAlignment extends Table { + + /** The schema alignment configuration for writes to this table. */ + SchemaAlignmentConfig schemaAlignmentConfig(); +} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/internal/connector/SchemaAlignmentConfig.java b/sql/catalyst/src/main/java/org/apache/spark/sql/internal/connector/SchemaAlignmentConfig.java new file mode 100644 index 000000000000..9275b57426d0 --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/internal/connector/SchemaAlignmentConfig.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.internal.connector; + +import org.apache.spark.sql.connector.catalog.Table; + +/** + * Schema alignment configuration for DSv2 batch/row-level writes to a {@link Table}, exposed via + * {@link ConfigurableSchemaAlignment}. + */ +public interface SchemaAlignmentConfig { + + /** The default data source v2 configuration. */ + SchemaAlignmentConfig DEFAULT = new SchemaAlignmentConfig() {}; + + /** + * When the {@code ANSI} store-assignment cast check runs for writes to a table, under + * {@code spark.sql.storeAssignmentPolicy=ANSI}. Has no effect under the {@code STRICT} policy. + */ + enum AnsiStoreAssignmentCastCheck { + /** (default) Reject incompatible casts at analysis. See {@code Cast.canANSIStoreAssign}. */ + AT_ANALYSIS, + + /** + * Insert an ANSI cast check, so malformed values or overflows fail at execution time instead + * of being rejected during analysis. + */ + AT_RUNTIME + } + + /** When the {@code ANSI} store-assignment cast check runs. */ + default AnsiStoreAssignmentCastCheck ansiStoreAssignmentCastCheck() { + return AnsiStoreAssignmentCastCheck.AT_ANALYSIS; + } +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 12d412b6b6df..c702a993a796 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -60,9 +60,11 @@ import org.apache.spark.sql.connector.catalog.functions.UnboundFunction import org.apache.spark.sql.connector.catalog.procedures.{BoundProcedure, ProcedureParameter, UnboundProcedure} import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform} import org.apache.spark.sql.errors.QueryCompilationErrors -import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, ExtractV2Table} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.{PartitionOverwriteMode, StoreAssignmentPolicy} +import org.apache.spark.sql.internal.connector.ConfigurableSchemaAlignment +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck import org.apache.spark.sql.internal.connector.V1Function import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -3945,10 +3947,7 @@ class Analyzer( case v2Write: V2WriteCommand if v2Write.table.resolved && v2Write.query.resolved && !v2Write.outputResolved && v2Write.pendingSchemaChanges.isEmpty => - val schemaAlignment = v2Write.table.collectFirst { - case r: DataSourceV2Relation => r.table.schemaAlignmentConfig() - }.getOrElse(SchemaAlignmentConfig.DEFAULT) - validateStoreAssignmentPolicy(schemaAlignment) + validateStoreAssignmentPolicy() TableOutputResolver.suitableForByNameCheck(v2Write.isByName, expected = v2Write.table.output, queryOutput = v2Write.query.output) // With schema evolution + coercion flag, missing top-level columns AND missing nested @@ -3965,11 +3964,15 @@ class Analyzer( case r: DataSourceV2Relation => GeneratedColumn.attachGenerationExpressions(r) case _ => v2Write.table.output } + val schemaAlignmentConfig = v2Write.table match { + case ExtractV2Table(table: ConfigurableSchemaAlignment) => + table.schemaAlignmentConfig() + case _ => SchemaAlignmentConfig.DEFAULT + } val (projection, autoFilledGenCols) = TableOutputResolver.resolveOutputColumnsWithGeneratedInfo( v2Write.table.name, expected, v2Write.query, v2Write.isByName, conf, - defaultValueFillMode, - deferAnsiCastValidationToRuntime = schemaAlignment.deferAnsiCastValidationToRuntime()) + defaultValueFillMode, schemaAlignmentConfig.ansiStoreAssignmentCastCheck()) if (projection != v2Write.query) { val cleanedTable = v2Write.table match { case r: DataSourceV2Relation => @@ -3985,10 +3988,9 @@ class Analyzer( } } - private def validateStoreAssignmentPolicy(schemaAlignment: SchemaAlignmentConfig): Unit = { - // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2 by default. - if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY && - !schemaAlignment.allowLegacyStoreAssignmentPolicy()) { + private def validateStoreAssignmentPolicy(): Unit = { + // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2. + if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY) { throw QueryCompilationErrors.legacyStoreAssignmentPolicyError() } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala index 042e0b08504c..5884430b6f8a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala @@ -28,9 +28,10 @@ import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getDefaultValueExprOrNullLit import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ -import org.apache.spark.sql.connector.catalog.SchemaAlignmentConfig import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck.AT_ANALYSIS import org.apache.spark.sql.types.{DataType, StructType} import org.apache.spark.util.ArrayImplicits._ @@ -58,6 +59,9 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { * (preserving existing fields). * @param coerceNestedTypes whether to coerce nested types to match the target type * for complex types + * @param ansiStoreAssignmentCastCheck when the ANSI store-assignment cast check runs: + * AT_ANALYSIS (default) rejects an unsafe cast at analysis + * time; AT_RUNTIME rejects invalid values at execution time. * @return aligned update assignments that match table attributes */ def alignUpdateAssignments( @@ -65,7 +69,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { assignments: Seq[Assignment], fromStar: Boolean, coerceNestedTypes: Boolean, - schemaAlignment: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT): Seq[Assignment] = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck = AT_ANALYSIS): Seq[Assignment] = { val errors = new mutable.ArrayBuffer[String]() @@ -78,7 +82,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { colPath = Seq(attr.name), coerceNestedTypes, fromStar, - schemaAlignment = schemaAlignment) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } if (errors.nonEmpty) { @@ -101,13 +105,16 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { * @param assignments insert assignments to align * @param coerceNestedTypes whether to coerce nested types to match the target type * for complex types + * @param ansiStoreAssignmentCastCheck when the ANSI store-assignment cast check runs: + * AT_ANALYSIS (default) rejects an unsafe cast at analysis + * time; AT_RUNTIME rejects invalid values at execution time. * @return aligned insert assignments that match table attributes */ def alignInsertAssignments( attrs: Seq[Attribute], assignments: Seq[Assignment], coerceNestedTypes: Boolean = false, - schemaAlignment: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT): Seq[Assignment] = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck = AT_ANALYSIS): Seq[Assignment] = { val errors = new mutable.ArrayBuffer[String]() @@ -142,7 +149,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate( "", value, actualAttr, conf, err => errors += err, colPath, coerceMode, - deferAnsiCastValidationToRuntime = schemaAlignment.deferAnsiCastValidationToRuntime()) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } Assignment(attr, resolvedValue) } @@ -166,7 +173,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { colPath: Seq[String], coerceNestedTypes: Boolean = false, updateStar: Boolean = false, - schemaAlignment: SchemaAlignmentConfig): Expression = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Expression = { val (exactAssignments, otherAssignments) = assignments.partition { assignment => assignment.key.semanticEquals(colExpr) @@ -194,24 +201,24 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { case _: StructType => // Expand assignments to leaf fields (fixNullExpansion is applied inside) applyNestedFieldAssignments(col, colExpr, value, addError, colPath, - coerceNestedTypes, schemaAlignment = schemaAlignment) + coerceNestedTypes, ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case _ => // For non-struct types, resolve directly val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate("", value, col, conf, addError, colPath, coerceMode, - deferAnsiCastValidationToRuntime = schemaAlignment.deferAnsiCastValidationToRuntime()) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } } else { val value = exactAssignments.head.value val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate("", value, col, conf, addError, colPath, coerceMode, - deferAnsiCastValidationToRuntime = schemaAlignment.deferAnsiCastValidationToRuntime()) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } } else { applyFieldAssignments(col, colExpr, fieldAssignments, addError, colPath, coerceNestedTypes, - schemaAlignment = schemaAlignment) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } } @@ -222,7 +229,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { addError: String => Unit, colPath: Seq[String], coerceNestedTypes: Boolean, - schemaAlignment: SchemaAlignmentConfig): Expression = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Expression = { col.dataType match { case structType: StructType => @@ -232,7 +239,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { } val updatedFieldExprs = fieldAttrs.zip(fieldExprs).map { case (fieldAttr, fieldExpr) => applyAssignments(fieldAttr, fieldExpr, assignments, addError, colPath :+ fieldAttr.name, - coerceNestedTypes, schemaAlignment = schemaAlignment) + coerceNestedTypes, ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } toNamedStruct(structType, updatedFieldExprs) @@ -251,7 +258,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { addError: String => Unit, colPath: Seq[String], coerceNestedTypes: Boolean, - schemaAlignment: SchemaAlignmentConfig): Expression = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Expression = { col.dataType match { case structType: StructType => @@ -285,15 +292,14 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { // Field is a struct, recurse applyNestedFieldAssignments(fieldAttr, targetFieldExpr, sourceFieldValue, addError, fieldPath, coerceNestedTypes, - schemaAlignment = schemaAlignment) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case _ => // Field is not a struct, resolve with TableOutputResolver val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate( "", sourceFieldValue, fieldAttr, conf, addError, fieldPath, coerceMode, - deferAnsiCastValidationToRuntime = - schemaAlignment.deferAnsiCastValidationToRuntime()) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } } val namedStruct = toNamedStruct(structType, updatedFieldExprs) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala index ca04b577c582..191e59285116 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala @@ -24,10 +24,11 @@ import org.apache.spark.sql.catalyst.plans.logical.{Assignment, DeleteAction, In import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.COMMAND import org.apache.spark.sql.catalyst.util.CharVarcharUtils -import org.apache.spark.sql.connector.catalog.SchemaAlignmentConfig import org.apache.spark.sql.errors.QueryCompilationErrors -import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, ExtractV2Table} import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy +import org.apache.spark.sql.internal.connector.ConfigurableSchemaAlignment +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck /** * A rule that resolves assignments in row-level commands. @@ -41,37 +42,37 @@ object ResolveRowLevelCommandAssignments extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning( _.containsPattern(COMMAND), ruleId) { case u: UpdateTable if !u.skipSchemaResolution && u.resolved && u.rewritable && !u.aligned => - val schemaAlignment = schemaAlignmentConfig(u.table) - validateStoreAssignmentPolicy(schemaAlignment) + validateStoreAssignmentPolicy() val newTable = cleanAttrMetadata(u.table) val newAssignments = AssignmentUtils.alignUpdateAssignments(u.table.output, u.assignments, - fromStar = false, coerceNestedTypes = false, schemaAlignment = schemaAlignment) + fromStar = false, coerceNestedTypes = false, + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck(u.table)) u.copy(table = newTable, assignments = newAssignments) case u: UpdateTable if !u.skipSchemaResolution && u.resolved && !u.aligned => resolveAssignments(u) case m: MergeIntoTable if m.rewritable && shouldAlignAssignments(m) && containsFinalSchema(m) => - val schemaAlignment = schemaAlignmentConfig(m.targetTable) - validateStoreAssignmentPolicy(schemaAlignment) + validateStoreAssignmentPolicy() val coerceNestedTypes = conf.coerceMergeNestedTypes && m.withSchemaEvolution + val castCheck = ansiStoreAssignmentCastCheck(m.targetTable) m.copy( targetTable = cleanAttrMetadata(m.targetTable), matchedActions = alignActions( m.targetTable.output, m.matchedActions, coerceNestedTypes, - schemaAlignment), + castCheck), notMatchedActions = alignActions( m.targetTable.output, m.notMatchedActions, coerceNestedTypes, - schemaAlignment), + castCheck), notMatchedBySourceActions = alignActions( m.targetTable.output, m.notMatchedBySourceActions, coerceNestedTypes, - schemaAlignment)) + castCheck)) case m: MergeIntoTable if shouldAlignAssignments(m) && containsFinalSchema(m) => resolveAssignments(m) @@ -85,16 +86,16 @@ object ResolveRowLevelCommandAssignments extends Rule[LogicalPlan] { !m.schemaEvolutionEnabled || (m.schemaEvolutionReady && m.pendingSchemaChanges.isEmpty) } - private def schemaAlignmentConfig(target: LogicalPlan): SchemaAlignmentConfig = + private def ansiStoreAssignmentCastCheck(target: LogicalPlan): AnsiStoreAssignmentCastCheck = { target.collectFirst { - case relation: DataSourceV2Relation => relation.table.schemaAlignmentConfig() - }.getOrElse(SchemaAlignmentConfig.DEFAULT) + case ExtractV2Table(table: ConfigurableSchemaAlignment) => + table.schemaAlignmentConfig().ansiStoreAssignmentCastCheck() + }.getOrElse(AnsiStoreAssignmentCastCheck.AT_ANALYSIS) + } - private def validateStoreAssignmentPolicy(schemaAlignment: SchemaAlignmentConfig): Unit = { - // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2, unless the - // target table opts into it via its SchemaAlignmentConfig. - if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY && - !schemaAlignment.allowLegacyStoreAssignmentPolicy()) { + private def validateStoreAssignmentPolicy(): Unit = { + // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2. + if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY) { throw QueryCompilationErrors.legacyStoreAssignmentPolicyError() } } @@ -141,16 +142,17 @@ object ResolveRowLevelCommandAssignments extends Rule[LogicalPlan] { attrs: Seq[Attribute], actions: Seq[MergeAction], coerceNestedTypes: Boolean, - schemaAlignment: SchemaAlignmentConfig): Seq[MergeAction] = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Seq[MergeAction] = { actions.map { case u @ UpdateAction(_, assignments, fromStar) => u.copy(assignments = AssignmentUtils.alignUpdateAssignments(attrs, assignments, - fromStar, coerceNestedTypes, schemaAlignment = schemaAlignment)) + fromStar, coerceNestedTypes, + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck)) case d: DeleteAction => d case i @ InsertAction(_, assignments) => i.copy(assignments = AssignmentUtils.alignInsertAssignments(attrs, assignments, - coerceNestedTypes, schemaAlignment = schemaAlignment)) + coerceNestedTypes, ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck)) case other => throw new AnalysisException( errorClass = "_LEGACY_ERROR_TEMP_3052", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala index 3a19935dfdbb..fd174ffaa381 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala @@ -36,6 +36,8 @@ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck.AT_ANALYSIS import org.apache.spark.sql.types.{ArrayType, DataType, DecimalType, IntegralType, MapType, StructType, UserDefinedType} object TableOutputResolver extends SQLConfHelper with Logging { @@ -95,10 +97,10 @@ object TableOutputResolver extends SQLConfHelper with Logging { byName: Boolean, conf: SQLConf, defaultValueFillMode: DefaultValueFillMode.Value = NONE, - deferAnsiCastValidationToRuntime: Boolean = false): LogicalPlan = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck = AT_ANALYSIS): LogicalPlan = { resolveOutputColumnsInternal( tableName, expected, query, byName, conf, defaultValueFillMode, - deferAnsiCastValidationToRuntime)._1 + ansiStoreAssignmentCastCheck)._1 } /** @@ -114,11 +116,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { byName: Boolean, conf: SQLConf, defaultValueFillMode: DefaultValueFillMode.Value = NONE, - deferAnsiCastValidationToRuntime: Boolean = false + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck = AT_ANALYSIS ): (LogicalPlan, Set[String]) = { resolveOutputColumnsInternal( tableName, expected, query, byName, conf, defaultValueFillMode, - deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck) } private def resolveOutputColumnsInternal( @@ -128,7 +130,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { byName: Boolean, conf: SQLConf, defaultValueFillMode: DefaultValueFillMode.Value, - deferAnsiCastValidationToRuntime: Boolean + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck ): (LogicalPlan, Set[String]) = { if (expected.size < query.output.size) { @@ -155,7 +157,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { Nil, defaultValueFillMode, enforceFullOutput = true, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } else { if (expected.size > query.output.size && !fillDefaultValue) { throw QueryCompilationErrors.cannotWriteNotEnoughColumnsToTableError( @@ -164,7 +166,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { resolveColumnsByPosition( tableName, query.output, expected, conf, errors += _, fillDefaultValue = fillDefaultValue, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } if (errors.nonEmpty) { @@ -188,7 +190,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { addError: String => Unit, colPath: Seq[String], defaultValueFillMode: DefaultValueFillMode.Value, - deferAnsiCastValidationToRuntime: Boolean = false): Expression = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck = AT_ANALYSIS): Expression = { val fillChildDefaultValue = defaultValueFillMode == RECURSE (value.dataType, col.dataType) match { @@ -196,7 +198,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { case (valueType, colType) if DataType.equalsIgnoreCompatibleNullability(valueType, colType) => val canWriteExpr = canWrite( tableName, valueType, colType, byName = true, conf, addError, colPath, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) if (canWriteExpr) { val nullsHandled = checkNullability(value, col, conf, colPath) applyColumnMetadata(nullsHandled, col) @@ -207,23 +209,23 @@ object TableOutputResolver extends SQLConfHelper with Logging { val resolvedValue = resolveStructType( tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) resolvedValue.getOrElse(value) case (valueType: ArrayType, colType: ArrayType) => val resolvedValue = resolveArrayType( tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) resolvedValue.getOrElse(value) case (valueType: MapType, colType: MapType) => val resolvedValue = resolveMapType( tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) resolvedValue.getOrElse(value) case _ => checkUpdate( - tableName, value, col, conf, addError, colPath, deferAnsiCastValidationToRuntime) + tableName, value, col, conf, addError, colPath, ansiStoreAssignmentCastCheck) } } @@ -234,7 +236,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { conf: SQLConf, addError: String => Unit, colPath: Seq[String], - deferAnsiCastValidationToRuntime: Boolean): Expression = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Expression = { val attrTypeHasCharVarchar = CharVarcharUtils.hasCharVarchar(attr.dataType) val attrTypeWithoutCharVarchar = if (attrTypeHasCharVarchar) { @@ -246,7 +248,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { val canWriteValue = canWrite( tableName, value.dataType, attrTypeWithoutCharVarchar, byName = true, conf, addError, colPath, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) if (canWriteValue) { val nullCheckedValue = checkNullability(value, attr, conf, colPath) @@ -370,16 +372,13 @@ object TableOutputResolver extends SQLConfHelper with Logging { conf: SQLConf, addError: String => Unit, colPath: Seq[String], - deferAnsiCastValidationToRuntime: Boolean): Boolean = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Boolean = { conf.storeAssignmentPolicy match { case StoreAssignmentPolicy.STRICT | StoreAssignmentPolicy.ANSI => - // Always delegate to DataTypeUtils.canWrite so structural checks (struct field names and - // nullability, array/map element nullability, field counts) still run. Only the atomic - // ANSI store-assignment cast check is relaxed, via deferAnsiCastValidationToRuntime. DataTypeUtils.canWrite( tableName, valueType, expectedType, byName, conf.resolver, colPath.quoted, conf.storeAssignmentPolicy, addError, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case _ => true } @@ -394,7 +393,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { colPath: Seq[String] = Nil, defaultValueFillMode: DefaultValueFillMode.Value, enforceFullOutput: Boolean = false, - deferAnsiCastValidationToRuntime: Boolean): (Seq[NamedExpression], Set[String]) = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck + ): (Seq[NamedExpression], Set[String]) = { // Names of generated columns that were auto-filled (not provided by the user). Only populated // for top-level columns, since generated columns cannot be nested. val autoFilledGenCols = mutable.Set.empty[String] @@ -445,21 +445,21 @@ object TableOutputResolver extends SQLConfHelper with Logging { resolveStructType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case (matchedType: ArrayType, expectedType: ArrayType) => resolveArrayType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case (matchedType: MapType, expectedType: MapType) => resolveMapType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case _ => checkField( tableName, actualExpectedCol, matchedCol, byName = true, conf, addError, newColPath, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } } } @@ -514,7 +514,8 @@ object TableOutputResolver extends SQLConfHelper with Logging { addError: String => Unit, colPath: Seq[String] = Nil, fillDefaultValue: Boolean = false, - deferAnsiCastValidationToRuntime: Boolean): (Seq[NamedExpression], Set[String]) = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck + ): (Seq[NamedExpression], Set[String]) = { // Names of generated columns that were auto-filled (not provided by the user). Only populated // for top-level columns, since generated columns cannot be nested. val autoFilledGenCols = mutable.Set.empty[String] @@ -555,20 +556,20 @@ object TableOutputResolver extends SQLConfHelper with Logging { resolveStructType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case (inputType: ArrayType, expectedType: ArrayType) => resolveArrayType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case (inputType: MapType, expectedType: MapType) => resolveMapType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) case _ => checkField(tableName, expectedCol, inputCol, byName = false, conf, addError, newColPath, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } } @@ -653,7 +654,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { colPath: Seq[String], fillDefaultValue: Boolean, enforceFullOutput: Boolean, - deferAnsiCastValidationToRuntime: Boolean): Option[NamedExpression] = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val fields = inputType.zipWithIndex.map { case (f, i) => Alias(GetStructField(nullCheckedInput, i, Some(f.name)), f.name)() @@ -663,11 +664,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val (resolved, _) = if (byName) { reorderColumnsByName(tableName, fields, toAttributes(expectedType), conf, addError, colPath, defaultValueMode, enforceFullOutput, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } else { resolveColumnsByPosition( tableName, fields, toAttributes(expectedType), conf, addError, colPath, fillDefaultValue, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } if (resolved.length == expectedType.length) { val struct = CreateStruct(resolved) @@ -699,7 +700,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { colPath: Seq[String], fillDefaultValue: Boolean, enforceFullOutput: Boolean, - deferAnsiCastValidationToRuntime: Boolean): Option[NamedExpression] = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val param = NamedLambdaVariable("element", inputType.elementType, inputType.containsNull) val fakeAttr = @@ -709,11 +710,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val defaultValueMode = if (fillDefaultValue) RECURSE else NONE reorderColumnsByName(tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, defaultValueMode, enforceFullOutput, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } else { resolveColumnsByPosition( tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, fillDefaultValue, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } if (res.length == 1) { val castedArray = @@ -745,7 +746,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { colPath: Seq[String], fillDefaultValue: Boolean, enforceFullOutput: Boolean, - deferAnsiCastValidationToRuntime: Boolean): Option[NamedExpression] = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Option[NamedExpression] = { val nullCheckedInput = checkNullability(input, expected, conf, colPath) val keyParam = NamedLambdaVariable("key", inputType.keyType, nullable = false) @@ -755,11 +756,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val (resKey, _) = if (byName) { reorderColumnsByName(tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, defaultValueFillMode, enforceFullOutput, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } else { resolveColumnsByPosition( tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, fillDefaultValue, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } val valueParam = @@ -770,11 +771,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val (resValue, _) = if (byName) { reorderColumnsByName(tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, defaultValueFillMode, enforceFullOutput, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } else { resolveColumnsByPosition( tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, fillDefaultValue, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) } if (resKey.length == 1 && resValue.length == 1) { @@ -875,7 +876,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { conf: SQLConf, addError: String => Unit, colPath: Seq[String], - deferAnsiCastValidationToRuntime: Boolean): Option[NamedExpression] = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck): Option[NamedExpression] = { val attrTypeHasCharVarchar = CharVarcharUtils.hasCharVarchar(tableAttr.dataType) val attrTypeWithoutCharVarchar = if (attrTypeHasCharVarchar) { @@ -887,7 +888,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { val canWriteExpr = canWrite( tableName, queryExpr.dataType, attrTypeWithoutCharVarchar, byName, conf, addError, colPath, - deferAnsiCastValidationToRuntime = deferAnsiCastValidationToRuntime) + ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) if (canWriteExpr) { val prepared = diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala index 485cc3b83ce4..d2183c62770f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala @@ -25,6 +25,8 @@ import org.apache.spark.sql.connector.catalog.Column import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy.{ANSI, STRICT} +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck.{AT_ANALYSIS, AT_RUNTIME} import org.apache.spark.sql.types._ import org.apache.spark.sql.types.DecimalType.{forType, fromDecimal} @@ -112,13 +114,13 @@ object DataTypeUtils { context: String, storeAssignmentPolicy: StoreAssignmentPolicy.Value, addError: String => Unit, - deferAnsiCastValidationToRuntime: Boolean = false): Boolean = { + ansiStoreAssignmentCastCheck: AnsiStoreAssignmentCastCheck = AT_ANALYSIS): Boolean = { (write, read) match { case (wArr: ArrayType, rArr: ArrayType) => // run compatibility check first to produce all error messages val typesCompatible = canWrite( tableName, wArr.elementType, rArr.elementType, byName, resolver, context + ".element", - storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) + storeAssignmentPolicy, addError, ansiStoreAssignmentCastCheck) if (wArr.containsNull && !rArr.containsNull) { throw QueryCompilationErrors.incompatibleDataToTableNullableArrayElementsError( @@ -135,10 +137,10 @@ object DataTypeUtils { // run compatibility check first to produce all error messages val keyCompatible = canWrite( tableName, wMap.keyType, rMap.keyType, byName, resolver, context + ".key", - storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) + storeAssignmentPolicy, addError, ansiStoreAssignmentCastCheck) val valueCompatible = canWrite( tableName, wMap.valueType, rMap.valueType, byName, resolver, context + ".value", - storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) + storeAssignmentPolicy, addError, ansiStoreAssignmentCastCheck) if (wMap.valueContainsNull && !rMap.valueContainsNull) { throw QueryCompilationErrors.incompatibleDataToTableNullableMapValuesError( @@ -156,7 +158,7 @@ object DataTypeUtils { val fieldContext = s"$context.${rField.name}" val typesCompatible = canWrite( tableName, wField.dataType, rField.dataType, byName, resolver, fieldContext, - storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) + storeAssignmentPolicy, addError, ansiStoreAssignmentCastCheck) if (byName && !nameMatch) { throw QueryCompilationErrors.incompatibleDataToTableUnexpectedColumnNameError( @@ -200,7 +202,7 @@ object DataTypeUtils { case (_: NullType, _) if storeAssignmentPolicy == ANSI => true case (w: AtomicType, r: AtomicType) if storeAssignmentPolicy == ANSI => - if (!Cast.canANSIStoreAssign(w, r) && !deferAnsiCastValidationToRuntime) { + if (!Cast.canANSIStoreAssign(w, r) && ansiStoreAssignmentCastCheck == AT_ANALYSIS) { throw QueryCompilationErrors.incompatibleDataToTableCannotSafelyCastError( tableName, context, w.catalogString, r.catalogString ) @@ -214,12 +216,18 @@ object DataTypeUtils { // If write-side data type is a user-defined type, check with its underlying data type. case (w, r) if w.isInstanceOf[UserDefinedType[_]] && !r.isInstanceOf[UserDefinedType[_]] => canWrite(tableName, w.asInstanceOf[UserDefinedType[_]].sqlType, r, byName, resolver, - context, storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) + context, storeAssignmentPolicy, addError, ansiStoreAssignmentCastCheck) // If read-side data type is a user-defined type, check with its underlying data type. case (w, r) if r.isInstanceOf[UserDefinedType[_]] && !w.isInstanceOf[UserDefinedType[_]] => canWrite(tableName, w, r.asInstanceOf[UserDefinedType[_]].sqlType, byName, resolver, - context, storeAssignmentPolicy, addError, deferAnsiCastValidationToRuntime) + context, storeAssignmentPolicy, addError, ansiStoreAssignmentCastCheck) + + // AT_RUNTIME defers to the inserted ANSI cast, which fails on malformed or overflowing + // values at runtime. Clearly invalid conversion are still rejected either in an earlier + // branch of this match clause, or in checkAnalysis. + case (_, _) if storeAssignmentPolicy == ANSI && ansiStoreAssignmentCastCheck == AT_RUNTIME => + true case (w, r) => throw QueryCompilationErrors.incompatibleDataToTableCannotSafelyCastError( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala index 04169c0e927d..e0ff83d73cd1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala @@ -19,9 +19,10 @@ package org.apache.spark.sql.connector.write import java.util -import org.apache.spark.sql.connector.catalog.{Column, SchemaAlignmentConfig, SupportsRead, SupportsRowLevelOperations, SupportsWrite, Table, TableCapability} +import org.apache.spark.sql.connector.catalog.{Column, SupportsRead, SupportsRowLevelOperations, SupportsWrite, Table, TableCapability} import org.apache.spark.sql.connector.catalog.constraints.Constraint import org.apache.spark.sql.connector.read.ScanBuilder +import org.apache.spark.sql.internal.connector.{ConfigurableSchemaAlignment, SchemaAlignmentConfig} import org.apache.spark.sql.util.CaseInsensitiveStringMap /** @@ -34,15 +35,20 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap */ private[sql] case class RowLevelOperationTable( table: Table with SupportsRowLevelOperations, - operation: RowLevelOperation) extends Table with SupportsRead with SupportsWrite { + operation: RowLevelOperation) + extends Table with SupportsRead with SupportsWrite with ConfigurableSchemaAlignment { override def name: String = table.name override def columns: Array[Column] = table.columns() override def capabilities: util.Set[TableCapability] = table.capabilities override def constraints(): Array[Constraint] = table.constraints() - override def schemaAlignmentConfig(): SchemaAlignmentConfig = table.schemaAlignmentConfig() override def toString: String = table.toString + override def schemaAlignmentConfig(): SchemaAlignmentConfig = table match { + case t: ConfigurableSchemaAlignment => t.schemaAlignmentConfig() + case _ => SchemaAlignmentConfig.DEFAULT + } + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { operation.newScanBuilder(options) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala index 056cc35e835f..820b200af6f2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala @@ -43,7 +43,7 @@ import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, Offset} import org.apache.spark.sql.connector.write._ import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.connector.{ColumnImpl, SupportsRuntimeCatalystFiltering, SupportsStreamingUpdateAsAppend} +import org.apache.spark.sql.internal.connector.{ColumnImpl, ConfigurableSchemaAlignment, SchemaAlignmentConfig, SupportsRuntimeCatalystFiltering, SupportsStreamingUpdateAsAppend} import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -67,7 +67,7 @@ abstract class InMemoryBaseTable( val numRowsPerSplit: Int = Int.MaxValue, override val schemaAlignmentConfig: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT) extends Table with SupportsRead with SupportsWrite with SupportsMetadataColumns - with SupportsSchemaEvolution { + with SupportsSchemaEvolution with ConfigurableSchemaAlignment { // Tracks the current version number of the table. protected var tableVersion: Int = 0 diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala index b79bd5fbb3d4..2beacf73b461 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTable.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.connector.expressions.filter.Predicate import org.apache.spark.sql.connector.read.{InputPartition, Scan, ScanBuilder} import org.apache.spark.sql.connector.write.{BatchWrite, DeltaBatchWrite, DeltaWrite, DeltaWriteBuilder, DeltaWriter, DeltaWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, RequiresDistributionAndOrdering, RowLevelOperation, RowLevelOperationBuilder, RowLevelOperationInfo, SupportsDelta, Write, WriteBuilder, WriterCommitMessage} import org.apache.spark.sql.connector.write.RowLevelOperation.Command +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.unsafe.types.UTF8String diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala index 5565dc190fce..20ce2af03ac6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTable.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.connector.distributions.{Distribution, Distributions import org.apache.spark.sql.connector.expressions.{SortOrder, Transform} import org.apache.spark.sql.connector.read._ import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsOverwrite, WriteBuilder, WriterCommitMessage} +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.{LongType, StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala index cc5c75dd779b..dc9253e42540 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.connector.catalog.procedures.{BoundProcedure, Proced import org.apache.spark.sql.connector.distributions.{Distribution, Distributions} import org.apache.spark.sql.connector.expressions.{SortOrder, Transform} import org.apache.spark.sql.connector.read.{LocalScan, Scan} +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig import org.apache.spark.sql.types.{DataTypes, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -219,11 +220,13 @@ class BasicInMemoryTableCatalog extends TableCatalog { advisoryPartitionSize: Option[Long], distributionStrictlyRequired: Boolean, numRowsPerSplit: Int, - id: String): InMemoryBaseTable = { + id: String, + schemaAlignmentConfig: SchemaAlignmentConfig = SchemaAlignmentConfig.DEFAULT + ): InMemoryBaseTable = { // scalastyle:on argcount new InMemoryTable(name, columns, partitioning, properties, constraints, distribution, ordering, requiredNumPartitions, advisoryPartitionSize, distributionStrictlyRequired, - numRowsPerSplit, id) + numRowsPerSplit, id, schemaAlignmentConfig) } override def alterTable(ident: Identifier, changes: TableChange*): Table = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalystRuntimeFilterCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalystRuntimeFilterCatalog.scala index e699f439223f..63389e3f941d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalystRuntimeFilterCatalog.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalystRuntimeFilterCatalog.scala @@ -22,6 +22,7 @@ import java.util import org.apache.spark.sql.connector.catalog.constraints.Constraint import org.apache.spark.sql.connector.distributions.Distribution import org.apache.spark.sql.connector.expressions.{SortOrder, Transform} +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig /** * Mix-in that constructs [[InMemoryCatalystRuntimeFilterTable]] from the shared in-memory @@ -41,7 +42,9 @@ trait InMemoryCatalystRuntimeFilterTableFactory { self: BasicInMemoryTableCatalo advisoryPartitionSize: Option[Long], distributionStrictlyRequired: Boolean, numRowsPerSplit: Int, - id: String): InMemoryBaseTable = { + id: String, + // InMemoryCatalystRuntimeFilterTable carries no schema-alignment config, so this is ignored. + schemaAlignmentConfig: SchemaAlignmentConfig): InMemoryBaseTable = { // scalastyle:on argcount new InMemoryCatalystRuntimeFilterTable( name, columns, partitioning, properties, constraints, distribution, ordering, diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableWithV2FilterCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableWithV2FilterCatalog.scala index be5a907f5212..0c0a79975ac1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableWithV2FilterCatalog.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableWithV2FilterCatalog.scala @@ -22,6 +22,7 @@ import java.util import org.apache.spark.sql.connector.catalog.constraints.Constraint import org.apache.spark.sql.connector.distributions.Distribution import org.apache.spark.sql.connector.expressions.{SortOrder, Transform} +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig class InMemoryTableWithV2FilterCatalog extends InMemoryTableCatalog { // scalastyle:off argcount @@ -37,7 +38,9 @@ class InMemoryTableWithV2FilterCatalog extends InMemoryTableCatalog { advisoryPartitionSize: Option[Long], distributionStrictlyRequired: Boolean, numRowsPerSplit: Int, - id: String): InMemoryBaseTable = { + id: String, + // InMemoryTableWithV2Filter carries no schema-alignment config, so this is ignored. + schemaAlignmentConfig: SchemaAlignmentConfig): InMemoryBaseTable = { // scalastyle:on argcount new InMemoryTableWithV2Filter( name, columns, partitioning, properties, constraints, distribution, ordering, diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala index 8f0233df5de4..33edb749cd6e 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.TypeUtils.toSQLType import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBaseSuite { override def storeAssignmentPolicy: SQLConf.StoreAssignmentPolicy.Value = @@ -152,10 +153,7 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase ) } - test("deferAnsiCastValidationToRuntime relaxes only the atomic cast, not struct field names") { - // deferAnsiCastValidationToRuntime relaxes the atomic STRING -> INT store-assignment cast on the - // mismatched field, but the field-name check must still reject the write. This guards against - // canWrite becoming too broad under the deferred-cast path (it must keep the structural checks). + test("AT_RUNTIME does not relax struct field ordering") { val writeStruct = StructType(Seq( StructField("first", FloatType, nullable = false), StructField("second", StringType, nullable = false))) @@ -168,7 +166,8 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase exception = intercept[AnalysisException]( DataTypeUtils.canWrite("", writeStruct, readStruct, byName = true, analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, - errMsg => errs += errMsg, deferAnsiCastValidationToRuntime = true) + errMsg => errs += errMsg, + ansiStoreAssignmentCastCheck = AnsiStoreAssignmentCastCheck.AT_RUNTIME) ), condition = "INCOMPATIBLE_DATA_FOR_TABLE.UNEXPECTED_COLUMN_NAME", parameters = Map( @@ -180,6 +179,91 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase ) } + test("AT_RUNTIME does not relax array element nullability") { + val errs = new mutable.ArrayBuffer[String]() + checkError( + exception = intercept[AnalysisException]( + DataTypeUtils.canWrite("", ArrayType(LongType, containsNull = true), + ArrayType(LongType, containsNull = false), byName = true, + analysis.caseSensitiveResolution, "arr", storeAssignmentPolicy, + errMsg => errs += errMsg, + ansiStoreAssignmentCastCheck = AnsiStoreAssignmentCastCheck.AT_RUNTIME)), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_ARRAY_ELEMENTS", + parameters = Map("tableName" -> "``", "colName" -> "`arr`")) + } + + test("AT_RUNTIME does not relax map value nullability") { + val errs = new mutable.ArrayBuffer[String]() + checkError( + exception = intercept[AnalysisException]( + DataTypeUtils.canWrite("", MapType(StringType, LongType, valueContainsNull = true), + MapType(StringType, LongType, valueContainsNull = false), byName = true, + analysis.caseSensitiveResolution, "m", storeAssignmentPolicy, + errMsg => errs += errMsg, + ansiStoreAssignmentCastCheck = AnsiStoreAssignmentCastCheck.AT_RUNTIME)), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_MAP_VALUES", + parameters = Map("tableName" -> "``", "colName" -> "`m`")) + } + + test("AT_RUNTIME does not relax struct field nullability") { + val writeType = StructType(Seq(StructField("x", LongType, nullable = true))) + val readType = StructType(Seq(StructField("x", LongType, nullable = false))) + val errs = new mutable.ArrayBuffer[String]() + checkError( + exception = intercept[AnalysisException]( + DataTypeUtils.canWrite("", writeType, readType, byName = true, + analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, + errMsg => errs += errMsg, + ansiStoreAssignmentCastCheck = AnsiStoreAssignmentCastCheck.AT_RUNTIME)), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_COLUMN", + parameters = Map("tableName" -> "``", "colName" -> "`t`.`x`")) + } + + test("AT_RUNTIME allows complex-to-string casts") { + Seq( + ArrayType(IntegerType), + MapType(StringType, IntegerType), + StructType(Seq(StructField("a", IntegerType)))).foreach { writeType => + assert( + DataTypeUtils.canWrite("", writeType, StringType, byName = true, + analysis.caseSensitiveResolution, "c", storeAssignmentPolicy, + _ => (), + ansiStoreAssignmentCastCheck = AnsiStoreAssignmentCastCheck.AT_RUNTIME), + s"$writeType -> string should be allowed under AT_RUNTIME") + } + } + + test("AT_RUNTIME does not relax UDT field-name validation") { + // A UDT backed by STRUCT written by name into STRUCT must be rejected on the + // field-name mismatch; deferring the ANSI cast check must not unwrap the UDT and silently swap + // fields positionally. + val udtType = new UserDefinedType[Any] { + override def sqlType: DataType = StructType(Seq( + StructField("b", IntegerType, nullable = false), + StructField("a", IntegerType, nullable = false))) + override def userClass: java.lang.Class[Any] = null + override def serialize(obj: Any): Any = null + override def deserialize(datum: Any): Any = null + } + val readType = StructType(Seq( + StructField("a", IntegerType, nullable = false), + StructField("b", IntegerType, nullable = false))) + val errs = new mutable.ArrayBuffer[String]() + checkError( + exception = intercept[AnalysisException]( + DataTypeUtils.canWrite("", udtType, readType, byName = true, + analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, + errMsg => errs += errMsg, + ansiStoreAssignmentCastCheck = AnsiStoreAssignmentCastCheck.AT_RUNTIME)), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.UNEXPECTED_COLUMN_NAME", + parameters = Map( + "tableName" -> "``", + "colName" -> "`t`", + "order" -> "0", + "expected" -> "`a`", + "found" -> "`b`")) + } + private val stringPoint2 = StructType(Seq( StructField("x", StringType, nullable = false), StructField("y", StringType, nullable = false))) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala index f18b0b05063d..6aeb62ff8955 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.connector.catalog -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Try} import org.apache.spark.SparkConf import org.apache.spark.SparkThrowable @@ -26,11 +26,13 @@ import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig +import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} /** - * A catalog that creates [[InMemoryRowLevelOperationTable]]s carrying a fixed + * A catalog that creates [[InMemoryRowLevelOperationTable]]s carrying the fixed * [[SchemaAlignmentConfig]] supplied by the concrete subclass. */ abstract class SchemaAlignmentTestCatalog extends InMemoryRowLevelOperationTableCatalog { @@ -52,11 +54,11 @@ abstract class SchemaAlignmentTestCatalog extends InMemoryRowLevelOperationTable } } -/** A catalog whose tables opt into every [[SchemaAlignmentConfig]] relaxation. */ +/** A catalog whose tables defer the ANSI store-assignment cast check to runtime. */ class RelaxedSchemaAlignmentCatalog extends SchemaAlignmentTestCatalog { override protected def tableConfig: SchemaAlignmentConfig = new SchemaAlignmentConfig { - override def allowLegacyStoreAssignmentPolicy(): Boolean = true - override def deferAnsiCastValidationToRuntime(): Boolean = true + override def ansiStoreAssignmentCastCheck(): AnsiStoreAssignmentCastCheck = + AnsiStoreAssignmentCastCheck.AT_RUNTIME } } @@ -82,47 +84,11 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { .set(s"spark.sql.catalog.$relaxed", classOf[RelaxedSchemaAlignmentCatalog].getName) .set(s"spark.sql.catalog.$strict", classOf[StrictSchemaAlignmentCatalog].getName) - private def withLegacyPolicy(f: => Unit): Unit = - withSQLConf( - SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString)(f) - private def withAnsiPolicy(f: => Unit): Unit = withSQLConf( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.ANSI.toString)(f) - private def legacyRejected(f: => Unit): Unit = - checkError( - exception = intercept[AnalysisException](f), - condition = "_LEGACY_ERROR_TEMP_1000", - parameters = Map("configKey" -> SQLConf.STORE_ASSIGNMENT_POLICY.key)) - - test("allowLegacyStoreAssignmentPolicy: INSERT under LEGACY policy") { - withTable(s"$relaxed.t", s"$strict.t") { - sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") - sql(s"CREATE TABLE $strict.t (id INT) USING foo") - withLegacyPolicy { - sql(s"INSERT INTO $relaxed.t VALUES (1)") - checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1)) - legacyRejected(sql(s"INSERT INTO $strict.t VALUES (1)")) - } - } - } - - test("allowLegacyStoreAssignmentPolicy: UPDATE under LEGACY policy") { - withTable(s"$relaxed.t", s"$strict.t") { - sql(s"CREATE TABLE $relaxed.t (id INT, data STRING) USING foo") - sql(s"CREATE TABLE $strict.t (id INT, data STRING) USING foo") - sql(s"INSERT INTO $relaxed.t VALUES (1, 'a')") - sql(s"INSERT INTO $strict.t VALUES (1, 'a')") - withLegacyPolicy { - sql(s"UPDATE $relaxed.t SET data = 'b' WHERE id = 1") - checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1, "b")) - legacyRejected(sql(s"UPDATE $strict.t SET data = 'b' WHERE id = 1")) - } - } - } - - test("deferAnsiCastValidationToRuntime: INSERT of an ANSI-incompatible cast") { + test("AT_RUNTIME: INSERT of an ANSI-incompatible cast") { withTable(s"$relaxed.t", s"$strict.t") { sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") sql(s"CREATE TABLE $strict.t (id INT) USING foo") @@ -143,7 +109,7 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } - test("deferAnsiCastValidationToRuntime: UPDATE with an ANSI-incompatible cast") { + test("AT_RUNTIME: UPDATE with an ANSI-incompatible cast") { withTable(s"$relaxed.t", s"$strict.t") { sql(s"CREATE TABLE $relaxed.t (id INT, data INT) USING foo") sql(s"CREATE TABLE $strict.t (id INT, data INT) USING foo") @@ -166,26 +132,7 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } - test("allowLegacyStoreAssignmentPolicy: MERGE under LEGACY policy") { - withTable(s"$relaxed.t", s"$strict.t") { - sql(s"CREATE TABLE $relaxed.t (id INT, data STRING) USING foo") - sql(s"CREATE TABLE $strict.t (id INT, data STRING) USING foo") - sql(s"INSERT INTO $relaxed.t VALUES (1, 'a')") - sql(s"INSERT INTO $strict.t VALUES (1, 'a')") - def merge(target: String): String = - s"""MERGE INTO $target t - |USING (SELECT 1 AS id, 'b' AS data) s - |ON t.id = s.id - |WHEN MATCHED THEN UPDATE SET t.data = s.data""".stripMargin - withLegacyPolicy { - sql(merge(s"$relaxed.t")) - checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row(1, "b")) - legacyRejected(sql(merge(s"$strict.t"))) - } - } - } - - test("deferAnsiCastValidationToRuntime: MERGE with an ANSI-incompatible cast") { + test("AT_RUNTIME: MERGE with an ANSI-incompatible cast") { withTable(s"$relaxed.t", s"$strict.t") { sql(s"CREATE TABLE $relaxed.t (id INT, data INT) USING foo") sql(s"CREATE TABLE $strict.t (id INT, data INT) USING foo") @@ -211,7 +158,7 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } - test("deferAnsiCastValidationToRuntime: structurally impossible casts are still rejected") { + test("AT_RUNTIME: structurally impossible casts are still rejected") { withTable(s"$relaxed.t") { sql(s"CREATE TABLE $relaxed.t (d DATE) USING foo") withAnsiPolicy { @@ -224,6 +171,26 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { } } + test("AT_RUNTIME: complex-to-string cast is deferred") { + withTable(s"$relaxed.t", s"$strict.t") { + sql(s"CREATE TABLE $relaxed.t (c STRING) USING foo") + sql(s"CREATE TABLE $strict.t (c STRING) USING foo") + withAnsiPolicy { + sql(s"INSERT INTO $relaxed.t VALUES (array(1, 2))") + checkAnswer(sql(s"SELECT * FROM $relaxed.t"), Row("[1, 2]")) + val strictInsert = s"INSERT INTO $strict.t VALUES (array(1, 2))" + checkError( + exception = intercept[AnalysisException](sql(strictInsert)), + condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST", + parameters = Map( + "tableName" -> s"`$strict`.`t`", + "colName" -> "`c`", + "srcType" -> "\"ARRAY\"", + "targetType" -> "\"STRING\"")) + } + } + } + test("schema alignment config survives ALTER TABLE ADD COLUMNS") { withTable(s"$relaxed.t", s"$strict.t") { sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") @@ -249,8 +216,6 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { withTable(s"$relaxed.t", s"$strict.t") { sql(s"CREATE TABLE $relaxed.t (id INT) USING foo") sql(s"CREATE TABLE $strict.t (id INT) USING foo") - // A schema-evolving write reconstructs the table (adds the `data` column) via the catalog's - // alterTable; the config must survive that reconstruction. val evolving = spark.createDataFrame( java.util.Arrays.asList(Row(1, 5)), new StructType().add("id", IntegerType).add("data", IntegerType)) @@ -285,44 +250,21 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { result } - /** - * Append `source` by name into a fresh table with `targetSchema` on both the relaxed and the - * strict catalog, and assert both reject it at analysis with the same error condition. The point - * is that [[SchemaAlignmentConfig.deferAnsiCastValidationToRuntime]] relaxes only the atomic ANSI - * store-assignment cast; it must not relax structural checks, so both catalogs behave the same. - */ private def assertBothReject( - targetSchema: StructType, source: DataFrame, condition: String): Unit = + targetSchema: StructType, source: DataFrame, errorClass: String): Unit = withAnsiPolicy { Seq(relaxed, strict).foreach { catalog => appendByName(catalog, targetSchema, source) match { case Failure(error: SparkThrowable) => - assert(error.getCondition == condition, - s"$catalog: expected $condition, got ${error.getCondition}") + assert(error.getCondition == errorClass, + s"$catalog: expected $errorClass, got ${error.getCondition}") case other => - fail(s"$catalog: expected rejection with $condition, got $other") + fail(s"$catalog: expected rejection with $errorClass, got $other") } } } - /** - * Append `source` by name into a fresh table with `targetSchema` on both the relaxed and the - * strict catalog, and assert both accept it and store exactly `source`'s rows (round trip). - */ - private def assertBothStore(targetSchema: StructType, source: DataFrame): Unit = - withAnsiPolicy { - val expected = source.collect().toSeq.map(_.toString).sorted - Seq(relaxed, strict).foreach { catalog => - appendByName(catalog, targetSchema, source) match { - case Success(rows) => - assert(rows.map(_.toString).sorted == expected, s"$catalog: got $rows") - case other => - fail(s"$catalog: expected success storing $expected, got $other") - } - } - } - - test("deferAnsiCastValidationToRuntime: renamed nested struct field is still rejected") { + test("AT_RUNTIME: renamed nested struct field is still rejected") { val target = new StructType() .add("s", new StructType().add("a", IntegerType).add("b", IntegerType)) val source = spark.createDataFrame( @@ -331,29 +273,4 @@ class SchemaAlignmentConfigSuite extends QueryTest with SharedSparkSession { assertBothReject(target, source, "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA") } - test("deferAnsiCastValidationToRuntime: nullable array element into non-null element type") { - val target = new StructType().add("a", ArrayType(IntegerType, containsNull = false)) - val source = spark.createDataFrame( - java.util.Arrays.asList(Row(Seq(1, 2))), - new StructType().add("a", ArrayType(IntegerType, containsNull = true))) - assertBothStore(target, source) - } - - test("deferAnsiCastValidationToRuntime: nullable map value into non-null value type") { - val target = new StructType() - .add("m", MapType(StringType, IntegerType, valueContainsNull = false)) - val source = spark.createDataFrame( - java.util.Arrays.asList(Row(Map("k" -> 1))), - new StructType().add("m", MapType(StringType, IntegerType, valueContainsNull = true))) - assertBothStore(target, source) - } - - test("deferAnsiCastValidationToRuntime: nullable child into non-null struct field") { - val target = new StructType() - .add("s", new StructType().add("a", IntegerType, nullable = false)) - val source = spark.createDataFrame( - java.util.Arrays.asList(Row(Row(1))), - new StructType().add("s", new StructType().add("a", IntegerType, nullable = true))) - assertBothStore(target, source) - } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala index a8c7e3b559aa..bcc4895616bd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlignAssignmentsSuiteBase.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, Column, ColumnDefaultValue, Identifier, SchemaAlignmentConfig, SupportsRowLevelOperations, TableCapability, TableCatalog, TableContext, TableWritePrivilege} +import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, Column, ColumnDefaultValue, Identifier, SupportsRowLevelOperations, TableCapability, TableCatalog, TableContext, TableWritePrivilege} import org.apache.spark.sql.connector.expressions.{LiteralValue, Transform} import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.datasources.v2.V2SessionCatalog @@ -48,7 +48,6 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -60,7 +59,6 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("i", "INT") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -75,7 +73,6 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -87,7 +84,6 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -100,7 +96,6 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -126,7 +121,6 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { nullable = false) when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -138,7 +132,6 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { .add("txt", "STRING") when(t.columns()).thenReturn(CatalogV2Util.structTypeToV2Columns(schema)) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) when(t.capabilities()).thenReturn(Collections.singleton(TableCapability.ACCEPT_ANY_SCHEMA)) t } @@ -150,7 +143,6 @@ abstract class AlignAssignmentsSuiteBase extends AnalysisTest { Column.create("b", BooleanType, true, null, null), Column.create("i", IntegerType, true, null, iDefault, null))) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala index 09f2abb00cf8..f960507b300e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{AlterColumns, AlterColumnSpe import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.TypeUtils.toSQLId import org.apache.spark.sql.connector.FakeV2Provider -import org.apache.spark.sql.connector.catalog.{CatalogManager, Column, ColumnDefaultValue, Identifier, SchemaAlignmentConfig, SupportsDelete, Table, TableCapability, TableCatalog, TableChange, TableContext, TableWritePrivilege, V1Table} +import org.apache.spark.sql.connector.catalog.{CatalogManager, Column, ColumnDefaultValue, Identifier, SupportsDelete, Table, TableCapability, TableCatalog, TableChange, TableContext, TableWritePrivilege, V1Table} import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME import org.apache.spark.sql.connector.expressions.{LiteralValue, Transform} import org.apache.spark.sql.errors.QueryExecutionErrors @@ -62,7 +62,6 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { when(t.columns()).thenReturn( Array(Column.create("i", IntegerType), Column.create("s", StringType))) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -71,7 +70,6 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { when(t.columns()).thenReturn( Array(Column.create("s", StringType), Column.create("i", IntegerType))) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -80,7 +78,6 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { when(t.columns()).thenReturn( Array(Column.create("i", IntegerType), Column.create("x", StringType, false))) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) when(t.name()).thenReturn("tab2") t } @@ -98,7 +95,6 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { when(t.columns()).thenReturn( Array(Column.create("c1", CharType(5)), Column.create("c2", VarcharType(5)))) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -110,7 +106,6 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { Column.create("i", BooleanType, true, null, default1, null), Column.create("s", IntegerType, true, null, default2, null))) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -122,7 +117,6 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { Column.create("i", StringType), Column.create("e", StringType, true, null, default, null))) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } @@ -132,7 +126,6 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { Column.create("s", StringType), Column.create("default", StringType))) when(t.partitioning()).thenReturn(Array.empty[Transform]) - when(t.schemaAlignmentConfig()).thenReturn(SchemaAlignmentConfig.DEFAULT) t } From 766c6309148ed37891d1ca696db5659302032f5d Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Mon, 21 Sep 2026 08:52:53 +0000 Subject: [PATCH 4/6] Formatting --- .../catalyst/analysis/AssignmentUtils.scala | 22 +++---- .../ResolveRowLevelCommandAssignments.scala | 7 +- .../analysis/TableOutputResolver.scala | 64 ++++++++----------- 3 files changed, 39 insertions(+), 54 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala index 5884430b6f8a..fa45dc80cc30 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala @@ -82,7 +82,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { colPath = Seq(attr.name), coerceNestedTypes, fromStar, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } if (errors.nonEmpty) { @@ -149,7 +149,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate( "", value, actualAttr, conf, err => errors += err, colPath, coerceMode, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } Assignment(attr, resolvedValue) } @@ -201,24 +201,22 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { case _: StructType => // Expand assignments to leaf fields (fixNullExpansion is applied inside) applyNestedFieldAssignments(col, colExpr, value, addError, colPath, - coerceNestedTypes, ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + coerceNestedTypes, ansiStoreAssignmentCastCheck) case _ => // For non-struct types, resolve directly val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate("", value, col, conf, addError, colPath, - coerceMode, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + coerceMode, ansiStoreAssignmentCastCheck) } } else { val value = exactAssignments.head.value val coerceMode = if (coerceNestedTypes) RECURSE else NONE TableOutputResolver.resolveUpdate("", value, col, conf, addError, - colPath, coerceMode, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + colPath, coerceMode, ansiStoreAssignmentCastCheck) } } else { applyFieldAssignments(col, colExpr, fieldAssignments, addError, colPath, coerceNestedTypes, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } } @@ -292,14 +290,12 @@ object AssignmentUtils extends SQLConfHelper with CastSupport { // Field is a struct, recurse applyNestedFieldAssignments(fieldAttr, targetFieldExpr, sourceFieldValue, addError, fieldPath, coerceNestedTypes, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) case _ => // Field is not a struct, resolve with TableOutputResolver val coerceMode = if (coerceNestedTypes) RECURSE else NONE - TableOutputResolver.resolveUpdate( - "", sourceFieldValue, fieldAttr, conf, addError, - fieldPath, coerceMode, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + TableOutputResolver.resolveUpdate("", sourceFieldValue, fieldAttr, conf, addError, + fieldPath, coerceMode, ansiStoreAssignmentCastCheck) } } val namedStruct = toNamedStruct(structType, updatedFieldExprs) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala index 191e59285116..78277200c307 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveRowLevelCommandAssignments.scala @@ -94,7 +94,7 @@ object ResolveRowLevelCommandAssignments extends Rule[LogicalPlan] { } private def validateStoreAssignmentPolicy(): Unit = { - // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2. + // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2 if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY) { throw QueryCompilationErrors.legacyStoreAssignmentPolicyError() } @@ -146,13 +146,12 @@ object ResolveRowLevelCommandAssignments extends Rule[LogicalPlan] { actions.map { case u @ UpdateAction(_, assignments, fromStar) => u.copy(assignments = AssignmentUtils.alignUpdateAssignments(attrs, assignments, - fromStar, coerceNestedTypes, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck)) + fromStar, coerceNestedTypes, ansiStoreAssignmentCastCheck)) case d: DeleteAction => d case i @ InsertAction(_, assignments) => i.copy(assignments = AssignmentUtils.alignInsertAssignments(attrs, assignments, - coerceNestedTypes, ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck)) + coerceNestedTypes, ansiStoreAssignmentCastCheck)) case other => throw new AnalysisException( errorClass = "_LEGACY_ERROR_TEMP_3052", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala index fd174ffaa381..b565cd350c5b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala @@ -157,7 +157,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { Nil, defaultValueFillMode, enforceFullOutput = true, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } else { if (expected.size > query.output.size && !fillDefaultValue) { throw QueryCompilationErrors.cannotWriteNotEnoughColumnsToTableError( @@ -198,7 +198,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { case (valueType, colType) if DataType.equalsIgnoreCompatibleNullability(valueType, colType) => val canWriteExpr = canWrite( tableName, valueType, colType, byName = true, conf, addError, colPath, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) if (canWriteExpr) { val nullsHandled = checkNullability(value, col, conf, colPath) applyColumnMetadata(nullsHandled, col) @@ -207,21 +207,18 @@ object TableOutputResolver extends SQLConfHelper with Logging { } case (valueType: StructType, colType: StructType) => val resolvedValue = resolveStructType( - tableName, value, valueType, col, colType, - byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, + fillChildDefaultValue, enforceFullOutput = false, ansiStoreAssignmentCastCheck) resolvedValue.getOrElse(value) case (valueType: ArrayType, colType: ArrayType) => val resolvedValue = resolveArrayType( - tableName, value, valueType, col, colType, - byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, + fillChildDefaultValue, enforceFullOutput = false, ansiStoreAssignmentCastCheck) resolvedValue.getOrElse(value) case (valueType: MapType, colType: MapType) => val resolvedValue = resolveMapType( - tableName, value, valueType, col, colType, - byName = true, conf, addError, colPath, fillChildDefaultValue, enforceFullOutput = false, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + tableName, value, valueType, col, colType, byName = true, conf, addError, colPath, + fillChildDefaultValue, enforceFullOutput = false, ansiStoreAssignmentCastCheck) resolvedValue.getOrElse(value) case _ => checkUpdate( @@ -247,8 +244,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { val canWriteValue = canWrite( tableName, value.dataType, attrTypeWithoutCharVarchar, - byName = true, conf, addError, colPath, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + byName = true, conf, addError, colPath, ansiStoreAssignmentCastCheck) if (canWriteValue) { val nullCheckedValue = checkNullability(value, attr, conf, colPath) @@ -377,8 +373,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { case StoreAssignmentPolicy.STRICT | StoreAssignmentPolicy.ANSI => DataTypeUtils.canWrite( tableName, valueType, expectedType, byName, conf.resolver, colPath.quoted, - conf.storeAssignmentPolicy, addError, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + conf.storeAssignmentPolicy, addError, ansiStoreAssignmentCastCheck) case _ => true } @@ -445,21 +440,21 @@ object TableOutputResolver extends SQLConfHelper with Logging { resolveStructType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) case (matchedType: ArrayType, expectedType: ArrayType) => resolveArrayType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) case (matchedType: MapType, expectedType: MapType) => resolveMapType( tableName, matchedCol, matchedType, actualExpectedCol, expectedType, byName = true, conf, addError, newColPath, childFillDefaultValue, enforceFullOutput, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) case _ => checkField( tableName, actualExpectedCol, matchedCol, byName = true, conf, addError, newColPath, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } } } @@ -556,20 +551,20 @@ object TableOutputResolver extends SQLConfHelper with Logging { resolveStructType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) case (inputType: ArrayType, expectedType: ArrayType) => resolveArrayType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) case (inputType: MapType, expectedType: MapType) => resolveMapType( tableName, inputCol, inputType, expectedCol, expectedType, byName = false, conf, addError, newColPath, fillDefaultValue, enforceFullOutput = true, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) case _ => checkField(tableName, expectedCol, inputCol, byName = false, conf, addError, newColPath, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } } @@ -663,12 +658,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { // Generated columns cannot be nested, so the auto-filled set is always empty here. val (resolved, _) = if (byName) { reorderColumnsByName(tableName, fields, toAttributes(expectedType), conf, addError, colPath, - defaultValueMode, enforceFullOutput, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + defaultValueMode, enforceFullOutput, ansiStoreAssignmentCastCheck) } else { resolveColumnsByPosition( tableName, fields, toAttributes(expectedType), conf, addError, colPath, fillDefaultValue, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } if (resolved.length == expectedType.length) { val struct = CreateStruct(resolved) @@ -709,12 +703,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { val (res, _) = if (byName) { val defaultValueMode = if (fillDefaultValue) RECURSE else NONE reorderColumnsByName(tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, - defaultValueMode, enforceFullOutput, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + defaultValueMode, enforceFullOutput, ansiStoreAssignmentCastCheck) } else { resolveColumnsByPosition( tableName, Seq(param), Seq(fakeAttr), conf, addError, colPath, fillDefaultValue, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } if (res.length == 1) { val castedArray = @@ -755,12 +748,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { // Generated columns cannot be nested, so the auto-filled set is always empty here. val (resKey, _) = if (byName) { reorderColumnsByName(tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, - defaultValueFillMode, enforceFullOutput, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + defaultValueFillMode, enforceFullOutput, ansiStoreAssignmentCastCheck) } else { resolveColumnsByPosition( tableName, Seq(keyParam), Seq(fakeKeyAttr), conf, addError, colPath, fillDefaultValue, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } val valueParam = @@ -770,12 +762,11 @@ object TableOutputResolver extends SQLConfHelper with Logging { // Generated columns cannot be nested, so the auto-filled set is always empty here. val (resValue, _) = if (byName) { reorderColumnsByName(tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, - defaultValueFillMode, enforceFullOutput, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + defaultValueFillMode, enforceFullOutput, ansiStoreAssignmentCastCheck) } else { resolveColumnsByPosition( tableName, Seq(valueParam), Seq(fakeValueAttr), conf, addError, colPath, fillDefaultValue, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + ansiStoreAssignmentCastCheck) } if (resKey.length == 1 && resValue.length == 1) { @@ -887,8 +878,7 @@ object TableOutputResolver extends SQLConfHelper with Logging { val canWriteExpr = canWrite( tableName, queryExpr.dataType, attrTypeWithoutCharVarchar, - byName, conf, addError, colPath, - ansiStoreAssignmentCastCheck = ansiStoreAssignmentCastCheck) + byName, conf, addError, colPath, ansiStoreAssignmentCastCheck) if (canWriteExpr) { val prepared = From 310371c931d53b46e596f33198770ac7123a96e3 Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Mon, 21 Sep 2026 10:05:40 +0000 Subject: [PATCH 5/6] Fix imports in Analyzer.scala --- .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 04b7d11e72d6..c22201c51a70 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -63,9 +63,7 @@ import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, ExtractV2Table} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.{PartitionOverwriteMode, StoreAssignmentPolicy} -import org.apache.spark.sql.internal.connector.ConfigurableSchemaAlignment -import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck -import org.apache.spark.sql.internal.connector.V1Function +import org.apache.spark.sql.internal.connector.{ConfigurableSchemaAlignment, SchemaAlignmentConfig, V1Function} import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util.ArrayImplicits._ From 2ff5f563fb76c2f40f0a34bc4f47d65c1f9015fc Mon Sep 17 00:00:00 2001 From: Johan Lasperas Date: Mon, 21 Sep 2026 13:34:38 +0000 Subject: [PATCH 6/6] Fix import in SchemaAlignmentConfigSuite.scala --- .../sql/connector/catalog/SchemaAlignmentConfigSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala index 6aeb62ff8955..4c18d7ed531e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/catalog/SchemaAlignmentConfigSuite.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig import org.apache.spark.sql.internal.connector.SchemaAlignmentConfig.AnsiStoreAssignmentCastCheck import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} +import org.apache.spark.sql.types.{IntegerType, StructType} /** * A catalog that creates [[InMemoryRowLevelOperationTable]]s carrying the fixed