diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index f21fb0198c..957c6d5eb9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -102,6 +102,9 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private final boolean intermingleTables; + @Nonnull + private final Map auxiliaryTypes; + private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, @Nonnull final Set invokedRoutines, @@ -109,7 +112,8 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, int version, boolean enableLongRows, boolean storeRowVersions, - boolean intermingleTables) { + boolean intermingleTables, + @Nonnull final Map auxiliaryTypes) { this.name = name; this.tables = ImmutableSet.copyOf(tables); this.invokedRoutines = ImmutableSet.copyOf(invokedRoutines); @@ -118,6 +122,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; this.intermingleTables = intermingleTables; + this.auxiliaryTypes = ImmutableMap.copyOf(auxiliaryTypes); this.metaDataSupplier = Suppliers.memoize(this::buildRecordMetadata); this.tableIndexMappingSupplier = Suppliers.memoize(this::computeTableIndexMapping); this.indexesSupplier = Suppliers.memoize(this::computeIndexes); @@ -133,7 +138,8 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, boolean enableLongRows, boolean storeRowVersions, boolean intermingleTables, - @Nonnull final RecordMetaData cachedMetadata) { + @Nonnull final RecordMetaData cachedMetadata, + @Nonnull final Map auxiliaryTypes) { this.name = name; this.version = version; this.tables = ImmutableSet.copyOf(tables); @@ -142,6 +148,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; this.intermingleTables = intermingleTables; + this.auxiliaryTypes = ImmutableMap.copyOf(auxiliaryTypes); this.metaDataSupplier = Suppliers.memoize(() -> cachedMetadata); this.tableIndexMappingSupplier = Suppliers.memoize(this::computeTableIndexMapping); this.indexesSupplier = Suppliers.memoize(this::computeIndexes); @@ -625,10 +632,10 @@ public RecordLayerSchemaTemplate build() { if (cachedMetadata != null) { return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), - new LinkedHashSet<>(invokedRoutines.values()), new LinkedHashSet<>(views.values()), version, enableLongRows, storeRowVersions, intermingleTables, cachedMetadata); + new LinkedHashSet<>(invokedRoutines.values()), new LinkedHashSet<>(views.values()), version, enableLongRows, storeRowVersions, intermingleTables, cachedMetadata, auxiliaryTypes); } else { return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), - new LinkedHashSet<>(invokedRoutines.values()), new LinkedHashSet<>(views.values()), version, enableLongRows, storeRowVersions, intermingleTables); + new LinkedHashSet<>(invokedRoutines.values()), new LinkedHashSet<>(views.values()), version, enableLongRows, storeRowVersions, intermingleTables, auxiliaryTypes); } } @@ -756,6 +763,7 @@ public Builder toBuilder() { .setIntermingleTables(intermingleTables) .addTables(getTables()) .addInvokedRoutines(getInvokedRoutines()) - .addViews(getViews()); + .addViews(getViews()) + .addAuxiliaryTypes(auxiliaryTypes.values()); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/StructTypeValidator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/StructTypeValidator.java new file mode 100644 index 0000000000..da154d26df --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/StructTypeValidator.java @@ -0,0 +1,139 @@ +/* + * StructTypeValidator.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.metadata; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.api.metadata.DataType; +import com.apple.foundationdb.relational.util.Assert; + +import javax.annotation.Nonnull; +import java.util.Locale; + +/** + * Utility class for validating struct type compatibility. + * Provides centralized logic for comparing struct types, with support for + * ignoring nullability differences and recursive validation of nested structs. + */ +@API(API.Status.EXPERIMENTAL) +public final class StructTypeValidator { + + private StructTypeValidator() { + // Utility class - prevent instantiation + } + + /** + * Check if two struct types are compatible, ignoring nullability differences. + * Two struct types are considered compatible if: + * - They have the same number of fields + * - Each corresponding field has the same type code (ignoring nullability) + * - If recursive=true, nested struct fields are recursively validated + * + * @param expected The expected struct type + * @param provided The provided struct type + * @param recursive If true, recursively validate nested struct types + * @return true if the struct types are compatible, false otherwise + */ + public static boolean areStructTypesCompatible(@Nonnull DataType.StructType expected, + @Nonnull DataType.StructType provided, + boolean recursive) { + final var expectedFields = expected.getFields(); + final var providedFields = provided.getFields(); + + // Check field count + if (!Integer.valueOf(expectedFields.size()).equals(providedFields.size())) { + return false; + } + + // Check each field type + for (int i = 0; i < expectedFields.size(); i++) { + final var expectedFieldType = expectedFields.get(i).getType(); + final var providedFieldType = providedFields.get(i).getType(); + + // Compare type codes (ignoring nullability) + if (!expectedFieldType.getCode().equals(providedFieldType.getCode())) { + return false; + } + + // Recursively validate nested structs if requested + if (recursive && expectedFieldType instanceof DataType.StructType && providedFieldType instanceof DataType.StructType) { + if (!areStructTypesCompatible((DataType.StructType) expectedFieldType, + (DataType.StructType) providedFieldType, + true)) { + return false; + } + } + } + + return true; + } + + /** + * Validate that two struct types are compatible, throwing an exception if they are not. + * This is a wrapper around {@link #areStructTypesCompatible} that throws an exception + * with a detailed error message if the types are incompatible. + * + * @param expected The expected struct type + * @param provided The provided struct type + * @param structName The name of the struct being validated (for error messages) + * @param recursive If true, recursively validate nested struct types + * @throws com.apple.foundationdb.relational.api.exceptions.RelationalException if the types are incompatible + */ + public static void validateStructTypesCompatible(@Nonnull DataType.StructType expected, + @Nonnull DataType.StructType provided, + @Nonnull String structName, + boolean recursive) { + final var expectedFields = expected.getFields(); + final var providedFields = provided.getFields(); + + // Check field count + if (!Integer.valueOf(expectedFields.size()).equals(providedFields.size())) { + Assert.failUnchecked(ErrorCode.CANNOT_CONVERT_TYPE, + String.format(Locale.ROOT, + "Struct type '%s' has incompatible signatures: expected %d fields but got %d fields", + structName, expectedFields.size(), providedFields.size())); + } + + // Check each field type + for (int i = 0; i < expectedFields.size(); i++) { + final var expectedFieldType = expectedFields.get(i).getType(); + final var providedFieldType = providedFields.get(i).getType(); + + // Compare type codes (ignoring nullability) + if (!expectedFieldType.getCode().equals(providedFieldType.getCode())) { + Assert.failUnchecked(ErrorCode.CANNOT_CONVERT_TYPE, + String.format(Locale.ROOT, + "Struct type '%s' has incompatible field at position %d: expected %s but got %s", + structName, i + 1, expectedFieldType.getCode(), providedFieldType.getCode())); + } + + // Recursively validate nested structs if requested + if (recursive && expectedFieldType instanceof DataType.StructType && providedFieldType instanceof DataType.StructType) { + // StructType extends Named, so we can always get the name + final var expectedStructName = ((DataType.StructType) expectedFieldType).getName(); + validateStructTypesCompatible((DataType.StructType) expectedFieldType, + (DataType.StructType) providedFieldType, + expectedStructName, + true); + } + } + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/TypeMetadataEnricher.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/TypeMetadataEnricher.java new file mode 100644 index 0000000000..22ae882483 --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/TypeMetadataEnricher.java @@ -0,0 +1,199 @@ +/* + * TypeMetadataEnricher.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.metadata; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.relational.api.metadata.DataType; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Descriptors.Descriptor; + +import javax.annotation.Nonnull; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Utility class for enriching DataType structures with metadata from semantic analysis and protobuf descriptors. + * + *

This class handles the merging of type information from multiple sources: + *

    + *
  • Field names from the planner's Type.Record (which handles aliases, star expansion, etc.)
    • + *
  • Type structure from semantic DataTypes (which preserves struct type names like "STRUCT_1")
    • + *
  • Additional enrichment from RecordMetaData descriptors for nested types
    • + *
+ * + *

The planner's Type.Record loses struct type names during optimization (they become null), + * but semantic analysis preserves them. This utility merges both sources to create complete + * type metadata for result sets. + */ +@API(API.Status.EXPERIMENTAL) +public final class TypeMetadataEnricher { + + private TypeMetadataEnricher() { + // Utility class - prevent instantiation + } + + /** + * Enrich a StructType by replacing UUID-based nested struct names with proper names from RecordMetaData descriptors. + * + *

This method handles the case where semantic DataTypes have UUID-based names for nested structs + * (because they were created from planner Types with null names). It matches nested structs to + * descriptors by structural signature and replaces the UUID names with descriptor names. + * + * @param structType The StructType to enrich (with field names already set correctly) + * @param recordMetaData Schema metadata for enriching nested types + * @return Enriched StructType with proper nested struct names + */ + @Nonnull + public static DataType.StructType enrichNestedStructs( + @Nonnull final DataType.StructType structType, + @Nonnull final RecordMetaData recordMetaData) { + + // Build descriptor cache for enriching nested structs + final Map descriptorCache = new HashMap<>(); + for (var recordTypeEntry : recordMetaData.getRecordTypes().values()) { + cacheDescriptorAndNested(recordTypeEntry.getDescriptor(), descriptorCache); + } + final var fileDescriptor = recordMetaData.getRecordTypes().values().iterator().next() + .getDescriptor().getFile(); + for (var messageType : fileDescriptor.getMessageTypes()) { + cacheDescriptorAndNested(messageType, descriptorCache); + } + + // Enrich nested structs recursively + final List enrichedFields = structType.getFields().stream() + .map(field -> enrichField(field, descriptorCache)) + .collect(java.util.stream.Collectors.toList()); + + return DataType.StructType.from(structType.getName(), enrichedFields, structType.isNullable()); + } + + /** + * Cache a descriptor and all its nested types, keyed by their structural signature. + * + * @param descriptor The protobuf descriptor to cache + * @param cache The cache map to populate + */ + private static void cacheDescriptorAndNested(@Nonnull final Descriptor descriptor, + @Nonnull final Map cache) { + // Create a structural signature for this descriptor (field names and count) + final String signature = createStructuralSignature(descriptor); + cache.put(signature, descriptor); + + // Process nested types + for (var nestedType : descriptor.getNestedTypes()) { + cacheDescriptorAndNested(nestedType, cache); + } + } + + /** + * Create a structural signature for a descriptor based on field names only. + * Field indices can vary between DataType and protobuf representations. + * + * @param descriptor The protobuf descriptor + * @return A signature string representing the structure + */ + @Nonnull + private static String createStructuralSignature(@Nonnull final Descriptor descriptor) { + return descriptor.getFields().stream() + .map(Descriptors.FieldDescriptor::getName) + .collect(java.util.stream.Collectors.joining(",")); + } + + /** + * Create a structural signature for a DataType.StructType based on field names only. + * + * @param structType The struct type + * @return A signature string representing the structure + */ + @Nonnull + private static String createStructuralSignature(@Nonnull final DataType.StructType structType) { + return structType.getFields().stream() + .map(DataType.StructType.Field::getName) + .collect(java.util.stream.Collectors.joining(",")); + } + + /** + * Recursively enrich a struct type with proper names from the descriptor cache. + * + * @param structType The struct type to enrich + * @param descriptorCache Cache of descriptors keyed by structural signature + * @return Enriched struct type with proper names from descriptors + */ + @Nonnull + private static DataType.StructType enrichStructType(@Nonnull final DataType.StructType structType, + @Nonnull final Map descriptorCache) { + // Enrich each field recursively + final List enrichedFields = structType.getFields().stream() + .map(field -> enrichField(field, descriptorCache)) + .collect(java.util.stream.Collectors.toList()); + + // Try to find a matching descriptor for this struct type + final String signature = createStructuralSignature(structType); + final Descriptor matchedDescriptor = descriptorCache.get(signature); + + // Use the descriptor's name if found, otherwise keep the existing name + final String enrichedName = matchedDescriptor != null ? matchedDescriptor.getName() : structType.getName(); + + return DataType.StructType.from(enrichedName, enrichedFields, structType.isNullable()); + } + + /** + * Enrich a field, recursively enriching any nested struct types. + * + * @param field The field to enrich + * @param descriptorCache Cache of descriptors keyed by structural signature + * @return Enriched field with proper type metadata + */ + @Nonnull + private static DataType.StructType.Field enrichField(@Nonnull final DataType.StructType.Field field, + @Nonnull final Map descriptorCache) { + final DataType enrichedType = enrichDataType(field.getType(), descriptorCache); + return DataType.StructType.Field.from(field.getName(), enrichedType, field.getIndex()); + } + + /** + * Enrich a DataType, handling structs, arrays, and primitives. + * + *

For struct types, looks up matching descriptors and enriches the struct name. + * For array types, recursively enriches the element type. + * For primitive types, returns as-is. + * + * @param dataType The data type to enrich + * @param descriptorCache Cache of descriptors keyed by structural signature + * @return Enriched data type with proper metadata + */ + @Nonnull + private static DataType enrichDataType(@Nonnull final DataType dataType, + @Nonnull final Map descriptorCache) { + if (dataType instanceof DataType.StructType) { + return enrichStructType((DataType.StructType) dataType, descriptorCache); + } else if (dataType instanceof DataType.ArrayType) { + final DataType.ArrayType arrayType = (DataType.ArrayType) dataType; + final DataType enrichedElementType = enrichDataType(arrayType.getElementType(), descriptorCache); + return DataType.ArrayType.from(enrichedElementType, arrayType.isNullable()); + } else { + // Primitive types don't need enrichment + return dataType; + } + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java index 93c0d4604c..46aefcdd9d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.util.Assert; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; @@ -207,6 +208,43 @@ public List underlyingTypes() { return Streams.stream(underlying()).map(Value::getResultType).collect(ImmutableList.toImmutableList()); } + /** + * Returns a StructType representing the semantic types of all expressions with their field names. + * + *

This wraps the individual expression data types into a single StructType that is + * structurally equivalent to the planner's Type.Record output. The StructType includes: + *

    + *
  • Field names from the expressions (handles aliases, star expansion, etc.)
    • + *
  • Type structure from semantic analysis (preserves struct type names)
    • + *
+ * + *

This StructType can be used directly for result set metadata after enriching nested + * struct names from RecordMetaData descriptors. + * + *

Note: The StructType name is a generated UUID since this is a general-purpose method. + * Contexts that need a specific name (e.g., "QUERY_RESULT" for top-level queries) should + * wrap or recreate the StructType with an appropriate name. + * + * @return A StructType with field names and semantic types from expressions + */ + @Nonnull + public DataType.StructType getStructType() { + final ImmutableList.Builder fieldsBuilder = ImmutableList.builder(); + int index = 0; + for (final Expression expression : underlying) { + // Use expression name if available, otherwise generate a name + final String fieldName = expression.getName() + .map(Identifier::toString) + .orElse("_" + index); + fieldsBuilder.add(DataType.StructType.Field.from(fieldName, expression.getDataType(), index)); + index++; + } + // Use UUID-based name since this is a general-purpose method + // Top-level contexts (like query results) will override with "QUERY_RESULT" + final String generatedName = "id" + java.util.UUID.randomUUID().toString().replace("-", "_"); + return DataType.StructType.from(generatedName, fieldsBuilder.build(), true); + } + @Nonnull public Stream stream() { return underlying.stream(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java index 232f7c786c..7bad379ce4 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java @@ -42,9 +42,11 @@ import com.apple.foundationdb.relational.api.WithMetadata; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; +import com.apple.foundationdb.relational.recordlayer.metadata.StructTypeValidator; import com.apple.foundationdb.relational.util.Assert; import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings; +import com.apple.foundationdb.relational.api.metadata.DataType; import com.google.common.collect.ImmutableList; import com.google.protobuf.ZeroCopyByteString; @@ -55,8 +57,11 @@ import java.sql.SQLException; import java.sql.Struct; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -98,6 +103,9 @@ public class MutablePlanGenerationContext implements QueryExecutionContext { @Nonnull private final ImmutableList.Builder equalityConstraints; + @Nonnull + private final Map dynamicStructDefinitions; + private void startStructLiteral() { literalsBuilder.startStructLiteral(); } @@ -282,6 +290,7 @@ public MutablePlanGenerationContext(@Nonnull PreparedParams preparedParams, forExplain = false; setContinuation(null); equalityConstraints = ImmutableList.builder(); + dynamicStructDefinitions = new HashMap<>(); } @Nonnull @@ -493,4 +502,28 @@ private static Type getObjectType(@Nullable final Object object) { } return Type.fromObject(object); } + + /** + * Registers or validates a dynamic struct definition created within the query. + * If this is the first time seeing this struct name, registers it. + * If the struct name was already registered, validates that the new definition matches the previous one. + * + * @param structName The name of the struct type + * @param structType The struct type definition + * @throws com.apple.foundationdb.relational.api.exceptions.RelationalException if a struct with this name + * already exists with an incompatible signature + */ + public void registerOrValidateDynamicStruct(@Nonnull String structName, @Nonnull DataType.StructType structType) { + final var normalizedName = structName.toUpperCase(Locale.ROOT); + final var existing = dynamicStructDefinitions.get(normalizedName); + + if (existing == null) { + // First time seeing this struct name, register it + dynamicStructDefinitions.put(normalizedName, structType); + } else { + // Struct name already exists, validate compatibility using centralized validator + // This now correctly ignores nullability and recursively validates nested structs + StructTypeValidator.validateStructTypesCompatible(existing, structType, structName, true); + } + } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index d5adc8953c..27826dc347 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -34,6 +34,7 @@ import com.apple.foundationdb.record.query.plan.cascades.SemanticException; import com.apple.foundationdb.record.query.plan.cascades.StableSelectorCostModel; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import com.apple.foundationdb.record.util.pair.NonnullPair; @@ -41,10 +42,12 @@ import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.api.exceptions.UncheckedRelationalException; +import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.api.metrics.RelationalMetric; import com.apple.foundationdb.relational.continuation.CompiledStatement; import com.apple.foundationdb.relational.continuation.TypedQueryArgument; import com.apple.foundationdb.relational.recordlayer.ContinuationImpl; +import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.query.cache.PhysicalPlanEquivalence; import com.apple.foundationdb.relational.recordlayer.query.cache.RelationalPlanCache; @@ -53,6 +56,7 @@ import com.apple.foundationdb.relational.util.Assert; import com.apple.foundationdb.relational.util.RelationalLoggingUtil; import com.google.common.base.VerifyException; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.protobuf.InvalidProtocolBufferException; import org.apache.logging.log4j.LogManager; @@ -61,6 +65,7 @@ import javax.annotation.Nonnull; import java.sql.SQLException; import java.util.Arrays; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -323,12 +328,30 @@ private QueryPlan.PhysicalQueryPlan generatePhysicalPlanForExecuteContinuation(@ planGenerationContext.setContinuation(continuationProto); final var continuationPlanConstraint = QueryPlanConstraint.fromProto(serializationContext, compiledStatement.getPlanConstraint()); + + final Type resultType = recordQueryPlan.getResultType().getInnerType(); + final DataType.StructType semanticStructType; + if (resultType instanceof Type.Record) { + final Type.Record recordType = (Type.Record) resultType; + final List fields = recordType.getFields().stream() + .map(field -> DataType.StructType.Field.from( + field.getFieldName(), + DataTypeUtils.toRelationalType(field.getFieldType()), + field.getFieldIndex())) + .collect(java.util.stream.Collectors.toList()); + semanticStructType = DataType.StructType.from("QUERY_RESULT", fields, true); + } else { + // Fallback for non-record types (shouldn't happen for SELECT results) + semanticStructType = DataType.StructType.from("QUERY_RESULT", ImmutableList.of(), true); + } + return new QueryPlan.ContinuedPhysicalQueryPlan(recordQueryPlan, typeRepository, continuationPlanConstraint, planGenerationContext, "EXECUTE CONTINUATION " + ast.getQueryCacheKey().getCanonicalQueryString(), currentPlanHashMode, - serializedPlanHashMode); + serializedPlanHashMode, + semanticStructType); } private void resetTimer() { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java index dd81fb418c..ddce3a9e5a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java @@ -73,7 +73,7 @@ import com.apple.foundationdb.relational.recordlayer.RecordLayerResultSet; import com.apple.foundationdb.relational.recordlayer.RecordLayerSchema; import com.apple.foundationdb.relational.recordlayer.ResumableIterator; -import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; +import com.apple.foundationdb.relational.recordlayer.metadata.TypeMetadataEnricher; import com.apple.foundationdb.relational.recordlayer.util.ExceptionUtil; import com.apple.foundationdb.relational.util.Assert; import com.google.common.base.Suppliers; @@ -128,6 +128,13 @@ public static class PhysicalQueryPlan extends QueryPlan { @Nonnull private final QueryExecutionContext queryExecutionContext; + /** + * Semantic type structure captured during semantic analysis. + * Complete StructType with field names and nested struct type names preserved. + */ + @Nonnull + private final DataType.StructType semanticStructType; + public PhysicalQueryPlan(@Nonnull final RecordQueryPlan recordQueryPlan, @Nullable final StatsMaps plannerStatsMaps, @Nonnull final TypeRepository typeRepository, @@ -135,7 +142,8 @@ public PhysicalQueryPlan(@Nonnull final RecordQueryPlan recordQueryPlan, @Nonnull final QueryPlanConstraint continuationConstraint, @Nonnull final QueryExecutionContext queryExecutionContext, @Nonnull final String query, - @Nonnull final PlanHashMode currentPlanHashMode) { + @Nonnull final PlanHashMode currentPlanHashMode, + @Nonnull final DataType.StructType semanticStructType) { super(query); this.recordQueryPlan = recordQueryPlan; this.plannerStatsMaps = plannerStatsMaps; @@ -145,6 +153,7 @@ public PhysicalQueryPlan(@Nonnull final RecordQueryPlan recordQueryPlan, this.queryExecutionContext = queryExecutionContext; this.currentPlanHashMode = currentPlanHashMode; this.planHashSupplier = Suppliers.memoize(() -> recordQueryPlan.planHash(currentPlanHashMode)); + this.semanticStructType = semanticStructType; } @Nonnull @@ -192,7 +201,8 @@ public PhysicalQueryPlan withExecutionContext(@Nonnull final QueryExecutionConte return this; } return new PhysicalQueryPlan(recordQueryPlan, plannerStatsMaps, typeRepository, constraint, - continuationConstraint, queryExecutionContext, query, queryExecutionContext.getPlanHashMode()); + continuationConstraint, queryExecutionContext, query, queryExecutionContext.getPlanHashMode(), + semanticStructType); } @Nonnull @@ -404,10 +414,13 @@ private RelationalResultSet executePhysicalPlan(@Nonnull final RecordLayerSchema parsedContinuation.getExecutionState(), executeProperties)); final var currentPlanHashMode = OptionsUtils.getCurrentPlanHashMode(options); - final var dataType = (DataType.StructType) DataTypeUtils.toRelationalType(type); + + // Enrich nested structs with proper names from RecordMetaData descriptors + final DataType.StructType resultDataType = TypeMetadataEnricher.enrichNestedStructs(semanticStructType, fdbRecordStore.getRecordMetaData()); + return executionContext.metricCollector.clock(RelationalMetric.RelationalEvent.CREATE_RESULT_SET_ITERATOR, () -> { final ResumableIterator iterator = RecordLayerIterator.create(cursor, messageFDBQueriedRecord -> new MessageTuple(messageFDBQueriedRecord.getMessage())); - return new RecordLayerResultSet(RelationalStructMetaData.of(dataType), iterator, connection, + return new RecordLayerResultSet(RelationalStructMetaData.of(resultDataType), iterator, connection, (continuation, reason) -> enrichContinuation(continuation, currentPlanHashMode, reason)); }); @@ -476,9 +489,10 @@ public ContinuedPhysicalQueryPlan(@Nonnull final RecordQueryPlan recordQueryPlan @Nonnull final QueryExecutionContext queryExecutionParameters, @Nonnull final String query, @Nonnull final PlanHashMode currentPlanHashMode, - @Nonnull final PlanHashMode serializedPlanHashMode) { + @Nonnull final PlanHashMode serializedPlanHashMode, + @Nonnull final DataType.StructType semanticStructType) { super(recordQueryPlan, null, typeRepository, QueryPlanConstraint.noConstraint(), - continuationConstraint, queryExecutionParameters, query, currentPlanHashMode); + continuationConstraint, queryExecutionParameters, query, currentPlanHashMode, semanticStructType); this.serializedPlanHashMode = serializedPlanHashMode; this.serializedPlanHashSupplier = Suppliers.memoize(() -> recordQueryPlan.planHash(serializedPlanHashMode)); } @@ -488,15 +502,29 @@ public PlanHashMode getSerializedPlanHashMode() { return serializedPlanHashMode; } - @SuppressWarnings("PMD.CompareObjectsWithEquals") + /** + * Returns a plan with updated execution context. + * + *

Note: This method is never called in production because ContinuedPhysicalQueryPlan instances + * are not cached - each EXECUTE CONTINUATION deserializes the plan fresh from the continuation blob. + * However, we must override to satisfy the Plan interface contract. + * + *

TODO: Refactor the class hierarchy to eliminate this dead code. Potential approaches: + *

    + *
  • Collapse ContinuedPhysicalQueryPlan into PhysicalQueryPlan with a flag
    • + *
  • Use composition instead of inheritance
    • + *
  • Make Plan.withExecutionContext() optional with default implementation
    • + *
+ * + * @param queryExecutionContext The new execution context (ignored - never called) + * @return This instance (since continuation plans are never cached) + */ @Override @Nonnull public PhysicalQueryPlan withExecutionContext(@Nonnull final QueryExecutionContext queryExecutionContext) { - if (queryExecutionContext == this.getQueryExecutionContext()) { - return this; - } - return new ContinuedPhysicalQueryPlan(getRecordQueryPlan(), getTypeRepository(), getContinuationConstraint(), - queryExecutionContext, query, queryExecutionContext.getPlanHashMode(), getSerializedPlanHashMode()); + // This method is never called in production - continuation plans bypass the cache. + // Return this to avoid maintaining dead code. + return this; } @Override @@ -549,18 +577,27 @@ public static class LogicalQueryPlan extends QueryPlan { @Nonnull private final String query; + /** + * Semantic type structure captured during semantic analysis. + * Preserves struct type names - will be merged with planner field names after planning. + */ + @Nonnull + private final DataType.StructType semanticStructType; + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") @Nonnull private Optional optimizedPlan; private LogicalQueryPlan(@Nonnull final RelationalExpression relationalExpression, @Nonnull final MutablePlanGenerationContext context, - @Nonnull final String query) { + @Nonnull final String query, + @Nonnull final DataType.StructType semanticStructType) { super(query); this.relationalExpression = relationalExpression; this.context = context; this.optimizedPlan = Optional.empty(); this.query = query; + this.semanticStructType = semanticStructType; } @Override @@ -609,7 +646,8 @@ public PhysicalQueryPlan optimize(@Nonnull CascadesPlanner planner, @Nonnull Pla optimizedPlan = Optional.of( new PhysicalQueryPlan(minimizedPlan, statsMaps, builder.build(), - constraint, continuationConstraint, context, query, currentPlanHashMode)); + constraint, continuationConstraint, context, query, currentPlanHashMode, + semanticStructType)); return optimizedPlan.get(); }); } @@ -657,8 +695,9 @@ public MutablePlanGenerationContext getGenerationContext() { @Nonnull public static LogicalQueryPlan of(@Nonnull final RelationalExpression relationalExpression, @Nonnull final MutablePlanGenerationContext context, - @Nonnull final String query) { - return new LogicalQueryPlan(relationalExpression, context, query); + @Nonnull final String query, + @Nonnull final DataType.StructType semanticStructType) { + return new LogicalQueryPlan(relationalExpression, context, query, semanticStructType); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 8178bd893a..8bacd0f49f 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -41,6 +41,7 @@ import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerColumn; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; +import com.apple.foundationdb.relational.recordlayer.metadata.StructTypeValidator; import com.apple.foundationdb.relational.recordlayer.query.Expression; import com.apple.foundationdb.relational.recordlayer.query.Expressions; import com.apple.foundationdb.relational.recordlayer.query.Identifier; @@ -799,6 +800,90 @@ public Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstru final var expressions = parseRecordFieldsUnderReorderings(ctx.expressionWithOptionalName()); if (ctx.ofTypeClause() != null) { final var recordId = visitUid(ctx.ofTypeClause().uid()); + + // When creating a named struct inline (e.g., STRUCT STRUCT_1(...)), validate that the + // constructed type is compatible with any schema-defined type with the same name. + // Look up the type by searching through all tables and their fields + final var schemaTemplate = getDelegate().getSchemaTemplate(); + Optional existingStructTypeMaybe = Optional.empty(); + + // Search through all tables to find a field with this struct type + for (final var table : schemaTemplate.getTables()) { + // Table datatype is always a StructType + final var structType = table.getDatatype(); + for (final var field : structType.getFields()) { + final var fieldType = field.getType(); + if (fieldType instanceof DataType.Named) { + final var namedType = (DataType.Named) fieldType; + if (namedType.getName().equalsIgnoreCase(recordId.getName())) { + // Found the type! Now resolve it to get the actual struct definition + if (namedType instanceof DataType.StructType) { + existingStructTypeMaybe = Optional.of((DataType.StructType) namedType); + break; + } + } + } + } + if (existingStructTypeMaybe.isPresent()) { + break; + } + } + + if (existingStructTypeMaybe.isPresent()) { + final var existingStructType = existingStructTypeMaybe.get(); + final var existingFields = existingStructType.getFields(); + final var providedExpressions = expressions.asList(); + + // Validate field count matches + if (!Integer.valueOf(existingFields.size()).equals(providedExpressions.size())) { + Assert.failUnchecked(ErrorCode.CANNOT_CONVERT_TYPE, + String.format("Cannot create struct '%s': expected %d fields but got %d", + recordId.getName(), existingFields.size(), providedExpressions.size())); + } + + // Validate each field type is compatible + for (int i = 0; i < existingFields.size(); i++) { + final var expectedField = existingFields.get(i); + final var providedExpression = providedExpressions.get(i); + final var expectedDataType = expectedField.getType(); + final var providedDataType = providedExpression.getDataType(); + + // Check if types are compatible + // For now, we check that the type codes match, ignoring nullability differences + // since a non-null value can always be used where a nullable field is expected + if (!expectedDataType.getCode().equals(providedDataType.getCode())) { + Assert.failUnchecked(ErrorCode.CANNOT_CONVERT_TYPE, + String.format("Cannot create struct '%s': field %d has incompatible type (expected %s but got %s)", + recordId.getName(), i + 1, expectedDataType.getCode(), providedDataType.getCode())); + } + // For struct types, we need to do deeper validation + if (expectedDataType instanceof DataType.StructType && providedDataType instanceof DataType.StructType) { + final var expectedStructType = (DataType.StructType) expectedDataType; + final var providedStructType = (DataType.StructType) providedDataType; + // Recursively validate struct fields using centralized validator + if (!StructTypeValidator.areStructTypesCompatible(expectedStructType, providedStructType, true)) { + Assert.failUnchecked(ErrorCode.CANNOT_CONVERT_TYPE, + String.format("Cannot create struct '%s': field %d has incompatible struct type", + recordId.getName(), i + 1)); + } + } + } + } + + // Also validate against dynamic struct definitions within this query + // Build a StructType from the provided expressions to validate consistency + final var providedFields = new ArrayList(); + final var providedExpressionsList = expressions.asList(); + for (int i = 0; i < providedExpressionsList.size(); i++) { + final var expression = providedExpressionsList.get(i); + final var fieldName = expression.getName().map(n -> n.getName()).orElse("_" + i); + providedFields.add(DataType.StructType.Field.from(fieldName, expression.getDataType(), i)); + } + final var providedStructType = DataType.StructType.from(recordId.getName(), providedFields, false); + + // Register or validate this dynamic struct definition + getDelegate().getPlanGenerationContext().registerOrValidateDynamicStruct(recordId.getName(), providedStructType); + final var resultValue = RecordConstructorValue.ofColumnsAndName(expressions.underlyingAsColumns(), recordId.getName()); return Expression.ofUnnamed(resultValue); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java index 901d606abd..205b13e7d2 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java @@ -83,14 +83,20 @@ public static QueryVisitor of(@Nonnull BaseVisitor baseVisitor) { @Override public QueryPlan.LogicalQueryPlan visitSelectStatement(@Nonnull RelationalParser.SelectStatementContext ctx) { final var logicalOperator = parseChild(ctx); - return QueryPlan.LogicalQueryPlan.of(logicalOperator.getQuantifier().getRangesOver().get(), getDelegate().getPlanGenerationContext(), "TODO"); + // Capture semantic type structure as StructType with field names + final var semanticStructType = logicalOperator.getOutput().getStructType(); + return QueryPlan.LogicalQueryPlan.of(logicalOperator.getQuantifier().getRangesOver().get(), + getDelegate().getPlanGenerationContext(), getDelegate().getPlanGenerationContext().getQuery(), semanticStructType); } @Nonnull @Override public QueryPlan.LogicalQueryPlan visitDmlStatement(@Nonnull RelationalParser.DmlStatementContext ctx) { final var logicalOperator = parseChild(ctx); - return QueryPlan.LogicalQueryPlan.of(logicalOperator.getQuantifier().getRangesOver().get(), getDelegate().getPlanGenerationContext(), "TODO"); + // Capture semantic type structure as StructType with field names + final var semanticStructType = logicalOperator.getOutput().getStructType(); + return QueryPlan.LogicalQueryPlan.of(logicalOperator.getQuantifier().getRangesOver().get(), + getDelegate().getPlanGenerationContext(), getDelegate().getPlanGenerationContext().getQuery(), semanticStructType); } @Nonnull @@ -548,7 +554,10 @@ public Object visitExecuteContinuationStatement(@Nonnull RelationalParser.Execut public QueryPlan.LogicalQueryPlan visitFullDescribeStatement(@Nonnull RelationalParser.FullDescribeStatementContext ctx) { getDelegate().getPlanGenerationContext().setForExplain(ctx.EXPLAIN() != null); final var logicalOperator = Assert.castUnchecked(ctx.describeObjectClause().accept(this), LogicalOperator.class); - return QueryPlan.LogicalQueryPlan.of(logicalOperator.getQuantifier().getRangesOver().get(), getDelegate().getPlanGenerationContext(), "TODO"); + // Capture semantic type structure as StructType with field names + final var semanticStructType = logicalOperator.getOutput().getStructType(); + return QueryPlan.LogicalQueryPlan.of(logicalOperator.getQuantifier().getRangesOver().get(), + getDelegate().getPlanGenerationContext(), getDelegate().getPlanGenerationContext().getQuery(), semanticStructType); } @Nonnull diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/StructDataMetadataTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/StructDataMetadataTest.java index 01f74d76ff..39a3b32d55 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/StructDataMetadataTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/StructDataMetadataTest.java @@ -20,21 +20,28 @@ package com.apple.foundationdb.relational.recordlayer; +import com.apple.foundationdb.relational.api.Continuation; import com.apple.foundationdb.relational.api.EmbeddedRelationalArray; import com.apple.foundationdb.relational.api.EmbeddedRelationalStruct; import com.apple.foundationdb.relational.api.KeySet; import com.apple.foundationdb.relational.api.Options; +import com.apple.foundationdb.relational.api.RelationalArray; import com.apple.foundationdb.relational.api.RelationalResultSet; import com.apple.foundationdb.relational.api.RelationalStruct; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.utils.RelationalAssertions; import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.function.ThrowingConsumer; import java.nio.charset.StandardCharsets; import java.sql.Array; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Set; @@ -48,7 +55,7 @@ public class StructDataMetadataTest { public static final EmbeddedRelationalExtension relationalExtension = new EmbeddedRelationalExtension(); private static final String TABLE_STRUCTURE = - "CREATE TYPE AS STRUCT struct_1 (a string) " + + "CREATE TYPE AS STRUCT struct_1 (a string, b bigint) " + " CREATE TABLE t (name string, st1 struct_1, PRIMARY KEY(name))" + " CREATE TYPE AS STRUCT struct_2 (c bigint, d struct_1) " + " CREATE TABLE nt (t_name string, st1 struct_2, PRIMARY KEY(t_name))" + @@ -79,7 +86,18 @@ public class StructDataMetadataTest { void setUp() throws SQLException { var m = EmbeddedRelationalStruct.newBuilder() .addString("NAME", "test_record_1") - .addStruct("ST1", EmbeddedRelationalStruct.newBuilder().addString("A", "Hello").build()) + .addStruct("ST1", EmbeddedRelationalStruct.newBuilder() + .addString("A", "Hello") + .addLong("B", 100L) + .build()) + .build(); + statement.executeInsert("T", m); + m = EmbeddedRelationalStruct.newBuilder() + .addString("NAME", "test_record_2") + .addStruct("ST1", EmbeddedRelationalStruct.newBuilder() + .addString("A", "World") + .addLong("B", 200L) + .build()) .build(); statement.executeInsert("T", m); @@ -94,7 +112,18 @@ void setUp() throws SQLException { .build(); statement.executeInsert("NT", m); - final var atBuilder = EmbeddedRelationalStruct.newBuilder(); + m = EmbeddedRelationalStruct.newBuilder() + .addString("T_NAME", "nt_record2") + .addStruct("ST1", EmbeddedRelationalStruct.newBuilder() + .addLong("C", 5678L) + .addStruct("D", EmbeddedRelationalStruct.newBuilder() + .addString("A", "Ciao") + .build()) + .build()) + .build(); + statement.executeInsert("NT", m); + + var atBuilder = EmbeddedRelationalStruct.newBuilder(); m = atBuilder.addString("A_NAME", "a_test_rec") .addArray("ST2", EmbeddedRelationalArray.newBuilder() .addStruct(EmbeddedRelationalStruct.newBuilder() @@ -108,6 +137,21 @@ void setUp() throws SQLException { .build()) .build(); statement.executeInsert("AT", m); + + atBuilder = EmbeddedRelationalStruct.newBuilder(); + m = atBuilder.addString("A_NAME", "another_test_rec") + .addArray("ST2", EmbeddedRelationalArray.newBuilder() + .addStruct(EmbeddedRelationalStruct.newBuilder() + .addBytes("C", "今日は".getBytes(StandardCharsets.UTF_8)) + .addBoolean("D", true) + .build()) + .addStruct(EmbeddedRelationalStruct.newBuilder() + .addBytes("C", "مرحبًا".getBytes(StandardCharsets.UTF_8)) + .addBoolean("D", false) + .build()) + .build()) + .build(); + statement.executeInsert("AT", m); } @Test @@ -119,41 +163,212 @@ void canReadSingleStruct() throws Exception { Assertions.assertNotNull(struct, "No struct found for column!"); Assertions.assertEquals("Hello", struct.getString(1), "Incorrect value for nested struct!"); Assertions.assertEquals("Hello", struct.getString("A"), "Incorrect value for nested struct!"); + Assertions.assertEquals(100L, struct.getLong(2), "Incorrect value for nested struct B field!"); + Assertions.assertEquals(100L, struct.getLong("B"), "Incorrect value for nested struct B field!"); //check that the JDBC attributes methods work properly - Assertions.assertArrayEquals(struct.getAttributes(), new Object[]{"Hello"}, "Incorrect attributes!"); + Assertions.assertArrayEquals(new Object[]{"Hello", 100L}, struct.getAttributes(), "Incorrect attributes!"); + } + } + + /** + * Helper method to test struct type metadata preservation across query execution and continuations. + * + * @param query The SQL query to execute + * @param assertOnMetaData Consumer to assert on the result set metadata + * @param numBaseQueryRuns Number of times to run the base query (tests PhysicalQueryPlan.withExecutionContext when > 1) + * @param numContinuationRuns Number of times to run the continuation (tests ContinuedPhysicalQueryPlan.withExecutionContext when > 1) + */ + private void canReadStructTypeName(String query, + ThrowingConsumer assertOnMetaData, + int numBaseQueryRuns, + int numContinuationRuns) throws Throwable { + // Only set maxRows if we're testing continuations + if (numContinuationRuns > 0) { + statement.setMaxRows(1); + } + + Continuation continuation = null; + + // Run base query the specified number of times + for (int i = 0; i < numBaseQueryRuns; i++) { + try (final RelationalResultSet resultSet = statement.executeQuery(query)) { + Assertions.assertTrue(resultSet.next(), "Did not find a record on base query run " + (i + 1)); + assertOnMetaData.accept(resultSet); + if (i == 0 && numContinuationRuns > 0) { + continuation = resultSet.getContinuation(); + } + } + } + + // Run continuation the specified number of times + for (int i = 0; i < numContinuationRuns; i++) { + try (final PreparedStatement ps = connection.prepareStatement("EXECUTE CONTINUATION ?")) { + ps.setBytes(1, continuation.serialize()); + try (final ResultSet resultSet = ps.executeQuery()) { + Assertions.assertTrue(resultSet.next(), "Did not find a record on continuation run " + (i + 1)); + assertOnMetaData.accept(resultSet.unwrap(RelationalResultSet.class)); + } + } } } + private void canReadStructTypeName(String query, ThrowingConsumer assertOnMetaData) throws Throwable { + canReadStructTypeName(query, assertOnMetaData, 1, 1); + } + @Test - void canReadProjectedStructTypeNameInNestedStar() throws Exception { - try (final RelationalResultSet resultSet = statement.executeQuery("SELECT (*) FROM T")) { - Assertions.assertTrue(resultSet.next(), "Did not find a record!"); + void canReadProjectedStructTypeNameInNestedStar() throws Throwable { + canReadStructTypeName("SELECT (*) FROM T", resultSet -> { RelationalStruct struct = resultSet.getStruct(1).getStruct("ST1"); Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName()); - } + }); } - // When projecting *, the underlying struct types are lost and replaced with a generic UUID type. - // This test should be replaced with the correct expected behavior once this is fixed. - // When projecting (*), everything works as expected, see `canReadProjectedStructTypeNameInNestedStar`. - // See https://github.com/FoundationDB/fdb-record-layer/issues/3743 @Test - void cannotReadProjectedStructTypeNameInUnnestedStar() throws Exception { - try (final RelationalResultSet resultSet = statement.executeQuery("SELECT * FROM T")) { - Assertions.assertTrue(resultSet.next(), "Did not find a record!"); + void canReadProjectedNestedStructTypeNameInNestedStar() throws Throwable { + canReadStructTypeName("SELECT (*) FROM NT", resultSet -> { + RelationalStruct struct = resultSet.getStruct(1).getStruct("ST1").getStruct("D"); + Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName()); + }); + } + + @Test + void canReadProjectedStructInArrayTypeNameInNestedStar() throws Throwable { + canReadStructTypeName("SELECT (*) FROM AT", resultSet -> { + RelationalArray array = resultSet.getStruct(1).getArray("ST2"); + Assertions.assertEquals("STRUCT", array.getMetaData().getElementTypeName()); + Assertions.assertEquals("STRUCT_3", array.getMetaData().getElementStructMetaData().getTypeName()); + }); + } + + @Test + void canReadProjectedStructTypeNameInUnnestedStar() throws Throwable { + canReadStructTypeName("SELECT * FROM T", resultSet -> { RelationalStruct struct = resultSet.getStruct("ST1"); - Assertions.assertNotEquals("STRUCT_1", struct.getMetaData().getTypeName()); - } + Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName()); + }); } @Test - void canReadProjectedStructTypeNameDirectlyProjected() throws Exception { - try (final RelationalResultSet resultSet = statement.executeQuery("SELECT ST1 FROM T")) { - Assertions.assertTrue(resultSet.next(), "Did not find a record!"); + void canReadProjectedNestedStructTypeNameInUnnestedStar() throws Throwable { + canReadStructTypeName("SELECT * FROM NT", resultSet -> { + RelationalStruct struct = resultSet.getStruct("ST1").getStruct("D"); + Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName()); + }); + } + + @Test + void canReadProjectedStructInArrayTypeNameInUnnestedStar() throws Throwable { + canReadStructTypeName("SELECT * FROM AT", resultSet -> { + RelationalArray array = resultSet.getArray("ST2"); + Assertions.assertEquals("STRUCT", array.getMetaData().getElementTypeName()); + Assertions.assertEquals("STRUCT_3", array.getMetaData().getElementStructMetaData().getTypeName()); + }); + } + + @Test + void canReadProjectedStructTypeNameDirectlyProjected() throws Throwable { + canReadStructTypeName("SELECT ST1 FROM T", resultSet -> { RelationalStruct struct = resultSet.getStruct("ST1"); Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName()); - } + }); + } + + @Test + void canReadProjectedNestedStructTypeNameDirectlyProjected() throws Throwable { + canReadStructTypeName("SELECT ST1 FROM NT", resultSet -> { + RelationalStruct struct = resultSet.getStruct("ST1").getStruct("D"); + Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName()); + }); + } + + @Test + void canReadProjectedStructInArrayTypeNameDirectlyProjected() throws Throwable { + canReadStructTypeName("SELECT * FROM AT", resultSet -> { + RelationalArray array = resultSet.getArray("ST2"); + Assertions.assertEquals("STRUCT", array.getMetaData().getElementTypeName()); + Assertions.assertEquals("STRUCT_3", array.getMetaData().getElementStructMetaData().getTypeName()); + }); + } + + @Test + void canReadProjectedDynamicStruct() throws Throwable { + canReadStructTypeName("SELECT STRUCT STRUCT_6(name, st1.a, st1) FROM T", resultSet -> { + RelationalStruct struct = resultSet.getStruct(1); + Assertions.assertEquals("STRUCT_6", struct.getMetaData().getTypeName()); + Assertions.assertEquals("STRUCT_1", struct.getStruct(3).getMetaData().getTypeName()); + }); + } + + @Test + void canReadProjectedStructWithDynamicStructInside() throws Throwable { + canReadStructTypeName("SELECT STRUCT STRUCT_6(name, STRUCT STRUCT_7(name, st1.a)) FROM T", resultSet -> { + RelationalStruct struct = resultSet.getStruct(1); + Assertions.assertEquals("STRUCT_6", struct.getMetaData().getTypeName()); + Assertions.assertEquals("STRUCT_7", struct.getStruct(2).getMetaData().getTypeName()); + }); + } + + @Test + void canReadAnonymousStructWithDynamicStructInside() throws Throwable { + canReadStructTypeName("SELECT (name, STRUCT STRUCT_7(name, st1.a)) FROM T", resultSet -> { + RelationalStruct struct = resultSet.getStruct(1); + Assertions.assertEquals("STRUCT_7", struct.getStruct(2).getMetaData().getTypeName()); + }); + } + + @Test + void canNameDynamicStructSameAsStaticStructIfSame() throws Throwable { + canReadStructTypeName("SELECT STRUCT STRUCT_1(name, st1.b) FROM T", resultSet -> { + RelationalStruct struct = resultSet.getStruct(1); + Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName()); + }); + } + + @Test + void canNameDynamicStructSameAsStaticStructIfSame2() throws Throwable { + canReadStructTypeName("SELECT STRUCT STRUCT_2(10l, STRUCT STRUCT_1(name, st1.b)) FROM T", resultSet -> { + RelationalStruct struct = resultSet.getStruct(1); + Assertions.assertEquals("STRUCT_2", struct.getMetaData().getTypeName()); + Assertions.assertEquals("STRUCT_1", struct.getStruct(2).getMetaData().getTypeName()); + }); + } + + @Test + void cannotNameDynamicStructSameAsStaticStructIfSignatureIsDifferent() throws Throwable { + RelationalAssertions.assertThrowsSqlException(() -> statement.executeQuery("SELECT STRUCT STRUCT_1(st1.b) FROM T")) + .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); + } + + @Test + void cannotHaveTwoDynamicStructsWithDifferentTypes() throws Throwable { + RelationalAssertions.assertThrowsSqlException(() -> statement.executeQuery("SELECT STRUCT STRUCT_19(st1.b), STRUCT STRUCT_19(name) FROM T")) + .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); + } + + @Test + void cannotHaveTwoDynamicStructsWithDifferentNestedTypes() throws Throwable { + RelationalAssertions.assertThrowsSqlException(() -> statement.executeQuery("SELECT STRUCT STRUCT_19(STRUCT STRUCT_20(st1.b)), STRUCT STRUCT_19(STRUCT STRUCT_20(name)) FROM T")) + .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); + } + + @Test + void cannotHaveTwoDynamicStructsOneWithSameAsTemplateOneDifferent() throws Throwable { + RelationalAssertions.assertThrowsSqlException(() -> statement.executeQuery("SELECT STRUCT STRUCT_1(name, st1.b), STRUCT STRUCT_1(name) FROM T")) + .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); + } + + @Test + void canUseDifferentDynamicStructsWithSameNameInTwoDifferentQueries() throws Throwable { + canReadStructTypeName("SELECT STRUCT STRUCT_18(name, st1.b) FROM T", resultSet -> { + RelationalStruct struct = resultSet.getStruct(1); + Assertions.assertEquals("STRUCT_18", struct.getMetaData().getTypeName()); + }); + canReadStructTypeName("SELECT STRUCT STRUCT_18(st1.b) FROM T", resultSet -> { + RelationalStruct struct = resultSet.getStruct(1); + Assertions.assertEquals("STRUCT_18", struct.getMetaData().getTypeName()); + }); } @Test @@ -255,4 +470,51 @@ void canReadRepeatedStructWithArray() throws SQLException { } } } + + @Test + void structTypeMetadataPreservedAcrossPlanCache() throws Throwable { + canReadStructTypeName("SELECT * FROM T WHERE NAME = 'test_record_1'", resultSet -> { + RelationalStruct struct = resultSet.getStruct("ST1"); + Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName(), + "Struct type name should be preserved across plan cache"); + }, 2, 0); + } + + @Test + void nestedStructTypeMetadataPreservedAcrossPlanCache() throws Throwable { + canReadStructTypeName("SELECT * FROM NT WHERE T_NAME = 'nt_record'", resultSet -> { + RelationalStruct struct = resultSet.getStruct("ST1"); + RelationalStruct nestedStruct = struct.getStruct("D"); + Assertions.assertEquals("STRUCT_1", nestedStruct.getMetaData().getTypeName(), + "Nested struct type name should be preserved across plan cache"); + }, 2, 0); + } + + @Test + void arrayStructTypeMetadataPreservedAcrossPlanCache() throws Throwable { + canReadStructTypeName("SELECT * FROM AT WHERE A_NAME = 'a_test_rec'", resultSet -> { + RelationalArray array = resultSet.getArray("ST2"); + Assertions.assertEquals("STRUCT_3", array.getMetaData().getElementStructMetaData().getTypeName(), + "Array element struct type name should be preserved across plan cache"); + }, 2, 0); + } + + @Test + void structTypeMetadataPreservedInContinuationAcrossPlanCache() throws Throwable { + canReadStructTypeName("SELECT * FROM T", resultSet -> { + RelationalStruct struct = resultSet.getStruct("ST1"); + Assertions.assertEquals("STRUCT_1", struct.getMetaData().getTypeName(), + "Struct type name should be preserved in continuation across plan cache"); + }, 1, 2); + } + + @Test + void nestedStructTypeMetadataPreservedInContinuationAcrossPlanCache() throws Throwable { + canReadStructTypeName("SELECT * FROM NT", resultSet -> { + RelationalStruct struct = resultSet.getStruct("ST1"); + RelationalStruct nestedStruct = struct.getStruct("D"); + Assertions.assertEquals("STRUCT_1", nestedStruct.getMetaData().getTypeName(), + "Nested struct type name should be preserved in continuation across plan cache"); + }, 1, 2); + } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlanTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlanTest.java new file mode 100644 index 0000000000..10678081e4 --- /dev/null +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlanTest.java @@ -0,0 +1,109 @@ +/* + * QueryPlanTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query; + +import com.apple.foundationdb.record.PlanHashable.PlanHashMode; +import com.apple.foundationdb.record.query.plan.QueryPlanConstraint; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; +import com.apple.foundationdb.relational.api.metadata.DataType; +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Unit tests for QueryPlan classes. + */ +class QueryPlanTest { + + /** + * Test that ContinuedPhysicalQueryPlan.withExecutionContext() returns the same instance. + * + *

This test exists primarily to satisfy code coverage requirements. In production, + * ContinuedPhysicalQueryPlan.withExecutionContext() is never called because continuation + * plans bypass the plan cache - each EXECUTE CONTINUATION statement deserializes the plan + * fresh from the continuation blob. + * + *

TODO: This test can be removed if/when the class hierarchy is refactored to eliminate + * the need for this dead code method (e.g., by collapsing ContinuedPhysicalQueryPlan into + * PhysicalQueryPlan with a flag, or making Plan.withExecutionContext() optional). + */ + @Test + void continuedPhysicalQueryPlanWithExecutionContextReturnsSameInstance() { + // Create minimal mocks to construct a ContinuedPhysicalQueryPlan + final RecordQueryPlan mockRecordQueryPlan = Mockito.mock(RecordQueryPlan.class); + final Type.Relation mockRelation = Mockito.mock(Type.Relation.class); + final Type.Record mockRecord = Type.Record.fromFields(ImmutableList.of( + Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("test_field")))); + + Mockito.when(mockRecordQueryPlan.getResultType()).thenReturn(mockRelation); + Mockito.when(mockRelation.getInnerType()).thenReturn(mockRecord); + + final TypeRepository typeRepository = TypeRepository.newBuilder().build(); + final QueryPlanConstraint constraint = QueryPlanConstraint.noConstraint(); + final QueryExecutionContext executionContext = new MutablePlanGenerationContext( + PreparedParams.empty(), + PlanHashMode.VC0, + "SELECT 1", + "SELECT 1", + 0); + + final ImmutableList semanticFieldTypes = ImmutableList.of( + DataType.Primitives.STRING.type()); + + // Wrap semantic types in a StructType + final DataType.StructType semanticStructType = DataType.StructType.from("QUERY_RESULT", + ImmutableList.of( + DataType.StructType.Field.from("field_0", DataType.Primitives.INTEGER.type(), 0), + DataType.StructType.Field.from("field_1", DataType.Primitives.STRING.type(), 1)), + true); + + // Create a ContinuedPhysicalQueryPlan instance + final QueryPlan.ContinuedPhysicalQueryPlan continuedPlan = new QueryPlan.ContinuedPhysicalQueryPlan( + mockRecordQueryPlan, + typeRepository, + constraint, + executionContext, + "EXECUTE CONTINUATION test", + PlanHashMode.VC0, + PlanHashMode.VL0, + semanticStructType); + + // Create a different execution context + final QueryExecutionContext differentContext = new MutablePlanGenerationContext( + PreparedParams.empty(), + PlanHashMode.VC0, + "SELECT 2", + "SELECT 2", + 1); + + // Call withExecutionContext and verify it returns the same instance + final QueryPlan.PhysicalQueryPlan result = continuedPlan.withExecutionContext(differentContext); + + assertSame(continuedPlan, result, + "ContinuedPhysicalQueryPlan.withExecutionContext() should return the same instance"); + } +} diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 615bb8fb55..7321c49d7d 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -1218,8 +1218,10 @@ void testNamingStructsDifferentTypesThrows() throws Exception { try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var statement = ddl.setSchemaAndGetConnection().createStatement()) { statement.executeUpdate("insert into t1 values (42, 100, 500, 101)"); - final var message = Assertions.assertThrows(SQLException.class, () -> statement.execute("select struct asd (a, 42, struct def (b, c), struct def(b, c, a)) as X from t1")).getMessage(); - Assertions.assertTrue(message.contains("value already present: DEF")); // we could improve this error message. + RelationalAssertions.assertThrowsSqlException( + () -> statement.execute("select struct asd (a, 42, struct def (b, c), struct def(b, c, a)) as X from t1")) + .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE) + .hasMessage("Struct type 'DEF' has incompatible signatures: expected 2 fields but got 3 fields"); } } }