diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.diagram b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.diagram index 8a658f0e1e..d3d56d6e5b 100644 --- a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.diagram +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.diagram @@ -11,6 +11,12 @@ Diagram( Sequence( NonTerminal('columnName'), NonTerminal('columnType'), + Optional( + Choice(0, + Terminal('INVISIBLE'), + Terminal('VISIBLE') + ) + ) ), Terminal(',') ), diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.rst index 48bba5bd9f..c756003b3a 100644 --- a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.rst +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.rst @@ -8,6 +8,10 @@ must be declared as a ``SINGLE ROW ONLY`` table, in which case only one row can This latter type of table can be useful for things like database configuration parameters, which the application wants to inform its interpretation of all data in the database. +Columns can be marked as ``INVISIBLE`` to hide them from ``SELECT *`` queries while still allowing +explicit selection by name. This is useful for backward compatibility when adding new columns, +or for system/computed columns that should not appear in normal queries. + Syntax ====== @@ -26,6 +30,12 @@ Parameters ``columnType`` The associated :ref:`type ` of the column +``INVISIBLE`` / ``VISIBLE`` + Optional visibility modifier for a column: + + - ``INVISIBLE``: Column is excluded from ``SELECT *`` queries but can be explicitly selected + - ``VISIBLE``: Column is included in ``SELECT *`` queries (default behavior) + ``primaryKeyColumnName`` The name of the column to be part of the primary key of the ``TABLE`` @@ -109,3 +119,73 @@ Attempting to insert a second row in a :sql:`SINGLE ROW ONLY` table will result * - :sql:`NULL` - :json:`0.0` - :json:`"X"` + +Table with invisible columns +----------------------------- + +Invisible columns are excluded from ``SELECT *`` but can be explicitly selected. +This is useful for adding columns without breaking existing queries that use ``SELECT *``. + +.. code-block:: sql + + CREATE SCHEMA TEMPLATE TEMP + CREATE TABLE T ( + id BIGINT, + name STRING, + secret STRING INVISIBLE, + PRIMARY KEY(id) + ) + + -- On a schema that uses the above schema template + INSERT INTO T VALUES + (1, 'Alice', 'password123'), + (2, 'Bob', 'secret456'); + + -- SELECT * excludes invisible columns + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`id` + - :sql:`name` + * - :json:`1` + - :json:`"Alice"` + * - :json:`2` + - :json:`"Bob"` + +.. code-block:: sql + + -- Explicitly selecting invisible columns includes them + SELECT id, name, secret FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`id` + - :sql:`name` + - :sql:`secret` + * - :json:`1` + - :json:`"Alice"` + - :json:`"password123"` + * - :json:`2` + - :json:`"Bob"` + - :json:`"secret456"` + +.. code-block:: sql + + -- Invisible columns in subqueries become visible when explicitly selected + SELECT * FROM (SELECT id, name, secret FROM T) sub; + +.. list-table:: + :header-rows: 1 + + * - :sql:`id` + - :sql:`name` + - :sql:`secret` + * - :json:`1` + - :json:`"Alice"` + - :json:`"password123"` + * - :json:`2` + - :json:`"Bob"` + - :json:`"secret456"` diff --git a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto index 0395c07fac..475cb3b189 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto @@ -65,6 +65,7 @@ message FieldOptions { optional int32 dimensions = 2 [default = 768]; } optional VectorOptions vectorOptions = 4; + optional bool invisible = 5 [default = false]; } extend google.protobuf.FieldOptions { diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/Column.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/Column.java index cff6ea4f9d..ac4aa29e25 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/Column.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/Column.java @@ -34,6 +34,24 @@ public interface Column extends Metadata { */ DataType getDataType(); + /** + * Returns whether this column is invisible. + *

+ * Invisible columns are a SQL feature that hides columns from {@code SELECT *} queries + * while still allowing them to be explicitly selected by name. + *

+ * Query behavior: + *

+ * + * @return {@code true} if the column is invisible, {@code false} otherwise + */ + default boolean isInvisible() { + return false; // Default to visible for backward compatibility + } + @Override default void accept(@Nonnull final Visitor visitor) { visitor.visit(this); diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java index bdc1a27f7c..e171a58399 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java @@ -1307,16 +1307,24 @@ public static class Field { private final int index; - private Field(@Nonnull final String name, @Nonnull final DataType type, int index) { + private final boolean invisible; + + private Field(@Nonnull final String name, @Nonnull final DataType type, int index, boolean invisible) { Assert.thatUnchecked(index >= 0); this.name = name; this.type = type; this.index = index; + this.invisible = invisible; } @Nonnull public static Field from(@Nonnull final String name, @Nonnull final DataType type, int index) { - return new Field(name, type, index); + return new Field(name, type, index, false); + } + + @Nonnull + public static Field from(@Nonnull final String name, @Nonnull final DataType type, int index, boolean invisible) { + return new Field(name, type, index, invisible); } @Nonnull @@ -1333,8 +1341,12 @@ public int getIndex() { return index; } + public boolean isInvisible() { + return invisible; + } + private int computeHashCode() { - return Objects.hash(name, index, type); + return Objects.hash(name, index, type, invisible); } @Override @@ -1354,6 +1366,7 @@ public boolean equals(Object other) { final var otherField = (Field) other; return name.equals(otherField.name) && index == otherField.index && + invisible == otherField.invisible && type.equals(otherField.type); } diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 8460fdd8e4..92756aab9d 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -126,7 +126,11 @@ tableDefinition ; columnDefinition - : colName=uid columnType ARRAY? columnConstraint? + : colName=uid columnType ARRAY? columnConstraint? columnVisibility? + ; + +columnVisibility + : VISIBLE | INVISIBLE ; // this is not aligned with SQL standard, but it eliminates ambiguities related to necessating a lookahead of 1 to resolve diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java index 7799db0db3..1eaf8d218a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java @@ -73,7 +73,7 @@ public static DataType toRelationalType(@Nonnull final Type type) { switch (typeCode) { case RECORD: final var record = (Type.Record) type; - final var columns = record.getFields().stream().map(field -> DataType.StructType.Field.from(field.getFieldName(), toRelationalType(field.getFieldType()), field.getFieldIndex())).collect(Collectors.toList()); + final var columns = record.getFields().stream().map(field -> DataType.StructType.Field.from(field.getFieldName(), toRelationalType(field.getFieldType()), field.getFieldIndex(), false)).collect(Collectors.toList()); return DataType.StructType.from(record.getName() == null ? ProtoUtils.uniqueTypeName() : record.getName(), columns, record.isNullable()); case ARRAY: final var asArray = (Type.Array) type; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerColumn.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerColumn.java index f6d1d1719c..bf03f52b55 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerColumn.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerColumn.java @@ -39,10 +39,13 @@ public class RecordLayerColumn implements Column { private final int index; - RecordLayerColumn(@Nonnull String name, @Nonnull DataType dataType, int index) { + private final boolean invisible; + + RecordLayerColumn(@Nonnull String name, @Nonnull DataType dataType, int index, boolean invisible) { this.name = name; this.dataType = dataType; this.index = index; + this.invisible = invisible; } @Nonnull @@ -61,6 +64,11 @@ public int getIndex() { return index; } + @Override + public boolean isInvisible() { + return invisible; + } + @Override public boolean equals(final Object object) { if (this == object) { @@ -70,17 +78,17 @@ public boolean equals(final Object object) { return false; } final RecordLayerColumn that = (RecordLayerColumn)object; - return index == that.index && Objects.equals(name, that.name) && Objects.equals(dataType, that.dataType); + return index == that.index && invisible == that.invisible && Objects.equals(name, that.name) && Objects.equals(dataType, that.dataType); } @Override public int hashCode() { - return Objects.hash(name, dataType, index); + return Objects.hash(name, dataType, index, invisible); } @Override public String toString() { - return name + ": " + dataType + " = " + index; + return name + ": " + dataType + " = " + index + (invisible ? " (invisible)" : ""); } public static final class Builder { @@ -89,8 +97,11 @@ public static final class Builder { private int index; + private boolean invisible; + private Builder() { this.index = -1; + this.invisible = false; } @Nonnull @@ -112,14 +123,20 @@ public Builder setIndex(int index) { return this; } + @Nonnull + public Builder setInvisible(boolean invisible) { + this.invisible = invisible; + return this; + } + public RecordLayerColumn build() { - return new RecordLayerColumn(name, dataType, index); + return new RecordLayerColumn(name, dataType, index, invisible); } } @Nonnull public static RecordLayerColumn from(@Nonnull final DataType.StructType.Field field) { - return new RecordLayerColumn(field.getName(), field.getType(), field.getIndex()); + return new RecordLayerColumn(field.getName(), field.getType(), field.getIndex(), field.isInvisible()); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java index 632059339a..aa5a1cd465 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java @@ -151,7 +151,7 @@ private Type.Record calculateRecordLayerType() { private DataType.StructType calculateDataType() { final var columnTypes = ImmutableList.builder(); for (final var column : columns) { - columnTypes.add(DataType.StructType.Field.from(column.getName(), column.getDataType(), column.getIndex())); + columnTypes.add(DataType.StructType.Field.from(column.getName(), column.getDataType(), column.getIndex(), column.isInvisible())); } /* * TODO (yhatem): note this is not entirely correct. Currently we're not setting nullable @@ -416,7 +416,7 @@ private static List normalize(@Nonnull final List generations = recordLayerTable.getGenerations(); + final var recordLayerTable = (RecordLayerTable) table; + final var type = recordLayerTable.getType(); + final var dataType = recordLayerTable.getDatatype(); + final var typeDescriptor = registerTypeDescriptors(type, dataType); + final var generations = recordLayerTable.getGenerations(); checkTableGenerations(generations); @@ -115,7 +117,7 @@ public void visit(@Nonnull final Table table) { // (yhatem) this is temporary, we use rec layer typing also as a bridge to PB serialization for now. @Nonnull - private String registerTypeDescriptors(@Nonnull final Type.Record type) { + private String registerTypeDescriptors(@Nonnull final Type.Record type, @Nonnull final DataType.StructType dataType) { final var builder = TypeRepository.newBuilder(); type.defineProtoType(builder); final var typeDescriptors = builder.build(); @@ -125,7 +127,15 @@ private String registerTypeDescriptors(@Nonnull final Type.Record type) { continue; } final var descriptor = typeDescriptors.getMessageDescriptor(descriptorName); - fileBuilder.addMessageType(descriptor.toProto()); + final var descriptorProto = descriptor.toProto(); + // Add invisible flag to field options only for the top-level table descriptor + final DescriptorProtos.DescriptorProto modifiedDescriptorProto; + if (descriptorName.equals(typeDescriptor)) { + modifiedDescriptorProto = addInvisibleFieldOptions(descriptorProto, dataType); + } else { + modifiedDescriptorProto = descriptorProto; + } + fileBuilder.addMessageType(modifiedDescriptorProto); descriptorNames.add(descriptorName); } for (final var enumName : typeDescriptors.getEnumTypes()) { @@ -139,6 +149,36 @@ private String registerTypeDescriptors(@Nonnull final Type.Record type) { return typeDescriptor; } + @Nonnull + private DescriptorProtos.DescriptorProto addInvisibleFieldOptions(@Nonnull final DescriptorProtos.DescriptorProto descriptorProto, + @Nonnull final DataType.StructType dataType) { + final var builder = descriptorProto.toBuilder(); + + // Build a set of invisible field names for O(1) lookup instead of O(N) stream filter per field + final var invisibleFieldNames = dataType.getFields().stream() + .filter(DataType.StructType.Field::isInvisible) + .map(DataType.StructType.Field::getName) + .collect(Collectors.toSet()); + + // Check each field against the set - O(N) total instead of O(N²) + for (int i = 0; i < builder.getFieldCount(); i++) { + final var fieldProto = builder.getField(i); + final var fieldName = fieldProto.getName(); + + if (invisibleFieldNames.contains(fieldName)) { + final var fieldOptions = RecordMetaDataOptionsProto.FieldOptions.newBuilder() + .setInvisible(true) + .build(); + final var modifiedFieldProto = fieldProto.toBuilder() + .setOptions(fieldProto.getOptions().toBuilder() + .setExtension(RecordMetaDataOptionsProto.field, fieldOptions)) + .build(); + builder.setField(i, modifiedFieldProto); + } + } + return builder.build(); + } + @Override public void startVisit(@Nonnull SchemaTemplate schemaTemplate) { fileBuilder.setName(schemaTemplate.getName()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java index f24c446da7..270812fe11 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java @@ -40,6 +40,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; import com.google.protobuf.Descriptors; import javax.annotation.Nonnull; @@ -130,19 +131,85 @@ private RecordLayerSchemaTemplate.Builder deserializeRecordMetaData() { } @Nonnull + @SuppressWarnings("PMD.UnusedFormalParameter") // userName kept for API consistency private RecordLayerTable.Builder generateTableBuilder(@Nonnull final String userName, @Nonnull final String storageName) { final RecordType recordType = recordMetaData.getRecordType(storageName); // todo (yhatem) we rely on the record type for deserialization from ProtoBuf for now, later on // we will avoid this step by having our own deserializers. + final var baseRecordLayerType = Type.Record.fromDescriptor(recordType.getDescriptor()); + + // Process nested records if needed, otherwise use the base type with the record name + final var recordLayerType = processNestedRecords(baseRecordLayerType, recordType); + + // Build table with invisible flags applied + return buildTableWithInvisibleFlags(recordLayerType, recordType); + } + + @Nonnull + private Type.Record processNestedRecords(@Nonnull final Type.Record baseRecordLayerType, + @Nonnull final RecordType recordType) { // todo (yhatem) this is hacky and must be cleaned up. We need to understand the actually field types so we can take decisions - // on higher level based on these types (wave3). - final var recordLayerType = Type.Record.fromDescriptorPreservingName(recordType.getDescriptor()); + // on higher level based on these types (wave3). + if (!baseRecordLayerType.getFields().stream().anyMatch(f -> f.getFieldType().isRecord())) { + // No nested records - just create a named Type.Record from the base type + return Type.Record.fromFieldsWithName(recordType.getName(), false, baseRecordLayerType.getFields()); + } + + // Handle nested records by preserving their names + ImmutableList.Builder newFields = ImmutableList.builder(); + for (int i = 0; i < baseRecordLayerType.getFields().size(); i++) { + final var protoField = recordType.getDescriptor().getFields().get(i); + final var field = baseRecordLayerType.getField(i); + if (field.getFieldType().isRecord()) { + Type.Record r = Type.Record.fromFieldsWithName( + protoField.getMessageType().getName(), + field.getFieldType().isNullable(), + ((Type.Record) field.getFieldType()).getFields() + ); + newFields.add(Type.Record.Field.of(r, field.getFieldNameOptional(), field.getFieldIndexOptional())); + } else { + newFields.add(field); + } + } + return Type.Record.fromFieldsWithName(recordType.getName(), false, newFields.build()); + } + + @Nonnull + private RecordLayerTable.Builder buildTableWithInvisibleFlags(@Nonnull final Type.Record recordLayerType, + @Nonnull final RecordType recordType) { + // Convert to DataType.StructType and apply invisible flags + final var dataType = (DataType.StructType) DataTypeUtils.toRelationalType(recordLayerType); + final var dataTypeWithInvisible = applyInvisibleFlags(dataType, recordType.getDescriptor()); + + // Build table directly from dataType to preserve invisible flags in columns return RecordLayerTable.Builder - .from(recordLayerType) - .setName(userName) + .from(dataTypeWithInvisible) + .setRecord(recordLayerType) .setPrimaryKey(recordType.getPrimaryKey()) - .addIndexes(recordType.getIndexes().stream().map(index -> RecordLayerIndex.from(Objects.requireNonNull(recordLayerType.getName()), Objects.requireNonNull(recordLayerType.getStorageName()), index)).collect(Collectors.toSet())); + .addIndexes(recordType.getIndexes().stream() + .map(index -> RecordLayerIndex.from( + Objects.requireNonNull(recordLayerType.getName()), + Objects.requireNonNull(recordLayerType.getStorageName()), + index)) + .collect(Collectors.toSet())); + } + + @Nonnull + private DataType.StructType applyInvisibleFlags(@Nonnull final DataType.StructType dataType, + @Nonnull final Descriptors.Descriptor descriptor) { + final var fieldsBuilder = ImmutableList.builder(); + for (int i = 0; i < dataType.getFields().size(); i++) { + final var field = dataType.getFields().get(i); + final var protoField = descriptor.getFields().get(i); + boolean isInvisible = false; + if (protoField.getOptions().hasExtension(com.apple.foundationdb.record.RecordMetaDataOptionsProto.field)) { + final var fieldOptions = protoField.getOptions().getExtension(com.apple.foundationdb.record.RecordMetaDataOptionsProto.field); + isInvisible = fieldOptions.hasInvisible() && fieldOptions.getInvisible(); + } + fieldsBuilder.add(DataType.StructType.Field.from(field.getName(), field.getType(), field.getIndex(), isInvisible)); + } + return DataType.StructType.from(dataType.getName(), fieldsBuilder.build(), dataType.isNullable()); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java index f73df9743f..c5d2b79886 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java @@ -69,18 +69,35 @@ public class Expression { @Nonnull private final Supplier underlying; + private final boolean invisible; + public Expression(@Nonnull Optional name, @Nonnull DataType dataType, @Nonnull Value expression) { - this(name, dataType, () -> expression); + this(name, dataType, () -> expression, false); } public Expression(@Nonnull Optional name, @Nonnull DataType dataType, @Nonnull Supplier valueSupplier) { + this(name, dataType, valueSupplier, false); + } + + public Expression(@Nonnull Optional name, + @Nonnull DataType dataType, + @Nonnull Value expression, + boolean invisible) { + this(name, dataType, () -> expression, invisible); + } + + public Expression(@Nonnull Optional name, + @Nonnull DataType dataType, + @Nonnull Supplier valueSupplier, + boolean invisible) { this.name = name; this.dataType = dataType; this.underlying = Suppliers.memoize(valueSupplier::get); + this.invisible = invisible; } @Nonnull @@ -98,12 +115,40 @@ public Value getUnderlying() { return underlying.get(); } + /** + * Returns whether this expression represents an invisible column. + *

+ * Invisible columns are excluded from {@code SELECT *} queries but can be explicitly selected. + * For example, given a table with an invisible column {@code secret}: + *

    + *
  • {@code SELECT * FROM t} - excludes {@code secret}
    • + *
  • {@code SELECT secret FROM t} - includes {@code secret} (made visible via {@link Expressions#makeVisible()})
    • + *
  • {@code SELECT * FROM (SELECT secret FROM t)} - includes {@code secret} from subquery
    • + *
+ *

+ * The invisible flag is set when columns are loaded from table metadata (via {@link LogicalOperator}) + * and is cleared when columns are explicitly selected (via {@link Expressions#makeVisible()}). + * + * @return {@code true} if this expression is invisible, {@code false} otherwise + */ + public boolean isInvisible() { + return invisible; + } + + @Nonnull + public Expression withInvisible(boolean invisible) { + if (this.invisible == invisible) { + return this; + } + return new Expression(getName(), getDataType(), getUnderlying(), invisible); + } + @Nonnull public Expression withName(@Nonnull Identifier name) { if (getName().isPresent() && getName().get().equals(name)) { return this; } - return new Expression(Optional.of(name), getDataType(), getUnderlying()); + return new Expression(Optional.of(name), getDataType(), getUnderlying(), invisible); } @Nonnull @@ -111,7 +156,7 @@ public Expression withUnderlying(@Nonnull Value underlying) { if (getUnderlying().semanticEquals(underlying, AliasMap.identitiesFor(underlying.getCorrelatedTo()))) { return this; } - return new Expression(getName(), DataTypeUtils.toRelationalType(underlying.getResultType()), underlying); + return new Expression(getName(), DataTypeUtils.toRelationalType(underlying.getResultType()), underlying, invisible); } @Nonnull @@ -123,7 +168,7 @@ public Expression clearQualifier() { if (!name.isQualified()) { return this; } - return new Expression(Optional.of(name.withoutQualifier()), getDataType(), getUnderlying()); + return new Expression(Optional.of(name.withoutQualifier()), getDataType(), getUnderlying(), invisible); } @Nonnull @@ -141,7 +186,7 @@ public Expression replaceQualifier(@Nonnull Function, Collect if (newNameMaybe.equals(name)) { return this; } - return new Expression(Optional.of(newNameMaybe), getDataType(), getUnderlying()); + return new Expression(Optional.of(newNameMaybe), getDataType(), getUnderlying(), invisible); } @Nonnull @@ -157,10 +202,10 @@ public Expression withQualifier(@Nonnull final Optional qualifier) { return this; } if (qualifier.isEmpty()) { - return new Expression(Optional.of(name.withoutQualifier()), getDataType(), getUnderlying()); + return new Expression(Optional.of(name.withoutQualifier()), getDataType(), getUnderlying(), invisible); } final var newName = name.withQualifier(qualifier.get().fullyQualifiedName()); - return new Expression(Optional.of(newName), getDataType(), getUnderlying()); + return new Expression(Optional.of(newName), getDataType(), getUnderlying(), invisible); } public boolean isAggregate() { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java index 93c0d4604c..19ef5893d7 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java @@ -71,7 +71,7 @@ public Iterator iterator() { public Expressions expanded() { return Expressions.of(underlying.stream() .flatMap(item -> item instanceof Star ? - ((Star) item).getExpansion().stream() : + ((Star) item).getExpansion().nonInvisible().stream() : Stream.of(item)).collect(ImmutableList.toImmutableList())); } @@ -156,6 +156,36 @@ public Expressions nonEphemeral() { return Expressions.of(stream().filter(e -> !(e instanceof EphemeralExpression)).collect(ImmutableList.toImmutableList())); } + @Nonnull + public Expressions nonInvisible() { + return Expressions.of(stream().filter(e -> !e.isInvisible()).collect(ImmutableList.toImmutableList())); + } + + /** + * Returns a new {@code Expressions} with all expressions marked as visible. + *

+ * This is used when columns are explicitly selected in a query to ensure they are visible + * in outer queries. For example: + * 

+     * SELECT id, secret FROM t    -- Both columns explicitly selected, marked visible
+     * SELECT * FROM (...)         -- Outer SELECT * includes both columns
+     *
+ *

+ * Without this, explicitly selected invisible columns would remain invisible and be filtered + * by outer {@code SELECT *} queries, which would be incorrect semantics. + * + * @return a new {@code Expressions} with all expressions marked as visible, or {@code this} + * if no expressions are invisible (optimization to avoid unnecessary allocation) + */ + @Nonnull + public Expressions makeVisible() { + // Optimization: if no expressions are invisible, avoid creating a new list + if (stream().noneMatch(Expression::isInvisible)) { + return this; + } + return Expressions.of(stream().map(e -> e.withInvisible(false)).collect(Collectors.toUnmodifiableList())); + } + @Nonnull public Expression getSingleItem() { Assert.thatUnchecked(size() == 1, "invalid attempt to get single item"); @@ -177,6 +207,12 @@ public Expressions replaceQualifier(@Nonnull Function, Collec .collect(ImmutableList.toImmutableList())); } + @Nonnull + public Expressions withQualifier(@Nonnull final Collection qualifier) { + return Expressions.of(underlying.stream().map(expression -> expression.withQualifier(qualifier)) + .collect(ImmutableList.toImmutableList())); + } + @Nonnull public Expressions withQualifier(@Nonnull final Identifier qualifier) { return Expressions.of(underlying.stream().map(expression -> expression.withQualifier(Optional.of(qualifier))) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java index 0f596a8750..b01c7558ce 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java @@ -252,11 +252,14 @@ public static LogicalOperator generateTableAccess(@Nonnull Identifier tableId, final var fieldType = type.getField(colCount); final var attributeExpression = FieldValue.ofFields(resultingQuantifier.getFlowedObjectValue(), FieldValue.FieldPath.ofSingle(FieldValue.ResolvedAccessor.of(fieldType, colCount))); - attributesBuilder.add(new Expression(Optional.of(attributeName), attributeType, attributeExpression)); + // Create expression with invisible flag set if column is invisible + final var expression = new Expression(Optional.of(attributeName), attributeType, attributeExpression, column.isInvisible()); + attributesBuilder.add(expression); colCount++; } final var attributes = Expressions.of(attributesBuilder.build()); - return LogicalOperator.newNamedOperator(tableId, attributes, resultingQuantifier); + final var qualifiedAttributes = attributes.withQualifier(tableId); + return new LogicalOperator(Optional.of(tableId), qualifiedAttributes, resultingQuantifier); } @Nonnull @@ -432,6 +435,8 @@ private static boolean canAvoidProjectingIndividualFields(@Nonnull Expressions o // must be a Star expression Iterables.size(output) == 1 && Iterables.getOnlyElement(output) instanceof Star && + // Cannot use optimization if there are invisible columns - they would be included in passthrough + logicalOperators.first().getOutput().stream().noneMatch(Expression::isInvisible) && // special case for CTEs where it is possible that a Star is referencing aliased columns of a named query // if these columns are aliased differently from the underlying query fragment, then we can only avoid // projecting individual columns (and lose their aliases) if and only if their names pairwise match the diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java index 728c6883c4..83c9cfc715 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java @@ -81,7 +81,7 @@ public static Stream pullUp(@Nonnull final Stream orderBy.getExpression() instanceof Star ? - ((Star) orderBy.getExpression()).getExpansion().stream().map(orderBy::withExpression) : + ((Star) orderBy.getExpression()).getExpansion().nonInvisible().stream().map(orderBy::withExpression) : Stream.of(orderBy)) .map(orderBy -> { final var orderByExpression = orderBy.getExpression(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index e28d62dfb5..2e27e8d22c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -299,7 +299,7 @@ public Star expandStar(@Nonnull Optional optionalQualifier, final var forEachOperators = operators.forEachOnly(); // Case 1: no qualifier, e.g. SELECT * FROM T, R; if (optionalQualifier.isEmpty()) { - final var expansion = forEachOperators.getExpressions().nonEphemeral(); + final var expansion = forEachOperators.getExpressions().nonEphemeral().nonInvisible(); return Star.overQuantifiers(Optional.empty(), Streams.stream(forEachOperators).map(LogicalOperator::getQuantifier) .map(Quantifier::getFlowedObjectValue).collect(ImmutableList.toImmutableList()), "unknown", expansion); } @@ -310,7 +310,7 @@ public Star expandStar(@Nonnull Optional optionalQualifier, .findFirst(); if (logicalTableMaybe.isPresent()) { return Star.overQuantifier(optionalQualifier, logicalTableMaybe.get().getQuantifier().getFlowedObjectValue(), - qualifier.getName(), logicalTableMaybe.get().getOutput().nonEphemeral()); + qualifier.getName(), logicalTableMaybe.get().getOutput().nonEphemeral().nonInvisible()); } // Case 2.1: represents a rare case where a logical operator contains a mix of columns that are qualified // differently. diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Star.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Star.java index d7e0989d8d..3488b501ee 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Star.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Star.java @@ -31,7 +31,6 @@ import com.apple.foundationdb.relational.util.Assert; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import javax.annotation.Nonnull; import java.util.Collection; @@ -49,19 +48,19 @@ public final class Star extends Expression { @Nonnull - private final List expansion; + private final Expressions expansion; private Star(@Nonnull Optional qualifier, @Nonnull DataType dataType, @Nonnull Value expression, - @Nonnull Iterable expansion) { + @Nonnull Expressions expansion) { super(qualifier, dataType, expression); Assert.thatUnchecked(expression.getResultType().isRecord()); Assert.thatUnchecked(dataType.getCode() == DataType.Code.STRUCT); - Assert.thatUnchecked(Iterables.size(expansion) == ((DataType.StructType) dataType).getFields().size()); - this.expansion = ImmutableList.copyOf(expansion); + Assert.thatUnchecked(expansion.size() == ((DataType.StructType) dataType).getFields().size()); + this.expansion = expansion; } @Nonnull - public List getExpansion() { + public Expressions getExpansion() { return expansion; } @@ -71,10 +70,11 @@ public Expression withQualifier(@Nonnull Collection qualifier) { if (getName().isEmpty()) { return this; } - final var name = getName().get(); - final var newNameMaybe = name.withQualifier(qualifier); - final var newExpansionMaybe = expansion.stream().map(expression -> expression.withQualifier(qualifier)).collect(ImmutableList.toImmutableList()); + final Identifier name = getName().get(); + final Identifier newNameMaybe = name.withQualifier(qualifier); + final Expressions newExpansionMaybe = expansion.withQualifier(qualifier); if (!newNameMaybe.equals(name) || !Objects.equals(newExpansionMaybe, expansion)) { + // Keep the same underlying value - it doesn't need to change when adding qualifiers return new Star(Optional.of(newNameMaybe), getDataType(), getUnderlying(), newExpansionMaybe); } return this; @@ -105,13 +105,13 @@ public Expression withUnderlying(@Nonnull Value underlying) { @Nonnull @Override public Expressions dereferenced(@Nonnull Literals literals) { - return Expressions.of(expansion).dereferenced(literals); + return expansion.dereferenced(literals); } @Nonnull @Override public EphemeralExpression asEphemeral() { - Assert.failUnchecked("attempt to create an ephermeral expression from a star"); + Assert.failUnchecked("attempt to create an ephemeral expression from a star"); return null; } @@ -128,8 +128,9 @@ public static Star overQuantifier(@Nonnull Optional qualifier, @Nonnull Value quantifier, @Nonnull String typeName, @Nonnull Expressions expansion) { - final var starType = createStarType(typeName, expansion); - return new Star(qualifier, starType, quantifier, expansion); + // Note: quantifier parameter is unused because expansion is already filtered to visible columns + // by SemanticAnalyzer, and we need to create the underlying from that filtered expansion + return createStar(qualifier, typeName, expansion); } @Nonnull @@ -137,27 +138,52 @@ public static Star overQuantifiers(@Nonnull Optional qualifier, @Nonnull List quantifiers, @Nonnull String typeName, @Nonnull Expressions expansion) { - final var underlyingStarType = quantifiers.size() == 1 ? quantifiers.get(0) : RecordConstructorValue.ofUnnamed(quantifiers); - final var starType = createStarType(typeName, expansion); - return new Star(qualifier, starType, underlyingStarType, expansion); + // Note: quantifiers parameter is unused because expansion is already filtered to visible columns + // by SemanticAnalyzer, and we need to create the underlying from that filtered expansion + return createStar(qualifier, typeName, expansion); } @Nonnull public static Star overIndividualExpressions(@Nonnull Optional qualifier, @Nonnull String typeName, @Nonnull Expressions expansion) { + return createStar(qualifier, typeName, expansion); + } + + @Nonnull + private static Star createStar(@Nonnull Optional qualifier, + @Nonnull String typeName, + @Nonnull Expressions expansion) { final var starType = createStarType(typeName, expansion); - return new Star(qualifier, starType, RecordConstructorValue.ofColumns(expansion.underlyingAsColumns()), expansion); + // The expansion is already filtered to visible columns only by SemanticAnalyzer, + // so we create the underlying value from the filtered expansion + final var filteredUnderlying = RecordConstructorValue.ofColumns(expansion.underlyingAsColumns()); + return new Star(qualifier, starType, filteredUnderlying, expansion); } + /** + * Creates the {@link DataType.StructType} for a star expression from the expansion. + *

+ * The {@code expansion} parameter must contain only visible columns. This invariant is enforced + * by {@link com.apple.foundationdb.relational.recordlayer.query.SemanticAnalyzer#expandStar}, + * which filters invisible columns via {@link Expressions#nonInvisible()} before creating the star. + * + * @param name the name for the struct type + * @param expansion the expressions to include in the star (must not contain invisible expressions) + * @return the struct type representing the star's schema + */ @Nonnull private static DataType.StructType createStarType(@Nonnull String name, @Nonnull Expressions expansion) { final ImmutableList.Builder fields = ImmutableList.builder(); int i = 0; for (final var expression : expansion) { + // Expansion is already filtered to visible columns only in SemanticAnalyzer.expandStar() + // Enforce this invariant with an assertion to catch bugs if filtering logic changes + Assert.thatUnchecked(!expression.isInvisible(), + "Star expansion should not contain invisible expressions"); fields.add(DataType.StructType.Field.from(expression.getName().map(Identifier::toString) - .orElseGet(() -> expression.getUnderlying().toString()), expression.getDataType(), i)); + .orElseGet(() -> expression.getUnderlying().toString()), expression.getDataType(), i, false)); i++; } return DataType.StructType.from(name, fields.build(), true); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 6602be62a5..8290d2ceab 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -144,6 +144,7 @@ public RecordLayerColumn visitColumnDefinition(@Nonnull RelationalParser.ColumnD final var columnId = visitUid(ctx.colName); final var isRepeated = ctx.ARRAY() != null; final var isNullable = ctx.columnConstraint() != null ? (Boolean) ctx.columnConstraint().accept(this) : true; + final var isInvisible = ctx.columnVisibility() != null && ctx.columnVisibility().INVISIBLE() != null; // TODO: We currently do not support NOT NULL for any type other than ARRAY. This is because there is no way to // specify not "nullability" at the RecordMetaData level. For ARRAY, specifying that is actually possible // by means of NullableArrayWrapper. In essence, we don't actually need a wrapper per se for non-array types, @@ -159,7 +160,7 @@ public RecordLayerColumn visitColumnDefinition(@Nonnull RelationalParser.ColumnD typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofPrimitiveType(ctx.columnType().primitiveType(), isNullable, isRepeated); } final var columnType = semanticAnalyzer.lookupType(typeInfo, metadataBuilder::findType); - return RecordLayerColumn.newBuilder().setName(columnId.getName()).setDataType(columnType).build(); + return RecordLayerColumn.newBuilder().setName(columnId.getName()).setDataType(columnType).setInvisible(isInvisible).build(); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index d82f3613e1..34a01ba921 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -189,6 +189,12 @@ public Object visitColumnDefinition(@Nonnull RelationalParser.ColumnDefinitionCo return getDelegate().visitColumnDefinition(ctx); } + @Nonnull + @Override + public Object visitColumnVisibility(@Nonnull RelationalParser.ColumnVisibilityContext ctx) { + return getDelegate().visitColumnVisibility(ctx); + } + @Nonnull @Override public DataType visitFunctionColumnType(@Nonnull final RelationalParser.FunctionColumnTypeContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index c374dd1105..16f7b76bbd 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -49,6 +49,7 @@ import com.apple.foundationdb.relational.recordlayer.query.OrderByExpression; import com.apple.foundationdb.relational.recordlayer.query.ParseHelpers; import com.apple.foundationdb.relational.recordlayer.query.SemanticAnalyzer; +import com.apple.foundationdb.relational.recordlayer.query.Star; import com.apple.foundationdb.relational.recordlayer.query.StringTrieNode; import com.apple.foundationdb.relational.recordlayer.query.TautologicalValue; import com.apple.foundationdb.relational.util.Assert; @@ -156,7 +157,11 @@ public Expression visitFullColumnNameExpressionAtom(@Nonnull RelationalParser.Fu public Expressions visitSelectElements(@Nonnull RelationalParser.SelectElementsContext selectElementsContext) { return Expressions.of(selectElementsContext.selectElement().stream() .map(selectElement -> Assert.castUnchecked(selectElement.accept(this), Expression.class)) - .collect(ImmutableList.toImmutableList())); + .collect(ImmutableList.toImmutableList())) + // Make all explicitly selected columns visible, even if they were marked invisible in the table. + // This ensures "SELECT secret FROM t" returns the invisible column, + // and "SELECT * FROM (SELECT secret FROM t)" doesn't filter it out. + .makeVisible(); } @Nonnull @@ -224,7 +229,7 @@ private static RecordLayerColumn toColumn(@Nonnull FieldValue.ResolvedAccessor f final var fields = new ArrayList<>(Collections.nCopies(columnCount, (DataType.StructType.Field) null)); for (final var child : columnIdTrie.getChildrenMap().entrySet()) { final var column = toColumn(child.getKey(), child.getValue()); - fields.set(column.getIndex(), DataType.StructType.Field.from(column.getName(), column.getDataType(), column.getIndex())); + fields.set(column.getIndex(), DataType.StructType.Field.from(column.getName(), column.getDataType(), column.getIndex(), column.isInvisible())); } builder.setDataType(DataType.StructType.from(columnName, fields, true)); return builder.build(); @@ -782,18 +787,18 @@ public Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstru if (ctx.uid() != null) { final var id = visitUid(ctx.uid()); if (ctx.STAR() == null) { - final var expression = getDelegate().getSemanticAnalyzer().resolveIdentifier(id, getDelegate().getCurrentPlanFragment()); - final var resultValue = RecordConstructorValue.ofUnnamed(List.of(expression.getUnderlying())); + final Expression expression = getDelegate().getSemanticAnalyzer().resolveIdentifier(id, getDelegate().getCurrentPlanFragment()); + final Value resultValue = RecordConstructorValue.ofUnnamed(List.of(expression.getUnderlying())); return expression.withUnderlying(resultValue); } else { - final var star = getDelegate().getSemanticAnalyzer().expandStar(Optional.of(id), getDelegate().getLogicalOperators()); - final var resultValue = star.getUnderlying(); + final Star star = getDelegate().getSemanticAnalyzer().expandStar(Optional.of(id), getDelegate().getLogicalOperators()); + final Value resultValue = star.getUnderlying(); return Expression.ofUnnamed(resultValue); } } if (ctx.STAR() != null) { - final var star = getDelegate().getSemanticAnalyzer().expandStar(Optional.empty(), getDelegate().getLogicalOperators()); - final var resultValue = star.getUnderlying(); + final Star star = getDelegate().getSemanticAnalyzer().expandStar(Optional.empty(), getDelegate().getLogicalOperators()); + final Value resultValue = star.getUnderlying(); return Expression.ofUnnamed(resultValue); } final var expressions = parseRecordFieldsUnderReorderings(ctx.expressionWithOptionalName()); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java index 6164015b59..41e0b4f61d 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java @@ -1382,6 +1382,63 @@ private static void checkColumnNullability(@Nonnull final SchemaTemplate templat Assertions.assertEquals(sqlType, maybeNullableArrayColumn.get().getDataType().getJdbcSqlCode()); } + @Test + void invisibleColumnsParsedCorrectly() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TABLE my_table (id bigint, name string, secret string INVISIBLE, age integer, password string INVISIBLE, PRIMARY KEY(id))"; + + shouldWorkWithInjectedFactory(stmt, new AbstractMetadataOperationsFactory() { + @Nonnull + @Override + public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { + Assertions.assertInstanceOf(RecordLayerSchemaTemplate.class, template); + final var tables = ((RecordLayerSchemaTemplate) template).getTables(); + Assertions.assertEquals(1, tables.size(), "should have only 1 table"); + + final var table = tables.iterator().next(); + final var columns = table.getColumns(); + Assertions.assertEquals(5, columns.size(), "should have 5 columns"); + + // Verify invisibility: id, name, age should be visible; secret, password should be invisible + Assertions.assertFalse(columns.stream().filter(c -> c.getName().equals("id")).findFirst().get().isInvisible(), "id should be visible"); + Assertions.assertFalse(columns.stream().filter(c -> c.getName().equals("name")).findFirst().get().isInvisible(), "name should be visible"); + Assertions.assertTrue(columns.stream().filter(c -> c.getName().equals("secret")).findFirst().get().isInvisible(), "secret should be invisible"); + Assertions.assertFalse(columns.stream().filter(c -> c.getName().equals("age")).findFirst().get().isInvisible(), "age should be visible"); + Assertions.assertTrue(columns.stream().filter(c -> c.getName().equals("password")).findFirst().get().isInvisible(), "password should be invisible"); + + return txn -> { + }; + } + }); + } + + @Test + void visibleColumnsParsedCorrectly() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TABLE my_table (id bigint VISIBLE, name string VISIBLE, PRIMARY KEY(id))"; + + shouldWorkWithInjectedFactory(stmt, new AbstractMetadataOperationsFactory() { + @Nonnull + @Override + public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { + Assertions.assertInstanceOf(RecordLayerSchemaTemplate.class, template); + final var tables = ((RecordLayerSchemaTemplate) template).getTables(); + Assertions.assertEquals(1, tables.size(), "should have only 1 table"); + + final var table = tables.iterator().next(); + final var columns = table.getColumns(); + Assertions.assertEquals(2, columns.size(), "should have 2 columns"); + + // Verify all are visible (explicit VISIBLE keyword) + Assertions.assertFalse(columns.stream().filter(c -> c.getName().equals("id")).findFirst().get().isInvisible(), "id should be visible"); + Assertions.assertFalse(columns.stream().filter(c -> c.getName().equals("name")).findFirst().get().isInvisible(), "name should be visible"); + + return txn -> { + }; + } + }); + } + @Nonnull private static String replaceLast(@Nonnull final String str, final char oldChar, @Nonnull final String replacement) { if (str.isEmpty()) { diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index fc11024c93..ddd8cfa107 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -358,4 +358,9 @@ public void versionsTests(YamlTest.Runner runner) throws Exception { public void viewsTest(YamlTest.Runner runner) throws Exception { runner.runYamsql("views.yamsql"); } + + @TestTemplate + public void invisibleColumns(YamlTest.Runner runner) throws Exception { + runner.runYamsql("invisible-columns.yamsql"); + } } diff --git a/yaml-tests/src/test/resources/invisible-columns.yamsql b/yaml-tests/src/test/resources/invisible-columns.yamsql new file mode 100644 index 0000000000..e03031c3e9 --- /dev/null +++ b/yaml-tests/src/test/resources/invisible-columns.yamsql @@ -0,0 +1,319 @@ +# +# invisible-columns.yamsql +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2021-2025 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +options: + # Invisible columns are a new feature in this version + supported_version: !current_version +--- +schema_template: + create table T1(id bigint, name string, secret string INVISIBLE, age integer, password string INVISIBLE, primary key(id)) + create table T2(id2 bigint, email string, token string INVISIBLE, active boolean, primary key(id2)) + create index T1_secret as select secret from t1 + create index T1_password as select password from t1 + create index T2_token as select token from t2 + create view V1 as select * from T1 + create view V2 as select id, name, secret from T1 + create view V3 as select t1.*, t2.* from T1 t1, T2 t2 where t1.id = t2.id2 + create function F1() as select * from T1 + create function F2() as select id, name, secret from T1 +--- +setup: + steps: + - query: insert into T1 values (1, 'alice', 'secret1', 25, 'pass1') + - query: insert into T1 values (2, 'bob', 'secret2', 30, 'pass2') + - query: insert into T1 values (3, 'charlie', 'secret3', 35, 'pass3') + - query: insert into T2 values (1, 'alice@example.com', 'token1', true) + - query: insert into T2 values (2, 'bob@example.com', 'token2', true) + - query: insert into T2 values (3, 'charlie@example.com', 'token3', false) +--- +test_block: + tests: + # Basic star expansion - invisible columns should be excluded + - + - query: select * from T1 + - result: [{id: 1, name: 'alice', age: 25}, {id: 2, name: 'bob', age: 30}, {id: 3, name: 'charlie', age: 35}] + + # Parenthesized star - should return nested struct excluding invisible columns + - + - query: select (*) from T1 + - result: [{{id: 1, name: 'alice', age: 25}}, {{id: 2, name: 'bob', age: 30}}, {{id: 3, name: 'charlie', age: 35}}] + + # Explicit selection of invisible columns should work + - + - query: select secret, password from T1 + - result: [{secret: 'secret1', password: 'pass1'}, {secret: 'secret2', password: 'pass2'}, {secret: 'secret3', password: 'pass3'}] + + # Mix of visible and invisible columns + - + - query: select id, secret, age from T1 where id = 1 + - result: [{id: 1, secret: 'secret1', age: 25}] + + # Star with additional explicit columns + - + - query: select *, secret from T1 where id = 1 + - result: [{id: 1, name: 'alice', age: 25, secret: 'secret1'}] + + # Parenthesized star with additional explicit columns + - + - query: select (*), secret from T1 where id = 1 + - result: [{{id: 1, name: 'alice', age: 25}, secret: 'secret1'}] + + # Qualified star expansion - should exclude invisible columns + - + - query: select T1.* from T1 where id = 1 + - result: [{id: 1, name: 'alice', age: 25}] + + # Parenthesized qualified star - should exclude invisible columns + - + - query: select (T1.*) from T1 where id = 1 + - result: [{{id: 1, name: 'alice', age: 25}}] + + # Qualified star with explicit invisible column + - + - query: select T1.*, T1.secret from T1 where id = 1 + - result: [{id: 1, name: 'alice', age: 25, secret: 'secret1'}] + + # Parenthesized qualified star with explicit invisible column + - + - query: select (T1.*), T1.secret from T1 where id = 1 + - result: [{{id: 1, name: 'alice', age: 25}, secret: 'secret1'}] + + # Referring to an invisible column does not make it visible + - + - query: select * from T1 where secret = 'secret1' + - result: [{id: 1, name: 'alice', age: 25}] + + # Parenthesized star - referring to invisible column does not make it visible + - + - query: select (*) from T1 where secret = 'secret1' + - result: [{{id: 1, name: 'alice', age: 25}}] + + # Table alias with star - should exclude invisible columns + - + - query: select t.* from T1 t where t.id = 1 + - result: [{id: 1, name: 'alice', age: 25}] + + # Parenthesized table alias star - should exclude invisible columns + - + - query: select (t.*) from T1 t where t.id = 1 + - result: [{{id: 1, name: 'alice', age: 25}}] + + # Table alias with explicit invisible column + - + - query: select t.id, t.secret, t.password from T1 t where t.id = 1 + - result: [{id: 1, secret: 'secret1', password: 'pass1'}] + + # Subquery with star - invisible columns should be excluded from outer query + - + - query: select * from (select * from T1 where id = 1) sub + - result: [{id: 1, name: 'alice', age: 25}] + + # Subquery with parenthesized star + - + - query: select (*) from (select * from T1 where id = 1) sub + - result: [{{id: 1, name: 'alice', age: 25}}] + + # Subquery explicitly selecting invisible columns + - + - query: select * from (select id, secret from T1 where id = 1) sub + - result: [{id: 1, secret: 'secret1'}] + + # Nested subquery with star expansion + - + - query: select * from (select * from (select * from T1 where id = 1) inner_sub) outer_sub + - result: [{id: 1, name: 'alice', age: 25}] + + # Nested subquery with parenthesized star expansion + - + - query: select (*) from (select (*) from (select (*) from T1 where id = 1) inner_sub) outer_sub + - result: [{{{{id: 1, name: 'alice', age: 25}}}}] + + # Subquery selecting invisible column, outer query referencing it + - + - query: select sub.secret from (select secret from T1 where id = 1) sub + - result: [{secret: 'secret1'}] + + # Join with star expansion - both tables should exclude invisible columns + - + - query: select * from T1, T2 where T1.id = T2.id2 and T1.id = 1 + - result: [{id: 1, name: 'alice', age: 25, id2: 1, email: 'alice@example.com', active: true}] + + # Join with parenthesized star expansion + - + - query: select (*) from T1, T2 where T1.id = T2.id2 and T1.id = 1 + - result: [{{id: 1, name: 'alice', age: 25, id2: 1, email: 'alice@example.com', active: true}}] + + # Join with qualified stars + - + - query: select T1.*, T2.* from T1, T2 where T1.id = T2.id2 and T1.id = 1 + - result: [{id: 1, name: 'alice', age: 25, id2: 1, email: 'alice@example.com', active: true}] + + # Join with parenthesized qualified stars + - + - query: select (T1.*), (T2.*) from T1, T2 where T1.id = T2.id2 and T1.id = 1 + - result: [{{id: 1, name: 'alice', age: 25}, {id2: 1, email: 'alice@example.com', active: true}}] + + # Join with one table's invisible column explicitly selected + - + - query: select T1.*, T2.token from T1, T2 where T1.id = T2.id2 and T1.id = 1 + - result: [{id: 1, name: 'alice', age: 25, token: 'token1'}] + + # Join with parenthesized qualified star and explicit invisible column + - + - query: select (T1.*), T2.token from T1, T2 where T1.id = T2.id2 and T1.id = 1 + - result: [{{id: 1, name: 'alice', age: 25}, token: 'token1'}] + + # Join with both tables' invisible columns + - + - query: select T1.secret, T2.token from T1, T2 where T1.id = T2.id2 and T1.id = 1 + - result: [{secret: 'secret1', token: 'token1'}] + + # View based on star - should exclude invisible columns + - + - query: select * from V1 + - result: [{id: 1, name: 'alice', age: 25}, {id: 2, name: 'bob', age: 30}, {id: 3, name: 'charlie', age: 35}] + + # Parenthesized star from view + - + - query: select (*) from V1 + - result: [{{id: 1, name: 'alice', age: 25}}, {{id: 2, name: 'bob', age: 30}}, {{id: 3, name: 'charlie', age: 35}}] + + # View that explicitly includes invisible column + - + - query: select * from V2 + - result: [{id: 1, name: 'alice', secret: 'secret1'}, {id: 2, name: 'bob', secret: 'secret2'}, {id: 3, name: 'charlie', secret: 'secret3'}] + + # Star from view that has star (should be already expanded, no invisible columns) + - + - query: select * from V1 where id = 1 + - result: [{id: 1, name: 'alice', age: 25}] + + # Parenthesized star from view that has star + - + - query: select (*) from V1 where id = 1 + - result: [{{id: 1, name: 'alice', age: 25}}] + + # View with join using star + - + - query: select * from V3 where id = 1 + - result: [{id: 1, name: 'alice', age: 25, id2: 1, email: 'alice@example.com', active: true}] + + # Parenthesized star from view with join + - + - query: select (*) from V3 where id = 1 + - result: [{{id: 1, name: 'alice', age: 25, id2: 1, email: 'alice@example.com', active: true}}] + + # Function returning star expansion - should exclude invisible columns + - + - query: select * from F1() + - result: [{id: 1, name: 'alice', age: 25}, {id: 2, name: 'bob', age: 30}, {id: 3, name: 'charlie', age: 35}] + + # Parenthesized star from function + - + - query: select (*) from F1() + - result: [{{id: 1, name: 'alice', age: 25}}, {{id: 2, name: 'bob', age: 30}}, {{id: 3, name: 'charlie', age: 35}}] + + # Function explicitly returning invisible column + - + - query: select * from F2() + - result: [{id: 1, name: 'alice', secret: 'secret1'}, {id: 2, name: 'bob', secret: 'secret2'}, {id: 3, name: 'charlie', secret: 'secret3'}] + + # Invisible columns in WHERE clause + - + - query: select id, name from T1 where secret = 'secret1' + - result: [{id: 1, name: 'alice'}] + + # Invisible columns in ORDER BY + - + - query: select id, name from T1 where id = 1 or id = 2 or id = 3 order by password desc + - result: [{id: 3, name: 'charlie'}, {id: 2, name: 'bob'}, {id: 1, name: 'alice'}] + + # Invisible columns in GROUP BY should work when explicitly selected + - + - query: select secret, count(*) from T1 group by secret order by secret + - result: [{'secret1', 1}, {'secret2', 1}, {'secret3', 1}] + + # DISTINCT with star - should exclude invisible columns + - + - query: select distinct * from T1 where id = 1 or id = 2 + - result: [{id: 1, name: 'alice', age: 25}, {id: 2, name: 'bob', age: 30}] + + # DISTINCT with parenthesized star + - + - query: select distinct (*) from T1 where id = 1 or id = 2 + - result: [{{id: 1, name: 'alice', age: 25}}, {{id: 2, name: 'bob', age: 30}}] + + # COUNT(*) should count all rows regardless of invisible columns + - + - query: select count(*) from T1 + - result: [{3}] + + # Aggregate functions on invisible columns + - + - query: select count(secret) as c1, count(password) as c2 from T1 + - result: [{c1: 3, c2: 3}] + + # UNION with star - both sides should exclude invisible columns + - + - query: select * from T1 where id = 1 union all select * from T1 where id = 2 + - unorderedResult: [{id: 1, name: 'alice', age: 25}, {id: 2, name: 'bob', age: 30}] + + # UNION with parenthesized star + - + - query: select (*) from T1 where id = 1 union all select (*) from T1 where id = 2 + - unorderedResult: [{{id: 1, name: 'alice', age: 25}}, {{id: 2, name: 'bob', age: 30}}] + + # UNION with one side explicitly selecting invisible column should work + - + - query: select id, name, age from T1 where id = 1 union all select id, name, age from T1 where id = 2 + - unorderedResult: [{id: 1, name: 'alice', age: 25}, {id: 2, name: 'bob', age: 30}] + + # CTE with star - should exclude invisible columns + - + - query: with cte as (select * from T1 where id = 1) select * from cte + - result: [{id: 1, name: 'alice', age: 25}] + + # CTE with parenthesized star + - + - query: with cte as (select (*) from T1 where id = 1) select (*) from cte + - result: [{{{id: 1, name: 'alice', age: 25}}}] + + # CTE explicitly selecting invisible column + - + - query: with cte as (select id, secret from T1 where id = 1) select * from cte + - result: [{id: 1, secret: 'secret1'}] + + # CASE expression with invisible columns + # TODO: This test fails with cache retrieval error in Embedded mode. This is a pre-existing bug + # that also affects regular (non-invisible) columns - see test-case-cache.yamsql on main branch. + # The issue is unrelated to the invisible columns feature. + # - + # - query: select id, case when secret like 'secret%' then 'valid' else 'invalid' end as status from T1 where id = 1 + # - result: [{id: 1, status: 'valid'}] + + # Invisible column in HAVING clause + - + - query: select count(*) as cnt from T1 group by secret having secret = 'secret1' + - result: [{cnt: 1}] + + # Invisible column in correlated subquery + - + - query: select id, name from T1 t1 where exists (select 1 from T2 t2 where t2.token = t1.secret and t2.id2 = t1.id) + - result: [] +...