add materialized column support

apps/framework-cli-e2e/test/utils/database-utils.ts

@@ -255,6 +255,8 @@ export interface ExpectedColumn {
   type: string | RegExp; // Allow regex for complex type matching
   nullable?: boolean;
   comment?: string;
+  codec?: string | RegExp;
+  materialized?: string | RegExp;
 }
 
 /**
@@ -433,6 +435,52 @@ export const validateTableSchema = async (
           `Column '${expectedCol.name}' comment mismatch: expected '${expectedCol.comment}', got '${actualCol.comment}'`,
         );
       }
+
+      // Codec validation (if specified)
+      if (expectedCol.codec !== undefined) {
+        const actualCodec = actualCol.codec_expression;
+        let codecMatches = false;
+
+        if (typeof expectedCol.codec === "string") {
+          // Exact string match
+          codecMatches = actualCodec === expectedCol.codec;
+        } else if (expectedCol.codec instanceof RegExp) {
+          // Regex match for complex codec expressions
+          codecMatches = expectedCol.codec.test(actualCodec);
+        }
+
+        if (!codecMatches) {
+          errors.push(
+            `Column '${expectedCol.name}' codec mismatch: expected '${expectedCol.codec}', got '${actualCodec}'`,
+          );
+        }
+      }
+
+      // Materialized validation (if specified)
+      if (expectedCol.materialized !== undefined) {
+        const actualMaterialized = actualCol.default_expression;
+        const actualDefaultType = actualCol.default_type;
+        let materializedMatches = false;
+
+        // Check that it's actually a MATERIALIZED column
+        if (actualDefaultType === "MATERIALIZED") {
+          if (typeof expectedCol.materialized === "string") {
+            // Exact string match
+            materializedMatches =
+              actualMaterialized === expectedCol.materialized;
+          } else if (expectedCol.materialized instanceof RegExp) {
+            // Regex match for complex expressions
+            materializedMatches =
+              expectedCol.materialized.test(actualMaterialized);
+          }
+        }
+
+        if (!materializedMatches) {
+          errors.push(
+            `Column '${expectedCol.name}' materialized mismatch: expected '${expectedCol.materialized}', got '${actualDefaultType === "MATERIALIZED" ? actualMaterialized : "(not materialized)"}'`,
+          );
+        }
+      }
     }
 
     // Check for unexpected columns (optional - could be made configurable)

apps/framework-cli-e2e/test/utils/schema-definitions.ts

@@ -421,6 +421,44 @@ export const TYPESCRIPT_TEST_SCHEMAS: ExpectedTableSchema[] = [
       { name: "payloadBasic", type: "JSON(count Int64, name String)" },
     ],
   },
+  // Codec compression test table
+  {
+    tableName: "CodecTest",
+    columns: [
+      { name: "id", type: "String" },
+      { name: "timestamp", type: /DateTime\('UTC'\)/, codec: /Delta.*LZ4/ },
+      { name: "log_blob", type: "JSON", codec: "ZSTD(3)" },
+      { name: "combination_hash", type: "Array(UInt64)", codec: "ZSTD(1)" },
+      { name: "temperature", type: "Float64", codec: /Gorilla.*ZSTD/ },
+      { name: "request_count", type: "Float64", codec: /DoubleDelta.*LZ4/ },
+      { name: "user_agent", type: "String", codec: "ZSTD(3)" },
+      { name: "tags", type: "Array(String)", codec: "LZ4" },
+      { name: "status_code", type: "Float64" },
+    ],
+  },
+  // Materialized column test table
+  {
+    tableName: "MaterializedTest",
+    columns: [
+      { name: "id", type: "String" },
+      { name: "timestamp", type: /DateTime\('UTC'\)/ },
+      { name: "userId", type: "String" },
+      {
+        name: "eventDate",
+        type: /Date(32)?/,
+        materialized: "toDate(timestamp)",
+      },
+      { name: "userHash", type: "UInt64", materialized: "cityHash64(userId)" },
+      { name: "log_blob", type: "JSON", codec: "ZSTD(3)" },
+      {
+        name: "combinationHash",
+        type: "Array(UInt64)",
+        materialized:
+          "arrayMap(kv -> cityHash64(kv.1, kv.2), JSONExtractKeysAndValuesRaw(toString(log_blob)))",
+        codec: "ZSTD(1)",
+      },
+    ],
+  },
 ];
 
 // ============ PYTHON TEMPLATE SCHEMA DEFINITIONS ============
@@ -805,6 +843,48 @@ export const PYTHON_TEST_SCHEMAS: ExpectedTableSchema[] = [
       { name: "payload_basic", type: "JSON(count Int64, name String)" },
     ],
   },
+  // Codec compression test table
+  {
+    tableName: "CodecTest",
+    columns: [
+      { name: "id", type: "String" },
+      { name: "timestamp", type: /DateTime\('UTC'\)/, codec: /Delta.*LZ4/ },
+      { name: "log_blob", type: "JSON", codec: "ZSTD(3)" },
+      { name: "combination_hash", type: "Array(UInt64)", codec: "ZSTD(1)" },
+      { name: "temperature", type: "Float64", codec: /Gorilla.*ZSTD/ },
+      { name: "request_count", type: "Float64", codec: /DoubleDelta.*LZ4/ },
+      { name: "user_agent", type: "String", codec: "ZSTD(3)" },
+      { name: "tags", type: "Array(String)", codec: "LZ4" },
+      { name: "status_code", type: "Float64" },
+    ],
+  },
+  // Materialized column test table
+  {
+    tableName: "MaterializedTest",
+    columns: [
+      { name: "id", type: "String" },
+      { name: "timestamp", type: /DateTime\('UTC'\)/ },
+      { name: "user_id", type: "String" },
+      {
+        name: "event_date",
+        type: /Date(32)?/,
+        materialized: "toDate(timestamp)",
+      },
+      {
+        name: "user_hash",
+        type: "UInt64",
+        materialized: "cityHash64(user_id)",
+      },
+      { name: "log_blob", type: "JSON", codec: "ZSTD(3)" },
+      {
+        name: "combination_hash",
+        type: "Array(UInt64)",
+        materialized:
+          "arrayMap(kv -> cityHash64(kv.1, kv.2), JSONExtractKeysAndValuesRaw(toString(log_blob)))",
+        codec: "ZSTD(1)",
+      },
+    ],
+  },
 ];
 
 // ============ HELPER FUNCTIONS ============

apps/framework-cli/src/cli/local_webserver.rs

@@ -3544,6 +3544,8 @@ mod tests {
                 annotations: vec![],
                 comment: None,
                 ttl: None,
+                codec: None,
+                materialized: None,
             }],
             order_by: OrderBy::Fields(vec!["id".to_string()]),
             partition_by: None,

apps/framework-cli/src/cli/routines/migrate.rs

@@ -763,6 +763,8 @@ mod tests {
                 annotations: vec![],
                 comment: None,
                 ttl: None,
+                codec: None,
+                materialized: None,
             }],
             order_by: OrderBy::Fields(vec!["id".to_string()]),
             partition_by: None,
@@ -797,6 +799,8 @@ mod tests {
             annotations: vec![],
             comment: None,
             ttl: None,
+            codec: None,
+            materialized: None,
         });
         table
     }
@@ -1140,6 +1144,8 @@ mod tests {
                     annotations: vec![],
                     comment: None,
                     ttl: None,
+                    codec: None,
+                    materialized: None,
                 },
                 after_column: None,
                 database: Some("bad_db".to_string()),
@@ -1157,6 +1163,8 @@ mod tests {
                     annotations: vec![],
                     comment: None,
                     ttl: None,
+                    codec: None,
+                    materialized: None,
                 },
                 after_column: Column {
                     name: "col".to_string(),
@@ -1168,6 +1176,8 @@ mod tests {
                     annotations: vec![],
                     comment: None,
                     ttl: None,
+                    codec: None,
+                    materialized: None,
                 },
                 database: Some("another_bad_db".to_string()),
                 cluster_name: None,

apps/framework-cli/src/framework/core/infra_reality_checker.rs

@@ -515,6 +515,8 @@ mod tests {
                 annotations: vec![],
                 comment: None,
                 ttl: None,
+                codec: None,
+                materialized: None,
             }],
             order_by: OrderBy::Fields(vec!["id".to_string()]),
             partition_by: None,
@@ -610,6 +612,8 @@ mod tests {
             annotations: vec![],
             comment: None,
             ttl: None,
+            codec: None,
+            materialized: None,
         });
 
         let mock_client = MockOlapClient {
@@ -679,6 +683,8 @@ mod tests {
             annotations: vec![],
             comment: None,
             ttl: None,
+            codec: None,
+            materialized: None,
         };
         actual_table.columns.push(timestamp_col.clone());
         infra_table.columns.push(timestamp_col);

apps/framework-cli/src/framework/core/infrastructure/table.rs

@@ -600,6 +600,10 @@ pub struct Column {
     pub comment: Option<String>, // Column comment for metadata storage
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub ttl: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub codec: Option<String>, // Compression codec expression (e.g., "ZSTD(3)", "Delta, LZ4")
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub materialized: Option<String>, // MATERIALIZED column expression (computed at write-time, physically stored)
 }
 
 #[derive(Debug, Clone, Eq, PartialEq, Hash)]
@@ -1114,6 +1118,8 @@ impl Column {
                 .collect(),
             comment: self.comment.clone(),
             ttl: self.ttl.clone(),
+            codec: self.codec.clone(),
+            materialized: self.materialized.clone(),
             special_fields: Default::default(),
         }
     }
@@ -1136,6 +1142,8 @@ impl Column {
             annotations,
             comment: proto.comment,
             ttl: proto.ttl,
+            codec: proto.codec,
+            materialized: proto.materialized,
         }
     }
 }
@@ -1515,6 +1523,8 @@ mod tests {
             annotations: vec![],
             comment: None,
             ttl: None,
+            codec: None,
+            materialized: None,
         };
 
         let json = serde_json::to_string(&nested_column).unwrap();
@@ -1535,6 +1545,8 @@ mod tests {
             annotations: vec![],
             comment: Some("[MOOSE_METADATA:DO_NOT_MODIFY] {\"version\":1,\"enum\":{\"name\":\"TestEnum\",\"members\":[]}}".to_string()),
             ttl: None,
+            codec: None,
+            materialized: None,
         };
 
         // Convert to proto and back
@@ -1558,6 +1570,8 @@ mod tests {
             annotations: vec![],
             comment: None,
             ttl: None,
+            codec: None,
+            materialized: None,
         };
 
         let proto = column_without_comment.to_proto();
@@ -1741,6 +1755,8 @@ mod tests {
                 annotations: vec![],
                 comment: None,
                 ttl: None,
+                codec: None,
+                materialized: None,
             },
             Column {
                 name: "name".to_string(),
@@ -1752,6 +1768,8 @@ mod tests {
                 annotations: vec![],
                 comment: None,
                 ttl: None,
+                codec: None,
+                materialized: None,
             },
         ];