diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/HMSPropertyManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/HMSPropertyManager.java index c1166bd28ed3..da03a69c3879 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/HMSPropertyManager.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/HMSPropertyManager.java @@ -25,7 +25,7 @@ import java.util.TreeMap; /** - * A property manager tailored for the HiveMetaStore. + * A property manager tailored for the HiveMetaStore operating on Iceberg tables. * It describes properties for cluster, database and table based on declared schemas. * A property is of the form: * + *

+ * This is meant as the base of a property manager that could be used to manage properties in the HiveMetaStore + * for Iceberg table maintenance operations. + * It is not meant to be a complete implementation, but rather a starting point for further development. + * A potential extension would be to add maintenance operation properties at the cluster and database levels to the schema + * so they could act as default values for table properties used by maintenance operations. */ public class HMSPropertyManager extends PropertyManager { private static final String CLUSTER_PREFIX = "cluster"; @@ -45,14 +51,13 @@ public class HMSPropertyManager extends PropertyManager { /** Table maintenance operation type. */ public enum MaintenanceOpType { - COMPACTION, - SNAPSHOT_EXPIRY, - STATS_REBUILD, - MV_BUILD, - MV_REFRESH, - SHUFFLE_TO_NEW_PART, - RECOMPRESS, - REORG + EXPIRE_SNAPSHOT, + REWRITE_DATA_FILES, + COMPRESS_DATA_FILES, + REWRITE_MANIFEST, + DELETE_ORPHAN_FILES, + REWRITE_POSITION_DELETE, + COMPUTE_TABLE_STATS } /** @@ -73,12 +78,7 @@ public static MaintenanceOpType findOpType(int ordinal) { /** Table maintenance operation status. */ public enum MaintenanceOpStatus { - MAINTENANCE_NEEDED, - SCHEDULED, - IN_PROGRESS, - DONE, - CLEANUP_NEEDED, - FAILED + INIT, CANCELED, SUBMITTED, COMPLETED, FAILED } /** The map from ordinal to OpStatus. */ @@ -101,14 +101,14 @@ public static MaintenanceOpStatus findOpStatus(int ordinal) { */ public static final PropertyType MAINTENANCE_OPERATION = new PropertyType("MaintenanceOperation"){ @Override public MaintenanceOpType cast(Object value) { - if (value instanceof MaintenanceOpType) { - return (MaintenanceOpType) value; + if (value instanceof MaintenanceOpType op) { + return op; } if (value == null) { return null; } - if (value instanceof Number) { - return findOpType(((Number) value).intValue()); + if (value instanceof Number n) { + return findOpType(n.intValue()); } return parse(value.toString()); } @@ -121,8 +121,8 @@ public static MaintenanceOpStatus findOpStatus(int ordinal) { } @Override public String format(Object value) { - if (value instanceof MaintenanceOpType) { - return value.toString(); + if (value instanceof MaintenanceOpType op) { + return op.toString(); } return null; } @@ -133,14 +133,14 @@ public static MaintenanceOpStatus findOpStatus(int ordinal) { */ public static final PropertyType MAINTENANCE_STATUS = new PropertyType("MaintenanceStatus"){ @Override public MaintenanceOpStatus cast(Object value) { - if (value instanceof MaintenanceOpStatus) { - return (MaintenanceOpStatus) value; + if (value instanceof MaintenanceOpStatus op) { + return op; } if (value == null) { return null; } - if (value instanceof Number) { - return findOpStatus(((Number) value).intValue()); + if (value instanceof Number n) { + return findOpStatus(n.intValue()); } return parse(value.toString()); } @@ -153,8 +153,8 @@ public static MaintenanceOpStatus findOpStatus(int ordinal) { } @Override public String format(Object value) { - if (value instanceof MaintenanceOpStatus) { - return value.toString(); + if (value instanceof MaintenanceOpStatus op) { + return op.toString(); } return null; } @@ -247,12 +247,12 @@ public static void declareTableProperty(String name, PropertyType type, Objec */ @Override public PropertySchema getSchema(String schemaName) { - switch(schemaName) { - case CLUSTER_PREFIX: return CLUSTER_SCHEMA; - case DATABASE_PREFIX: return DATABASE_SCHEMA; - case TABLE_PREFIX: return TABLE_SCHEMA; - default: return null; - } + return switch (schemaName) { + case CLUSTER_PREFIX -> CLUSTER_SCHEMA; + case DATABASE_PREFIX -> DATABASE_SCHEMA; + case TABLE_PREFIX -> TABLE_SCHEMA; + default -> null; + }; } /** diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyManager.java index c12445339459..96d4156d51a5 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyManager.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyManager.java @@ -75,14 +75,14 @@ public abstract class PropertyManager { /** A Jexl engine for convenience. */ static final JexlEngine JEXL; static { - JexlFeatures features = new JexlFeatures() + JexlFeatures features = JexlFeatures.createDefault() .sideEffect(false) .sideEffectGlobal(false); - JexlPermissions p = JexlPermissions.RESTRICTED + JexlPermissions permissions = JexlPermissions.RESTRICTED .compose("org.apache.hadoop.hive.metastore.properties.*"); JEXL = new JexlBuilder() .features(features) - .permissions(p) + .permissions(permissions) .create(); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyMap.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyMap.java index 06a7f7aa31bd..5918bc1c5828 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyMap.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyMap.java @@ -135,7 +135,7 @@ public PropertyMap(PropertyMap src, Map input) { /** * Deserialization ctor. - *

Used through deserializtion through reflection by the serialization proxy.

+ *

Used through deserialization through reflection by the serialization proxy.

* @param input the input stream * @throws IOException if IO fail */ diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyStore.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyStore.java index 0bc6beabf6c0..911575d17937 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyStore.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyStore.java @@ -25,7 +25,7 @@ /** * The PropertyStore is the persistent container of property maps. - * Maps are addressed in the store by their key - their name prepended by their manager"s namespace. + * Maps are addressed in the store by their key - their name prepended by their manager"s namespace. */ public abstract class PropertyStore { diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyType.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyType.java index 930b625f593e..89a1b0c0baac 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyType.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/PropertyType.java @@ -182,8 +182,8 @@ public String parse(String str) { @Nullable @Override public String format(Object value) { - if (value instanceof Boolean) { - return ((Boolean) value) ? "true" : "false"; + if (value instanceof Boolean b) { + return b.toString(); } return null; } @@ -201,8 +201,8 @@ public Boolean read(DataInput in) throws IOException { public static final PropertyType INTEGER = new PropertyType("integer") { @Override public Integer cast(Object value) { - if (value instanceof Number) { - return ((Number) value).intValue(); + if (value instanceof Number n) { + return n.intValue(); } if (value == null) { return null; @@ -241,8 +241,8 @@ public Integer read(DataInput in) throws IOException { public static final PropertyType LONG = new PropertyType("long"){ @Override public Long cast(Object value) { - if (value instanceof Number) { - return ((Number) value).longValue(); + if (value instanceof Number n) { + return n.longValue(); } if (value == null) { return null; @@ -281,11 +281,11 @@ public Long read(DataInput in) throws IOException { public static final PropertyType DATETIME = new PropertyType("date"){ @Override public Date cast(Object value) { - if (value instanceof Number) { - return new Date(((Number) value).longValue()); + if (value instanceof Number n) { + return new Date(n.longValue()); } - if (value instanceof Date) { - return (Date) value; + if (value instanceof Date d) { + return d; } if (value == null) { return null; @@ -322,8 +322,8 @@ public Long read(DataInput in) throws IOException { public static final PropertyType DOUBLE = new PropertyType("double"){ @Override public Double cast(Object value) { - if (value instanceof Number) { - return ((Number) value).doubleValue(); + if (value instanceof Number n) { + return n.doubleValue(); } if (value == null) { return null; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/SerializationProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/SerializationProxy.java index 8321d0ee276a..9a90f223cb58 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/SerializationProxy.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/SerializationProxy.java @@ -33,6 +33,7 @@ import java.io.ObjectOutput; import java.io.ObjectOutputStream; import java.io.OutputStream; +import java.io.Serial; import java.io.Serializable; import java.lang.reflect.Constructor; import java.lang.reflect.Executable; @@ -75,6 +76,7 @@ */ public class SerializationProxy implements Externalizable { /** Serial version. */ + @Serial private static final long serialVersionUID = 202212281757L; /** The logger. */ public static final Logger LOGGER = LoggerFactory.getLogger(SerializationProxy.class); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/package-info.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/package-info.java new file mode 100644 index 000000000000..4f237331a112 --- /dev/null +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/properties/package-info.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Type-safe, schema-driven property management for Hive metastore objects (clusters, databases, tables). + * + *

This package allows declaring, validating, and persisting custom properties with runtime type + * checking and serialization support.

+ * + *

Core Components

+ * + *

{@link org.apache.hadoop.hive.metastore.properties.PropertySchema}

+ *

Defines the blueprint for what properties can be set on a metastore object.

+ *
    + *
  • Declares allowed properties and their types (STRING, INTEGER, BOOLEAN, DATE, JSON, etc.)
    • + *
  • Manages default values for properties
    • + *
  • Tracks schema version — incremented when properties are added or removed
    • + *
  • Generates a digest (UUID) for schema identity and caching
    • + *
+ * + *

{@link org.apache.hadoop.hive.metastore.properties.PropertyType}

+ *

Abstract type system for property values with parsing, formatting, and binary serialization. + * Built-in types: {@code STRING}, {@code BOOLEAN}, {@code INTEGER}, {@code LONG}, {@code DOUBLE}, + * {@code DATETIME} (ISO8601/UTC), {@code JSON} (Gson-backed). Custom types can be registered via + * {@link org.apache.hadoop.hive.metastore.properties.PropertyType#register(PropertyType)}.

+ * + *

{@link org.apache.hadoop.hive.metastore.properties.PropertyMap}

+ *

Holds the actual property values for a specific metastore object instance, validated against + * a {@code PropertySchema}. Uses copy-on-write semantics gated by a dirty flag for thread-safe, + * efficient sharing.

+ * + *

{@link org.apache.hadoop.hive.metastore.properties.PropertyManager}

+ *

High-level per-session manager that ties schemas into one namespace and drives transactional + * reads and writes. Tracks dirty maps to batch persistence store updates. Integrates with JEXL + * for dynamic property expressions.

+ * + *

{@link org.apache.hadoop.hive.metastore.properties.PropertyStore}

+ *

Persistence layer for loading and saving property maps to the metastore database. + * Can be wrapped by {@link org.apache.hadoop.hive.metastore.properties.CachingPropertyStore} + * for performance.

+ * + *

Workflow Example

+ * 
{@code
+ * // Define a schema (typically done at startup)
+ * PropertySchema tableSchema = new PropertySchema("table-schema", 1, new TreeMap<>());
+ * tableSchema.declareProperty("owner", PropertyType.STRING, "admin");
+ * tableSchema.declareProperty("is_external", PropertyType.BOOLEAN, false);
+ * tableSchema.declareProperty("created_time", PropertyType.DATETIME);
+ *
+ * // Create a property map for an object, set values, and persist
+ * PropertyMap tableProps = manager.newPropertyMap(tableSchema);
+ * tableProps.put("owner", "alice");
+ * tableProps.put("is_external", true);
+ * manager.persistProperties(tableProps);
+ * }
+ * + *

Design Patterns

+ *
    + *
  • Thread safety: schemas use {@code AtomicInteger} version counters; maps use + * copy-on-write with dirty flags; digest computation is double-checked locked.
    • + *
  • Serialization: a custom {@link org.apache.hadoop.hive.metastore.properties.SerializationProxy} + * handles transient fields safely; both {@code DataInput}/{@code DataOutput} and Java object + * serialization are supported.
    • + *
  • Schema versioning: version and digest UUID allow detecting schema changes and + * support schema evolution without breaking existing data.
    • + *
+ */ +package org.apache.hadoop.hive.metastore.properties; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java index adea578f2392..fd3caea34cf1 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java @@ -52,7 +52,100 @@ import java.util.TreeMap; /** - * The property-maps servlet. + * HTTP servlet exposing the {@link org.apache.hadoop.hive.metastore.properties.PropertyManager} + * API over JSON. + * + *

The servlet is mounted at a configurable path and port (see + * {@code MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT} and + * {@code MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH}). The last segment of the request URI is + * used as the property manager namespace, so the same servlet instance can serve multiple + * independent namespaces.

+ * + *

Namespace and Schema resolution

+ *

The last URI segment is the namespace. Each namespace must be registered in advance + * by calling {@link org.apache.hadoop.hive.metastore.properties.PropertyManager#declare(String, Class)} + * with a concrete {@link org.apache.hadoop.hive.metastore.properties.PropertyManager} subclass. + * That subclass is instantiated per request via its {@code (String, PropertyStore)} constructor.

+ * + *

Within a namespace, the {@link org.apache.hadoop.hive.metastore.properties.PropertySchema} is + * not fixed — it is resolved dynamically by the manager from the number of dot-separated + * key fragments. For example, {@link org.apache.hadoop.hive.metastore.properties.HMSPropertyManager} + * (registered under {@code "hms"}) uses:

+ * + * + * + * + * + * + *
HMSPropertyManager key-to-schema mapping
FragmentsExample keySchema
1{@code name}CLUSTER_SCHEMA
2{@code db.name}DATABASE_SCHEMA
3+{@code db.table.name}TABLE_SCHEMA
+ *

A namespace with no registered manager throws {@code NoSuchObjectException} (HTTP 400).

+ * + *

HTTP Methods

+ * + *

GET — fetch properties by key

+ *

One or more {@code key} query parameters select individual property values. + * Returns a JSON object mapping each found key to its string value.

+ * 
{@code
+ * GET /properties/mynamespace?key=db1.table1.owner&key=db1.table1.created_time
+ * → { "db1.table1.owner": "alice", "db1.table1.created_time": "2024-01-15T08:00:00.00Z" }
+ * }
+ * + *

PUT — set properties

+ *

Request body must be a JSON object mapping qualified keys to values. All updates are + * committed atomically; any error triggers a rollback.

+ * 
{@code
+ * PUT /properties/mynamespace
+ * { "db1.table1.owner": "bob", "db1.table1.is_external": "true" }
+ * }
+ * + *

POST — query or script

+ *

The request body is a single JSON action object or a JSON array of action objects. + * Each action must have a {@code "method"} field (defaults to {@code "selectProperties"} if absent). + * Returns a single result object, or an array when multiple actions are submitted. + * All actions in one request share a transaction; any failure rolls back the entire batch.

+ * + *

Supported methods:

+ *
+ *
{@code fetchProperties}
+ *
Fetches exact values for a list of fully-qualified keys. + * 
{@code
+ * { "method": "fetchProperties", "keys": ["db1.table1.owner", "db1.table1.tags"] }
+ * → { "db1.table1.owner": "alice" }
+ *   }
+ * + *
{@code selectProperties}
+ *
Selects property maps whose key starts with {@code prefix}, optionally filtered by a + * JEXL {@code predicate} expression and projected to a subset of property names via + * {@code selection}. + * 
{@code
+ * { "method": "selectProperties",
+ *   "prefix": "db1.",
+ *   "predicate": "owner == 'alice'",
+ *   "selection": ["owner", "is_external"] }
+ * → { "db1.table1": { "owner": "alice", "is_external": "true" } }
+ *   }
+ * + *
{@code script}
+ *
Executes a JEXL script via {@link org.apache.hadoop.hive.metastore.properties.PropertyManager#runScript}. + * 
{@code
+ * { "method": "script", "source": "select('db1.', null, null)" }
+ *   }
+ * + *
{@code echo}
+ *
Returns the action object unchanged. Useful for health checks and round-trip testing.
+ *
+ * + *

Error handling

+ *

{@link org.apache.hadoop.hive.metastore.properties.PropertyException} and + * {@code NoSuchObjectException} map to HTTP 400 (Bad Request); + * all other exceptions map to HTTP 500 (Internal Server Error).

+ * + *

Configuration

+ *
    + *
  • {@code MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT} — listening port (negative disables the servlet)
    • + *
  • {@code MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH} — URL path prefix
    • + *
  • {@code MetastoreConf.ConfVars.PROPERTIES_SERVLET_AUTH} — authentication type
    • + *
*/ public class PropertyServlet extends HttpServlet { /** The common prefix for errors. */ diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index dc25fe3410a8..e2e0906e7d01 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -88,7 +88,7 @@ 3.4.2 4.0.3 2.18.6 - 3.3 + 3.7.0 5.5.1 4.13.2 5.13.3