HDDS-15872. Ozone Bucket Support StoragePolicy and allowFallbackStoragePolicy.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -297,7 +297,9 @@ public final class OzoneConsts {
   public static final String ACLS = "acls";
   public static final String MAX_NUM_OF_BUCKETS = "maxNumOfBuckets";
   public static final String HAS_SNAPSHOT = "hasSnapshot";
-  public static final String STORAGE_TYPE = "storageType";
+  public static final String STORE_TYPE = "storeType";
+  public static final String STORAGE_POLICY = "storagePolicy";
+  public static final String ALLOW_FALLBACK_STORAGE_POLICY = "allowFallbackStoragePolicy";
   public static final String RESOURCE_TYPE = "resourceType";
   public static final String IS_VERSION_ENABLED = "isVersionEnabled";
   public static final String CREATION_TIME = "creationTime";

hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java

@@ -21,7 +21,6 @@
 import java.io.IOException;
 import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
 import org.apache.hadoop.hdds.client.OzoneQuota;
-import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.client.BucketArgs;
 import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -80,7 +79,7 @@ public void execute(OzoneClient client, OzoneAddress address)
     }
 
     BucketArgs.Builder bb =
-        new BucketArgs.Builder().setStorageType(StorageType.DEFAULT)
+        new BucketArgs.Builder()
             .setVersioning(false).setOwner(ownerName);
     if (allowedBucketLayout != null) {
       bb.setBucketLayout(allowedBucketLayout);

hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/LinkBucketHandler.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.ozone.shell.bucket;
 
 import java.io.IOException;
-import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.ozone.client.BucketArgs;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
@@ -58,7 +57,6 @@ public void execute(OzoneClient client, OzoneAddress address)
       throws IOException {
 
     BucketArgs.Builder bb = new BucketArgs.Builder()
-        .setStorageType(StorageType.DEFAULT)
         .setVersioning(false)
         .setSourceVolume(source.getVolumeName())
         .setSourceBucket(source.getBucketName());

hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/tenant/TenantBucketLinkHandler.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.ozone.shell.tenant;
 
 import java.io.IOException;
-import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.ozone.client.BucketArgs;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
@@ -54,7 +53,6 @@ protected void execute(OzoneClient client, OzoneAddress address)
       throws IOException {
 
     BucketArgs.Builder bb = new BucketArgs.Builder()
-        .setStorageType(StorageType.DEFAULT)
         .setVersioning(false)
         .setSourceVolume(source.getVolumeName())
         .setSourceBucket(source.getBucketName());

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java

@@ -25,7 +25,7 @@
 import java.util.Map;
 import net.jcip.annotations.Immutable;
 import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
-import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.hdds.client.StoragePolicy;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -46,10 +46,14 @@ public final class BucketArgs {
    */
   private final boolean versioning;
   /**
-   * Type of storage to be used for this bucket.
-   * [RAM_DISK, SSD, DISK, ARCHIVE]
+   * Storage policy to be used for this bucket.
    */
-  private final StorageType storageType;
+  private final StoragePolicy storagePolicy;
+  /**
+   * Whether creation is allowed to fall back to the policy's fallback tier
+   * when the primary tier is unavailable.
+   */
+  private final Boolean allowFallbackStoragePolicy;
 
   /**
    * Custom key/value metadata.
@@ -77,7 +81,8 @@ public final class BucketArgs {
   private BucketArgs(Builder b) {
     acls = b.acls == null ? ImmutableList.of() : ImmutableList.copyOf(b.acls);
     versioning = b.versioning;
-    storageType = b.storageType;
+    storagePolicy = b.storagePolicy;
+    allowFallbackStoragePolicy = b.allowFallbackStoragePolicy;
     metadata = b.metadata == null ? ImmutableMap.of() : ImmutableMap.copyOf(b.metadata);
     bucketEncryptionKey = b.bucketEncryptionKey;
     sourceVolume = b.sourceVolume;
@@ -98,11 +103,21 @@ public boolean getVersioning() {
   }
 
   /**
-   * Returns the type of storage to be used.
-   * @return StorageType
+   * Returns the storage policy to be used.
+   * @return StoragePolicy
    */
-  public StorageType getStorageType() {
-    return storageType;
+  public StoragePolicy getStoragePolicy() {
+    return storagePolicy;
+  }
+
+  /**
+   * Returns whether creation is allowed to fall back to the policy's
+   * fallback tier when the primary tier is unavailable. Returns {@code null}
+   * when the flag was not explicitly set by the caller.
+   * @return allowFallbackStoragePolicy
+   */
+  public Boolean getAllowFallbackStoragePolicy() {
+    return allowFallbackStoragePolicy;
   }
 
   /**
@@ -190,7 +205,8 @@ public String getOwner() {
    */
   public static class Builder {
     private boolean versioning;
-    private StorageType storageType;
+    private StoragePolicy storagePolicy;
+    private Boolean allowFallbackStoragePolicy;
     private List<OzoneAcl> acls;
     private Map<String, String> metadata;
     private String bucketEncryptionKey;
@@ -212,8 +228,13 @@ public BucketArgs.Builder setVersioning(boolean versionFlag) {
       return this;
     }
 
-    public BucketArgs.Builder setStorageType(StorageType storage) {
-      this.storageType = storage;
+    public BucketArgs.Builder setStoragePolicy(StoragePolicy storage) {
+      this.storagePolicy = storage;
+      return this;
+    }
+
+    public BucketArgs.Builder setAllowFallbackStoragePolicy(Boolean allowFallback) {
+      this.allowFallbackStoragePolicy = allowFallback;
       return this;
     }
 

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -42,8 +42,8 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.client.StoragePolicy;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.OzoneAcl;
@@ -90,10 +90,15 @@ public class OzoneBucket extends WithMetadata {
   private ReplicationConfig defaultReplication;
 
   /**
-   * Type of storage to be used for this bucket.
-   * [RAM_DISK, SSD, DISK, ARCHIVE]
+   * Storage policy of the bucket.
    */
-  private StorageType storageType;
+  private StoragePolicy storagePolicy;
+
+  /**
+   * Whether fallback to the policy's fallback storage tier is allowed when
+   * the primary tier is unavailable.
+   */
+  private Boolean allowFallbackStoragePolicy;
 
   /**
    * Bucket Version flag.
@@ -169,7 +174,8 @@ protected OzoneBucket(Builder builder) {
     // Bucket level replication is not configured by default.
     this.defaultReplication = builder.defaultReplicationConfig != null ?
         builder.defaultReplicationConfig.getReplicationConfig() : null;
-    this.storageType = builder.storageType;
+    this.storagePolicy = builder.storagePolicy;
+    this.allowFallbackStoragePolicy = builder.allowFallbackStoragePolicy;
     this.versioning = builder.versioning;
     if (builder.conf != null) {
       this.listCacheSize = HddsClientUtils.getListCacheSize(builder.conf);
@@ -233,12 +239,24 @@ public List<OzoneAcl> getAcls() throws IOException {
   }
 
   /**
-   * Returns StorageType of the Bucket.
+   * Returns the StoragePolicy of the Bucket.
    *
-   * @return storageType
+   * @return storagePolicy (may be {@code null} for buckets read from records
+   *     that predate storage-policy support).
    */
-  public StorageType getStorageType() {
-    return storageType;
+  public StoragePolicy getStoragePolicy() {
+    return storagePolicy;
+  }
+
+  /**
+   * Returns whether the bucket allows creation to fall back to the storage
+   * policy's fallback tier when the primary tier is unavailable.
+   *
+   * @return allowFallbackStoragePolicy (may be {@code null} when the flag
+   *     was never set).
+   */
+  public Boolean getAllowFallbackStoragePolicy() {
+    return allowFallbackStoragePolicy;
   }
 
   /**
@@ -350,13 +368,13 @@ public boolean setAcl(List<OzoneAcl> acls) throws IOException {
   }
 
   /**
-   * Sets/Changes the storage type of the bucket.
-   * @param newStorageType Storage type to be set
+   * Sets/Changes the storage policy of the bucket.
+   * @param newStoragePolicy Storage policy to be set
    * @throws IOException
    */
-  public void setStorageType(StorageType newStorageType) throws IOException {
-    proxy.setBucketStorageType(volumeName, name, newStorageType);
-    storageType = newStorageType;
+  public void setStoragePolicy(StoragePolicy newStoragePolicy) throws IOException {
+    proxy.setBucketStoragePolicy(volumeName, name, newStoragePolicy);
+    storagePolicy = newStoragePolicy;
   }
 
   /**
@@ -1387,7 +1405,8 @@ public static class Builder extends WithMetadata.Builder {
     private String volumeName;
     private String name;
     private DefaultReplicationConfig defaultReplicationConfig;
-    private StorageType storageType;
+    private StoragePolicy storagePolicy;
+    private Boolean allowFallbackStoragePolicy;
     private Boolean versioning;
     private long usedBytes;
     private long usedNamespace;
@@ -1433,8 +1452,13 @@ public Builder setDefaultReplicationConfig(
       return this;
     }
 
-    public Builder setStorageType(StorageType storageType) {
-      this.storageType = storageType;
+    public Builder setStoragePolicy(StoragePolicy storagePolicy) {
+      this.storagePolicy = storagePolicy;
+      return this;
+    }
+
+    public Builder setAllowFallbackStoragePolicy(Boolean allowFallbackStoragePolicy) {
+      this.allowFallbackStoragePolicy = allowFallbackStoragePolicy;
       return this;
     }
 

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -26,8 +26,8 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.client.StoragePolicy;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneFsServerDefaults;
@@ -283,14 +283,14 @@ void setBucketVersioning(String volumeName, String bucketName,
       throws IOException;
 
   /**
-   * Sets the Storage Class of a Bucket.
+   * Sets the Storage Policy of a Bucket.
    * @param volumeName Name of the Volume
    * @param bucketName Name of the Bucket
-   * @param storageType StorageType to be set
+   * @param storagePolicy StoragePolicy to be set
    * @throws IOException
    */
-  void setBucketStorageType(String volumeName, String bucketName,
-                            StorageType storageType)
+  void setBucketStoragePolicy(String volumeName, String bucketName,
+                              StoragePolicy storagePolicy)
       throws IOException;
 
   /**

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -67,15 +67,16 @@
 import org.apache.hadoop.fs.Syncable;
 import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.OzoneStoragePolicy;
 import org.apache.hadoop.hdds.client.RatisReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfigValidator;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.client.StoragePolicy;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.StorageUnit;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.ContainerClientMetrics;
 import org.apache.hadoop.hdds.scm.OzoneClientConfig;
@@ -637,8 +638,9 @@ public void createBucket(
     }
 
     boolean isVersionEnabled = bucketArgs.getVersioning();
-    StorageType storageType = bucketArgs.getStorageType() == null ?
-        StorageType.DEFAULT : bucketArgs.getStorageType();
+    StoragePolicy storagePolicy = bucketArgs.getStoragePolicy() == null ?
+        OzoneStoragePolicy.getDefaultPolicy() : bucketArgs.getStoragePolicy();
+    Boolean allowFallbackStoragePolicy = bucketArgs.getAllowFallbackStoragePolicy();
     BucketLayout bucketLayout = bucketArgs.getBucketLayout();
     BucketEncryptionKeyInfo bek = null;
     if (bucketArgs.getEncryptionKey() != null) {
@@ -651,7 +653,8 @@ public void createBucket(
         .setBucketName(bucketName)
         .setIsVersionEnabled(isVersionEnabled)
         .addAllMetadata(bucketArgs.getMetadata())
-        .setStorageType(storageType)
+        .setStoragePolicy(storagePolicy)
+        .setAllowFallbackStoragePolicy(allowFallbackStoragePolicy)
         .setSourceVolume(bucketArgs.getSourceVolume())
         .setSourceBucket(bucketArgs.getSourceBucket())
         .setQuotaInBytes(bucketArgs.getQuotaInBytes())
@@ -687,11 +690,11 @@ public void createBucket(
         ? "with bucket layout " + bucketLayout
         : "with server-side default bucket layout";
     LOG.info("Creating Bucket: {}/{}, {}, {} as owner, Versioning {}, " +
-            "Storage Type set to {} and Encryption set to {}, " +
+            "Storage Policy set to {} (allowFallback={}) and Encryption set to {}, " +
             "Replication Type set to {}, Namespace Quota set to {}, " +
             "Space Quota set to {} ",
         volumeName, bucketName, layoutMsg, owner, isVersionEnabled,
-        storageType, bek != null, replicationType,
+        storagePolicy, allowFallbackStoragePolicy, bek != null, replicationType,
         bucketArgs.getQuotaInNamespace(), bucketArgs.getQuotaInBytes());
 
     ozoneManagerClient.createBucket(builder.build());
@@ -1220,16 +1223,16 @@ public void setBucketVersioning(
   }
 
   @Override
-  public void setBucketStorageType(
-      String volumeName, String bucketName, StorageType storageType)
+  public void setBucketStoragePolicy(
+      String volumeName, String bucketName, StoragePolicy storagePolicy)
       throws IOException {
     verifyVolumeName(volumeName);
     verifyBucketName(bucketName);
-    Objects.requireNonNull(storageType, "storageType == null");
+    Objects.requireNonNull(storagePolicy, "storagePolicy == null");
     OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
     builder.setVolumeName(volumeName)
         .setBucketName(bucketName)
-        .setStorageType(storageType);
+        .setStoragePolicy(storagePolicy);
     ozoneManagerClient.setBucketProperty(builder.build());
   }
 
@@ -1323,7 +1326,8 @@ public OzoneBucket getBucketDetails(
     return OzoneBucket.newBuilder(conf, this)
         .setVolumeName(bucketInfo.getVolumeName())
         .setName(bucketInfo.getBucketName())
-        .setStorageType(bucketInfo.getStorageType())
+        .setStoragePolicy(bucketInfo.getStoragePolicy())
+        .setAllowFallbackStoragePolicy(bucketInfo.getAllowFallbackStoragePolicy())
         .setVersioning(bucketInfo.getIsVersionEnabled())
         .setCreationTime(bucketInfo.getCreationTime())
         .setModificationTime(bucketInfo.getModificationTime())
@@ -1356,7 +1360,8 @@ public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
             OzoneBucket.newBuilder(conf, this)
                 .setVolumeName(bucket.getVolumeName())
                 .setName(bucket.getBucketName())
-                .setStorageType(bucket.getStorageType())
+                .setStoragePolicy(bucket.getStoragePolicy())
+                .setAllowFallbackStoragePolicy(bucket.getAllowFallbackStoragePolicy())
                 .setVersioning(bucket.getIsVersionEnabled())
                 .setCreationTime(bucket.getCreationTime())
                 .setModificationTime(bucket.getModificationTime())