diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 859e133ff562..a578bf581b32 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -297,7 +297,9 @@ public final class OzoneConsts { public static final String ACLS = "acls"; public static final String MAX_NUM_OF_BUCKETS = "maxNumOfBuckets"; public static final String HAS_SNAPSHOT = "hasSnapshot"; - public static final String STORAGE_TYPE = "storageType"; + public static final String STORE_TYPE = "storeType"; + public static final String STORAGE_POLICY = "storagePolicy"; + public static final String ALLOW_FALLBACK_STORAGE_POLICY = "allowFallbackStoragePolicy"; public static final String RESOURCE_TYPE = "resourceType"; public static final String IS_VERSION_ENABLED = "isVersionEnabled"; public static final String CREATION_TIME = "creationTime"; diff --git a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java index e74fb0b1b431..ab842f184e5c 100644 --- a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java +++ b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.OzoneQuota; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -80,7 +79,7 @@ public void execute(OzoneClient client, OzoneAddress address) } BucketArgs.Builder bb = - new BucketArgs.Builder().setStorageType(StorageType.DEFAULT) + new BucketArgs.Builder() .setVersioning(false).setOwner(ownerName); if (allowedBucketLayout != null) { bb.setBucketLayout(allowedBucketLayout); diff --git a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/LinkBucketHandler.java b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/LinkBucketHandler.java index e95b0fc36757..5f4b5398f6e9 100644 --- a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/LinkBucketHandler.java +++ b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/LinkBucketHandler.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.shell.bucket; import java.io.IOException; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -58,7 +57,6 @@ public void execute(OzoneClient client, OzoneAddress address) throws IOException { BucketArgs.Builder bb = new BucketArgs.Builder() - .setStorageType(StorageType.DEFAULT) .setVersioning(false) .setSourceVolume(source.getVolumeName()) .setSourceBucket(source.getBucketName()); diff --git a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/tenant/TenantBucketLinkHandler.java b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/tenant/TenantBucketLinkHandler.java index 198437018d73..c8a1a5d18364 100644 --- a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/tenant/TenantBucketLinkHandler.java +++ b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/tenant/TenantBucketLinkHandler.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.shell.tenant; import java.io.IOException; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -54,7 +53,6 @@ protected void execute(OzoneClient client, OzoneAddress address) throws IOException { BucketArgs.Builder bb = new BucketArgs.Builder() - .setStorageType(StorageType.DEFAULT) .setVersioning(false) .setSourceVolume(source.getVolumeName()) .setSourceBucket(source.getBucketName()); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java index f4173963dd6e..e63bf5dfa9b2 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java @@ -25,7 +25,7 @@ import java.util.Map; import net.jcip.annotations.Immutable; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; -import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -46,10 +46,14 @@ public final class BucketArgs { */ private final boolean versioning; /** - * Type of storage to be used for this bucket. - * [RAM_DISK, SSD, DISK, ARCHIVE] + * Storage policy to be used for this bucket. */ - private final StorageType storageType; + private final StoragePolicy storagePolicy; + /** + * Whether creation is allowed to fall back to the policy's fallback tier + * when the primary tier is unavailable. + */ + private final Boolean allowFallbackStoragePolicy; /** * Custom key/value metadata. @@ -77,7 +81,8 @@ public final class BucketArgs { private BucketArgs(Builder b) { acls = b.acls == null ? ImmutableList.of() : ImmutableList.copyOf(b.acls); versioning = b.versioning; - storageType = b.storageType; + storagePolicy = b.storagePolicy; + allowFallbackStoragePolicy = b.allowFallbackStoragePolicy; metadata = b.metadata == null ? ImmutableMap.of() : ImmutableMap.copyOf(b.metadata); bucketEncryptionKey = b.bucketEncryptionKey; sourceVolume = b.sourceVolume; @@ -98,11 +103,21 @@ public boolean getVersioning() { } /** - * Returns the type of storage to be used. - * @return StorageType + * Returns the storage policy to be used. + * @return StoragePolicy */ - public StorageType getStorageType() { - return storageType; + public StoragePolicy getStoragePolicy() { + return storagePolicy; + } + + /** + * Returns whether creation is allowed to fall back to the policy's + * fallback tier when the primary tier is unavailable. Returns {@code null} + * when the flag was not explicitly set by the caller. + * @return allowFallbackStoragePolicy + */ + public Boolean getAllowFallbackStoragePolicy() { + return allowFallbackStoragePolicy; } /** @@ -190,7 +205,8 @@ public String getOwner() { */ public static class Builder { private boolean versioning; - private StorageType storageType; + private StoragePolicy storagePolicy; + private Boolean allowFallbackStoragePolicy; private List acls; private Map metadata; private String bucketEncryptionKey; @@ -212,8 +228,13 @@ public BucketArgs.Builder setVersioning(boolean versionFlag) { return this; } - public BucketArgs.Builder setStorageType(StorageType storage) { - this.storageType = storage; + public BucketArgs.Builder setStoragePolicy(StoragePolicy storage) { + this.storagePolicy = storage; + return this; + } + + public BucketArgs.Builder setAllowFallbackStoragePolicy(Boolean allowFallback) { + this.allowFallbackStoragePolicy = allowFallback; return this; } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 9b5f663b7f09..1999c8144ca7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -42,8 +42,8 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; @@ -90,10 +90,15 @@ public class OzoneBucket extends WithMetadata { private ReplicationConfig defaultReplication; /** - * Type of storage to be used for this bucket. - * [RAM_DISK, SSD, DISK, ARCHIVE] + * Storage policy of the bucket. */ - private StorageType storageType; + private StoragePolicy storagePolicy; + + /** + * Whether fallback to the policy's fallback storage tier is allowed when + * the primary tier is unavailable. + */ + private Boolean allowFallbackStoragePolicy; /** * Bucket Version flag. @@ -169,7 +174,8 @@ protected OzoneBucket(Builder builder) { // Bucket level replication is not configured by default. this.defaultReplication = builder.defaultReplicationConfig != null ? builder.defaultReplicationConfig.getReplicationConfig() : null; - this.storageType = builder.storageType; + this.storagePolicy = builder.storagePolicy; + this.allowFallbackStoragePolicy = builder.allowFallbackStoragePolicy; this.versioning = builder.versioning; if (builder.conf != null) { this.listCacheSize = HddsClientUtils.getListCacheSize(builder.conf); @@ -233,12 +239,24 @@ public List getAcls() throws IOException { } /** - * Returns StorageType of the Bucket. + * Returns the StoragePolicy of the Bucket. * - * @return storageType + * @return storagePolicy (may be {@code null} for buckets read from records + * that predate storage-policy support). */ - public StorageType getStorageType() { - return storageType; + public StoragePolicy getStoragePolicy() { + return storagePolicy; + } + + /** + * Returns whether the bucket allows creation to fall back to the storage + * policy's fallback tier when the primary tier is unavailable. + * + * @return allowFallbackStoragePolicy (may be {@code null} when the flag + * was never set). + */ + public Boolean getAllowFallbackStoragePolicy() { + return allowFallbackStoragePolicy; } /** @@ -350,13 +368,13 @@ public boolean setAcl(List acls) throws IOException { } /** - * Sets/Changes the storage type of the bucket. - * @param newStorageType Storage type to be set + * Sets/Changes the storage policy of the bucket. + * @param newStoragePolicy Storage policy to be set * @throws IOException */ - public void setStorageType(StorageType newStorageType) throws IOException { - proxy.setBucketStorageType(volumeName, name, newStorageType); - storageType = newStorageType; + public void setStoragePolicy(StoragePolicy newStoragePolicy) throws IOException { + proxy.setBucketStoragePolicy(volumeName, name, newStoragePolicy); + storagePolicy = newStoragePolicy; } /** @@ -1387,7 +1405,8 @@ public static class Builder extends WithMetadata.Builder { private String volumeName; private String name; private DefaultReplicationConfig defaultReplicationConfig; - private StorageType storageType; + private StoragePolicy storagePolicy; + private Boolean allowFallbackStoragePolicy; private Boolean versioning; private long usedBytes; private long usedNamespace; @@ -1433,8 +1452,13 @@ public Builder setDefaultReplicationConfig( return this; } - public Builder setStorageType(StorageType storageType) { - this.storageType = storageType; + public Builder setStoragePolicy(StoragePolicy storagePolicy) { + this.storagePolicy = storagePolicy; + return this; + } + + public Builder setAllowFallbackStoragePolicy(Boolean allowFallbackStoragePolicy) { + this.allowFallbackStoragePolicy = allowFallbackStoragePolicy; return this; } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index a8e75eed7469..49d5b82cf3ec 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -26,8 +26,8 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneFsServerDefaults; @@ -283,14 +283,14 @@ void setBucketVersioning(String volumeName, String bucketName, throws IOException; /** - * Sets the Storage Class of a Bucket. + * Sets the Storage Policy of a Bucket. * @param volumeName Name of the Volume * @param bucketName Name of the Bucket - * @param storageType StorageType to be set + * @param storagePolicy StoragePolicy to be set * @throws IOException */ - void setBucketStorageType(String volumeName, String bucketName, - StorageType storageType) + void setBucketStoragePolicy(String volumeName, String bucketName, + StoragePolicy storagePolicy) throws IOException; /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 01b7b8d97964..0218522b1d69 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -67,15 +67,16 @@ import org.apache.hadoop.fs.Syncable; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfigValidator; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ContainerClientMetrics; import org.apache.hadoop.hdds.scm.OzoneClientConfig; @@ -637,8 +638,9 @@ public void createBucket( } boolean isVersionEnabled = bucketArgs.getVersioning(); - StorageType storageType = bucketArgs.getStorageType() == null ? - StorageType.DEFAULT : bucketArgs.getStorageType(); + StoragePolicy storagePolicy = bucketArgs.getStoragePolicy() == null ? + OzoneStoragePolicy.getDefaultPolicy() : bucketArgs.getStoragePolicy(); + Boolean allowFallbackStoragePolicy = bucketArgs.getAllowFallbackStoragePolicy(); BucketLayout bucketLayout = bucketArgs.getBucketLayout(); BucketEncryptionKeyInfo bek = null; if (bucketArgs.getEncryptionKey() != null) { @@ -651,7 +653,8 @@ public void createBucket( .setBucketName(bucketName) .setIsVersionEnabled(isVersionEnabled) .addAllMetadata(bucketArgs.getMetadata()) - .setStorageType(storageType) + .setStoragePolicy(storagePolicy) + .setAllowFallbackStoragePolicy(allowFallbackStoragePolicy) .setSourceVolume(bucketArgs.getSourceVolume()) .setSourceBucket(bucketArgs.getSourceBucket()) .setQuotaInBytes(bucketArgs.getQuotaInBytes()) @@ -687,11 +690,11 @@ public void createBucket( ? "with bucket layout " + bucketLayout : "with server-side default bucket layout"; LOG.info("Creating Bucket: {}/{}, {}, {} as owner, Versioning {}, " + - "Storage Type set to {} and Encryption set to {}, " + + "Storage Policy set to {} (allowFallback={}) and Encryption set to {}, " + "Replication Type set to {}, Namespace Quota set to {}, " + "Space Quota set to {} ", volumeName, bucketName, layoutMsg, owner, isVersionEnabled, - storageType, bek != null, replicationType, + storagePolicy, allowFallbackStoragePolicy, bek != null, replicationType, bucketArgs.getQuotaInNamespace(), bucketArgs.getQuotaInBytes()); ozoneManagerClient.createBucket(builder.build()); @@ -1220,16 +1223,16 @@ public void setBucketVersioning( } @Override - public void setBucketStorageType( - String volumeName, String bucketName, StorageType storageType) + public void setBucketStoragePolicy( + String volumeName, String bucketName, StoragePolicy storagePolicy) throws IOException { verifyVolumeName(volumeName); verifyBucketName(bucketName); - Objects.requireNonNull(storageType, "storageType == null"); + Objects.requireNonNull(storagePolicy, "storagePolicy == null"); OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); builder.setVolumeName(volumeName) .setBucketName(bucketName) - .setStorageType(storageType); + .setStoragePolicy(storagePolicy); ozoneManagerClient.setBucketProperty(builder.build()); } @@ -1323,7 +1326,8 @@ public OzoneBucket getBucketDetails( return OzoneBucket.newBuilder(conf, this) .setVolumeName(bucketInfo.getVolumeName()) .setName(bucketInfo.getBucketName()) - .setStorageType(bucketInfo.getStorageType()) + .setStoragePolicy(bucketInfo.getStoragePolicy()) + .setAllowFallbackStoragePolicy(bucketInfo.getAllowFallbackStoragePolicy()) .setVersioning(bucketInfo.getIsVersionEnabled()) .setCreationTime(bucketInfo.getCreationTime()) .setModificationTime(bucketInfo.getModificationTime()) @@ -1356,7 +1360,8 @@ public List listBuckets(String volumeName, String bucketPrefix, OzoneBucket.newBuilder(conf, this) .setVolumeName(bucket.getVolumeName()) .setName(bucket.getBucketName()) - .setStorageType(bucket.getStorageType()) + .setStoragePolicy(bucket.getStoragePolicy()) + .setAllowFallbackStoragePolicy(bucket.getAllowFallbackStoragePolicy()) .setVersioning(bucket.getIsVersionEnabled()) .setCreationTime(bucket.getCreationTime()) .setModificationTime(bucket.getModificationTime()) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/hdds/protocol/StorageType.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/hdds/protocol/StorageType.java deleted file mode 100644 index c455302427b5..000000000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/hdds/protocol/StorageType.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.protocol; - -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; - -/** - * Ozone specific storage types. - * - * @deprecated Ozone buckets should not have a StorageType attribute. - * This class is specific to `OzoneBucket` and is planned for removal in future versions. - * It is recommended to use `{@link org.apache.hadoop.fs.StorageType}` instead for - * any storage type requirements. - */ -@Deprecated -public enum StorageType { - RAM_DISK, - SSD, - DISK, - ARCHIVE; - - public static final StorageType DEFAULT = DISK; - - public StorageTypeProto toProto() { - switch (this) { - case DISK: - return StorageTypeProto.DISK; - case SSD: - return StorageTypeProto.SSD; - case ARCHIVE: - return StorageTypeProto.ARCHIVE; - case RAM_DISK: - return StorageTypeProto.RAM_DISK; - default: - throw new IllegalStateException( - "BUG: StorageType not found, type=" + this); - } - } - - public static StorageType valueOf(StorageTypeProto type) { - switch (type) { - case DISK: - return DISK; - case SSD: - return SSD; - case ARCHIVE: - return ARCHIVE; - case RAM_DISK: - return RAM_DISK; - default: - throw new IllegalStateException( - "BUG: StorageTypeProto not found, type=" + type); - } - } -} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java index 8eed2630ead6..c25173e26142 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java @@ -22,7 +22,8 @@ import java.util.Map; import java.util.Objects; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; -import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.Auditable; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketArgs; @@ -45,10 +46,15 @@ public final class OmBucketArgs extends WithMetadata implements Auditable { */ private final Boolean isVersionEnabled; /** - * Type of storage to be used for this bucket. - * [RAM_DISK, SSD, DISK, ARCHIVE] + * Storage policy for the bucket. */ - private final StorageType storageType; + private final StoragePolicy storagePolicy; + /** + * Whether creation is allowed to fall back to the storage policy's + * fallback tier when the primary tier is unavailable. {@code null} when + * the flag was not set. + */ + private final Boolean allowFallbackStoragePolicy; /** * Bucket encryption key info if encryption is enabled. @@ -73,7 +79,8 @@ private OmBucketArgs(Builder b) { this.volumeName = b.volumeName; this.bucketName = b.bucketName; this.isVersionEnabled = b.isVersionEnabled; - this.storageType = b.storageType; + this.storagePolicy = b.storagePolicy; + this.allowFallbackStoragePolicy = b.allowFallbackStoragePolicy; this.ownerName = b.ownerName; this.defaultReplicationConfig = b.defaultReplicationConfig; this.quotaInBytesSet = b.quotaInBytesSet; @@ -109,11 +116,20 @@ public Boolean getIsVersionEnabled() { } /** - * Returns the type of storage to be used. - * @return StorageType + * Returns the storage policy for this bucket. + * @return StoragePolicy (may be {@code null} when not set). + */ + public StoragePolicy getStoragePolicy() { + return storagePolicy; + } + + /** + * Returns whether creation is allowed to fall back to the storage policy's + * fallback tier when the primary tier is unavailable. + * @return allowFallbackStoragePolicy (may be {@code null} when not set). */ - public StorageType getStorageType() { - return storageType; + public Boolean getAllowFallbackStoragePolicy() { + return allowFallbackStoragePolicy; } /** @@ -190,8 +206,12 @@ public Map toAuditMap() { getMetadata().get(OzoneConsts.GDPR_FLAG)); auditMap.put(OzoneConsts.IS_VERSION_ENABLED, String.valueOf(this.isVersionEnabled)); - if (this.storageType != null) { - auditMap.put(OzoneConsts.STORAGE_TYPE, this.storageType.name()); + if (this.storagePolicy != null) { + auditMap.put(OzoneConsts.STORAGE_POLICY, this.storagePolicy.getName()); + } + if (this.allowFallbackStoragePolicy != null) { + auditMap.put(OzoneConsts.ALLOW_FALLBACK_STORAGE_POLICY, + String.valueOf(this.allowFallbackStoragePolicy)); } if (this.ownerName != null) { auditMap.put(OzoneConsts.OWNER, this.ownerName); @@ -227,7 +247,8 @@ public static class Builder extends WithMetadata.Builder { private String volumeName; private String bucketName; private Boolean isVersionEnabled; - private StorageType storageType; + private StoragePolicy storagePolicy; + private Boolean allowFallbackStoragePolicy; private boolean quotaInBytesSet = false; private long quotaInBytes; private boolean quotaInNamespaceSet = false; @@ -275,8 +296,13 @@ public Builder addAllMetadata(Map map) { return this; } - public Builder setStorageType(StorageType storage) { - this.storageType = storage; + public Builder setStoragePolicy(StoragePolicy storage) { + this.storagePolicy = storage; + return this; + } + + public Builder setAllowFallbackStoragePolicy(Boolean allowFallback) { + this.allowFallbackStoragePolicy = allowFallback; return this; } @@ -339,8 +365,11 @@ public BucketArgs getProtobuf() { if (isVersionEnabled != null) { builder.setIsVersionEnabled(isVersionEnabled); } - if (storageType != null) { - builder.setStorageType(storageType.toProto()); + if (storagePolicy != null) { + builder.setStoragePolicy(OzoneStoragePolicy.toProto(storagePolicy)); + } + if (allowFallbackStoragePolicy != null) { + builder.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy); } if (quotaInBytesSet && ( quotaInBytes > 0 || quotaInBytes == OzoneConsts.QUOTA_RESET)) { @@ -381,8 +410,11 @@ public static Builder builderFromProtobuf(BucketArgs bucketArgs) { if (bucketArgs.hasIsVersionEnabled()) { builder.setIsVersionEnabled(bucketArgs.getIsVersionEnabled()); } - if (bucketArgs.hasStorageType()) { - builder.setStorageType(StorageType.valueOf(bucketArgs.getStorageType())); + if (bucketArgs.hasStoragePolicy()) { + builder.setStoragePolicy(OzoneStoragePolicy.fromProto(bucketArgs.getStoragePolicy())); + } + if (bucketArgs.hasAllowFallbackStoragePolicy()) { + builder.setAllowFallbackStoragePolicy(bucketArgs.getAllowFallbackStoragePolicy()); } if (bucketArgs.hasOwnerName()) { builder.setOwnerName(bucketArgs.getOwnerName()); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java index 639dbaa9f392..df21bc6131a9 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java @@ -25,7 +25,9 @@ import java.util.Objects; import java.util.stream.Collectors; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; -import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; +import org.apache.hadoop.hdds.client.StoragePolicy; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.CopyObject; import org.apache.hadoop.hdds.utils.db.DelegatedCodec; @@ -63,10 +65,15 @@ public final class OmBucketInfo extends WithObjectID implements Auditable, CopyO */ private final boolean isVersionEnabled; /** - * Type of storage to be used for this bucket. - * [RAM_DISK, SSD, DISK, ARCHIVE] + * Storage policy for this bucket. May be {@code null} for records read + * from serialized bucket rows that predate storage-policy support. */ - private final StorageType storageType; + private final StoragePolicy storagePolicy; + /** + * Whether creation is allowed to fall back to the storage policy's + * fallback tier when the primary tier is unavailable. + */ + private final Boolean allowFallbackStoragePolicy; /** * Creation time of bucket. */ @@ -119,7 +126,8 @@ private OmBucketInfo(Builder b) { this.bucketName = b.bucketName; this.acls = b.acls.build(); this.isVersionEnabled = b.isVersionEnabled; - this.storageType = b.storageType; + this.storagePolicy = b.storagePolicy; + this.allowFallbackStoragePolicy = b.allowFallbackStoragePolicy; this.creationTime = b.creationTime; this.modificationTime = b.modificationTime; this.bekInfo = b.bekInfo; @@ -174,11 +182,20 @@ public boolean getIsVersionEnabled() { } /** - * Returns the type of storage to be used. - * @return StorageType + * Returns the storage policy for this bucket. + * @return StoragePolicy (may be {@code null} for older serialized rows). */ - public StorageType getStorageType() { - return storageType; + public StoragePolicy getStoragePolicy() { + return storagePolicy; + } + + /** + * Returns whether creation is allowed to fall back to the storage policy's + * fallback tier when the primary tier is unavailable. + * @return allowFallbackStoragePolicy (may be {@code null} when not set). + */ + public Boolean getAllowFallbackStoragePolicy() { + return allowFallbackStoragePolicy; } /** @@ -338,8 +355,11 @@ public Map toAuditMap() { (this.acls != null) ? this.acls.toString() : null); auditMap.put(OzoneConsts.IS_VERSION_ENABLED, String.valueOf(this.isVersionEnabled)); - auditMap.put(OzoneConsts.STORAGE_TYPE, - (this.storageType != null) ? this.storageType.name() : null); + auditMap.put(OzoneConsts.STORAGE_POLICY, + (this.storagePolicy != null) ? this.storagePolicy.getName() : null); + auditMap.put(OzoneConsts.ALLOW_FALLBACK_STORAGE_POLICY, + (this.allowFallbackStoragePolicy != null) + ? String.valueOf(this.allowFallbackStoragePolicy) : null); auditMap.put(OzoneConsts.CREATION_TIME, String.valueOf(this.creationTime)); auditMap.put(OzoneConsts.BUCKET_ENCRYPTION_KEY, (bekInfo != null) ? bekInfo.getKeyName() : null); @@ -377,7 +397,8 @@ public Builder toBuilder() { return new Builder(this) .setVolumeName(volumeName) .setBucketName(bucketName) - .setStorageType(storageType) + .setStoragePolicy(storagePolicy) + .setAllowFallbackStoragePolicy(allowFallbackStoragePolicy) .setIsVersionEnabled(isVersionEnabled) .setCreationTime(creationTime) .setModificationTime(modificationTime) @@ -406,7 +427,8 @@ public Builder toBuilder() { */ public OmBucketInfo withOperationalPropertiesFrom(OmBucketInfo source) { return toBuilder() - .setStorageType(source.getStorageType()) + .setStoragePolicy(source.getStoragePolicy()) + .setAllowFallbackStoragePolicy(source.getAllowFallbackStoragePolicy()) .setIsVersionEnabled(source.getIsVersionEnabled()) .setBucketEncryptionKey(source.getEncryptionKeyInfo()) .setUsedBytes(source.getUsedBytes()) @@ -430,7 +452,8 @@ public static class Builder extends WithObjectID.Builder { private String bucketName; private final AclListBuilder acls; private boolean isVersionEnabled; - private StorageType storageType = StorageType.DISK; + private StoragePolicy storagePolicy; + private Boolean allowFallbackStoragePolicy; private long creationTime; private long modificationTime; private BucketEncryptionKeyInfo bekInfo; @@ -491,8 +514,13 @@ public Builder setIsVersionEnabled(boolean versionFlag) { return this; } - public Builder setStorageType(StorageType storage) { - this.storageType = storage; + public Builder setStoragePolicy(StoragePolicy storage) { + this.storagePolicy = storage; + return this; + } + + public Builder setAllowFallbackStoragePolicy(Boolean allowFallback) { + this.allowFallbackStoragePolicy = allowFallback; return this; } @@ -607,8 +635,9 @@ protected void validate() { Objects.requireNonNull(volumeName, "volumeName == null"); Objects.requireNonNull(bucketName, "bucketName == null"); Objects.requireNonNull(acls, "acls == null"); - Objects.requireNonNull(storageType, "storageType == null"); Objects.requireNonNull(tags, "tags == null"); + // storagePolicy may be null for records read from older serialized + // bucket rows written before storage-policy support existed. } @Override @@ -626,7 +655,12 @@ public BucketInfo getProtobuf() { .setBucketName(bucketName) .addAllAcls(OzoneAclUtil.toProtobuf(acls)) .setIsVersionEnabled(isVersionEnabled) - .setStorageType(storageType.toProto()) + // Emit legacy `storageType = DISK` so pre-storage-policy peers whose + // BucketInfo proto still has `required storageType = 5` can deserialize + // messages produced by this OM. The field is deprecated on our side; + // storage policy lives in `storagePolicy` (field 24). Do not read this + // field back into OmBucketInfo. + .setStorageType(HddsProtos.StorageTypeProto.DISK) .setCreationTime(creationTime) .setModificationTime(modificationTime) .setObjectID(getObjectID()) @@ -657,6 +691,12 @@ public BucketInfo getProtobuf() { if (owner != null) { bib.setOwner(owner); } + if (storagePolicy != null) { + bib.setStoragePolicy(OzoneStoragePolicy.toProto(storagePolicy)); + } + if (allowFallbackStoragePolicy != null) { + bib.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy); + } return bib.build(); } @@ -683,7 +723,6 @@ public static Builder builderFromProtobuf(BucketInfo bucketInfo, .setAcls(bucketInfo.getAclsList().stream().map( OzoneAcl::fromProtobuf).collect(Collectors.toList())) .setIsVersionEnabled(bucketInfo.getIsVersionEnabled()) - .setStorageType(StorageType.valueOf(bucketInfo.getStorageType())) .setCreationTime(bucketInfo.getCreationTime()) .setUsedBytes(bucketInfo.getUsedBytes()) .setModificationTime(bucketInfo.getModificationTime()) @@ -729,6 +768,12 @@ public static Builder builderFromProtobuf(BucketInfo bucketInfo, if (bucketInfo.hasOwner()) { obib.setOwner(bucketInfo.getOwner()); } + if (bucketInfo.hasStoragePolicy()) { + obib.setStoragePolicy(OzoneStoragePolicy.fromProto(bucketInfo.getStoragePolicy())); + } + if (bucketInfo.hasAllowFallbackStoragePolicy()) { + obib.setAllowFallbackStoragePolicy(bucketInfo.getAllowFallbackStoragePolicy()); + } return obib; } @@ -762,7 +807,8 @@ public String getObjectInfo() { "volume='" + volumeName + "'" + ", bucket='" + bucketName + "'" + ", isVersionEnabled='" + isVersionEnabled + "'" + - ", storageType='" + storageType + "'" + + ", storagePolicy='" + storagePolicy + "'" + + ", allowFallbackStoragePolicy='" + allowFallbackStoragePolicy + "'" + ", creationTime='" + creationTime + "'" + ", usedBytes='" + usedBytes + "'" + ", usedNamespace='" + usedNamespace + "'" + @@ -789,7 +835,8 @@ public boolean equals(Object o) { bucketName.equals(that.bucketName) && Objects.equals(acls, that.acls) && Objects.equals(isVersionEnabled, that.isVersionEnabled) && - storageType == that.storageType && + Objects.equals(storagePolicy, that.storagePolicy) && + Objects.equals(allowFallbackStoragePolicy, that.allowFallbackStoragePolicy) && getObjectID() == that.getObjectID() && getUpdateID() == that.getUpdateID() && usedBytes == that.usedBytes && @@ -817,7 +864,8 @@ public String toString() { ", bucketName='" + bucketName + "'" + ", acls=" + acls + ", isVersionEnabled=" + isVersionEnabled + - ", storageType=" + storageType + + ", storagePolicy=" + storagePolicy + + ", allowFallbackStoragePolicy=" + allowFallbackStoragePolicy + ", creationTime=" + creationTime + ", bekInfo=" + bekInfo + ", sourceVolume='" + sourceVolume + "'" + diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java index 716ad4d40ad9..0d6b53cba242 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java @@ -139,7 +139,7 @@ public String toString() { public Map toAuditMap() { Map auditMap = new LinkedHashMap<>(); auditMap.put(OzoneConsts.RESOURCE_TYPE, this.getResourceType().value); - auditMap.put(OzoneConsts.STORAGE_TYPE, this.getStoreType().value); + auditMap.put(OzoneConsts.STORE_TYPE, this.getStoreType().value); auditMap.put(OzoneConsts.VOLUME, this.getVolumeName()); auditMap.put(OzoneConsts.BUCKET, this.getBucketName()); auditMap.put(OzoneConsts.KEY, this.getKeyName()); diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java index 3cd3d53508d3..9ed80cb8093f 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java @@ -26,9 +26,9 @@ import java.util.Collections; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; @@ -47,7 +47,7 @@ public void protobufConversion() { .setVolumeName("vol1") .setCreationTime(1L) .setIsVersionEnabled(false) - .setStorageType(StorageType.ARCHIVE) + .setStoragePolicy(OzoneStoragePolicy.COLD) .build(); assertEquals(bucket, @@ -74,7 +74,7 @@ public void testClone() { .setVolumeName("vol1") .setCreationTime(Time.now()) .setIsVersionEnabled(false) - .setStorageType(StorageType.ARCHIVE) + .setStoragePolicy(OzoneStoragePolicy.COLD) .setAcls(Collections.singletonList(OzoneAcl.of( IAccessAuthorizer.ACLIdentityType.USER, "defaultUser", @@ -104,7 +104,7 @@ public void testWithOperationalPropertiesFromPreservesLinkIdentity() { .setVolumeName("vol1") .setBucketName("source") .setBucketLayout(BucketLayout.OBJECT_STORE) - .setStorageType(StorageType.SSD) + .setStoragePolicy(OzoneStoragePolicy.HOT) .setIsVersionEnabled(true) .setQuotaInBytes(1000) .setQuotaInNamespace(10) @@ -135,7 +135,7 @@ public void testWithOperationalPropertiesFromPreservesLinkIdentity() { assertEquals(456L, resolvedLink.getModificationTime()); assertEquals(BucketLayout.OBJECT_STORE, resolvedLink.getBucketLayout()); - assertEquals(StorageType.SSD, resolvedLink.getStorageType()); + assertEquals(OzoneStoragePolicy.HOT, resolvedLink.getStoragePolicy()); assertTrue(resolvedLink.getIsVersionEnabled()); assertEquals(1000, resolvedLink.getQuotaInBytes()); assertEquals(10, resolvedLink.getQuotaInNamespace()); @@ -173,7 +173,7 @@ public void getProtobufMessageEC() { OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder().setBucketName("bucket").setVolumeName("vol1") .setCreationTime(Time.now()).setIsVersionEnabled(false) - .setStorageType(StorageType.ARCHIVE).setAcls(Collections + .setStoragePolicy(OzoneStoragePolicy.COLD).setAcls(Collections .singletonList(OzoneAcl.of( IAccessAuthorizer.ACLIdentityType.USER, "defaultUser", OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE_ACL @@ -192,7 +192,7 @@ public void getProtobufMessageEC() { .setVolumeName("vol1") .setCreationTime(Time.now()) .setIsVersionEnabled(false) - .setStorageType(StorageType.ARCHIVE) + .setStoragePolicy(OzoneStoragePolicy.COLD) .setAcls(Collections.singletonList(OzoneAcl.of( IAccessAuthorizer.ACLIdentityType.USER, "defaultUser", OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE_ACL diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot index 7affbcba16c3..75f0464d72f8 100644 --- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot @@ -48,8 +48,8 @@ Test ozone shell ${result} = Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1 --space-quota 10TB --namespace-quota 100 Should Be Empty ${result} Execute ozone sh bucket info ${protocol}${server}/${volume}/bb1 > ${TMP_JSON} - ${result} = Execute jq -r '. | select(.name=="bb1") | .storageType' ${TMP_JSON} - Should Be Equal ${result} DISK + ${result} = Execute jq -r '. | select(.name=="bb1") | .storagePolicy' ${TMP_JSON} + Should Be Equal ${result} WARM ${result} = Execute jq -r '. | select(.name=="bb1") | .quotaInBytes' ${TMP_JSON} Should Be Equal ${result} 10995116277760 ${result} = Execute jq -r '. | select(.name=="bb1") | .quotaInNamespace' ${TMP_JSON} diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java index 7c55483595ef..11ed92709a0f 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java @@ -21,7 +21,6 @@ import java.util.concurrent.Callable; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; @@ -94,7 +93,6 @@ private void createBucket(long index) throws Exception { OmBucketInfo bucketInfo = new OmBucketInfo.Builder() .setBucketName(generateBucketName(index)) .setVolumeName(volumeName) - .setStorageType(StorageType.DISK) .build(); bucketCreationTimer.time(() -> { diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index b0739c62eaa6..b72cfa8f763f 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -66,7 +66,6 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; @@ -3464,7 +3463,6 @@ public void setBucketVerificationOnLinkBucket() throws Exception { String linkBucketName = "link-bucket"; nonS3Volume.createBucket(OzoneConsts.BUCKET); BucketArgs.Builder bb = new BucketArgs.Builder() - .setStorageType(StorageType.DEFAULT) .setVersioning(false) .setSourceVolume(NON_S3_VOLUME_NAME) .setSourceBucket(OzoneConsts.BUCKET); @@ -3494,7 +3492,6 @@ public void testDanglingBucket() throws Exception { String linkBucket = "link-bucket-dangling"; nonS3Volume.createBucket(sourceBucket); BucketArgs.Builder bb = new BucketArgs.Builder() - .setStorageType(StorageType.DEFAULT) .setVersioning(false) .setSourceVolume(NON_S3_VOLUME_NAME) .setSourceBucket(sourceBucket); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index a0d09264adbe..ca5f075fcafb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -91,7 +91,6 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; @@ -1782,7 +1781,6 @@ void testRenameDestinationParentDoesNotExist() throws Exception { void testBucketDefaultsShouldNotBeInheritedToFileForNonEC() throws Exception { BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(BucketLayout.LEGACY); builder.setDefaultReplicationConfig( new DefaultReplicationConfig(StandaloneReplicationConfig.getInstance( @@ -1811,7 +1809,6 @@ void testBucketDefaultsShouldNotBeInheritedToFileForNonEC() void testBucketDefaultsShouldBeInheritedToFileForEC() throws Exception { BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(BucketLayout.LEGACY); builder.setDefaultReplicationConfig( new DefaultReplicationConfig( @@ -2013,7 +2010,6 @@ private void createLinkBucket(String linkVolume, String linkBucket, private Path createAndGetBucketPath() throws IOException { BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(bucketLayout); BucketArgs omBucketArgs = builder.build(); String vol = UUID.randomUUID().toString(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index 02fa289c0234..a5460caf6176 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -84,7 +84,6 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.ErrorInjector; import org.apache.hadoop.hdds.scm.OzoneClientConfig; @@ -1069,7 +1068,6 @@ public void testOzoneStreamCapabilityForHsyncHflush() throws Exception { public void testECStreamCapability() throws Exception { // create EC bucket to be used by OzoneFileSystem BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); builder.setDefaultReplicationConfig( new DefaultReplicationConfig( diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java index c4073167faac..a73f62d544b7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.ozone.DataTestUtil; @@ -93,7 +92,6 @@ void init() throws Exception { // create EC bucket to be used by OzoneFileSystem BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); builder.setDefaultReplicationConfig( new DefaultReplicationConfig( diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java index 45a855335e0d..d700de09ee46 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -132,7 +131,6 @@ void testEcFileChecksum(List missingIndexes, double checksumSizeInMB) t client.getObjectStore().createVolume(volumeName); BucketArgs.Builder bucketArgs = BucketArgs.newBuilder() - .setStorageType(StorageType.DISK) .setBucketLayout(BucketLayout.LEGACY); createBucket(client, volumeName, bucketArgs.build(), legacyBucket); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/DataTestUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/DataTestUtil.java index b1d10d99b321..f5c2c42bff7b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/DataTestUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/DataTestUtil.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; @@ -70,7 +69,6 @@ public static OzoneBucket createVolumeAndBucket(OzoneClient client, String volumeName, String bucketName, BucketLayout bucketLayout) throws IOException { BucketArgs omBucketArgs; BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); if (bucketLayout != null) { builder.setBucketLayout(bucketLayout); } @@ -85,7 +83,6 @@ public static OzoneBucket createVolumeAndBucket(OzoneClient client, throws IOException { BucketArgs omBucketArgs; BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); if (bucketLayout != null) { builder.setBucketLayout(bucketLayout); } @@ -207,7 +204,6 @@ public static OzoneBucket createBucket(OzoneClient client, public static OzoneBucket createLinkedBucket(OzoneClient client, String vol, String sourceBucketName, String linkedBucketName) throws IOException { BucketArgs.Builder bb = new BucketArgs.Builder() - .setStorageType(StorageType.DEFAULT) .setVersioning(false) .setSourceVolume(vol) .setSourceBucket(sourceBucketName); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 00a419ff0c0f..494e5425ad85 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -116,6 +116,7 @@ import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec; import org.apache.hadoop.hdds.client.OzoneQuota; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; @@ -124,7 +125,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.OzoneClientConfig; @@ -753,18 +753,18 @@ public void testCreateBucketWithVersioning() } @Test - public void testCreateBucketWithStorageType() + public void testCreateBucketWithStoragePolicy() throws IOException { String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); store.createVolume(volumeName); OzoneVolume volume = store.getVolume(volumeName); BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.SSD); + builder.setStoragePolicy(OzoneStoragePolicy.HOT); volume.createBucket(bucketName, builder.build()); OzoneBucket bucket = volume.getBucket(bucketName); assertEquals(bucketName, bucket.getName()); - assertEquals(StorageType.SSD, bucket.getStorageType()); + assertEquals(OzoneStoragePolicy.HOT, bucket.getStoragePolicy()); } @Test @@ -813,14 +813,14 @@ public void testCreateBucketWithAllArgument() OzoneVolume volume = store.getVolume(volumeName); BucketArgs.Builder builder = BucketArgs.newBuilder(); builder.setVersioning(true) - .setStorageType(StorageType.SSD) + .setStoragePolicy(OzoneStoragePolicy.HOT) .addAcl(userAcl) .setDefaultReplicationConfig(new DefaultReplicationConfig(repConfig)); volume.createBucket(bucketName, builder.build()); OzoneBucket bucket = volume.getBucket(bucketName); assertEquals(bucketName, bucket.getName()); assertTrue(bucket.getVersioning()); - assertEquals(StorageType.SSD, bucket.getStorageType()); + assertEquals(OzoneStoragePolicy.HOT, bucket.getStoragePolicy()); assertThat(bucket.getAcls()).contains(userAcl); assertEquals(repConfig, bucket.getReplicationConfig()); } @@ -1000,7 +1000,7 @@ public void testAclDeDuplication() } @Test - public void testSetBucketStorageType() + public void testSetBucketStoragePolicy() throws IOException { String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); @@ -1008,10 +1008,10 @@ public void testSetBucketStorageType() OzoneVolume volume = store.getVolume(volumeName); volume.createBucket(bucketName); OzoneBucket bucket = volume.getBucket(bucketName); - bucket.setStorageType(StorageType.SSD); + bucket.setStoragePolicy(OzoneStoragePolicy.HOT); OzoneBucket newBucket = volume.getBucket(bucketName); assertEquals(bucketName, newBucket.getName()); - assertEquals(StorageType.SSD, newBucket.getStorageType()); + assertEquals(OzoneStoragePolicy.HOT, newBucket.getStoragePolicy()); } @ParameterizedTest @@ -2065,7 +2065,6 @@ public void testVolumeUsedNamespace() throws IOException { OzoneVolume volumeWithLinkedBucket = store.getVolume(targetVolName); String targetBucketName = UUID.randomUUID().toString(); BucketArgs.Builder argsBuilder = new BucketArgs.Builder() - .setStorageType(StorageType.DEFAULT) .setVersioning(false) .setSourceVolume(volumeName) .setSourceBucket(bucketName); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java index 71a9ab3c02ac..a3d9153a379c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java @@ -21,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import java.util.UUID; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.DataTestUtil; @@ -86,8 +85,7 @@ public void testCreateBucketWithOlderClient() throws Exception { .setBucketInfo( OzoneManagerProtocolProtos.BucketInfo.newBuilder() .setVolumeName(volumeName).setBucketName(buckName) - .setIsVersionEnabled(false).setStorageType( - StorageTypeProto.DISK) + .setIsVersionEnabled(false) .build()) .build()).build(); createBucketReq = createBucketReq.toBuilder() diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java index 642be2a8b6ce..90657d33200c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.UUID; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; @@ -74,7 +73,7 @@ void init() throws Exception { ObjectStore objectStore = client.getObjectStore(); OzoneVolume volume = objectStore.getVolume(VOLUME_NAME); BucketArgs omBucketArgs = BucketArgs.newBuilder() - .setStorageType(StorageType.DISK).setOwner(user1.getShortUserName()).build(); + .setOwner(user1.getShortUserName()).build(); volume.createBucket("bucket1", omBucketArgs); volume.createBucket("bucket2", omBucketArgs); volume.createBucket("bucket3", omBucketArgs); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java index 6929dac516ab..d2b7facac132 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java @@ -34,7 +34,6 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -92,7 +91,6 @@ void init() throws Exception { // create buckets BucketArgs omBucketArgs; BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); omBucketArgs = builder.build(); @@ -109,7 +107,6 @@ void init() throws Exception { emptyFsoOzoneBucket = ozoneVolume.getBucket(fsoBucketName); builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(BucketLayout.LEGACY); omBucketArgs = builder.build(); String legacyBucketName = uniqueObjectName("bucket"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java index 7904666b9515..68edf9295e8a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java @@ -33,7 +33,6 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; @@ -172,7 +171,6 @@ public void testMultiPartCompleteUpload() throws Exception { // Test-1: Upload MPU to an OBS layout with Directory Exists String legacyBuckName = UUID.randomUUID().toString(); BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(BucketLayout.OBJECT_STORE); BucketArgs omBucketArgs = builder.build(); volume.createBucket(legacyBuckName, omBucketArgs); @@ -189,7 +187,6 @@ public void testMultiPartCompleteUpload() throws Exception { // Test-2: Upload MPU to an LEGACY layout with Directory Exists legacyBuckName = UUID.randomUUID().toString(); builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); builder.setBucketLayout(BucketLayout.LEGACY); omBucketArgs = builder.build(); volume.createBucket(legacyBuckName, omBucketArgs); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAFollowerReadWithAllRunning.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAFollowerReadWithAllRunning.java index 0e97b2494209..00c3d7484f7a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAFollowerReadWithAllRunning.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAFollowerReadWithAllRunning.java @@ -44,8 +44,8 @@ import java.util.List; import java.util.stream.Stream; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneTestUtils; @@ -442,7 +442,7 @@ public void testAllBucketOperations() throws Exception { OzoneVolume retVolume = createAndCheckVolume(volumeName); BucketArgs bucketArgs = - BucketArgs.newBuilder().setStorageType(StorageType.DISK) + BucketArgs.newBuilder().setStoragePolicy(OzoneStoragePolicy.WARM) .setVersioning(true).build(); @@ -454,7 +454,7 @@ public void testAllBucketOperations() throws Exception { assertEquals(volumeName, ozoneBucket.getVolumeName()); assertEquals(bucketName, ozoneBucket.getName()); assertTrue(ozoneBucket.getVersioning()); - assertEquals(StorageType.DISK, ozoneBucket.getStorageType()); + assertEquals(OzoneStoragePolicy.WARM, ozoneBucket.getStoragePolicy()); assertFalse(ozoneBucket.getCreationTime().isAfter(Instant.now())); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithAllRunning.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithAllRunning.java index 746f99d64958..3aa990d04d0f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithAllRunning.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithAllRunning.java @@ -52,8 +52,8 @@ import javax.management.MBeanServer; import javax.management.ObjectName; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; @@ -258,7 +258,7 @@ public void testAllBucketOperations() throws Exception { OzoneVolume retVolume = createAndCheckVolume(volumeName); BucketArgs bucketArgs = - BucketArgs.newBuilder().setStorageType(StorageType.DISK) + BucketArgs.newBuilder().setStoragePolicy(OzoneStoragePolicy.WARM) .setVersioning(true).build(); @@ -270,7 +270,7 @@ public void testAllBucketOperations() throws Exception { assertEquals(volumeName, ozoneBucket.getVolumeName()); assertEquals(bucketName, ozoneBucket.getName()); assertTrue(ozoneBucket.getVersioning()); - assertEquals(StorageType.DISK, ozoneBucket.getStorageType()); + assertEquals(OzoneStoragePolicy.WARM, ozoneBucket.getStoragePolicy()); assertFalse(ozoneBucket.getCreationTime().isAfter(Instant.now())); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java index 9df0d34e62c4..7ae83f706113 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java @@ -30,7 +30,6 @@ import java.util.Arrays; import java.util.List; import java.util.UUID; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.BucketArgs; @@ -80,7 +79,7 @@ public void testKeyDeleteAndRenameWithoutPermission() throws Exception { ObjectStore objectStore = client.getObjectStore(); OzoneVolume volume = objectStore.getVolume(VOLUME_NAME); BucketArgs omBucketArgs = - BucketArgs.newBuilder().setStorageType(StorageType.DISK).build(); + BucketArgs.newBuilder().build(); // create bucket with user1 volume.createBucket("bucket1", omBucketArgs); @@ -225,7 +224,7 @@ public void testKeyDefaultACL() throws Exception { ObjectStore objectStore = client.getObjectStore(); OzoneVolume volume = objectStore.getVolume(volumeName); BucketArgs omBucketArgs = - BucketArgs.newBuilder().setStorageType(StorageType.DISK).build(); + BucketArgs.newBuilder().build(); String bucketName = "bucket"; volume.createBucket(bucketName, omBucketArgs); OzoneBucket ozoneBucket = volume.getBucket(bucketName); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 1ebb8b391c53..94b3020f8e36 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -799,7 +799,11 @@ message BucketInfo { required string bucketName = 2; repeated OzoneAclInfo acls = 3; required bool isVersionEnabled = 4 [default = false]; - required hadoop.hdds.StorageTypeProto storageType = 5 [deprecated = true, default = DISK]; + // Legacy field kept for wire compatibility with older peers whose schema + // still has `required storageType = 5`. New OM emits DISK on write via + // OmBucketInfo.getProtobuf(); new OM ignores it on read. Storage policy + // lives in `storagePolicy` (field 24). + optional hadoop.hdds.StorageTypeProto storageType = 5 [deprecated = true, default = DISK]; optional uint64 creationTime = 6; repeated hadoop.hdds.KeyValue metadata = 7; optional BucketEncryptionInfoProto beinfo = 8; @@ -818,6 +822,8 @@ message BucketInfo { optional uint64 snapshotUsedBytes = 21; optional uint64 snapshotUsedNamespace = 22; repeated hadoop.hdds.KeyValue tags = 23; + optional hadoop.hdds.StoragePolicyProto storagePolicy = 24; + optional bool allowFallbackStoragePolicy = 25; } enum BucketLayoutProto { @@ -884,7 +890,8 @@ message BucketArgs { required string volumeName = 1; required string bucketName = 2; optional bool isVersionEnabled = 5; - optional hadoop.hdds.StorageTypeProto storageType = 6; + // Legacy field kept for wire compatibility. Same rationale as BucketInfo.storageType. + optional hadoop.hdds.StorageTypeProto storageType = 6 [deprecated = true]; repeated hadoop.hdds.KeyValue metadata = 7; optional uint64 quotaInBytes = 8; optional uint64 quotaInNamespace = 9; @@ -892,6 +899,8 @@ message BucketArgs { optional hadoop.hdds.DefaultReplicationConfig defaultReplicationConfig = 11; optional BucketEncryptionInfoProto bekInfo = 12; repeated hadoop.hdds.KeyValue tags = 13; + optional hadoop.hdds.StoragePolicyProto storagePolicy = 14; + optional bool allowFallbackStoragePolicy = 15; } message PrefixInfo { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index a88e5fb73334..2911ba47c530 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -25,7 +25,7 @@ import java.util.Objects; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; -import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; @@ -164,14 +164,24 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut bucketInfoBuilder.setModificationTime( setBucketPropertyRequest.getModificationTime()); - //Check StorageType to update - StorageType storageType = omBucketArgs.getStorageType(); - if (storageType != null) { - bucketInfoBuilder.setStorageType(storageType); - LOG.debug("Updating bucket storage type for bucket: {} in volume: {}", + //Check StoragePolicy to update + StoragePolicy storagePolicy = omBucketArgs.getStoragePolicy(); + if (storagePolicy != null) { + bucketInfoBuilder.setStoragePolicy(storagePolicy); + LOG.debug("Updating bucket storage policy for bucket: {} in volume: {}", bucketName, volumeName); } + //Check allowFallbackStoragePolicy to update + Boolean allowFallbackStoragePolicy = + omBucketArgs.getAllowFallbackStoragePolicy(); + if (allowFallbackStoragePolicy != null) { + bucketInfoBuilder.setAllowFallbackStoragePolicy( + allowFallbackStoragePolicy); + LOG.debug("Updating bucket allowFallbackStoragePolicy for bucket: {}" + + " in volume: {}", bucketName, volumeName); + } + //Check Versioning to update Boolean versioning = omBucketArgs.getIsVersionEnabled(); if (versioning != null) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java index 6252836b57de..bcc83616e4fd 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java @@ -48,10 +48,10 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; import org.apache.hadoop.hdds.server.ServerUtils; @@ -220,7 +220,7 @@ void testGetBucketInfo() throws Exception { OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() .setVolumeName(volumeName) .setBucketName(bucketName) - .setStorageType(StorageType.DISK) + .setStoragePolicy(OzoneStoragePolicy.WARM) .setIsVersionEnabled(false) .build(); // Note: the helper method createBucket() in this scope won't create the @@ -243,7 +243,7 @@ void testGetBucketInfo() throws Exception { OmBucketInfo result = bucketManager.getBucketInfo(volumeName, bucketName); assertEquals(volumeName, result.getVolumeName()); assertEquals(bucketName, result.getBucketName()); - assertEquals(StorageType.DISK, result.getStorageType()); + assertEquals(OzoneStoragePolicy.WARM, result.getStoragePolicy()); assertFalse(result.getIsVersionEnabled()); } @@ -253,7 +253,7 @@ private void createBucket(OMMetadataManager metadataManager, } @Test - public void testSetBucketPropertyChangeStorageType() throws Exception { + public void testSetBucketPropertyChangeStoragePolicy() throws Exception { String volume = volumeName(); createSampleVol(volume); @@ -262,23 +262,24 @@ public void testSetBucketPropertyChangeStorageType() throws Exception { OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() .setVolumeName(volume) .setBucketName("bucket-one") - .setStorageType(StorageType.DISK) + .setStoragePolicy(OzoneStoragePolicy.WARM) .build(); createBucket(metaMgr, bucketInfo); OmBucketInfo result = bucketManager.getBucketInfo( volume, "bucket-one"); - assertEquals(StorageType.DISK, - result.getStorageType()); + assertEquals(OzoneStoragePolicy.WARM, + result.getStoragePolicy()); OmBucketArgs bucketArgs = OmBucketArgs.newBuilder() .setVolumeName(volume) .setBucketName("bucket-one") - .setStorageType(StorageType.SSD) + .setStoragePolicy(OzoneStoragePolicy.HOT) + .setAllowFallbackStoragePolicy(true) .build(); writeClient.setBucketProperty(bucketArgs); OmBucketInfo updatedResult = bucketManager.getBucketInfo( volume, "bucket-one"); - assertEquals(StorageType.SSD, - updatedResult.getStorageType()); + assertEquals(OzoneStoragePolicy.HOT, + updatedResult.getStoragePolicy()); } @Test @@ -401,7 +402,7 @@ public void testLinkedBucketResolution() throws Exception { .setQuotaInNamespace(24 * 1024) .setUsedBytes(10 * 1024) .setUsedNamespace(5 * 1024) - .setStorageType(StorageType.SSD) + .setStoragePolicy(OzoneStoragePolicy.HOT) .setIsVersionEnabled(true) .addAllMetadata(singletonMap("CustomKey", "CustomValue")) .build(); @@ -470,8 +471,8 @@ public void testLinkedBucketResolution() throws Exception { bucketInfo.getMetadata(), storedLinkBucket.getMetadata()); assertEquals( - bucketInfo.getStorageType(), - storedLinkBucket.getStorageType()); + bucketInfo.getStoragePolicy(), + storedLinkBucket.getStoragePolicy()); assertEquals( bucketInfo.getIsVersionEnabled(), storedLinkBucket.getIsVersionEnabled()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index e7df8077cc37..4ed8b9ecdaa0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -49,7 +49,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; @@ -437,7 +436,6 @@ private void createBucket(OMMetadataManager omMetadataManager, OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder() .setVolumeName(volume) .setBucketName(bucket) - .setStorageType(StorageType.DISK) .setIsVersionEnabled(false) .build(); OMRequestTestUtils.addBucketToOM(omMetadataManager, omBucketInfo); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 1ffcc32f0acb..211135de1371 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -79,7 +79,6 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -392,7 +391,6 @@ private void addBucketsToCache(String volumeName, String bucketName) { OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder() .setVolumeName(volumeName) .setBucketName(bucketName) - .setStorageType(StorageType.DISK) .setIsVersionEnabled(false) .build(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java index 8b3027d12cd1..bb8e818ecaca 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java @@ -38,7 +38,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.OzoneConsts; @@ -468,8 +467,7 @@ private OMBucketCreateResponse createBucket(String volumeName, String bucketName, long transactionID) { BucketInfo.Builder bucketInfo = - newBucketInfoBuilder(bucketName, volumeName) - .setStorageType(StorageTypeProto.DISK); + newBucketInfoBuilder(bucketName, volumeName); OzoneManagerProtocolProtos.OMRequest omRequest = OMRequestTestUtils.newCreateBucketRequest(bucketInfo).build(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java index 0a7ad4862352..b62b333c799b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java @@ -768,7 +768,7 @@ public static BucketInfo.Builder newBucketInfoBuilder( return BucketInfo.newBuilder() .setBucketName(bucketName) .setVolumeName(volumeName) - .setStorageType(HddsProtos.StorageTypeProto.SSD) + .setStoragePolicy(HddsProtos.StoragePolicyProto.HOT) .setIsVersionEnabled(false) .addAllMetadata(getMetadataList()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java index 38036912b96b..6f09272c7025 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java @@ -33,7 +33,6 @@ import java.nio.file.Path; import java.util.UUID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; import org.apache.hadoop.ipc_.Server; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -106,8 +105,7 @@ public void testUserInfoInCaseOfHadoopTransport() throws Exception { String volumeName = UUID.randomUUID().toString(); BucketInfo.Builder bucketInfo = newBucketInfoBuilder(bucketName, volumeName) - .setIsVersionEnabled(true) - .setStorageType(StorageTypeProto.DISK); + .setIsVersionEnabled(true); OMRequest omRequest = newCreateBucketRequest(bucketInfo).build(); OMBucketCreateRequest omBucketCreateRequest = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java index 96a60b647d2b..64ba9df4e086 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StoragePolicyProto; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -260,7 +260,7 @@ public void testValidateAndUpdateCacheCrossSpaceQuota() throws Exception { .setVolumeName(volumeName) .setIsVersionEnabled(false) .setQuotaInBytes(99999L) - .setStorageType(StorageTypeProto.SSD) + .setStoragePolicy(StoragePolicyProto.HOT) .addAllMetadata(OMRequestTestUtils.getMetadataList()).build(); OzoneManagerProtocolProtos.CreateBucketRequest.Builder req = OzoneManagerProtocolProtos.CreateBucketRequest.newBuilder(); @@ -390,7 +390,7 @@ public void testIgnoreClientACL(boolean ignoreClientACLs) throws Exception { OzoneManagerProtocolProtos.BucketInfo.newBuilder() .setBucketName(bucketName) .setVolumeName(volumeName) - .setStorageType(HddsProtos.StorageTypeProto.SSD) + .setStoragePolicy(HddsProtos.StoragePolicyProto.HOT) .setIsVersionEnabled(false) .setQuotaInBytes(5000L) .addAcls(OzoneAcl.toProtobuf(OzoneAcl.parseAcl(acl))); @@ -484,8 +484,10 @@ protected void doValidateAndUpdateCache(String volumeName, String bucketName, assertTrue(dbBucketInfo.getAcls().containsAll(bucketInfoFromProto.getAcls())); assertEquals(bucketInfoFromProto.getIsVersionEnabled(), dbBucketInfo.getIsVersionEnabled()); - assertEquals(bucketInfoFromProto.getStorageType(), - dbBucketInfo.getStorageType()); + assertEquals(bucketInfoFromProto.getStoragePolicy(), + dbBucketInfo.getStoragePolicy()); + assertEquals(bucketInfoFromProto.getAllowFallbackStoragePolicy(), + dbBucketInfo.getAllowFallbackStoragePolicy()); assertEquals(bucketInfoFromProto.getMetadata(), dbBucketInfo.getMetadata()); assertEquals(bucketInfoFromProto.getEncryptionKeyInfo(), @@ -506,7 +508,9 @@ protected void verifyRequest(OMRequest modifiedOmRequest, assertEquals(original.getBucketName(), updated.getBucketName()); assertEquals(original.getVolumeName(), updated.getVolumeName()); assertEquals(original.getIsVersionEnabled(), updated.getIsVersionEnabled()); - assertEquals(original.getStorageType(), updated.getStorageType()); + assertEquals(original.getStoragePolicy(), updated.getStoragePolicy()); + assertEquals(original.getAllowFallbackStoragePolicy(), + updated.getAllowFallbackStoragePolicy()); assertEquals(original.getMetadataList(), updated.getMetadataList()); assertNotEquals(original.getCreationTime(), updated.getCreationTime()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java index 107e697971f8..21f739f9bb47 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java @@ -170,8 +170,8 @@ protected void doValidateAndUpdateCache(String volumeName, String bucketName, assertTrue(dbBucketInfo.getAcls().containsAll(bucketInfoFromProto.getAcls())); assertEquals(bucketInfoFromProto.getIsVersionEnabled(), dbBucketInfo.getIsVersionEnabled()); - assertEquals(bucketInfoFromProto.getStorageType(), - dbBucketInfo.getStorageType()); + assertEquals(bucketInfoFromProto.getStoragePolicy(), + dbBucketInfo.getStoragePolicy()); assertEquals(bucketInfoFromProto.getMetadata(), dbBucketInfo.getMetadata()); assertEquals(bucketInfoFromProto.getEncryptionKeyInfo(), diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 1a9e20a59859..043ce17a8eff 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.Pipeline; @@ -305,7 +304,6 @@ private OmBucketInfo aBucketInfo() { .setVolumeName(TEST_VOLUME_NAME) .setBucketName(TEST_BUCKET_NAME) .setIsVersionEnabled(false) - .setStorageType(StorageType.DEFAULT) .build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java index 0a5cc3301861..c4f5a73f3e89 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java @@ -26,7 +26,6 @@ import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -138,7 +137,6 @@ private void addBucketToDB(String volumeName, String bucketName) .setVolumeName(volumeName) .setBucketName(bucketName) .setObjectID(System.currentTimeMillis()) - .setStorageType(StorageType.DISK) .setIsVersionEnabled(false) .build(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/OMKeyResponseTests.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/OMKeyResponseTests.java index 99842c9cfb3d..ad73e999f544 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/OMKeyResponseTests.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/OMKeyResponseTests.java @@ -26,7 +26,6 @@ import java.util.UUID; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -97,7 +96,6 @@ public void setup() throws Exception { .setVolumeName(volumeName) .setBucketName(bucketName) .setObjectID(System.currentTimeMillis()) - .setStorageType(StorageType.DISK) .setIsVersionEnabled(false) .build(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartResponseTests.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartResponseTests.java index 70b8ea0b05e0..6dc8d09ec0c1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartResponseTests.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartResponseTests.java @@ -27,7 +27,6 @@ import java.util.UUID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -472,7 +471,6 @@ public void addBucketToDB(String volumeName, String bucketName) .setVolumeName(volumeName) .setBucketName(bucketName) .setObjectID(System.currentTimeMillis()) - .setStorageType(StorageType.DISK) .setIsVersionEnabled(false) .build(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java index 6130b2b7fb41..0756fb6ba926 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; -import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -31,8 +31,11 @@ public class BucketObjectDBInfo extends ObjectDBInfo { @JsonProperty("volumeName") private String volumeName; - @JsonProperty("storageType") - private StorageType storageType; + @JsonProperty("storagePolicy") + private StoragePolicy storagePolicy; + + @JsonProperty("allowFallbackStoragePolicy") + private Boolean allowFallbackStoragePolicy; @JsonProperty("versioning") private boolean isVersioningEnabled; @@ -78,7 +81,9 @@ public BucketObjectDBInfo(OmBucketInfo omBucketInfo) { this.sourceBucket = omBucketInfo.getSourceBucket(); this.sourceVolume = omBucketInfo.getSourceVolume(); this.isVersioningEnabled = omBucketInfo.getIsVersionEnabled(); - this.storageType = omBucketInfo.getStorageType(); + this.storagePolicy = omBucketInfo.getStoragePolicy(); + this.allowFallbackStoragePolicy = + omBucketInfo.getAllowFallbackStoragePolicy(); this.defaultReplicationConfig = omBucketInfo.getDefaultReplicationConfig(); this.bucketLayout = omBucketInfo.getBucketLayout(); this.owner = omBucketInfo.getOwner(); @@ -95,12 +100,12 @@ public void setVolumeName(String volumeName) { this.volumeName = volumeName; } - public StorageType getStorageType() { - return storageType; + public StoragePolicy getStoragePolicy() { + return storagePolicy; } - public void setStorageType(StorageType storageType) { - this.storageType = storageType; + public Boolean getAllowFallbackStoragePolicy() { + return allowFallbackStoragePolicy; } public long getUsedBytes() { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java index 114647c2c547..071ae0e3d2c6 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java @@ -35,7 +35,6 @@ import java.util.List; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; @@ -548,8 +547,6 @@ void assertBasicInfoBucketOne(NSSummaryEndpoint endpoint) throws Exception { assertEquals(5, bucket.getCountStats().getNumTotalKey()); assertEquals(VOL, ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); assertEquals(getBucketLayout(), ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); assertEquals(BUCKET_ONE, @@ -565,8 +562,6 @@ void assertBasicInfoBucketTwo(NSSummaryEndpoint endpoint) throws Exception { assertEquals(2, bucket.getCountStats().getNumTotalKey()); assertEquals(VOL, ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); assertEquals(getBucketLayout(), ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); assertEquals(BUCKET_TWO, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java index 7bd37ab10276..91372541a340 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java @@ -32,7 +32,6 @@ import java.util.List; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -427,8 +426,6 @@ private void assertBucket(NSSummaryEndpoint endpoint, String path, assertEquals(numKeys, bucket.getCountStats().getNumTotalKey()); assertEquals(volName, ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); assertEquals(layout, ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); assertEquals(bucketName, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index db9301e4795e..704ecaa81d6e 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -66,10 +66,10 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ExtendedDatanodeDetailsProto; @@ -518,7 +518,7 @@ public void setUp() throws Exception { .setQuotaInBytes(OzoneConsts.GB) .setUsedBytes(OzoneConsts.MB) .setQuotaInNamespace(5) - .setStorageType(StorageType.DISK) + .setStoragePolicy(OzoneStoragePolicy.WARM) .setUsedNamespace(3) .setBucketLayout(BucketLayout.LEGACY) .setOwner("TestUser2") @@ -541,7 +541,7 @@ public void setUp() throws Exception { .setQuotaInBytes(OzoneConsts.GB) .setUsedBytes(100 * OzoneConsts.MB) .setQuotaInNamespace(5) - .setStorageType(StorageType.SSD) + .setStoragePolicy(OzoneStoragePolicy.HOT) .setUsedNamespace(3) .setBucketLayout(BucketLayout.OBJECT_STORE) .setOwner("TestUser2") @@ -1106,7 +1106,7 @@ private void testBucketResponse(BucketObjectDBInfo bucketMetadata) switch (bucketName) { case "bucketOne": assertEquals("sampleVol2", bucketMetadata.getVolumeName()); - assertEquals(StorageType.DISK, bucketMetadata.getStorageType()); + assertEquals(OzoneStoragePolicy.WARM, bucketMetadata.getStoragePolicy()); assertNull(bucketMetadata.getSourceVolume()); assertNull(bucketMetadata.getSourceBucket()); assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); @@ -1128,7 +1128,7 @@ private void testBucketResponse(BucketObjectDBInfo bucketMetadata) break; case "bucketTwo": assertEquals("sampleVol2", bucketMetadata.getVolumeName()); - assertEquals(StorageType.SSD, bucketMetadata.getStorageType()); + assertEquals(OzoneStoragePolicy.HOT, bucketMetadata.getStoragePolicy()); assertNull(bucketMetadata.getSourceVolume()); assertNull(bucketMetadata.getSourceBucket()); assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 122158262fa7..22e85954ae23 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -28,8 +28,8 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneFsServerDefaults; @@ -193,8 +193,8 @@ public void setBucketVersioning(String volumeName, String bucketName, } @Override - public void setBucketStorageType(String volumeName, String bucketName, - StorageType storageType) throws IOException { + public void setBucketStoragePolicy(String volumeName, String bucketName, + StoragePolicy storagePolicy) throws IOException { } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java index 414c397aeee6..9fa15f6f3f0d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java @@ -27,8 +27,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -110,7 +110,7 @@ public OzoneVolumeStub build() { @Override public void createBucket(String bucketName) throws OMException { createBucket(bucketName, new BucketArgs.Builder() - .setStorageType(StorageType.DEFAULT) + .setStoragePolicy(OzoneStoragePolicy.getDefaultPolicy()) .setVersioning(false) .build()); } @@ -129,7 +129,8 @@ public void createBucket(String bucketName, BucketArgs bucketArgs) throws OMExce RatisReplicationConfig.getInstance( HddsProtos.ReplicationFactor.THREE))) .setBucketLayout(bucketArgs.getBucketLayout()) - .setStorageType(bucketArgs.getStorageType()) + .setStoragePolicy(bucketArgs.getStoragePolicy()) + .setAllowFallbackStoragePolicy(bucketArgs.getAllowFallbackStoragePolicy()) .setVersioning(bucketArgs.getVersioning()) .setOwner(bucketArgs.getOwner()) .setCreationTime(Time.now())