Skip to content

HDDS-15606. Support S3 object metadata headers (Cache-Control, Expires, Content-Encoding) end-to-end in S3 Gateway #10786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Gargi-jais11 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

HDDS-15606. Support S3 object metadata headers (Cache-Control, Expires, Content-Encoding) end-to-end in S3 Gateway #10786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 106 additions & 0 deletions ...tion-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
Expand Down Expand Up @@ -2890,6 +2892,110 @@ private void verifyBucketOwnershipVerificationAccessDenied(Executable function)
}
}

static Stream<Arguments> standardObjectHeaderContentEncodingCases() {
return Stream.of(
Arguments.of("gzip", "gzip"),
Arguments.of("deflate, gzip", "deflate, gzip"),
Arguments.of("gzip, aws-chunked", "gzip"),
Arguments.of("aws-chunked, gzip", "gzip"),
Arguments.of("aws-chunked", null),
Arguments.of("aws-chunked, aws-chunked", null));
}

/**
* ceph s3-tests coverage for standard object headers persisted on PUT and returned on HEAD/GET.
*/
@Nested
class StandardObjectHeaderTests {

private static final String CONTENT = "bar";
private static final String CACHE_CONTROL = "public, max-age=14400";

/**
* s3-tests: test_object_write_cache_control.
*/
@Test
public void testObjectWriteCacheControl() {
final String bucketName = getBucketName("cache-control");
final String keyName = getKeyName("cache-control");
s3Client.createBucket(b -> b.bucket(bucketName));

s3Client.putObject(b -> b.bucket(bucketName).key(keyName).cacheControl(CACHE_CONTROL),
RequestBody.fromString(CONTENT));

HeadObjectResponse head = s3Client.headObject(b -> b.bucket(bucketName).key(keyName));
assertEquals(CACHE_CONTROL, head.cacheControl());

GetObjectResponse getObject = s3Client.getObject(b -> b.bucket(bucketName).key(keyName)).response();
assertEquals(CACHE_CONTROL, getObject.cacheControl());
}

/**
* s3-tests: test_object_write_expires.
*/
@Test
public void testObjectWriteExpires() {
final String bucketName = getBucketName("expires");
final String keyName = getKeyName("expires");
final Instant expires = Instant.now().plusSeconds(6000).truncatedTo(ChronoUnit.SECONDS);
s3Client.createBucket(b -> b.bucket(bucketName));

s3Client.putObject(b -> b.bucket(bucketName).key(keyName).expires(expires),
RequestBody.fromString(CONTENT));

HeadObjectResponse head = s3Client.headObject(b -> b.bucket(bucketName).key(keyName));
assertEquals(expires, head.expires());

GetObjectResponse getObject = s3Client.getObject(b -> b.bucket(bucketName).key(keyName)).response();
assertEquals(expires, getObject.expires());
}

/**
* s3-tests: test_object_content_encoding_aws_chunked.
*/
@ParameterizedTest
@MethodSource("org.apache.hadoop.ozone.s3.awssdk.v2.AbstractS3SDKV2Tests#standardObjectHeaderContentEncodingCases")
public void testObjectContentEncodingAwsChunked(String requestEncoding,
String expectedEncoding) {
final String bucketName = getBucketName("content-encoding");
final String keyName = getKeyName("encoding");
s3Client.createBucket(b -> b.bucket(bucketName));

s3Client.putObject(b -> b.bucket(bucketName).key(keyName)
.contentEncoding(requestEncoding),
RequestBody.fromString(CONTENT));

HeadObjectResponse head = s3Client.headObject(b -> b.bucket(bucketName).key(keyName));
assertEquals(expectedEncoding, head.contentEncoding());

GetObjectResponse getObject = s3Client.getObject(b -> b.bucket(bucketName).key(keyName)).response();
assertEquals(expectedEncoding, getObject.contentEncoding());
}

@Test
public void testObjectWriteContentLanguageAndDisposition() {
final String bucketName = getBucketName("lang-disp");
final String keyName = getKeyName("lang-disp");
final String language = "en-CA";
final String disposition = "attachment; filename=\"test.txt\"";
s3Client.createBucket(b -> b.bucket(bucketName));

s3Client.putObject(b -> b.bucket(bucketName).key(keyName)
.contentLanguage(language)
.contentDisposition(disposition),
RequestBody.fromString(CONTENT));

HeadObjectResponse head = s3Client.headObject(b -> b.bucket(bucketName).key(keyName));
assertEquals(language, head.contentLanguage());
assertEquals(disposition, head.contentDisposition());

GetObjectResponse getObject =
s3Client.getObject(b -> b.bucket(bucketName).key(keyName)).response();
assertEquals(language, getObject.contentLanguage());
assertEquals(disposition, getObject.contentDisposition());
}
}

/**
* Integration tests for ListBuckets (GET / ListAllMyBuckets).
*/
Expand Down
17 changes: 14 additions & 3 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,15 +120,26 @@ public abstract class EndpointBase {

protected static final String ETAG_CUSTOM = "etag-custom";
protected static final String CONTENT_TYPE_CUSTOM = "content-type-custom";
protected static final String CACHE_CONTROL_CUSTOM = "cache-control-custom";
protected static final String EXPIRES_CUSTOM = "expires-custom";
protected static final String CONTENT_ENCODING_CUSTOM = "content-encoding-custom";
protected static final String CONTENT_LANGUAGE_CUSTOM = "content-language-custom";
protected static final String CONTENT_DISPOSITION_CUSTOM = "content-disposition-custom";

// System metadata key -> custom key. A user x-amz-meta-{etag,content-type}
// collides with the system ETag / Content-Type stored under the same key, so
// it is remapped on write and rebuilt on read; the system value is returned
// via its own ETag / Content-Type response header.
private static final Map<String, String> RESERVED_METADATA_KEYS =
ImmutableMap.of(
ETAG, ETAG_CUSTOM,
HttpHeaders.CONTENT_TYPE, CONTENT_TYPE_CUSTOM);
ImmutableMap.<String, String>builder()
.put(ETAG, ETAG_CUSTOM)
.put(HttpHeaders.CONTENT_TYPE, CONTENT_TYPE_CUSTOM)
.put(HttpHeaders.CACHE_CONTROL, CACHE_CONTROL_CUSTOM)
.put(HttpHeaders.EXPIRES, EXPIRES_CUSTOM)
.put(HttpHeaders.CONTENT_ENCODING, CONTENT_ENCODING_CUSTOM)
.put(HttpHeaders.CONTENT_LANGUAGE, CONTENT_LANGUAGE_CUSTOM)
.put(HttpHeaders.CONTENT_DISPOSITION, CONTENT_DISPOSITION_CUSTOM)
.build();

// Custom key -> lower-cased header, to rebuild remapped user metadata on read.
private static final Map<String, String> REBUILT_RESERVED_KEYS =
Expand Down
57 changes: 48 additions & 9 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_DIRECTIVE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Utils.normalizeContentEncoding;
import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes;
import static org.apache.hadoop.ozone.s3.util.S3Utils.validateSignatureHeader;
import static org.apache.hadoop.ozone.s3.util.S3Utils.wrapInQuotes;
Expand Down Expand Up @@ -272,6 +273,7 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr
Map<String, String> customMetadata =
getCustomMetadataFromHeaders(getHeaders().getRequestHeaders());
putContentType(customMetadata);
putStandardObjectHeaders(customMetadata);
Map<String, String> tags = getTaggingFromHeaders(getHeaders());

long putLength;
Expand Down Expand Up @@ -475,14 +477,8 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath)
responseBuilder.header(HttpHeaders.CONTENT_TYPE, contentType);

for (Map.Entry<String, String> entry : overrideQueryParameter.entrySet()) {
String headerValue = getHeaders().getHeaderString(entry.getKey());
String queryValue = queryParams.getFirst(entry.getValue());
if (queryValue != null) {
headerValue = queryValue;
}
if (headerValue != null) {
responseBuilder.header(entry.getKey(), headerValue);
}
addStoredObjectHeader(responseBuilder, keyDetails, queryParams, entry.getKey(),
entry.getValue());
}

addLastModifiedDate(responseBuilder, keyDetails);
Expand Down Expand Up @@ -522,6 +518,46 @@ private void putContentType(Map<String, String> metadata) {
}
}

/**
* Store standard S3 object headers from the PUT request in key metadata.
*/
private void putStandardObjectHeaders(Map<String, String> metadata) {
for (Map.Entry<String, String> entry : overrideQueryParameter.entrySet()) {
String headerName = entry.getKey();
String value = getHeaders().getHeaderString(headerName);
if (HttpHeaders.CONTENT_ENCODING.equals(headerName)) {
value = normalizeContentEncoding(value);
}
if (value != null) {
metadata.put(headerName, value);
}
}
}

private void addStoredObjectHeader(ResponseBuilder responseBuilder, OzoneKey key,
MultivaluedMap<String, String> queryParams, String headerName,
String responseOverrideParam) {
String headerValue = queryParams.getFirst(responseOverrideParam);
if (headerValue == null) {
headerValue = key.getMetadata().get(headerName);
}
if (HttpHeaders.CONTENT_ENCODING.equals(headerName)) {
headerValue = normalizeContentEncoding(headerValue);
}
if (headerValue != null) {
responseBuilder.header(headerName, headerValue);
}
}

private void addStoredObjectHeaders(ResponseBuilder responseBuilder, OzoneKey key) {
MultivaluedMap<String, String> queryParams =
getContext().getUriInfo().getQueryParameters();
for (Map.Entry<String, String> entry : overrideQueryParameter.entrySet()) {
addStoredObjectHeader(responseBuilder, key, queryParams, entry.getKey(),
entry.getValue());
}
}

/** Returns the object's stored Content-Type, or the default if absent. */
private static String contentTypeOf(OzoneKey key) {
String contentType = key.getMetadata().get(HttpHeaders.CONTENT_TYPE);
Expand Down Expand Up @@ -606,6 +642,7 @@ public Response head(
.header(HttpHeaders.CONTENT_TYPE, contentTypeOf(key))
.header(STORAGE_CLASS_HEADER, s3StorageType.toString());
addEntityTagHeader(response, key);
addStoredObjectHeaders(response, key);

addLastModifiedDate(response, key);
addTagCountIfAny(response, key);
Expand Down Expand Up @@ -719,6 +756,7 @@ public Response initializeMultipartUpload(
Map<String, String> customMetadata =
getCustomMetadataFromHeaders(getHeaders().getRequestHeaders());
putContentType(customMetadata);
putStandardObjectHeaders(customMetadata);

Map<String, String> tags = getTaggingFromHeaders(getHeaders());

Expand Down Expand Up @@ -1143,8 +1181,9 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
} else if (metadataCopyDirective.equals(CopyDirective.REPLACE.name())) {
// Replace the metadata with the metadata form the request headers
customMetadata = getCustomMetadataFromHeaders(getHeaders().getRequestHeaders());
// REPLACE: Content-Type comes from the request, not the source.
// REPLACE: Content-Type and standard object headers come from the request.
putContentType(customMetadata);
putStandardObjectHeaders(customMetadata);
} else {
OS3Exception ex = newError(INVALID_ARGUMENT, metadataCopyDirective);
ex.setErrorMessage("An error occurred (InvalidArgument) " +
Expand Down
17 changes: 17 additions & 0 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.util.Arrays;
import java.util.Base64;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
Expand Down Expand Up @@ -73,6 +74,22 @@ public static String s3urlEncode(String str)
return urlEncode(str).replace("+", "%20");
}

/**
* Returns the persisted S3 {@code Content-Encoding} value, with {@code aws-chunked}
* removed per AWS semantics.
*/
public static String normalizeContentEncoding(String contentEncoding) {
if (contentEncoding == null || contentEncoding.isEmpty()) {
return null;
}
String normalized = Arrays.stream(contentEncoding.split(","))
.map(String::trim)
.filter(value -> !value.isEmpty())
.filter(value -> !AWS_CHUNKED.equals(value))
.collect(Collectors.joining(", "));
return normalized.isEmpty() ? null : normalized;
}

private S3Utils() {
// no instances
}
Expand Down
20 changes: 17 additions & 3 deletions hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,12 +212,12 @@ public void getKeyWithTag() throws IOException, OS3Exception {
}

@Test
public void inheritRequestHeader() throws IOException, OS3Exception {
public void storedObjectHeadersOnGetAndHead() throws IOException, OS3Exception {
setDefaultHeader();
assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
clearRequestHeaderMocks();

Response response = get(rest, BUCKET_NAME, KEY_NAME);

// Content-Type is not inherited from the request; key1 has none stored.
assertEquals("binary/octet-stream",
response.getHeaderString("Content-Type"));
assertEquals(CONTENT_LANGUAGE1,
Expand All @@ -230,11 +230,16 @@ public void inheritRequestHeader() throws IOException, OS3Exception {
response.getHeaderString("Content-Disposition"));
assertEquals(CONTENT_ENCODING1,
response.getHeaderString("Content-Encoding"));

assertEquals(CONTENT_ENCODING1,
rest.head(BUCKET_NAME, KEY_NAME).getHeaderString("Content-Encoding"));
}

@Test
public void overrideResponseHeader() throws IOException, OS3Exception {
setDefaultHeader();
assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
clearRequestHeaderMocks();

MultivaluedMap<String, String> queryParameter = rest.getContext().getUriInfo().getQueryParameters();
// overrider request header
Expand Down Expand Up @@ -364,6 +369,15 @@ private void setDefaultHeader() {
.when(headers).getHeaderString("Content-Encoding");
}

private void clearRequestHeaderMocks() {
doReturn(null).when(headers).getHeaderString("Content-Type");
doReturn(null).when(headers).getHeaderString("Content-Language");
doReturn(null).when(headers).getHeaderString("Expires");
doReturn(null).when(headers).getHeaderString("Cache-Control");
doReturn(null).when(headers).getHeaderString("Content-Disposition");
doReturn(null).when(headers).getHeaderString("Content-Encoding");
}

@Test
public void testGetWhenKeyIsDirectoryAndDoesNotEndWithASlash()
throws IOException {
Expand Down
Loading