Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions lib/Config.js
Original file line number Diff line number Diff line change
Expand Up @@ -1832,6 +1832,16 @@ class Config extends EventEmitter {

this.testingMode = config.testingMode || false;

// When true, replication behaves like in a cloudserver older than 9.4:
// no multi-destination and no cascade.
if (config.replicationCascadeCompatMode !== undefined) {
assert(
typeof config.replicationCascadeCompatMode === 'boolean',
'bad config: replicationCascadeCompatMode must be a boolean',
);
}
this.replicationCascadeCompatMode = config.replicationCascadeCompatMode === true;

this.maxScannedLifecycleListingEntries = constants.maxScannedLifecycleListingEntries;
if (config.maxScannedLifecycleListingEntries !== undefined) {
// maxScannedLifecycleListingEntries > 2 is required as a minimum because we must
Expand Down
64 changes: 64 additions & 0 deletions lib/api/apiUtils/bucket/getReplicationConfiguration.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,69 @@ const config = require('../../../Config').config;
const parseXML = require('../../../utilities/parseXML');
const ReplicationConfiguration = require('arsenal').models.ReplicationConfiguration;

/**
* We tell if a rule replicates to a CRR site: a replication endpoint with no
* type. A storageClass that is not an endpoint (none, or STANDARD) means the
* default endpoint.
* @param {object[]} rules - The rules of the replication configuration
* @return {boolean} true when at least one rule replicates to a CRR site
*/
function _hasCrrSite(rules) {
const { replicationEndpoints } = config;
const defaultEndpoint = replicationEndpoints.find(endpoint => endpoint.default) ?? replicationEndpoints[0];
const storageClasses = rules.flatMap(rule => (rule.storageClass ?? '').split(','));
return storageClasses.some(storageClass => {
const site = storageClass.split(':')[0];
const endpoint = replicationEndpoints.find(endpoint => endpoint.site === site) ?? defaultEndpoint;
return endpoint !== undefined && endpoint.type === undefined;
});
}

/**
* We rewrite the replication configuration like a cloudserver older than
* 9.4 stores it: one destination bucket for all the rules. An old
* cloudserver crashes on the new shape.
* We refuse what it refuses: a single role for a CRR site, rules to
* different buckets, overlapping prefixes. Like it, we ignore the account
* of a rule.
* @param {object} replicationConfig - The configuration in the current shape
* @return {object} { refused: { code, description } }, or { replicationConfig } in the old shape
*/
function toLegacyConfiguration(replicationConfig) {
const { role, rules, preferredReadLocation } = replicationConfig;
const refuse = (code, description) => ({ refused: { code, description } });
if (role.split(',').length !== 2 && _hasCrrSite(rules)) {
return refuse(
'InvalidArgument',
'Invalid Role specified in replication configuration: ' +
'Role must be a comma-separated list of two IAM roles',
);
}
if (rules.some(rule => rule.destination !== rules[0].destination)) {
return refuse('InvalidRequest', 'The destination bucket must be the same for all rules');
}
const prefixes = rules.map(rule => rule.prefix ?? '');
for (let i = 0; i < prefixes.length; i++) {
const overlapping = prefixes.slice(0, i).find(p => p.startsWith(prefixes[i]) || prefixes[i].startsWith(p));
if (overlapping !== undefined) {
return refuse('InvalidRequest', `Found overlapping prefixes '${overlapping}' and '${prefixes[i]}'`);
}
}
return {
replicationConfig: {
role,
destination: rules[0].destination,
rules: rules.map(({ prefix, enabled, id, storageClass }) => ({
prefix: prefix ?? '',
enabled,
id,
storageClass,
})),
preferredReadLocation,
},
};
}

// Handle the steps for returning a valid replication configuration object.
function getReplicationConfiguration(xml, log, cb) {
return parseXML(xml, log, (err, result) => {
Expand All @@ -27,5 +90,6 @@ function getReplicationConfigurationXML(config) {

module.exports = {
getReplicationConfiguration,
toLegacyConfiguration,
getReplicationConfigurationXML,
};
4 changes: 4 additions & 0 deletions lib/api/apiUtils/object/bumpMicroVersionId.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ function bumpMicroVersionId(objectMD, force) {
if (!force && !objectMD?.replicationInfo) {
return;
}
// A cloudserver older than 9.4 doesn't set a microVersionId here.
if (config.replicationCascadeCompatMode) {
return;
}

const { instanceId, replicationGroupId } = config;

Expand Down
59 changes: 57 additions & 2 deletions lib/api/apiUtils/object/getReplicationInfo.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,17 @@ const { constants, models } = require('arsenal');
const { replicationBackends } = constants;
const { ReplicationConfiguration } = models;

/**
* A cloud site has the location type aws_s3, azure or gcp. Any other site
* is another cloudserver (CRR).
* @param {object} s3config - Cloudserver configuration object
* @param {string} site - The site name
* @return {boolean} true for a cloud backend
*/
function isCloudSite(s3config, site) {
return !!replicationBackends[s3config.locationConstraints[site]?.type];
}

/**
* Apply the default replication endpoint as a fallback storageClass
* for rules that don't specify one. Returns a new rules array; rules
Expand Down Expand Up @@ -45,6 +56,43 @@ function _canUserReplicate(authInfo) {
return !!props?.canReplicate;
}

/**
* We rewrite the replicationInfo like a cloudserver older than 9.4 writes
* it: destination, site and roles at the top, not in each backend. A
* backbeat older than 9.5.0 only reads them at the top.
* Example: one rule replicates to the bucket arn:aws:s3:::backup on site sfb.
* We return { role: 'src-role,dst-role', destination: 'arn:aws:s3:::backup',
* storageClass: 'sfb', storageType: '',
* backends: [{ site: 'sfb', status: 'PENDING', dataStoreVersionId: '' }] }.
* @param {object} replicationInfo - The replicationInfo in the current format
* @param {object} config - The bucket replication configuration
* @param {object} rule - The rule that applies to the object
* @param {object} s3config - Cloudserver configuration object
* @param {function} isCloud - Tells if a site is a cloud backend
* @return {object} The replicationInfo in the old format
*/
function _toLegacyFormat(replicationInfo, config, rule, s3config, isCloud) {
const backends = replicationInfo.backends.map(({ site, status, dataStoreVersionId }) => ({
site,
status,
dataStoreVersionId,
}));
return {
status: replicationInfo.status,
backends,
content: replicationInfo.content,
destination: config.destination,
storageClass: rule.storageClass,
role: config.role,
storageType: backends
.filter(backend => isCloud(backend.site))
.map(backend => s3config.locationConstraints[backend.site].type)
.join(','),
dataStoreVersionId: '',
isNFS: replicationInfo.isNFS,
};
}

/**
* Get the object replicationInfo to replicate data and metadata, or only
* metadata if the operation only changes metadata or the object is 0 bytes.
Expand Down Expand Up @@ -88,7 +136,7 @@ function getReplicationInfo(params) {
return undefined;
}

const isCloud = site => !!replicationBackends[s3config.locationConstraints[site]?.type];
const isCloud = site => isCloudSite(s3config, site);
const rules = _withDefaultStorageClass(config.rules || [], s3config);
const allBackends = ReplicationConfiguration.resolveBackends(
{ ...config, rules },
Expand Down Expand Up @@ -116,7 +164,7 @@ function getReplicationInfo(params) {
content.push(operationType);
}

return {
const replicationInfo = {
status: 'PENDING',
backends,
content,
Expand All @@ -125,6 +173,13 @@ function getReplicationInfo(params) {
// `undefined` so JSON/BSON drop the field on persist (saves bytes).
isReplica: undefined,
};
if (!s3config.replicationCascadeCompatMode) {
return replicationInfo;
}
// No overlapping prefixes in compatibility mode, so one rule applies.
const rule = rules.find(r => r.enabled && objKey.startsWith(r.prefix ?? ''));
return _toLegacyFormat(replicationInfo, config, rule, s3config, isCloud);
}

module.exports = getReplicationInfo;
module.exports.isCloudSite = isCloudSite;
15 changes: 14 additions & 1 deletion lib/api/bucketPutReplication.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
const metadata = require('../metadata/wrapper');
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
const { getReplicationConfiguration } = require('./apiUtils/bucket/getReplicationConfiguration');
const { getReplicationConfiguration, toLegacyConfiguration } = require('./apiUtils/bucket/getReplicationConfiguration');
const validateConfiguration = require('./apiUtils/bucket/validateReplicationConfig');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const monitoring = require('../utilities/monitoringHandler');
const { config: s3config } = require('../Config');

// The error response when a bucket does not have versioning 'Enabled'.
const versioningNotEnabledError = errorInstances.InvalidRequest.customizeDescription(
Expand Down Expand Up @@ -36,6 +37,18 @@
[
// Validate the request XML and return the replication configuration.
next => getReplicationConfiguration(post, log, next),
// In compatibility mode, we store the configuration like a
// cloudserver older than 9.4, or we refuse it.
(config, next) => {
if (!s3config.replicationCascadeCompatMode) {
return next(null, config);
}
const { refused, replicationConfig } = toLegacyConfiguration(config);
if (refused) {
return next(errorInstances[refused.code].customizeDescription(refused.description));
}
return next(null, replicationConfig);
},
Comment on lines +42 to +51
// Check bucket user privileges and ensure versioning is 'Enabled'.
(config, next) =>
// TODO: Validate that destination bucket exists and has versioning.
Expand Down
17 changes: 9 additions & 8 deletions lib/metadata/acl.js
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,15 @@ const acl = {
objectMD.originOp = 's3:ObjectAcl:Put';

// Rebuild replication info from the current bucket config to
// pick up any new destinations. A "role" on the entry means
// backbeat uses IAM role-based auth on the destination
// (CRR). No role means credentials live in the location
// configuration (cloud, including scality-to-scality via
// stored S3 creds), and ACL replication is not supported —
// preserve the existing status instead of resetting to
// PENDING.
const hasDestRole = b => !!b.role;
// pick up any new destinations. On a CRR destination backbeat
// uses IAM role-based auth. On a cloud destination the
// credentials live in the location configuration (cloud,
// including scality-to-scality via stored S3 creds), and ACL
// replication is not supported: we preserve the existing
// status instead of resetting to PENDING.
// We check the location type, not the backend role: in the old
// format a backend has no role.
const hasDestRole = b => !getReplicationInfo.isCloudSite(config, b.site);
const sameBackend = (a, b) => a.site === b.site && a.destination === b.destination && a.role === b.role;

const replicationInfo = getReplicationInfo({
Expand Down
15 changes: 12 additions & 3 deletions lib/routes/routeBackbeat.js
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,9 @@ function putData(request, response, bucketInfo, objMd, log, callback) {
return callback(errorInstances.BadRequest.customizeDescription(errMessage));
}

const incomingVersionIdEncoded = request.query?.versionId;
// In compatibility mode we always write the data, like a cloudserver older
// than 9.4: the check below needs a microVersionId and we store none.
const incomingVersionIdEncoded = config.replicationCascadeCompatMode ? undefined : request.query?.versionId;
if (incomingVersionIdEncoded !== undefined && objMd) {
// objMd is the specific version requested via the versionId query param.
// Its existence means the data is already at the destination. Return 409 with the
Expand Down Expand Up @@ -564,7 +566,11 @@ function getCanonicalIdsByAccountId(accountId, log, cb) {
function putMetadata(request, response, bucketInfo, objMd, log, callback) {
const { bucketName, objectKey } = request;

const encodedMicroVersionId = request.headers['x-scal-micro-version-id'];
// In compatibility mode we ignore x-scal-micro-version-id, so we don't
// compare microVersionId and we don't replicate the replica further.
const encodedMicroVersionId = config.replicationCascadeCompatMode
? undefined
: request.headers['x-scal-micro-version-id'];
// When the header is present this is a conditional write: accept only if
// the incoming microVersionId is newer than the one already stored.
// '' is a valid value meaning the source has no microVersionId (null revision).
Expand Down Expand Up @@ -829,7 +835,10 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
}

const isReplicationWrite = !!headers['x-scal-replication-content'];
if (isReplicationWrite) {
if (config.replicationCascadeCompatMode) {
// A cloudserver older than 9.4 does not store isReplica, so we remove it.
delete omVal.replicationInfo?.isReplica;
} else if (isReplicationWrite) {
const isMDOnly = headers['x-scal-replication-content'] === 'METADATA';
const objSize = omVal['content-length'] || 0;

Expand Down
34 changes: 34 additions & 0 deletions tests/unit/api/apiUtils/getReplicationInfo.js
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,40 @@ describe('getReplicationInfo helper', () => {
});
});

describe('replication compatibility mode', () => {
const compatS3Config = { ...TEST_CONFIG, replicationCascadeCompatMode: true };

it('should write what a cloudserver older than 9.4 writes', () => {
const replicationConfig = {
role: TWO_PART_ROLE,
destination: 'arn:aws:s3:::backup',
rules: [{ prefix: '', enabled: true, storageClass: 'crr-site' }],
};
assert.deepStrictEqual(_getObjectReplicationInfo(compatS3Config, replicationConfig), {
status: 'PENDING',
backends: [{ site: 'crr-site', status: 'PENDING', dataStoreVersionId: '' }],
content: ['METADATA'],
destination: 'arn:aws:s3:::backup',
storageClass: 'crr-site',
role: TWO_PART_ROLE,
storageType: '',
dataStoreVersionId: '',
isNFS: undefined,
});
});

it('should write the rule storageClass as it is, and the types of the cloud backends', () => {
const replicationConfig = {
role: TWO_PART_ROLE,
destination: 'tosomewhere',
rules: [{ prefix: '', enabled: true, storageClass: 'awsbackend:preferred_read,azurebackend' }],
};
const replicationInfo = _getObjectReplicationInfo(compatS3Config, replicationConfig);
assert.strictEqual(replicationInfo.storageClass, 'awsbackend:preferred_read,azurebackend');
assert.strictEqual(replicationInfo.storageType, 'aws_s3,azure');
});
});

// --- V2 Format Tests (multi-rule matching) ---
describe('V2 format (multi-rule matching)', () => {
const V2_ROLE = 'arn:aws:iam::123456:role/src-role,arn:aws:iam::111111:role/dst-role';
Expand Down
Loading
Loading