From 9e3612210cdd4f182311c110e79cb2ce4579d975 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Tue, 27 Oct 2026 17:39:25 +0100 Subject: [PATCH 1/5] S3UTILS-256 Write the S3C 10.1 replication format in crrExistingObjects crrExistingObjects picked its write path from the bucket rule format. Rules without Filter (V1) got the old single-destination format: a new backend had no bucket and no role. Since S3C 10.1, cloudserver writes the multi-destination format for every bucket, whatever the rule format, so backbeat failed with BadRole and the object stayed PENDING (S3C-11599). The rule format only changes where the prefix is. s3utils 1.19 runs on S3C 10.1 and later only. - one write path for every bucket: bucket and role per backend, from the bucket rules (arsenal resolveBackends, like cloudserver on a PUT) - top-level status PENDING: the backbeat queue populator only queues PENDING objects. PROCESSING left objects of V2 buckets never replicated (S3C-11596) - compute content when it is empty (object put before the rule) - rule without StorageClass: SITE_NAME is the S3C default site. Fails with "missing SITE_NAME" when it's not set - SITE_NAME unset: every site of the rules, instead of the site of the first rule - keys outside every rule prefix are skipped: backbeat refuses them anyway - match object backends on site, bucket and role: one site can have several buckets since S3C 10.1. Backends written by S3C 9.5 match on the site - don't add a rule site that TARGET_REPLICATION_STATUS doesn't pick, keep the backends this run doesn't pick as they are - top-level role: source role from the bucket config. Keep the object role and destination while a backend written by S3C 9.5 is kept: backbeat needs them for it - FORCE_USING_CONFIGURATION is ignored: the bucket config is always used (S3UTILS-208) - warn when SITE_NAME is not a site of the bucket rules - keep the unknown-site check (S3UTILS-250), the SITE_NAME requeue of a site the rules don't name anymore, and isReplica - STORAGE_TYPE is ignored --- CRR/ReplicationStatusUpdater.js | 487 ++++++++----- README.md | 92 ++- crrExistingObjects.js | 7 +- tests/functional/crrExistingObjects.js | 1 - tests/unit/CRR/ReplicationStatusUpdater.js | 788 +++++++++++++-------- tests/unit/CRR/crrExistingObjects.js | 7 +- 6 files changed, 902 insertions(+), 480 deletions(-) diff --git a/CRR/ReplicationStatusUpdater.js b/CRR/ReplicationStatusUpdater.js index 4ff3885a..392db269 100644 --- a/CRR/ReplicationStatusUpdater.js +++ b/CRR/ReplicationStatusUpdater.js @@ -32,7 +32,6 @@ class ReplicationStatusUpdater { * @param {string} [params.keyMarker] - (Optional) Key marker for resuming object listing. * @param {string} [params.versionIdMarker] - (Optional) Version ID marker for resuming object listing. * @param {boolean} [params.currentVersionOnly] - (Optional) Whether to process only the current version of objects. - * @param {boolean} [params.forceUsingConfiguration] - (Optional) Force reset replication target to bucket's configuration. * @param {boolean} [params.allowNewSite] - (Optional) Accept a site that is not in the bucket * replication rules and not in the object replication info. */ @@ -53,7 +52,6 @@ class ReplicationStatusUpdater { keyMarker, versionIdMarker, currentVersionOnly, - forceUsingConfiguration, allowNewSite, } = params; @@ -73,7 +71,6 @@ class ReplicationStatusUpdater { this.inputKeyMarker = keyMarker; this.inputVersionIdMarker = versionIdMarker; this.currentVersionOnly = currentVersionOnly; - this.forceUsingConfiguration = forceUsingConfiguration; this.allowNewSite = allowNewSite; this.log = log; @@ -133,41 +130,50 @@ class ReplicationStatusUpdater { /** - * Returns true if the replication config uses the V1 format. - * V2 rules carry a Filter element; V1 rules do not. + * Tells if TARGET_REPLICATION_STATUS picks a backend with this status. * @private - * @param {Object} repConfig - The replication configuration from GetBucketReplicationCommand. - * @returns {boolean} + * @param {string} [status] - Status of the object backend. Undefined when the object + * was never copied to this site and bucket, and that's what NEW picks. + * @returns {boolean} True if we must copy the object to this site and bucket again. */ - _isV1Format(repConfig) { - return repConfig.Rules.every(r => !r.Filter); - } - - /** - * Computes the aggregate top-level replication status from all backends per the V2 rules: - * any FAILED → FAILED; else any PENDING → PROCESSING; else COMPLETED. - * @private - * @param {Array} backends - The backends array from replicationInfo. - * @returns {string} - */ - _computeTopLevelStatus(backends) { - if (backends.some(b => b.status === 'FAILED')) { return 'FAILED'; } - if (backends.some(b => b.status === 'PENDING')) { return 'PROCESSING'; } - return 'COMPLETED'; + _statusShouldBeProcessed(status) { + return this.replicationStatusToProcess.some(filter => + (filter === 'NEW' ? !status : status === filter)); } /** - * Determines if an object should be updated based on its replication metadata properties. + * Tells if an object backend and a candidate are the same: same site and same bucket. + * The site is not enough: since S3C 10.1, two bucket replication rules can send to the + * same site with two buckets. Example: site sf, bucket d1, and site sf, bucket d2. + * A backend written by S3C 9.5 has no bucket: its bucket is the top-level destination + * of the object. A backend of a public cloud site has no bucket, and the object has no + * top-level destination: there, the site is enough. + * When sameRole is false, a backend still matches after the role changed. We need it to + * send the object again with the new role: backbeat keeps refusing the old one (BadRole). + * Example: the bucket Role had a typo, and the customer fixed it. The object backend is + * site sf, bucket d1, role typo-role. The rule now gives role replication-role. * @private - * @param {ObjectMD} objMD - The metadata of the object. - * @param {string} site - The destination site name. - * @returns {boolean} True if the object should be updated. + * @param {Object} existing - Backend of the object. + * @param {Object} candidate - Backend built from the bucket rules. + * @param {Object} [replicationInfo] - Replication info of the object. + * @param {boolean} [sameRole] - When true, we also compare the role. + * @returns {boolean} True if both are the same. */ - _objectShouldBeUpdated(objMD, site) { - const info = objMD.getReplicationInfo(); - const siteStatus = info && objMD.getReplicationSiteStatus({ site }); - return this.replicationStatusToProcess.some(filter => - filter === 'NEW' ? (!info || !siteStatus) : (info && siteStatus === filter)); + _isSameBackend(existing, candidate, replicationInfo, sameRole) { + // With the SITE_NAME fallback (step 3 of _markObjectPending), the candidate is the + // object backend itself. So it is the same, even with no bucket to compare (S3C 9.5). + if (existing === candidate) { + return true; + } + if (existing.site !== candidate.site) { + return false; + } + if (!existing.destination) { + return !replicationInfo?.destination + || candidate.destination === replicationInfo.destination; + } + return existing.destination === candidate.destination + && (!sameRole || existing.role === candidate.role); } /** @@ -220,19 +226,23 @@ class ReplicationStatusUpdater { * * "names X": a rule has StorageClass=X. * + * We only check the objects we write (_markObjectPending, step 6). The table below + * is for TARGET_REPLICATION_STATUS=NEW. With FAILED, COMPLETED or PENDING, we only + * write an object that already has a backend for SITE_NAME, so the check passes. + * * | Rules | Object sites | SITE_NAME | Result | * |------------------------|------------------|-----------|----------------------------------------| * | no StorageClass | none | any value | processed: can't check | * | all name lab-9512 | none | lab-9512 | processed | - * | all name lab-9512 | none | foo | run stopped | + * | all name lab-9512 | none | foo | skipped: no rule gives site foo | * | mixed (one without) | none | any value | processed: can't check | * | no StorageClass | lab-9512 | lab-9512 | processed: requeue to its own site | * | no StorageClass | lab-9512 | foo | run stopped | * | one names lab-94 | lab-9512 | lab-94 | processed: site from the rule | * | one names lab-94 | lab-9512 | lab-9512 | processed: requeue to its own site | - * | one names lab-94 | lab-9512 | foo | run stopped | + * | one names lab-94 | lab-9512 | foo | skipped: no rule gives site foo | * | one names foo | lab-9512 | foo | processed: site from the rule | - * | one names lab-9512 | none | unset | processed with lab-9512 (_markPending) | + * | all name a site | none | unset | processed: every rule site (_markPending) | * | no StorageClass | none | unset | error "missing SITE_NAME" (_markPending) | * | no StorageClass | 9.5 replica (dr) | primary | run stopped: needs ALLOW_NEW_SITE=true | * @@ -319,108 +329,36 @@ class ReplicationStatusUpdater { }); } - /** - * Marks an object as pending for replication. - * @private - * @param {string} bucket - The bucket name. - * @param {string} key - The object key. - * @param {string} versionId - The object version ID. - * @param {string} storageClass - The storage class for replication. - * @param {Object} repConfig - The replication configuration. - * @param {Function} cb - Callback function. - * @returns {void} - */ - _markObjectPending(bucket, key, versionId, storageClass, repConfig, cb) { - this._updateObjectMD(bucket, key, versionId, objMD => { - if (!this._objectShouldBeUpdated(objMD, storageClass)) { - return { skip: true }; - } - // unknown site -> write nothing and stop the run: SITE_NAME is likely - // wrong for every bucket (see _isKnownSite) - if (!this.allowNewSite - && !this._isKnownSite(objMD, storageClass, repConfig.Rules)) { - if (!this._unknownSite) { - this._unknownSite = { bucket, key, versionId }; - this.log.error('unknown replication site, run stopped. SITE_NAME is not a site ' - + 'of this object or of the bucket rules. Expected for the S3C default site ' - + '(rules without StorageClass), e.g. DR failback or renamed DR site. ' - + 'Objects processed before this point may already have this SITE_NAME: ' - + 'if SITE_NAME is wrong, run removeReplicationSite (dry run) with ' - + `SITE_TO_REMOVE=${storageClass} on these buckets to find and remove it. ` - + 'If it reset objects, re-run with the right SITE_NAME from the start of ' - + 'this run (same bucket list and markers): the reset objects are before ' - + 'the resume point. Otherwise re-run from the resume point with the right ' - + 'SITE_NAME, or with ALLOW_NEW_SITE=true', { - bucket, - key, - versionId, - site: storageClass, - ruleSites: this._getConfiguredSites(repConfig.Rules), - objectSites: this._getObjectSites(objMD), - }); - } - return { skip: true }; - } - - let replicationInfo = objMD.getReplicationInfo(); - const { Rules, Role } = repConfig; - const destination = Rules[0].Destination.Bucket; - - if (!replicationInfo || !replicationInfo.status) { - const ops = objMD.getContentLength() === 0 ? ['METADATA'] : ['METADATA', 'DATA']; - replicationInfo = { - status: 'PENDING', - backends: [], - content: ops, - destination, - storageClass: '', - role: Role, - storageType: '', - dataStoreVersionId: '', - }; - objMD.setReplicationInfo(replicationInfo); - } - - if (this.forceUsingConfiguration) { - const ri = objMD.getReplicationInfo(); - ri.destination = destination; - objMD.setReplicationRoles(Role); - } - - if (!objMD.getReplicationSiteStatus({ site: storageClass })) { - const ri = objMD.getReplicationInfo(); - ri.storageClass = ri.storageClass - ? `${ri.storageClass},${storageClass}` : storageClass; - if (this.storageType) { - ri.storageType = ri.storageType - ? `${ri.storageType},${this.storageType}` : this.storageType; - } - const backends = objMD.getReplicationBackends(); - backends.push({ site: storageClass, status: 'PENDING', dataStoreVersionId: '' }); - objMD.setReplicationBackends(backends); - } - - objMD.setReplicationSiteStatus({ site: storageClass }, 'PENDING'); - objMD.setReplicationStatus('PENDING'); - return { skip: false }; - }, cb); - } - /** * Adapts AWS SDK replication config to arsenal's ReplicationConfigurationMetadata shape. + * A bucket replication rule without StorageClass sends the objects to the S3C default + * site. That site is in the federation config (env_replication_endpoints), and we + * can't read it from here. So we use SITE_NAME as the site of these rules. Without a + * site, resolveBackends ignores the rule. * @private * @param {Object} repConfig - The full replication configuration. * @returns {Object} */ _buildArsenalConfig(repConfig) { const { Role, Rules } = repConfig; + // When a rule names SITE_NAME in its StorageClass, we assume the default site is + // another site, and the rules without StorageClass get no site. Otherwise we would + // send their objects to the wrong site. + // Example: r1 has no StorageClass (bucket dr-bucket, default site dr), r2 uses site + // sf, bucket sf-bucket. With SITE_NAME=sf, r1 is left out: we only requeue site sf, + // bucket sf-bucket. With SITE_NAME=dr, r1 gets site dr. + // If the default site is SITE_NAME too, we get it wrong: r1 is left out without a + // word. Workaround: set StorageClass sf on r1 for the run, then put the bucket + // replication config back. + const isRuleSite = this._getConfiguredSites(Rules).includes(this.siteName); + const defaultSite = isRuleSite ? undefined : this.siteName; return { role: Role, destination: Rules[0]?.Destination.Bucket ?? '', rules: Rules.map(r => ({ enabled: r.Status === 'Enabled', prefix: r.Filter?.Prefix ?? r.Prefix ?? '', - storageClass: r.Destination.StorageClass, + storageClass: r.Destination.StorageClass || defaultSite, destination: r.Destination.Bucket, account: r.Destination.Account, priority: r.Priority, @@ -430,8 +368,91 @@ class ReplicationStatusUpdater { } /** - * Marks an object as pending for replication using the V2 multi-destination format. - * Writes per-backend destination and role; strips V1-only top-level storageClass/storageType/destination. + * Stops the run when we don't know SITE_NAME (see _isKnownSite), unless + * ALLOW_NEW_SITE is set. We stop the whole run, not only this object: if SITE_NAME is + * wrong here, it's probably wrong for every bucket. + * @private + * @param {ObjectMD} objMD - Object metadata. + * @param {string} bucket - The bucket name. + * @param {string} key - The object key. + * @param {string} versionId - The object version ID. + * @param {Array} rules - Bucket replication rules. + * @returns {boolean} True if the run is stopped and the object must be skipped. + */ + _stopOnUnknownSite(objMD, bucket, key, versionId, rules) { + if (!this.siteName || this.allowNewSite || this._isKnownSite(objMD, this.siteName, rules)) { + return false; + } + if (!this._unknownSite) { + this._unknownSite = { bucket, key, versionId }; + this.log.error('unknown replication site, run stopped. SITE_NAME is not a site ' + + 'of this object or of the bucket rules. Expected for the S3C default site ' + + '(rules without StorageClass), e.g. DR failback or renamed DR site. ' + + 'Objects processed before this point may already have this SITE_NAME: ' + + 'if SITE_NAME is wrong, run removeReplicationSite (dry run) with ' + + `SITE_TO_REMOVE=${this.siteName} on these buckets to find and remove it. ` + + 'If it reset objects, re-run with the right SITE_NAME from the start of ' + + 'this run (same bucket list and markers): the reset objects are before ' + + 'the resume point. Otherwise re-run from the resume point with the right ' + + 'SITE_NAME, or with ALLOW_NEW_SITE=true', { + bucket, + key, + versionId, + site: this.siteName, + ruleSites: this._getConfiguredSites(rules), + objectSites: this._getObjectSites(objMD), + }); + } + return true; + } + + /** + * Tells backbeat what to send: the metadata, and the data if the object has some. + * An object put before the bucket had a replication rule has an empty list (content []). + * Then we compute it from the object size: with an empty list, backbeat would first + * send the metadata only. + * @private + * @param {Object} [replicationInfo] - The current object replication info. + * @param {ObjectMD} objMD - Object metadata. + * @returns {Array} 'METADATA', and 'DATA' for an object with data. + */ + _getReplicationContent(replicationInfo, objMD) { + if (replicationInfo?.content?.length > 0) { + return replicationInfo.content; + } + if (objMD.getContentLength() === 0) { + return ['METADATA']; + } + return ['METADATA', 'DATA']; + } + + /** + * Marks an object as pending for replication. + * + * We write the same format as cloudserver since S3C 10.1: each backend has its own + * destination bucket and role. Cloudserver does this for every bucket, V1 rules + * (Prefix) or V2 rules (Filter), so we do the same. + * + * "The rules" in the comments below means the replication rules of the bucket + * (GetBucketReplication). + * + * _markPending only calls this for keys that match a rule prefix. If a rule has no + * StorageClass, it also checks that SITE_NAME is set. + * + * Example used in the comments below: + * bucket Role: '...:role/crr-src,...:role/crr-dst' + * bucket rules (V1 or V2, same result): + * r1 Prefix '' Destination { Bucket: 'arn:aws:s3:::backup-sf', StorageClass: 'sf' } + * r2 Prefix 'docs/' Destination { Bucket: 'arn:aws:s3:::backup-dr2', StorageClass: 'dr2' } + * TARGET_REPLICATION_STATUS=NEW, SITE_NAME not set + * object 'docs/a.pdf', already replicated to sf: + * replicationInfo: { status: 'COMPLETED', role: '...:role/crr-src', + * content: ['DATA', 'METADATA'], isNFS: null, + * backends: [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '', + * destination: 'arn:aws:s3:::backup-sf', role: '...:role/crr-dst' }] } + * result: we leave sf as it is and add dr2 as PENDING. Backbeat copies the object + * to site dr2, bucket backup-dr2, and nowhere else. + * * @private * @param {string} bucket - The bucket name. * @param {string} key - The object key. @@ -440,52 +461,148 @@ class ReplicationStatusUpdater { * @param {Function} cb - Callback function. * @returns {void} */ - _markObjectPendingV2(bucket, key, versionId, repConfig, cb) { + _markObjectPending(bucket, key, versionId, repConfig, cb) { this._updateObjectMD(bucket, key, versionId, objMD => { - const arsenalConfig = this._buildArsenalConfig(repConfig); + // 1. We read the object backends: where the object was already sent, and how it + // went. We need them because TARGET_REPLICATION_STATUS is about the object: what + // already happened to it for each site and bucket. The bucket replication rules + // don't know that. + // Example: [site sf, bucket backup-sf, COMPLETED] const prev = objMD.getReplicationInfo(); - const existingBackends = prev?.backends; - + const existingBackends = prev?.backends || []; + + // 2. We build the candidates: where the bucket replication rules send this key + // today (site, bucket and role). We use resolveBackends, like cloudserver on a + // PUT. We need them for every status, not only NEW: a requeue must also use the + // current bucket and role, not the ones the object was sent with (S3UTILS-208). + // The role of a candidate is the destination role of the bucket Role, with the + // account of Destination.Account when the rule has one. + // resolveBackends sets every candidate to PENDING: that's not a decision yet, + // step 5 picks the ones we really send again. + // Example: two candidates, one per rule that matches the key: + // site sf, bucket backup-sf, role ...crr-dst + // site dr2, bucket backup-dr2, role ...crr-dst + const arsenalConfig = this._buildArsenalConfig(repConfig); let candidateBackends = ReplicationConfiguration.resolveBackends( arsenalConfig, key, () => false, existingBackends, ); + // 3. When SITE_NAME is set, we keep only the candidates for that site. We need it + // to requeue one site without touching the others. Example: resend everything + // to dr2 after dr2 lost its data. + // Example: SITE_NAME=dr2 keeps [site dr2, bucket backup-dr2]. if (this.siteName) { candidateBackends = candidateBackends.filter(b => b.site === this.siteName); + if (candidateBackends.length === 0) { + // No rule sends this key to SITE_NAME anymore, but the object may still + // have a backend for it. We need this to retry old replications to a site + // the rules moved away from: we take that backend as it is, with its own + // bucket and role. + // Example: the rules now only name dr3. With SITE_NAME=sf, we take the + // object backend for site sf. + candidateBackends = existingBackends.filter(b => b.site === this.siteName); + } } - const backendsToUpdate = candidateBackends.filter(b => - this._objectShouldBeUpdated(objMD, b.site) - ); + // 4. For each candidate, we find the object backend for the same site and bucket + // (see _isSameBackend), and so its status. We need it to apply + // TARGET_REPLICATION_STATUS. + // No backend means the object was never sent to this site and bucket: NEW picks + // it. That's also the case when a rule moved to another bucket on the same site. + // We look for the same role first: two rules can send to the same site and + // bucket with two Destination.Account, and each one must find its own backend. + // Then we look without the role: the role changed since the object was sent. + // Example: sf has the object backend, COMPLETED. dr2 has no backend. + // Example: say the sf backend has role ...crr-typo, because the bucket Role had + // a typo back then. We still find it, and PENDING picks it. + // + // 5. TARGET_REPLICATION_STATUS picks the candidates to send again. + // Example: NEW picks dr2. COMPLETED would pick sf. + const pairs = candidateBackends.map(candidate => { + const existing = existingBackends.find(e => this._isSameBackend(e, candidate, prev, true)) + || existingBackends.find(e => this._isSameBackend(e, candidate, prev)); + const selected = this._statusShouldBeProcessed(existing?.status); + return { candidate, existing, selected }; + }); - if (backendsToUpdate.length === 0) { + // Nothing picked: we write nothing, and the object counts as skipped. + if (!pairs.some(p => p.selected)) { return { skip: true }; } - const sourceRole = ReplicationConfiguration.resolveSourceRole(repConfig.Role); - const updatedSites = new Set(backendsToUpdate.map(b => b.site)); - // resolveBackends can't match V1-format existing backends (no destination/role), - // so it resets dataStoreVersionId to '' for those — restore it from the original. - const candidateSites = new Set(candidateBackends.map(b => b.site)); - const updatedBackends = candidateBackends.map(c => { - if (updatedSites.has(c.site)) { return c; } - const orig = existingBackends?.find(e => e.site === c.site); - return orig - ? { ...c, status: orig.status, dataStoreVersionId: orig.dataStoreVersionId ?? c.dataStoreVersionId } - : c; + // 6. A SITE_NAME that is in neither the bucket replication rules nor the object is + // probably a typo. We stop before writing anything: a backend for a site that + // no backbeat process handles stays PENDING forever (S3UTILS-250). + // We check only the objects we write: an object we skip gets no backend, so a + // wrong SITE_NAME can't hurt it. Otherwise a FAILED retry run would stop on an + // object that has nothing to retry. + // Example: SITE_NAME is not set, so there is nothing to check. + // Example: say r1 has no StorageClass, and SITE_NAME=sff. r1 gets site sff + // (see _buildArsenalConfig), and NEW picks site sff, bucket backup-sf. The + // object only knows site sf: we stop the run. + if (this._stopOnUnknownSite(objMD, bucket, key, versionId, repConfig.Rules)) { + return { skip: true }; + } + + // 7. We build the backends to write. Only what we picked changes: this run must not + // touch the rest of the object. + // We keep the object backends in their order and add the new ones at the end. + // Backbeat finds a backend written by S3C 9.5 by its site only, so it takes the + // first backend of that site: it must stay before a new one for the same site. + // Example: [site sf, bucket backup-sf, COMPLETED (as it was); + // site dr2, bucket backup-dr2, PENDING] + const allBackends = existingBackends.map(existing => { + const pair = pairs.find(p => p.existing === existing); + // Picked: send it again, to the site and bucket of the rule, with the role + // of the rule. Not picked, or no rule for it: leave it as it is. + return pair?.selected ? { ...pair.candidate, status: 'PENDING' } : existing; }); - // Preserve backends for sites not targeted by this run (e.g. when SITE_NAME is set), - // so they are not silently dropped from the metadata. - const preservedBackends = (existingBackends || []).filter(b => !candidateSites.has(b.site)); - const allBackends = [...updatedBackends, ...preservedBackends]; - - objMD.setReplicationInfo({ - status: this._computeTopLevelStatus(allBackends), - role: this.forceUsingConfiguration ? sourceRole : (prev?.role || sourceRole), + for (const { candidate, existing, selected } of pairs) { + // Picked and not on the object yet: add it at the end. + if (selected && !existing) { + allBackends.push({ ...candidate, status: 'PENDING' }); + } + } + + // 8. We build the replication info to write. + // Role: backbeat sends a backend with ','. + // So we only write the source role, from the bucket config, like cloudserver on + // a PUT. + // But a backend written by S3C 9.5 has no bucket and no role: backbeat takes + // them from the top-level destination and role. While we keep such a backend, we + // keep the destination and the destination role of the object. S3C 9.5 allowed + // one bucket and one Role per bucket config, so all these backends share them. + // The source role still comes from the bucket config: backbeat uses it for the + // new backends too, and an old one would make them fail (BadRole). + // Example: { status: 'PENDING', role: '...crr-src', backends: [sf, dr2], + // content: ['DATA', 'METADATA'], isNFS: null } + // Example: say sf is a backend written by S3C 9.5, and the object role is + // '...old-src,...old-dst'. We write role '...crr-src,...old-dst'. + const keepsOldBackend = allBackends.some(b => !b.role); + const sourceRole = ReplicationConfiguration.resolveSourceRole(repConfig.Role); + // Undefined when the object role has no destination role. Example: a public + // cloud site in S3C 9.5 has a single role. Then we write the source role alone. + const oldDestinationRole = keepsOldBackend ? prev?.role?.split(',')[1] : undefined; + const replicationInfo = { + // PENDING: backbeat only picks up objects that are PENDING. Backbeat sets + // PROCESSING itself later, when one site is done and others aren't. + status: 'PENDING', + role: oldDestinationRole ? `${sourceRole},${oldDestinationRole}` : sourceRole, backends: allBackends, - content: prev?.content ?? (objMD.getContentLength() === 0 ? ['METADATA'] : ['METADATA', 'DATA']), + content: this._getReplicationContent(prev, objMD), isNFS: prev?.isNFS ?? null, - }); + }; + if (keepsOldBackend && prev?.destination) { + replicationInfo.destination = prev.destination; + } + // A replica that we send further (cascading) is still a replica. + if (prev?.isReplica) { + replicationInfo.isReplica = true; + } + + // 9. We save it. _updateObjectMD gives the object a new microVersionId: we need it so + // the destination accepts this update as newer than what it has. + objMD.setReplicationInfo(replicationInfo); return { skip: false }; }, cb); } @@ -533,43 +650,51 @@ class ReplicationStatusUpdater { (repConfig, next) => { const { Rules } = repConfig; - if (this._isV1Format(repConfig)) { - const storageClass = this.siteName || Rules[0].Destination.StorageClass; - if (!storageClass) { - const errMsg = 'missing SITE_NAME environment variable, must be set to' - + ' the value of "site" property in the CRR configuration'; - this.log.error(errMsg); - return next(new Error(errMsg)); - } - if (!this.siteName) { - this.log.warn(`missing SITE_NAME environment variable, triggering replication to the ${storageClass} storage class`); - } - return eachLimit(versions, this.workers, (i, apply) => { - const { Key, VersionId, IsLatest } = i; - if (this._unknownSite) { - // run stopped on an unknown site: don't start new objects - apply(); - return; - } - if (this.currentVersionOnly && !IsLatest) { - ++this._nSkipped; - apply(); - return; - } - this._markObjectPending(bucket, Key, VersionId, storageClass, repConfig, apply); - }, next); + // A bucket replication rule without StorageClass sends the objects to the S3C + // default site, which we can't read from here: SITE_NAME must give it (see + // _buildArsenalConfig). Without it, we stop with "missing SITE_NAME". + const hasRuleWithoutSite = Rules.some(r => + r.Status === 'Enabled' && !r.Destination.StorageClass + ); + if (!this.siteName && hasRuleWithoutSite) { + const errMsg = 'missing SITE_NAME environment variable, must be set to' + + ' the value of "site" property in the CRR configuration'; + this.log.error(errMsg); + return next(new Error(errMsg)); + } + // We warn when no rule sends objects to SITE_NAME: no rule names it, and no rule + // uses the default site. Then we only requeue the objects that already have a + // backend for SITE_NAME, and skip the others. The warning tells the user why few + // or no objects are updated. We say it once per bucket: _markPending runs for + // each listing page. + // Example: the rules name sf and dr2, and SITE_NAME=dr3. + const ruleSites = this._getConfiguredSites(Rules); + if (this.siteName && !hasRuleWithoutSite && !ruleSites.includes(this.siteName) + && this._siteWarnedBucket !== bucket) { + this._siteWarnedBucket = bucket; + this.log.warn('SITE_NAME is not a site of the bucket rules: only objects that ' + + 'already replicate to it are requeued, the other objects are skipped', { + bucket, + site: this.siteName, + ruleSites, + }); } - return eachLimit(versions, this.workers, (i, apply) => { const { Key, VersionId, IsLatest } = i; + if (this._unknownSite) { + // The run stopped on an unknown site: we don't start new objects. + apply(); + return; + } if (this.currentVersionOnly && !IsLatest) { ++this._nSkipped; apply(); return; } + // We skip a key that no enabled rule covers: backbeat refuses it anyway + // ("replication disabled for object"). const hasMatchingRule = Rules.some(r => r.Status === 'Enabled' && - (!this.siteName || r.Destination.StorageClass === this.siteName) && Key.startsWith(r.Filter?.Prefix ?? r.Prefix ?? '') ); if (!hasMatchingRule) { @@ -577,7 +702,7 @@ class ReplicationStatusUpdater { apply(); return; } - this._markObjectPendingV2(bucket, Key, VersionId, repConfig, apply); + this._markObjectPending(bucket, Key, VersionId, repConfig, apply); }, next); }, ], cb); diff --git a/README.md b/README.md index 2e4396fc..13834bef 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,63 @@ Example: Only scan keys beginning with "foo/" +Keys that match no enabled bucket replication rule prefix are skipped: +Backbeat refuses them anyway ("replication disabled for object"). -#### STORAGE_TYPE +#### SITE_NAME -Comma-separated list of the destination storage location types. This is used -only if a replication destination is a public cloud. +Replication site to requeue. The script only touches the backends of the +object for this site, and leaves the other sites as they are. -The recognized statuses are: +Examples, with these bucket rules: -* **aws_s3**: The destination storage location type is Amazon S3. +* r1: no prefix, `StorageClass` sf, bucket D1 +* r2: prefix `docs/`, `StorageClass` dr2, bucket D2 -* **azure**: The destination storage location type is Microsoft Azure Blob - Storage. +`SITE_NAME=dr2` -* **gcp**: The destination storage location type is Google Cloud Storage. +Requeue site dr2 only. Keys under `docs/` go to bucket D2. The other keys +are skipped: no rule sends them to dr2. The sf backends stay as they are. -Examples: +`SITE_NAME` not set + +Requeue every site of the rules that match the key. `docs/a.pdf` goes to +sf (D1) and dr2 (D2), `a.pdf` goes to sf (D1). + +`SITE_NAME=dr3` + +No rule sends objects to dr3. The script logs the warning "SITE_NAME is +not a site of the bucket rules", once per bucket. It skips the objects +with no backend for dr3. With `TARGET_REPLICATION_STATUS=FAILED`, +`COMPLETED` or `PENDING`, it still requeues the objects that have a dr3 +backend with that status, to their own bucket. The run doesn't stop. -`STORAGE_TYPE=aws_s3` +A rule without `StorageClass` sends the objects to the S3C default site, +set in the federation config (`env_replication_endpoints`). The script +can't read it, so it uses `SITE_NAME` as that site: -The destination storage location type is AWS. +* `SITE_NAME` is required. Without it, the script fails with "missing + SITE_NAME environment variable". +* Except when another rule names `SITE_NAME` in its `StorageClass`: then + the script assumes the default site is another site, and skips the rule + without `StorageClass`. -`STORAGE_TYPE=aws_s3,azure,gcp` +Example: the default site is sf, rule r1 has no `StorageClass` (bucket +D1), rule r2 has `StorageClass` sf (bucket D2). With `SITE_NAME=sf`, the +objects of r1 on site sf, bucket D1 are skipped. To requeue them, add +`StorageClass` sf to r1, run the script, then put the bucket replication +config back. + +#### FORCE_USING_CONFIGURATION + +Ignored since S3C 10.1: the script always uses the current bucket +replication configuration (destination bucket and roles), like cloudserver +does on a PUT. + +#### STORAGE_TYPE -The destination storage type is a one-to-many configuration replicating to AWS, -Azure, and GCP destination storage locations. +Ignored since S3C 10.1: the object metadata no longer stores the location +type, Backbeat reads it from the location config of the site. #### WORKERS @@ -167,19 +199,25 @@ Example: #### ALLOW_NEW_SITE A backend for a site that no replication processor handles leaves the -object PENDING forever. So the script checks `SITE_NAME` when it has -something reliable to compare against: +object PENDING forever. So before it writes an object, the script checks +`SITE_NAME` when it has something reliable to compare against. Objects it +skips are not checked: they get no backend. So only +`TARGET_REPLICATION_STATUS=NEW` can stop on an unknown site: with +`FAILED`, `COMPLETED` or `PENDING`, the script only writes objects that +already have a backend for `SITE_NAME`. The checks are: * the object already replicates: `SITE_NAME` must be one of its sites (backend sites, or `storageClass` on older objects), or a site named in the `StorageClass` of the bucket replication rules -* the object was never replicated and every bucket rule names its site in - `StorageClass`: `SITE_NAME` must be one of those sites Otherwise the object is accepted: a rule without `StorageClass` uses the S3C default replication site, set in the federation config (`env_replication_endpoints`), which the script can't read. +When every bucket rule names its site in `StorageClass` and `SITE_NAME` is +none of them, the script doesn't stop: no rule gives the object a backend +for `SITE_NAME`, so the object is skipped (see [SITE_NAME](#site_name)). + At the first object with an unknown site, the script writes nothing for it, stops, logs the object and the sites it compared against, logs the bucket list, `KEY_MARKER` and `VERSION_ID_MARKER` to resume from that @@ -219,6 +257,10 @@ Example: #### Notes on S3C versions +* s3utils 1.19 and later run on S3C 10.1 and later only. They write the + object metadata like cloudserver 9.4+: each backend has its own + destination bucket and role, whatever the bucket rule format (`Prefix` + or `Filter`). * S3C 9.5.2 / 9.5.4: Backbeat routes on the replication `storageClass`. Re-triggering one site with `SITE_NAME` also re-sends the objects under the `sourceCheckIfSizeGreaterThanMB` limit of the Backbeat config @@ -290,6 +332,20 @@ don't set it in `StorageClass`, also set: export ALLOW_NEW_SITE=true ``` +#### Replicate existing objects to the new bucket of a rule + +When a rule now sends its objects to another bucket on the same site, the +objects already replicated have a backend for the old bucket only. `NEW` +picks the new bucket and keeps the backend of the old one: + +``` +export TARGET_REPLICATION_STATUS=NEW +export MAX_UPDATES=10000 +``` + +`COMPLETED` doesn't send them to the new bucket: it only requeues the +buckets the objects were already sent to, and that a rule still uses. + #### Re-sync a DR site back to the primary site When objects have been lost from the primary site you can re-sync diff --git a/crrExistingObjects.js b/crrExistingObjects.js index e5d13ac3..196bf66c 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -15,7 +15,6 @@ const { SITE_NAME } = process.env; let { STORAGE_TYPE } = process.env; let { TARGET_REPLICATION_STATUS } = process.env; const { TARGET_PREFIX } = process.env; -const FORCE_USING_CONFIGURATION = process.env.FORCE_USING_CONFIGURATION === 'true' || process.env.FORCE_USING_CONFIGURATION === '1' || false; const WORKERS = (process.env.WORKERS && Number.parseInt(process.env.WORKERS, 10)) || 10; const MAX_UPDATES = (process.env.MAX_UPDATES @@ -78,6 +77,11 @@ log.info('Objects with replication status ' + `${replicationStatusToProcess.join(' or ')} ` + 'will be reset to PENDING to trigger CRR'); +if (process.env.FORCE_USING_CONFIGURATION) { + log.warn('FORCE_USING_CONFIGURATION is ignored: the script always uses the bucket ' + + 'replication configuration'); +} + const replicationStatusUpdater = new ReplicationStatusUpdater({ buckets: BUCKETS, replicationStatusToProcess, @@ -94,7 +98,6 @@ const replicationStatusUpdater = new ReplicationStatusUpdater({ keyMarker: KEY_MARKER, versionIdMarker: VERSION_ID_MARKER, currentVersionOnly: CURRENT_VERSION_ONLY, - forceUsingConfiguration: FORCE_USING_CONFIGURATION, allowNewSite: ALLOW_NEW_SITE, }, log); diff --git a/tests/functional/crrExistingObjects.js b/tests/functional/crrExistingObjects.js index 99d22fb0..9ac6798c 100644 --- a/tests/functional/crrExistingObjects.js +++ b/tests/functional/crrExistingObjects.js @@ -99,7 +99,6 @@ describe('crrExistingObjects', () => { storageType: '', listingLimit: 1000, currentVersionOnly: false, - forceUsingConfiguration: true, // these tests check key encoding: SITE_NAME is not the bucket's // default replication site, so allow it explicitly allowNewSite: true, diff --git a/tests/unit/CRR/ReplicationStatusUpdater.js b/tests/unit/CRR/ReplicationStatusUpdater.js index d4feba3b..18410f40 100644 --- a/tests/unit/CRR/ReplicationStatusUpdater.js +++ b/tests/unit/CRR/ReplicationStatusUpdater.js @@ -14,6 +14,15 @@ const { const logger = new werelogs.Logger('ReplicationStatusUpdater::tests', 'debug', 'debug'); +// The backend we write for the default fixture rule (V1, StorageClass aws-location). +const newBackend = { + site: 'aws-location', + status: 'PENDING', + dataStoreVersionId: '', + destination: 'arn:aws:s3:::destination', + role: 'arn:aws:iam::root:role/s3-replication-role', +}; + describe('ReplicationStatusUpdater', () => { it('should process bucket for CRR', done => { const crr = initializeCrrWithMocks({ @@ -55,28 +64,15 @@ describe('ReplicationStatusUpdater', () => { }, expect.any(Function)); expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1); - const expectedReplicationInfo = { + // We write a bucket and a role on each backend, for a V1 rule too. + const { replicationInfo } = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); + expect(replicationInfo).toEqual({ status: 'PENDING', - backends: [ - { - site: 'aws-location', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'aws-location', role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: '', - dataStoreVersionId: '', - }; - expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledWith( - expect.objectContaining({ - Body: expect.stringContaining(JSON.stringify(expectedReplicationInfo)), - }), - expect.any(Function), - ); + backends: [newBackend], + content: ['METADATA', 'DATA'], + isNFS: null, + }); assert.strictEqual(crr._nProcessed, 1); assert.strictEqual(crr._nSkipped, 0); @@ -143,6 +139,34 @@ describe('ReplicationStatusUpdater', () => { }); }); + // An object written by S3C 9.5: its backends have no bucket and no role. + const oldFormat = (backends, storageClass, storageType) => ({ + status: backends.some(b => b.status === 'FAILED') ? 'FAILED' : 'COMPLETED', + backends, + content: ['METADATA', 'DATA'], + destination: 'arn:aws:s3:::destination', + storageClass, + role: 'arn:aws:iam::root:role/s3-replication-role', + storageType, + dataStoreVersionId: '', + }); + // What we write when we replace every S3C 9.5 backend: no top-level destination. + const replacedOldFormat = { + status: 'PENDING', + role: 'arn:aws:iam::root:role/s3-replication-role', + backends: [newBackend], + content: ['METADATA', 'DATA'], + isNFS: null, + }; + // What we write when we keep an S3C 9.5 backend: we keep the top-level destination. + const fromOldFormat = backends => ({ + status: 'PENDING', + role: 'arn:aws:iam::root:role/s3-replication-role', + backends, + content: ['METADATA', 'DATA'], + isNFS: null, + destination: 'arn:aws:s3:::destination', + }); [ { description: 'for an object with a null replication info', @@ -150,19 +174,10 @@ describe('ReplicationStatusUpdater', () => { replicationStatusToProcess: ['NEW'], expectedReplicationInfo: { status: 'PENDING', - backends: [ - { - site: 'aws-location', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'aws-location', role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - dataStoreVersionId: '', + backends: [newBackend], + content: ['METADATA', 'DATA'], + isNFS: null, }, }, { description: 'for an object with empty replication info', @@ -179,182 +194,49 @@ describe('ReplicationStatusUpdater', () => { replicationStatusToProcess: ['NEW'], expectedReplicationInfo: { status: 'PENDING', - backends: [ - { - site: 'aws-location', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'aws-location', role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - dataStoreVersionId: '', + backends: [newBackend], + content: ['METADATA', 'DATA'], + isNFS: null, }, }, { description: 'for an object with a failed replication', - replicationInfo: { - status: 'FAILED', - backends: [ - { - site: 'aws-location', - status: 'FAILED', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'aws-location', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - dataStoreVersionId: '', - }, + replicationInfo: oldFormat( + [{ site: 'aws-location', status: 'FAILED', dataStoreVersionId: '' }], 'aws-location', 'aws_s3'), replicationStatusToProcess: ['FAILED'], - expectedReplicationInfo: { - status: 'PENDING', - backends: [ - { - site: 'aws-location', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'aws-location', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - dataStoreVersionId: '', - }, + // We replace the only S3C 9.5 backend, so we drop the top-level destination. + expectedReplicationInfo: replacedOldFormat, }, { description: 'for an object with a completed replication', - replicationInfo: { - status: 'COMPLETED', - backends: [ - { - site: 'aws-location', - status: 'COMPLETED', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'aws-location', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - dataStoreVersionId: '', - }, + replicationInfo: oldFormat( + [{ site: 'aws-location', status: 'COMPLETED', dataStoreVersionId: '' }], 'aws-location', 'aws_s3'), replicationStatusToProcess: ['COMPLETED'], - expectedReplicationInfo: { - status: 'PENDING', - backends: [ - { - site: 'aws-location', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'aws-location', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - dataStoreVersionId: '', - }, + expectedReplicationInfo: replacedOldFormat, }, { description: 'of a single site for an object with multiple replication destinations', - replicationInfo: { - status: 'FAILED', - backends: [ - { - site: 'azure-location', - status: 'COMPLETED', - dataStoreVersionId: '', - }, - { - site: 'aws-location', - status: 'FAILED', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'azure-location,aws-location', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'azure,aws_s3', - dataStoreVersionId: '', - }, + replicationInfo: oldFormat([ + { site: 'azure-location', status: 'COMPLETED', dataStoreVersionId: '' }, + { site: 'aws-location', status: 'FAILED', dataStoreVersionId: '' }, + ], 'azure-location,aws-location', 'azure,aws_s3'), replicationStatusToProcess: ['FAILED'], - expectedReplicationInfo: { - status: 'PENDING', - backends: [ - { - site: 'azure-location', - status: 'COMPLETED', - dataStoreVersionId: '', - }, { - site: 'aws-location', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'azure-location,aws-location', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'azure,aws_s3', - dataStoreVersionId: '', - }, + // The backends keep their order: we replace aws-location in place. + expectedReplicationInfo: fromOldFormat([ + { site: 'azure-location', status: 'COMPLETED', dataStoreVersionId: '' }, + newBackend, + ]), }, { description: 'of a single non initialized site for an object with multiple replication destinations', - replicationInfo: { - status: 'FAILED', - backends: [ - { - site: 'azure-location', - status: 'COMPLETED', - dataStoreVersionId: '', - }, - { - site: 'azure-location-2', - status: 'FAILED', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'azure-location,azure-location-2', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'azure,azure', - dataStoreVersionId: '', - }, + replicationInfo: oldFormat([ + { site: 'azure-location', status: 'COMPLETED', dataStoreVersionId: '' }, + { site: 'azure-location-2', status: 'FAILED', dataStoreVersionId: '' }, + ], 'azure-location,azure-location-2', 'azure,azure'), replicationStatusToProcess: ['NEW'], - expectedReplicationInfo: { - status: 'PENDING', - backends: [ - { - site: 'azure-location', - status: 'COMPLETED', - dataStoreVersionId: '', - }, { - site: 'azure-location-2', - status: 'FAILED', - dataStoreVersionId: '', - }, { - site: 'aws-location', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA', 'DATA'], - destination: 'arn:aws:s3:::destination', - storageClass: 'azure-location,azure-location-2,aws-location', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'azure,azure,aws_s3', - dataStoreVersionId: '', - }, + // The backends keep their order: we add the new one at the end. + expectedReplicationInfo: fromOldFormat([ + { site: 'azure-location', status: 'COMPLETED', dataStoreVersionId: '' }, + { site: 'azure-location-2', status: 'FAILED', dataStoreVersionId: '' }, + newBackend, + ]), }, ].forEach(params => { it(`should trigger replication ${params.description}`, done => { @@ -398,12 +280,8 @@ describe('ReplicationStatusUpdater', () => { expect(crr.cloudserverclient.getMetadata).toHaveBeenCalledTimes(1); expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1); - expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledWith( - expect.objectContaining({ - Body: expect.stringContaining(JSON.stringify(params.expectedReplicationInfo)), - }), - expect.any(Function), - ); + const { replicationInfo } = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); + expect(replicationInfo).toEqual(params.expectedReplicationInfo); assert.strictEqual(crr._nProcessed, 1); assert.strictEqual(crr._nSkipped, 0); @@ -585,58 +463,86 @@ describe('ReplicationStatusUpdater with specifics', () => { }); }); -describe('ReplicationStatusUpdater with forceUsingConfiguration', () => { - it('should overwrite replication destination and role when forceUsingConfiguration is true', done => { - // Deep copy objectMd from utils to avoid affecting the original - const objectMdWithOldReplication = JSON.parse(JSON.stringify(objectMd)); - - // Only modify the replicationInfo part - objectMdWithOldReplication.replicationInfo = { - status: 'COMPLETED', - backends: [ - { - site: 'sf', - status: 'COMPLETED', - dataStoreVersionId: '', - }, - ], - content: ['DATA', 'METADATA'], - destination: 'arn:aws:s3:::destination2', - storageClass: 'sf', - role: 'arn:aws:iam::123456789012:role/src-resource,arn:aws:iam::123456789012:role/dest-resource', - storageType: '', - dataStoreVersionId: '', - }; - +describe('ReplicationStatusUpdater top-level role', () => { + function initCrr(params, replicationInfo) { const crr = initializeCrrWithMocks({ buckets: ['bucket0'], workers: 10, replicationStatusToProcess: ['NEW'], - forceUsingConfiguration: true, + ...params, }, logger); + crr.cloudserverclient.getMetadata = jest.fn((p, cb) => { + const md = JSON.parse(getMetadataRes.Body); + md.replicationInfo = replicationInfo; + cb(null, { Body: JSON.stringify(md) }); + }); + return crr; + } - // Override getMetadata to return object with old replication info - crr.cloudserverclient.getMetadata = jest.fn((params, cb) => cb(null, { - Body: JSON.stringify(objectMdWithOldReplication), - })); + it('should take the source role from the bucket config', done => { + // Cloudserver wrote this object when the bucket had another source role. + const crr = initCrr({ replicationStatusToProcess: ['COMPLETED'] }, { + status: 'COMPLETED', + role: 'arn:aws:iam::root:role/old-source-role', + backends: [{ ...newBackend, status: 'COMPLETED' }], + content: ['DATA', 'METADATA'], + isNFS: null, + }); + crr.run(err => { + assert.ifError(err); + const { replicationInfo } = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); + expect(replicationInfo.role).toBe('arn:aws:iam::root:role/s3-replication-role'); + expect(replicationInfo.backends).toEqual([newBackend]); + done(); + }); + }); + it('should keep the destination and destination role while a backend written by S3C 9.5 is kept', done => { + // Site sf is an S3C 9.5 backend that no rule names: we keep it, so we keep the + // top-level destination and destination role it needs. The source role comes from + // the bucket config: backbeat uses it for the new backend too. + const crr = initCrr({}, { + status: 'COMPLETED', + backends: [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '' }], + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::destination2', + storageClass: 'sf', + role: 'arn:aws:iam::123456789012:role/src-resource,arn:aws:iam::123456789012:role/dest-resource', + storageType: '', + dataStoreVersionId: '', + }); crr.run(err => { assert.ifError(err); + const { replicationInfo } = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); + expect(replicationInfo.role).toBe( + 'arn:aws:iam::root:role/s3-replication-role,arn:aws:iam::123456789012:role/dest-resource'); + expect(replicationInfo.destination).toBe('arn:aws:s3:::destination2'); + expect(replicationInfo.backends).toEqual([ + { site: 'sf', status: 'COMPLETED', dataStoreVersionId: '' }, + newBackend, + ]); + done(); + }); + }); - expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1); - - // Verify that putMetadata was called with updated destination and role from bucket config - const putMetadataCall = crr.cloudserverclient.putMetadata.mock.calls[0][0]; - const updatedMetadata = JSON.parse(putMetadataCall.Body); - - // Check that replicationInfo contains the bucket configuration values - expect(updatedMetadata.replicationInfo.destination).toBe('arn:aws:s3:::destination'); - expect(updatedMetadata.replicationInfo.role).toBe('arn:aws:iam::root:role/s3-replication-role'); - - assert.strictEqual(crr._nProcessed, 1); - assert.strictEqual(crr._nSkipped, 0); - assert.strictEqual(crr._nUpdated, 1); - assert.strictEqual(crr._nErrors, 0); + it('should write only the source role when the S3C 9.5 object role has no destination role', done => { + // A public cloud site in S3C 9.5 has a single role: there is no destination role to + // keep, so we write the source role alone. + const crr = initCrr({}, { + status: 'COMPLETED', + backends: [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '' }], + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::destination2', + storageClass: 'sf', + role: 'arn:aws:iam::123456789012:role/src-resource', + storageType: '', + dataStoreVersionId: '', + }); + crr.run(err => { + assert.ifError(err); + const { replicationInfo } = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); + expect(replicationInfo.role).toBe('arn:aws:iam::root:role/s3-replication-role'); + expect(replicationInfo.destination).toBe('arn:aws:s3:::destination2'); done(); }); }); @@ -984,8 +890,8 @@ describe('ReplicationStatusUpdater V2 format', () => { dataStoreVersionId: '', }); - // All pending → top-level PROCESSING - expect(repInfo.status).toBe('PROCESSING'); + // The backbeat queue populator only queues PENDING objects. + expect(repInfo.status).toBe('PENDING'); assert.strictEqual(crr._nProcessed, 1); assert.strictEqual(crr._nUpdated, 1); @@ -1176,7 +1082,7 @@ describe('ReplicationStatusUpdater V2 format', () => { }); }); - it('should compute PROCESSING top-level status for docs key with mixed backend statuses in V2', done => { + it('should set top-level status PENDING for docs key with mixed backend statuses in V2', done => { const listVersionDocsKey = { IsTruncated: false, Versions: [{ @@ -1223,8 +1129,8 @@ describe('ReplicationStatusUpdater V2 format', () => { const body = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); const repInfo = body.replicationInfo; - // dest-A COMPLETED, dest-B PENDING → PROCESSING - expect(repInfo.status).toBe('PROCESSING'); + // dest-A is COMPLETED and dest-B is PENDING: we write PENDING so backbeat queues dest-B. + expect(repInfo.status).toBe('PENDING'); expect(repInfo.backends).toHaveLength(2); const destA = repInfo.backends.find(b => b.site === 'dest-A'); const destB = repInfo.backends.find(b => b.site === 'dest-B'); @@ -1234,10 +1140,10 @@ describe('ReplicationStatusUpdater V2 format', () => { }); }); - it('should upgrade V1-format skipped backends to V2 shape when processing mixed status', done => { + it('should keep a V1-format backend unchanged when this run does not select it', done => { // dest-A is COMPLETED in V1 format (no per-backend destination/role) — skipped by NEW filter // dest-B is NEW and matches 'docs/' prefix — should be set to PENDING - // After processing, dest-A must have V2-shaped destination/role and its COMPLETED status preserved + // We leave dest-A as it is: this run doesn't pick it. const listVersionDocsKey = { IsTruncated: false, Versions: [{ @@ -1288,11 +1194,9 @@ describe('ReplicationStatusUpdater V2 format', () => { const destA = repInfo.backends.find(b => b.site === 'dest-A'); const destB = repInfo.backends.find(b => b.site === 'dest-B'); - // dest-A: status preserved, V2 fields populated from config - expect(destA.status).toBe('COMPLETED'); - expect(destA.dataStoreVersionId).toBe('v1-stored-id'); - expect(destA.destination).toBe('arn:aws:s3:::bucket-a'); - expect(destA.role).toBe('arn:aws:iam::222222222222:role/repRule'); + // We keep dest-A, so we keep its bucket: the top-level destination. + expect(destA).toEqual({ site: 'dest-A', status: 'COMPLETED', dataStoreVersionId: 'v1-stored-id' }); + expect(repInfo.destination).toBe('arn:aws:s3:::bucket-a'); // dest-B: newly queued expect(destB.status).toBe('PENDING'); @@ -1300,30 +1204,30 @@ describe('ReplicationStatusUpdater V2 format', () => { }); }); - it('should update per-backend destination and role when forceUsingConfiguration is true in V2', done => { - // Use COMPLETED filter so the object (dest-A COMPLETED) is re-processed with forceUsingConfiguration + it('should copy to the new bucket of a rule with NEW, and keep the old backend', done => { + // The object was sent to old-bucket, and the rule for dest-A now uses bucket-a. The + // object is not on bucket-a yet, so NEW picks it. const crr = initializeCrrWithMocks({ buckets: ['bucket0'], workers: 10, - replicationStatusToProcess: ['COMPLETED'], - forceUsingConfiguration: true, + replicationStatusToProcess: ['NEW'], }, logger, { GetBucketReplicationCommand: getBucketReplicationV2Res, }); - // Object has dest-A with stale top-level role and stale backend destination/role + const oldBackend = { + site: 'dest-A', + status: 'COMPLETED', + destination: 'arn:aws:s3:::old-bucket', + role: 'arn:aws:iam::999999999999:role/oldRole', + dataStoreVersionId: '', + }; crr.cloudserverclient.getMetadata = jest.fn((p, cb) => { const md = JSON.parse(getMetadataRes.Body); md.replicationInfo = { status: 'COMPLETED', role: 'arn:aws:iam::OLD:role/staleSourceRole', - backends: [{ - site: 'dest-A', - status: 'COMPLETED', - destination: 'arn:aws:s3:::old-bucket', - role: 'arn:aws:iam::999999999999:role/oldRole', - dataStoreVersionId: '', - }], + backends: [oldBackend], content: ['METADATA', 'DATA'], }; cb(null, { Body: JSON.stringify(md) }); @@ -1336,10 +1240,49 @@ describe('ReplicationStatusUpdater V2 format', () => { const body = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); const repInfo = body.replicationInfo; + // The source role comes from the bucket config, not from the object. expect(repInfo.role).toBe('arn:aws:iam::8765432:role/sourceRole'); - const destA = repInfo.backends.find(b => b.site === 'dest-A'); - expect(destA.destination).toBe('arn:aws:s3:::bucket-a'); - expect(destA.role).toBe('arn:aws:iam::222222222222:role/repRule'); + expect(repInfo.backends).toEqual([ + oldBackend, + { + site: 'dest-A', + status: 'PENDING', + dataStoreVersionId: '', + destination: 'arn:aws:s3:::bucket-a', + role: 'arn:aws:iam::222222222222:role/repRule', + }, + ]); + done(); + }); + }); + + it('should not move objects to the new bucket of a rule with COMPLETED', done => { + // Same object: bucket-a has no backend yet, so COMPLETED doesn't pick it, and no rule + // uses old-bucket anymore. We write nothing. + const crr = initializeCrrWithMocks({ + buckets: ['bucket0'], + workers: 10, + replicationStatusToProcess: ['COMPLETED'], + }, logger, { + GetBucketReplicationCommand: getBucketReplicationV2Res, + }); + crr.cloudserverclient.getMetadata = jest.fn((p, cb) => { + const md = JSON.parse(getMetadataRes.Body); + md.replicationInfo = { + status: 'COMPLETED', + role: 'arn:aws:iam::8765432:role/sourceRole', + backends: [{ + site: 'dest-A', status: 'COMPLETED', dataStoreVersionId: '', + destination: 'arn:aws:s3:::old-bucket', role: 'arn:aws:iam::999999999999:role/oldRole', + }], + content: ['METADATA', 'DATA'], + }; + cb(null, { Body: JSON.stringify(md) }); + }); + crr.run(err => { + assert.ifError(err); + expect(crr.cloudserverclient.putMetadata).not.toHaveBeenCalled(); + assert.strictEqual(crr._nSkipped, 1); done(); }); }); @@ -1449,14 +1392,16 @@ describe('ReplicationStatusUpdater site validation', () => { return crr; } - it('should stop the run when every rule names its site and SITE_NAME is not one of them', done => { - // default fixture: rule with StorageClass=aws-location, object never replicated - const crr = initCrr({ siteName: 'foo' }); + it('should not stop a FAILED run on an object that has nothing to retry', done => { + // The rule has no StorageClass, so it gets site foo, which the object doesn't know. + // But FAILED picks nothing on this object: we write nothing, so we don't stop. + const crr = initCrr({ siteName: 'foo', replicationStatusToProcess: ['FAILED'] }, + defaultEndpointReplicationRes, replicatedReplicationInfo); crr.run(err => { - assert(err); + assert.ifError(err); expect(crr.cloudserverclient.putMetadata).not.toHaveBeenCalled(); - assert.strictEqual(crr._nUpdated, 0); - assert.strictEqual(crr._unknownSite.bucket, 'bucket0'); + assert.strictEqual(crr._nSkipped, 1); + assert.strictEqual(crr._unknownSite, null); done(); }); }); @@ -1641,9 +1586,300 @@ describe('ReplicationStatusUpdater site validation', () => { assert.ifError(err); expect(crr.cloudserverclient.putMetadata).toHaveBeenCalledTimes(1); const { replicationInfo } = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); - assert.strictEqual(replicationInfo.storageClass, 'destination,foo'); + expect(replicationInfo.backends.map(b => b.site)).toEqual(['destination', 'foo']); assert.strictEqual(crr._nUpdated, 1); done(); }); }); }); + +describe('ReplicationStatusUpdater writes the multi-destination format for every rule format', () => { + const srcRole = 'arn:aws:iam::123456789012:role/src'; + const dstRole = 'arn:aws:iam::123456789012:role/dst'; + // A V1 rule (Prefix, no Filter). Without storageClass, it uses the S3C default site. + function v1Rule(destination, storageClass, prefix = '') { + const rule = { + ID: `r-${destination}`, + Prefix: prefix, + Status: 'Enabled', + Destination: { Bucket: `arn:aws:s3:::${destination}` }, + }; + if (storageClass) { + rule.Destination.StorageClass = storageClass; + } + return rule; + } + const replication = (...rules) => ({ ReplicationConfiguration: { Role: `${srcRole},${dstRole}`, Rules: rules } }); + + function initCrr(params, replicationRes, replicationInfo) { + const crr = initializeCrrWithMocks({ + buckets: ['bucket0'], + workers: 10, + replicationStatusToProcess: ['NEW'], + ...params, + }, logger, { GetBucketReplicationCommand: replicationRes }); + if (replicationInfo) { + crr.cloudserverclient.getMetadata = jest.fn((p, cb) => { + const md = JSON.parse(getMetadataRes.Body); + md.replicationInfo = replicationInfo; + cb(null, { Body: JSON.stringify(md) }); + }); + } + return crr; + } + const written = crr => JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body).replicationInfo; + + // An object that cloudserver 9.4+ wrote for site us-east-2. + const cloudserverReplicationInfo = { + status: 'PENDING', + backends: [{ + site: 'us-east-2', status: 'PENDING', dataStoreVersionId: '', + destination: 'arn:aws:s3:::dst-1', role: dstRole, + }], + content: ['DATA', 'METADATA'], + role: srcRole, + isNFS: null, + }; + + it('should add a site with its bucket and role on a V1 rule (S3C-11599)', done => { + const crr = initCrr({ siteName: 'sf' }, replication(v1Rule('dst-1', 'sf')), cloudserverReplicationInfo); + crr.run(err => { + assert.ifError(err); + expect(written(crr)).toEqual({ + status: 'PENDING', + role: srcRole, + backends: [ + cloudserverReplicationInfo.backends[0], + { site: 'sf', status: 'PENDING', dataStoreVersionId: '', destination: 'arn:aws:s3:::dst-1', role: dstRole }, + ], + content: ['DATA', 'METADATA'], + isNFS: null, + }); + done(); + }); + }); + + it('should fail without SITE_NAME when a rule has no StorageClass', done => { + const rule = { ...v1Rule('dst-1'), Filter: { Prefix: '' } }; + delete rule.Prefix; + const crr = initCrr({}, replication(rule)); + crr.run(err => { + assert(err); + expect(err.message).toMatch(/missing SITE_NAME/); + expect(crr.cloudserverclient.putMetadata).not.toHaveBeenCalled(); + done(); + }); + }); + + it('should not use SITE_NAME for a rule without StorageClass when another rule names it', done => { + // r1 has no StorageClass (bucket dr-bucket), r2 uses site sf, bucket sf-bucket. + // r2 names SITE_NAME=sf, so r1 must not get site sf. + const crr = initCrr({ siteName: 'sf' }, replication(v1Rule('dr-bucket'), v1Rule('sf-bucket', 'sf'))); + crr.run(err => { + assert.ifError(err); + expect(written(crr).backends.map(b => [b.site, b.destination])).toEqual([ + ['sf', 'arn:aws:s3:::sf-bucket'], + ]); + done(); + }); + }); + + it('should use SITE_NAME for a rule without StorageClass when no rule names it', done => { + // Same rules, with SITE_NAME=dr: no rule names dr, so r1 gets site dr. We don't + // requeue r2 (site sf). + const crr = initCrr({ siteName: 'dr', allowNewSite: true }, + replication(v1Rule('dr-bucket'), v1Rule('sf-bucket', 'sf'))); + crr.run(err => { + assert.ifError(err); + expect(written(crr).backends).toEqual([{ + site: 'dr', status: 'PENDING', dataStoreVersionId: '', destination: 'arn:aws:s3:::dr-bucket', role: dstRole, + }]); + done(); + }); + }); + + it('should mark every rule site when SITE_NAME is not set', done => { + const crr = initCrr({}, replication(v1Rule('dst-1', 'sf'), v1Rule('dst-2', 'dr2', 'key'))); + crr.run(err => { + assert.ifError(err); + expect(written(crr).backends.map(b => [b.site, b.destination])).toEqual([ + ['sf', 'arn:aws:s3:::dst-1'], + ['dr2', 'arn:aws:s3:::dst-2'], + ]); + done(); + }); + }); + + it('should skip a key that no rule prefix matches', done => { + const crr = initCrr({ siteName: 'sf' }, replication(v1Rule('dst-1', 'sf', 'other/'))); + crr.run(err => { + assert.ifError(err); + expect(crr.cloudserverclient.getMetadata).not.toHaveBeenCalled(); + assert.strictEqual(crr._nSkipped, 1); + done(); + }); + }); + + it('should not add a rule site that TARGET_REPLICATION_STATUS does not select', done => { + const replicationInfo = { + ...cloudserverReplicationInfo, + status: 'COMPLETED', + backends: [{ ...cloudserverReplicationInfo.backends[0], site: 'sf', status: 'COMPLETED' }], + }; + const crr = initCrr({ replicationStatusToProcess: ['COMPLETED'] }, + replication(v1Rule('dst-1', 'sf'), v1Rule('dst-2', 'dr2')), replicationInfo); + crr.run(err => { + assert.ifError(err); + expect(written(crr).backends.map(b => [b.site, b.status])).toEqual([['sf', 'PENDING']]); + done(); + }); + }); + + describe('one site with two buckets (one rule per bucket)', () => { + const twoDestinations = replication(v1Rule('d1', 'sf'), v1Rule('d2', 'sf')); + const backend = (destination, status) => ({ + site: 'sf', status, dataStoreVersionId: '', destination: `arn:aws:s3:::${destination}`, role: dstRole, + }); + const withBackends = backends => ({ ...cloudserverReplicationInfo, status: 'FAILED', backends }); + + [ + [backend('d1', 'COMPLETED'), backend('d2', 'FAILED')], + [backend('d2', 'FAILED'), backend('d1', 'COMPLETED')], + ].forEach(backends => { + it(`should retry only the FAILED bucket (object backends ${backends.map(b => b.destination.slice(13))})`, + done => { + const crr = initCrr({ replicationStatusToProcess: ['FAILED'] }, twoDestinations, + withBackends(backends)); + crr.run(err => { + assert.ifError(err); + const statuses = Object.fromEntries( + written(crr).backends.map(b => [b.destination.slice(13), b.status])); + expect(statuses).toEqual({ d1: 'COMPLETED', d2: 'PENDING' }); + done(); + }); + }); + }); + }); + + it('should warn once per bucket when SITE_NAME is not a site of the rules', done => { + // The rules only name sf, and the object only replicates to us-east-2: we requeue nothing. + const crr = initCrr({ siteName: 'dr9', allowNewSite: true }, + replication(v1Rule('dst-1', 'sf')), cloudserverReplicationInfo); + const warnSpy = jest.spyOn(crr.log, 'warn'); + crr.run(err => { + assert.ifError(err); + const siteWarnings = warnSpy.mock.calls.filter(c => c[0].startsWith('SITE_NAME is not a site')); + expect(siteWarnings).toHaveLength(1); + expect(crr.cloudserverclient.putMetadata).not.toHaveBeenCalled(); + assert.strictEqual(crr._nSkipped, 1); + assert.strictEqual(crr._unknownSite, null); + warnSpy.mockRestore(); + done(); + }); + }); + + describe('object written by S3C 9.5, two buckets on its site', () => { + // The object was replicated to site sf, bucket d1. Its backend has no bucket: the + // bucket is the top-level destination. The rules now use site sf with bucket d1, + // and site sf with bucket d2. + const oldFormat = { + status: 'COMPLETED', + backends: [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '' }], + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::d1', + storageClass: 'sf', + role: `${srcRole},${dstRole}`, + storageType: '', + dataStoreVersionId: '', + }; + const oldBackend = oldFormat.backends[0]; + const sites = backends => backends.map(b => `${b.site}|${(b.destination || '-').split(':::').pop()}|${b.status}`); + + it('should send the object to the new bucket with NEW, and keep the old backend first', done => { + const crr = initCrr({}, + replication(v1Rule('d1', 'sf'), v1Rule('d2', 'sf'), v1Rule('x', 'dr2')), oldFormat); + crr.run(err => { + assert.ifError(err); + // The old backend is the one for bucket d1. Bucket d2 and site dr2 have none, so + // NEW picks them. We write the old backend once. + expect(sites(written(crr).backends)).toEqual(['sf|-|COMPLETED', 'sf|d2|PENDING', 'dr2|x|PENDING']); + expect(written(crr).backends[0]).toEqual(oldBackend); + done(); + }); + }); + + it('should requeue only the old bucket with COMPLETED, in place', done => { + const crr = initCrr({ replicationStatusToProcess: ['COMPLETED'] }, + replication(v1Rule('d1', 'sf'), v1Rule('d2', 'sf')), oldFormat); + crr.run(err => { + assert.ifError(err); + expect(sites(written(crr).backends)).toEqual(['sf|d1|PENDING']); + done(); + }); + }); + }); + + describe('the destination role changed after the object was sent', () => { + // The object was sent to site sf, bucket d1, with a destination role that the bucket + // Role no longer gives. Backbeat refused it (BadRole), so the backend stayed PENDING. + const typoRole = 'arn:aws:iam::123456789012:role/typo'; + const backend = (role, status) => ({ + site: 'sf', status, dataStoreVersionId: '', destination: 'arn:aws:s3:::d1', role, + }); + const withBackends = backends => ({ ...cloudserverReplicationInfo, backends }); + const withAccount = (rule, account, id) => ({ + ...rule, ID: id, Destination: { ...rule.Destination, Account: account }, + }); + + it('should requeue the backend in place with the new role', done => { + const crr = initCrr({ replicationStatusToProcess: ['PENDING'] }, + replication(v1Rule('d1', 'sf')), withBackends([backend(typoRole, 'PENDING')])); + crr.run(err => { + assert.ifError(err); + expect(written(crr).backends).toEqual([backend(dstRole, 'PENDING')]); + done(); + }); + }); + + it('should not add a second backend with NEW', done => { + const crr = initCrr({}, replication(v1Rule('d1', 'sf')), + withBackends([backend(typoRole, 'PENDING')])); + crr.run(err => { + assert.ifError(err); + expect(crr.cloudserverclient.putMetadata).not.toHaveBeenCalled(); + assert.strictEqual(crr._nSkipped, 1); + done(); + }); + }); + + it('should match each backend on its own role when two rules use the same bucket', done => { + const roleA = 'arn:aws:iam::111111111111:role/dst'; + const roleB = 'arn:aws:iam::222222222222:role/dst'; + const crr = initCrr({ replicationStatusToProcess: ['FAILED'] }, + replication(withAccount(v1Rule('d1', 'sf'), '111111111111', 'rA'), + withAccount(v1Rule('d1', 'sf'), '222222222222', 'rB')), + withBackends([backend(roleA, 'COMPLETED'), backend(roleB, 'FAILED')])); + crr.run(err => { + assert.ifError(err); + expect(written(crr).backends).toEqual([backend(roleA, 'COMPLETED'), backend(roleB, 'PENDING')]); + done(); + }); + }); + }); + + it('should keep isReplica on a replica that replicates further (cascading)', done => { + const replicationInfo = { + ...cloudserverReplicationInfo, + status: 'COMPLETED', + backends: [{ ...cloudserverReplicationInfo.backends[0], site: 'sf', status: 'COMPLETED' }], + isReplica: true, + }; + const crr = initCrr({ siteName: 'sf', replicationStatusToProcess: ['COMPLETED'] }, + replication(v1Rule('dst-1', 'sf')), replicationInfo); + crr.run(err => { + assert.ifError(err); + expect(written(crr)).toEqual(expect.objectContaining({ status: 'PENDING', isReplica: true })); + done(); + }); + }); +}); diff --git a/tests/unit/CRR/crrExistingObjects.js b/tests/unit/CRR/crrExistingObjects.js index d200fa62..27575db3 100644 --- a/tests/unit/CRR/crrExistingObjects.js +++ b/tests/unit/CRR/crrExistingObjects.js @@ -3,12 +3,14 @@ const werelogs = require('werelogs'); const mockFatal = jest.fn(); const mockInfo = jest.fn(); const mockError = jest.fn(); +const mockWarn = jest.fn(); jest.mock('werelogs', () => ({ Logger: jest.fn().mockImplementation(() => ({ fatal: mockFatal, info: mockInfo, error: mockError, + warn: mockWarn, })), configure: jest.fn(), })); @@ -41,6 +43,7 @@ describe('crrExistingObjects', () => { mockFatal.mockClear(); mockInfo.mockClear(); mockError.mockClear(); + mockWarn.mockClear(); mockCrrRun.mockClear(); mockCrrStop.mockClear(); @@ -99,9 +102,10 @@ describe('crrExistingObjects', () => { keyMarker: 'testKeyMarker', versionIdMarker: 'testVersionIdMarker', currentVersionOnly: true, - forceUsingConfiguration: true, allowNewSite: false, }, expect.anything()); + // FORCE_USING_CONFIGURATION is set: we ignore it and warn. + expect(mockWarn).toHaveBeenCalledWith(expect.stringContaining('FORCE_USING_CONFIGURATION is ignored')); expect(mockFatal).not.toHaveBeenCalled(); expect(mockError).not.toHaveBeenCalled(); @@ -137,7 +141,6 @@ describe('crrExistingObjects', () => { keyMarker: undefined, versionIdMarker: undefined, currentVersionOnly: false, - forceUsingConfiguration: false, allowNewSite: false, }, expect.anything()); From 49053fe8c9da7a475261889587403a21094778bf Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Thu, 29 Oct 2026 10:54:21 +0100 Subject: [PATCH 2/5] S3UTILS-256 bump package json --- CRR/ReplicationStatusUpdater.js | 33 ++++++++++++---------- package.json | 2 +- tests/unit/CRR/ReplicationStatusUpdater.js | 17 ----------- 3 files changed, 19 insertions(+), 33 deletions(-) diff --git a/CRR/ReplicationStatusUpdater.js b/CRR/ReplicationStatusUpdater.js index 392db269..001d8ee7 100644 --- a/CRR/ReplicationStatusUpdater.js +++ b/CRR/ReplicationStatusUpdater.js @@ -148,18 +148,20 @@ class ReplicationStatusUpdater { * A backend written by S3C 9.5 has no bucket: its bucket is the top-level destination * of the object. A backend of a public cloud site has no bucket, and the object has no * top-level destination: there, the site is enough. - * When sameRole is false, a backend still matches after the role changed. We need it to - * send the object again with the new role: backbeat keeps refusing the old one (BadRole). + * We don't compare the role. A bucket has one owner on a site, so we don't need the + * role to tell two backends apart. And we need a backend to match after the role + * changed: then we send the object again with the new role, because backbeat keeps + * refusing the old one (BadRole). * Example: the bucket Role had a typo, and the customer fixed it. The object backend is - * site sf, bucket d1, role typo-role. The rule now gives role replication-role. + * site sf, bucket d1, role typo-role. The rule now gives role replication-role. We + * still find the backend. * @private * @param {Object} existing - Backend of the object. * @param {Object} candidate - Backend built from the bucket rules. * @param {Object} [replicationInfo] - Replication info of the object. - * @param {boolean} [sameRole] - When true, we also compare the role. * @returns {boolean} True if both are the same. */ - _isSameBackend(existing, candidate, replicationInfo, sameRole) { + _isSameBackend(existing, candidate, replicationInfo) { // With the SITE_NAME fallback (step 3 of _markObjectPending), the candidate is the // object backend itself. So it is the same, even with no bucket to compare (S3C 9.5). if (existing === candidate) { @@ -172,8 +174,7 @@ class ReplicationStatusUpdater { return !replicationInfo?.destination || candidate.destination === replicationInfo.destination; } - return existing.destination === candidate.destination - && (!sameRole || existing.role === candidate.role); + return existing.destination === candidate.destination; } /** @@ -509,18 +510,20 @@ class ReplicationStatusUpdater { // TARGET_REPLICATION_STATUS. // No backend means the object was never sent to this site and bucket: NEW picks // it. That's also the case when a rule moved to another bucket on the same site. - // We look for the same role first: two rules can send to the same site and - // bucket with two Destination.Account, and each one must find its own backend. - // Then we look without the role: the role changed since the object was sent. - // Example: sf has the object backend, COMPLETED. dr2 has no backend. - // Example: say the sf backend has role ...crr-typo, because the bucket Role had - // a typo back then. We still find it, and PENDING picks it. + // We match on the site and the bucket, not on the role. A bucket has one owner + // on a site, so we don't need the role to tell two backends apart. And the role + // can change after the object was sent: we still need to find its backend, so + // we can send the object again with the new role. + // Example: site sf, bucket backup-sf has an object backend, COMPLETED. Site dr2, + // bucket backup-dr2 has no backend. + // Example: say the backend for site sf, bucket backup-sf has role ...crr-typo, + // because the bucket Role had a typo back then. The rule now gives role + // ...crr-dst. We still find the backend, and PENDING picks it. // // 5. TARGET_REPLICATION_STATUS picks the candidates to send again. // Example: NEW picks dr2. COMPLETED would pick sf. const pairs = candidateBackends.map(candidate => { - const existing = existingBackends.find(e => this._isSameBackend(e, candidate, prev, true)) - || existingBackends.find(e => this._isSameBackend(e, candidate, prev)); + const existing = existingBackends.find(e => this._isSameBackend(e, candidate, prev)); const selected = this._statusShouldBeProcessed(existing?.status); return { candidate, existing, selected }; }); diff --git a/package.json b/package.json index 28262913..7e3d9c45 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3utils", - "version": "1.19.2", + "version": "1.19.3", "engines": { "node": ">= 22" }, diff --git a/tests/unit/CRR/ReplicationStatusUpdater.js b/tests/unit/CRR/ReplicationStatusUpdater.js index 18410f40..99a0d0be 100644 --- a/tests/unit/CRR/ReplicationStatusUpdater.js +++ b/tests/unit/CRR/ReplicationStatusUpdater.js @@ -1827,9 +1827,6 @@ describe('ReplicationStatusUpdater writes the multi-destination format for every site: 'sf', status, dataStoreVersionId: '', destination: 'arn:aws:s3:::d1', role, }); const withBackends = backends => ({ ...cloudserverReplicationInfo, backends }); - const withAccount = (rule, account, id) => ({ - ...rule, ID: id, Destination: { ...rule.Destination, Account: account }, - }); it('should requeue the backend in place with the new role', done => { const crr = initCrr({ replicationStatusToProcess: ['PENDING'] }, @@ -1851,20 +1848,6 @@ describe('ReplicationStatusUpdater writes the multi-destination format for every done(); }); }); - - it('should match each backend on its own role when two rules use the same bucket', done => { - const roleA = 'arn:aws:iam::111111111111:role/dst'; - const roleB = 'arn:aws:iam::222222222222:role/dst'; - const crr = initCrr({ replicationStatusToProcess: ['FAILED'] }, - replication(withAccount(v1Rule('d1', 'sf'), '111111111111', 'rA'), - withAccount(v1Rule('d1', 'sf'), '222222222222', 'rB')), - withBackends([backend(roleA, 'COMPLETED'), backend(roleB, 'FAILED')])); - crr.run(err => { - assert.ifError(err); - expect(written(crr).backends).toEqual([backend(roleA, 'COMPLETED'), backend(roleB, 'PENDING')]); - done(); - }); - }); }); it('should keep isReplica on a replica that replicates further (cascading)', done => { From a0bc2aa2a59515af5230a329ed7a956bd53331db Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Thu, 29 Oct 2026 18:04:02 +0100 Subject: [PATCH 3/5] S3UTILS-256 address review comments - remove the "every rule names its site" check in _isKnownSite: we only write a site that a rule names or that the object already has - warn when STORAGE_TYPE is set, and drop the unused storageType parameter - warn about FORCE_USING_CONFIGURATION only for true or 1, like before - README: s3utils 1.19.3+ runs on S3C 10.1+ and Zenko 2.16+, remove the S3C 9.5.2 / 9.5.4 note --- CRR/ReplicationStatusUpdater.js | 19 ++++++++----------- README.md | 12 ++++-------- crrExistingObjects.js | 12 ++++++------ tests/functional/crrExistingObjects.js | 1 - tests/unit/CRR/ReplicationStatusUpdater.js | 1 - tests/unit/CRR/crrExistingObjects.js | 9 ++++++--- 6 files changed, 24 insertions(+), 30 deletions(-) diff --git a/CRR/ReplicationStatusUpdater.js b/CRR/ReplicationStatusUpdater.js index 001d8ee7..53b3041f 100644 --- a/CRR/ReplicationStatusUpdater.js +++ b/CRR/ReplicationStatusUpdater.js @@ -24,7 +24,6 @@ class ReplicationStatusUpdater { * @param {Object} log - The logging object used for logging purposes within the instance. * * @param {string} [params.siteName] - (Optional) Name of the destination site. - * @param {string} [params.storageType] - (Optional) Type of the destination site (aws_s3, azure...). * @param {string} [params.targetPrefix] - (Optional) Prefix to target for replication. * @param {number} [params.listingLimit] - (Optional) Limit for listing objects. * @param {number} [params.maxUpdates] - (Optional) Maximum number of updates to perform. @@ -44,7 +43,6 @@ class ReplicationStatusUpdater { secretKey, endpoint, siteName, - storageType, targetPrefix, listingLimit, maxUpdates, @@ -63,7 +61,6 @@ class ReplicationStatusUpdater { this.secretKey = secretKey; this.endpoint = endpoint; this.siteName = siteName; - this.storageType = storageType; this.targetPrefix = targetPrefix; this.listingLimit = listingLimit; this.maxUpdates = maxUpdates; @@ -219,11 +216,13 @@ class ReplicationStatusUpdater { * Checks that the site is a known replication destination. * An unknown site creates a backend no replication processor handles -> the * object stays PENDING forever. - * If there is something reliable to compare against, the site must be in it: - * - the object already replicates: its sites plus the rule sites - * - every rule names its site (StorageClass): the rule sites - * Otherwise accept. A rule without StorageClass uses the S3C default site from - * the federation config (env_replication_endpoints). This script can't read it. + * When the object already replicates, the site must be one of its sites or one of + * the rule sites. Otherwise we accept it: a rule without StorageClass uses the S3C + * default site from the federation config (env_replication_endpoints), and this + * script can't read it. + * We don't need a check for "every rule names its site": then we only write a site + * that a rule names (_markObjectPending, step 2), or a site the object already has + * (step 3). Both are known. * * "names X": a rule has StorageClass=X. * @@ -259,9 +258,7 @@ class ReplicationStatusUpdater { _isKnownSite(objMD, site, rules) { const ruleSites = this._getConfiguredSites(rules); const objectSites = this._getObjectSites(objMD); - const canCheck = objectSites.length > 0 - || rules.every(rule => rule.Destination && rule.Destination.StorageClass); - return !canCheck || ruleSites.concat(objectSites).includes(site); + return objectSites.length === 0 || ruleSites.concat(objectSites).includes(site); } /** diff --git a/README.md b/README.md index 13834bef..5cfde181 100644 --- a/README.md +++ b/README.md @@ -257,14 +257,10 @@ Example: #### Notes on S3C versions -* s3utils 1.19 and later run on S3C 10.1 and later only. They write the - object metadata like cloudserver 9.4+: each backend has its own - destination bucket and role, whatever the bucket rule format (`Prefix` - or `Filter`). -* S3C 9.5.2 / 9.5.4: Backbeat routes on the replication `storageClass`. - Re-triggering one site with `SITE_NAME` also re-sends the objects under - the `sourceCheckIfSizeGreaterThanMB` limit of the Backbeat config - (100 MB by default) to the other sites of the object. +* s3utils 1.19.3 and later run on S3C 10.1 and later, and on Zenko 2.16 + and later, only. They write the object metadata like S3C 10.1 does: each + backend has its own destination bucket and role, whatever the bucket + rule format (`Prefix` or `Filter`). On S3C 9.5, use s3utils 1.18. * S3C 10.1: requeueing objects that already exist at the destination can leave them PENDING, because of the microVersionId format written by s3utils 1.18 (S3C-11591). diff --git a/crrExistingObjects.js b/crrExistingObjects.js index 196bf66c..4e9dba46 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -12,7 +12,6 @@ const log = new werelogs.Logger('s3utils::crrExistingObjects'); const BUCKETS = process.argv[2] ? process.argv[2].split(',') : null; const { SITE_NAME } = process.env; -let { STORAGE_TYPE } = process.env; let { TARGET_REPLICATION_STATUS } = process.env; const { TARGET_PREFIX } = process.env; const WORKERS = (process.env.WORKERS @@ -52,9 +51,6 @@ if (!SECRET_KEY) { log.fatal('SECRET_KEY not defined'); process.exit(1); } -if (!STORAGE_TYPE) { - STORAGE_TYPE = ''; -} if (!TARGET_REPLICATION_STATUS) { TARGET_REPLICATION_STATUS = 'NEW'; } @@ -77,10 +73,15 @@ log.info('Objects with replication status ' + `${replicationStatusToProcess.join(' or ')} ` + 'will be reset to PENDING to trigger CRR'); -if (process.env.FORCE_USING_CONFIGURATION) { +// FORCE_USING_CONFIGURATION was only on with 'true' or '1': we warn in these cases only. +if (['true', '1'].includes(process.env.FORCE_USING_CONFIGURATION)) { log.warn('FORCE_USING_CONFIGURATION is ignored: the script always uses the bucket ' + 'replication configuration'); } +if (process.env.STORAGE_TYPE) { + log.warn('STORAGE_TYPE is ignored: backbeat reads the location type from the location ' + + 'config of the site'); +} const replicationStatusUpdater = new ReplicationStatusUpdater({ buckets: BUCKETS, @@ -90,7 +91,6 @@ const replicationStatusUpdater = new ReplicationStatusUpdater({ secretKey: SECRET_KEY, endpoint: ENDPOINT, siteName: SITE_NAME, - storageType: STORAGE_TYPE, targetPrefix: TARGET_PREFIX, listingLimit: LISTING_LIMIT, maxUpdates: MAX_UPDATES, diff --git a/tests/functional/crrExistingObjects.js b/tests/functional/crrExistingObjects.js index 9ac6798c..4c993f0d 100644 --- a/tests/functional/crrExistingObjects.js +++ b/tests/functional/crrExistingObjects.js @@ -96,7 +96,6 @@ describe('crrExistingObjects', () => { secretKey: accountSource.accountSecretKey, endpoint, siteName: SITE_NAME, - storageType: '', listingLimit: 1000, currentVersionOnly: false, // these tests check key encoding: SITE_NAME is not the bucket's diff --git a/tests/unit/CRR/ReplicationStatusUpdater.js b/tests/unit/CRR/ReplicationStatusUpdater.js index 99a0d0be..7adafb96 100644 --- a/tests/unit/CRR/ReplicationStatusUpdater.js +++ b/tests/unit/CRR/ReplicationStatusUpdater.js @@ -254,7 +254,6 @@ describe('ReplicationStatusUpdater', () => { cb(null, { Body: JSON.stringify(objectMd) }); }); crr.siteName = 'aws-location'; - crr.storageType = 'aws_s3'; crr.replicationStatusToProcess = params.replicationStatusToProcess; crr.run(err => { assert.ifError(err); diff --git a/tests/unit/CRR/crrExistingObjects.js b/tests/unit/CRR/crrExistingObjects.js index 27575db3..966ee3ec 100644 --- a/tests/unit/CRR/crrExistingObjects.js +++ b/tests/unit/CRR/crrExistingObjects.js @@ -94,7 +94,6 @@ describe('crrExistingObjects', () => { secretKey: 'testSecretKey', endpoint: 'http://fake.endpoint', siteName: 'testSite', - storageType: 'testStorage', targetPrefix: 'testPrefix', listingLimit: 2000, maxUpdates: 100, // or a more specific expectation @@ -104,8 +103,9 @@ describe('crrExistingObjects', () => { currentVersionOnly: true, allowNewSite: false, }, expect.anything()); - // FORCE_USING_CONFIGURATION is set: we ignore it and warn. + // FORCE_USING_CONFIGURATION and STORAGE_TYPE are set: we ignore them and warn. expect(mockWarn).toHaveBeenCalledWith(expect.stringContaining('FORCE_USING_CONFIGURATION is ignored')); + expect(mockWarn).toHaveBeenCalledWith(expect.stringContaining('STORAGE_TYPE is ignored')); expect(mockFatal).not.toHaveBeenCalled(); expect(mockError).not.toHaveBeenCalled(); @@ -120,11 +120,15 @@ describe('crrExistingObjects', () => { process.env.ACCESS_KEY = 'testAccessKey'; process.env.SECRET_KEY = 'testSecretKey'; process.env.ENDPOINT = 'http://fake.endpoint'; + // FORCE_USING_CONFIGURATION=false never turned it on: we don't warn. + process.env.FORCE_USING_CONFIGURATION = 'false'; + delete process.env.STORAGE_TYPE; require('../../../crrExistingObjects'); const ReplicationStatusUpdater = require('../../../CRR/ReplicationStatusUpdater'); + expect(mockWarn).not.toHaveBeenCalled(); expect(ReplicationStatusUpdater).toHaveBeenCalledTimes(1); expect(ReplicationStatusUpdater).toHaveBeenCalledWith({ buckets: ['bucket1', 'bucket2'], @@ -133,7 +137,6 @@ describe('crrExistingObjects', () => { accessKey: 'testAccessKey', secretKey: 'testSecretKey', endpoint: 'http://fake.endpoint', - storageType: '', targetPrefix: undefined, listingLimit: 1000, maxUpdates: undefined, From c2da17b6c6a66692595b84451c009bc0bb67b555 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Thu, 29 Oct 2026 22:07:53 +0100 Subject: [PATCH 4/5] S3UTILS-256 address review comments - README: name TARGET_REPLICATION_STATUS with its values, rephrase the single ALLOW_NEW_SITE check without a bullet - explain when the candidate is the object backend itself in _isSameBackend, and check the bucket first - say where the run stops in _stopOnUnknownSite - remove the role note from step 4, it is in _isSameBackend - rename pairs to objectTargets --- CRR/ReplicationStatusUpdater.js | 44 +++++++++++++++++++-------------- README.md | 33 ++++++++++++++----------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/CRR/ReplicationStatusUpdater.js b/CRR/ReplicationStatusUpdater.js index 53b3041f..c8463978 100644 --- a/CRR/ReplicationStatusUpdater.js +++ b/CRR/ReplicationStatusUpdater.js @@ -159,19 +159,25 @@ class ReplicationStatusUpdater { * @returns {boolean} True if both are the same. */ _isSameBackend(existing, candidate, replicationInfo) { - // With the SITE_NAME fallback (step 3 of _markObjectPending), the candidate is the - // object backend itself. So it is the same, even with no bucket to compare (S3C 9.5). + // When no rule sends the key to SITE_NAME anymore, step 3 of _markObjectPending + // takes the candidates from the object backends themselves. Then the candidate is + // the object backend, the same JavaScript object, and it is the same backend. + // We need this check for a backend written by S3C 9.5: it has no bucket, so the + // checks below would compare the top-level destination with nothing, and fail. + // Example: the object has destination backup-sf and the backend [site sf, no + // bucket]. The rules now only name dr3. With SITE_NAME=sf, step 3 takes that + // backend as the candidate, and we find it here. if (existing === candidate) { return true; } if (existing.site !== candidate.site) { return false; } - if (!existing.destination) { - return !replicationInfo?.destination - || candidate.destination === replicationInfo.destination; + if (existing.destination) { + return existing.destination === candidate.destination; } - return existing.destination === candidate.destination; + return !replicationInfo?.destination + || candidate.destination === replicationInfo.destination; } /** @@ -369,6 +375,11 @@ class ReplicationStatusUpdater { * Stops the run when we don't know SITE_NAME (see _isKnownSite), unless * ALLOW_NEW_SITE is set. We stop the whole run, not only this object: if SITE_NAME is * wrong here, it's probably wrong for every bucket. + * This function only decides for one object: it skips it, and sets _unknownSite. The + * callers stop the run. _markPending doesn't start new objects once _unknownSite is + * set, and the listing stops at this page and keeps its markers as the resume point. + * Objects already started still finish: we write the ones with a known site, and we + * skip the ones with an unknown site here. * @private * @param {ObjectMD} objMD - Object metadata. * @param {string} bucket - The bucket name. @@ -507,26 +518,21 @@ class ReplicationStatusUpdater { // TARGET_REPLICATION_STATUS. // No backend means the object was never sent to this site and bucket: NEW picks // it. That's also the case when a rule moved to another bucket on the same site. - // We match on the site and the bucket, not on the role. A bucket has one owner - // on a site, so we don't need the role to tell two backends apart. And the role - // can change after the object was sent: we still need to find its backend, so - // we can send the object again with the new role. // Example: site sf, bucket backup-sf has an object backend, COMPLETED. Site dr2, // bucket backup-dr2 has no backend. - // Example: say the backend for site sf, bucket backup-sf has role ...crr-typo, - // because the bucket Role had a typo back then. The rule now gives role - // ...crr-dst. We still find the backend, and PENDING picks it. // - // 5. TARGET_REPLICATION_STATUS picks the candidates to send again. + // 5. TARGET_REPLICATION_STATUS picks the candidates to send again. Each object + // target holds the candidate, the object backend we found for it, and whether + // we pick it. // Example: NEW picks dr2. COMPLETED would pick sf. - const pairs = candidateBackends.map(candidate => { + const objectTargets = candidateBackends.map(candidate => { const existing = existingBackends.find(e => this._isSameBackend(e, candidate, prev)); const selected = this._statusShouldBeProcessed(existing?.status); return { candidate, existing, selected }; }); // Nothing picked: we write nothing, and the object counts as skipped. - if (!pairs.some(p => p.selected)) { + if (!objectTargets.some(t => t.selected)) { return { skip: true }; } @@ -552,12 +558,12 @@ class ReplicationStatusUpdater { // Example: [site sf, bucket backup-sf, COMPLETED (as it was); // site dr2, bucket backup-dr2, PENDING] const allBackends = existingBackends.map(existing => { - const pair = pairs.find(p => p.existing === existing); + const target = objectTargets.find(t => t.existing === existing); // Picked: send it again, to the site and bucket of the rule, with the role // of the rule. Not picked, or no rule for it: leave it as it is. - return pair?.selected ? { ...pair.candidate, status: 'PENDING' } : existing; + return target?.selected ? { ...target.candidate, status: 'PENDING' } : existing; }); - for (const { candidate, existing, selected } of pairs) { + for (const { candidate, existing, selected } of objectTargets) { // Picked and not on the object yet: add it at the end. if (selected && !existing) { allBackends.push({ ...candidate, status: 'PENDING' }); diff --git a/README.md b/README.md index 5cfde181..9f7a2af4 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ sf (D1) and dr2 (D2), `a.pdf` goes to sf (D1). No rule sends objects to dr3. The script logs the warning "SITE_NAME is not a site of the bucket rules", once per bucket. It skips the objects -with no backend for dr3. With `TARGET_REPLICATION_STATUS=FAILED`, +with no backend for dr3. With `TARGET_REPLICATION_STATUS` having the value `FAILED`, `COMPLETED` or `PENDING`, it still requeues the objects that have a dr3 backend with that status, to their own bucket. The run doesn't stop. @@ -203,16 +203,17 @@ object PENDING forever. So before it writes an object, the script checks `SITE_NAME` when it has something reliable to compare against. Objects it skips are not checked: they get no backend. So only `TARGET_REPLICATION_STATUS=NEW` can stop on an unknown site: with -`FAILED`, `COMPLETED` or `PENDING`, the script only writes objects that -already have a backend for `SITE_NAME`. The checks are: - -* the object already replicates: `SITE_NAME` must be one of its sites - (backend sites, or `storageClass` on older objects), or a site named in - the `StorageClass` of the bucket replication rules - -Otherwise the object is accepted: a rule without `StorageClass` uses the -S3C default replication site, set in the federation config -(`env_replication_endpoints`), which the script can't read. +`TARGET_REPLICATION_STATUS` having the value `FAILED`, `COMPLETED` or +`PENDING`, the script only writes objects that already have a backend for +`SITE_NAME`. + +The script checks objects that already replicate: `SITE_NAME` must be one +of the sites of the object (backend sites, or `storageClass` on older +objects), or a site named in the `StorageClass` of the bucket replication +rules. It accepts an object that was never replicated: a rule without +`StorageClass` uses the S3C default replication site, set in the +federation config (`env_replication_endpoints`), which the script can't +read. When every bucket rule names its site in `StorageClass` and `SITE_NAME` is none of them, the script doesn't stop: no rule gives the object a backend @@ -331,16 +332,18 @@ export ALLOW_NEW_SITE=true #### Replicate existing objects to the new bucket of a rule When a rule now sends its objects to another bucket on the same site, the -objects already replicated have a backend for the old bucket only. `NEW` -picks the new bucket and keeps the backend of the old one: +objects already replicated have a backend for the old bucket only. +`TARGET_REPLICATION_STATUS=NEW` picks the new bucket and keeps the backend +of the old one: ``` export TARGET_REPLICATION_STATUS=NEW export MAX_UPDATES=10000 ``` -`COMPLETED` doesn't send them to the new bucket: it only requeues the -buckets the objects were already sent to, and that a rule still uses. +Note: `TARGET_REPLICATION_STATUS=COMPLETED` doesn't send them to the new +bucket: it only requeues the buckets the objects were already sent to, and +that a rule still uses. #### Re-sync a DR site back to the primary site From bb23826b8c817aa88085efb727c28022c0a33bde Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Fri, 30 Oct 2026 10:28:07 +0100 Subject: [PATCH 5/5] S3UTILS-256 split crrExistingObjects metadata update and write one format - split _markObjectPending into _upgradeBackends, _buildPlan and _buildReplicationInfo, with one table of the cases - rewrite the backends written by S3C 9.5 with the top-level bucket and role of the object, and stop writing the top-level destination: backbeat uses the same bucket and role - match an object backend on site and bucket only, without the special cases for S3C 9.5 backends - shorten the comments of the helpers --- CRR/ReplicationStatusUpdater.js | 414 ++++++++++----------- README.md | 5 +- tests/unit/CRR/ReplicationStatusUpdater.js | 71 ++-- 3 files changed, 242 insertions(+), 248 deletions(-) diff --git a/CRR/ReplicationStatusUpdater.js b/CRR/ReplicationStatusUpdater.js index c8463978..5bf4a9e2 100644 --- a/CRR/ReplicationStatusUpdater.js +++ b/CRR/ReplicationStatusUpdater.js @@ -140,44 +140,21 @@ class ReplicationStatusUpdater { /** * Tells if an object backend and a candidate are the same: same site and same bucket. - * The site is not enough: since S3C 10.1, two bucket replication rules can send to the - * same site with two buckets. Example: site sf, bucket d1, and site sf, bucket d2. - * A backend written by S3C 9.5 has no bucket: its bucket is the top-level destination - * of the object. A backend of a public cloud site has no bucket, and the object has no - * top-level destination: there, the site is enough. - * We don't compare the role. A bucket has one owner on a site, so we don't need the - * role to tell two backends apart. And we need a backend to match after the role - * changed: then we send the object again with the new role, because backbeat keeps - * refusing the old one (BadRole). - * Example: the bucket Role had a typo, and the customer fixed it. The object backend is - * site sf, bucket d1, role typo-role. The rule now gives role replication-role. We - * still find the backend. + * Since S3C 10.1, one site can have two buckets. A backend with no bucket is a cloud + * site written by S3C 10.1: the site is enough (backends written by S3C 9.5 got their + * bucket in _upgradeBackends). + * We don't compare the role: a bucket has one owner on a site, and a backend must + * still match after the bucket Role changed, so we send it again with the new role. + * Example: the object backend is site sf, bucket d1, role typo-role. The rule now gives + * role replication-role. We still find the backend. * @private * @param {Object} existing - Backend of the object. * @param {Object} candidate - Backend built from the bucket rules. - * @param {Object} [replicationInfo] - Replication info of the object. * @returns {boolean} True if both are the same. */ - _isSameBackend(existing, candidate, replicationInfo) { - // When no rule sends the key to SITE_NAME anymore, step 3 of _markObjectPending - // takes the candidates from the object backends themselves. Then the candidate is - // the object backend, the same JavaScript object, and it is the same backend. - // We need this check for a backend written by S3C 9.5: it has no bucket, so the - // checks below would compare the top-level destination with nothing, and fail. - // Example: the object has destination backup-sf and the backend [site sf, no - // bucket]. The rules now only name dr3. With SITE_NAME=sf, step 3 takes that - // backend as the candidate, and we find it here. - if (existing === candidate) { - return true; - } - if (existing.site !== candidate.site) { - return false; - } - if (existing.destination) { - return existing.destination === candidate.destination; - } - return !replicationInfo?.destination - || candidate.destination === replicationInfo.destination; + _isSameBackend(existing, candidate) { + return existing.site === candidate.site + && (!existing.destination || existing.destination === candidate.destination); } /** @@ -220,21 +197,13 @@ class ReplicationStatusUpdater { /** * Checks that the site is a known replication destination. - * An unknown site creates a backend no replication processor handles -> the - * object stays PENDING forever. - * When the object already replicates, the site must be one of its sites or one of - * the rule sites. Otherwise we accept it: a rule without StorageClass uses the S3C - * default site from the federation config (env_replication_endpoints), and this - * script can't read it. - * We don't need a check for "every rule names its site": then we only write a site - * that a rule names (_markObjectPending, step 2), or a site the object already has - * (step 3). Both are known. - * - * "names X": a rule has StorageClass=X. - * - * We only check the objects we write (_markObjectPending, step 6). The table below - * is for TARGET_REPLICATION_STATUS=NEW. With FAILED, COMPLETED or PENDING, we only - * write an object that already has a backend for SITE_NAME, so the check passes. + * A site that no backbeat process handles leaves the object PENDING forever. + * When the object already replicates, the site must be one of its sites or a rule + * site. Otherwise we accept it: a rule without StorageClass uses the S3C default + * site, and we can't read it. + * We only check the objects we write. The table is for TARGET_REPLICATION_STATUS=NEW: + * with the other statuses, we only write objects that already have a backend for + * SITE_NAME. "names X" means a rule has StorageClass=X. * * | Rules | Object sites | SITE_NAME | Result | * |------------------------|------------------|-----------|----------------------------------------| @@ -252,9 +221,8 @@ class ReplicationStatusUpdater { * | no StorageClass | none | unset | error "missing SITE_NAME" (_markPending) | * | no StorageClass | 9.5 replica (dr) | primary | run stopped: needs ALLOW_NEW_SITE=true | * - * ALLOW_NEW_SITE=true skips the check. It's needed when the object must get a - * site it doesn't know and that is the S3C default site: DR failback with - * replicas written by S3C 9.5, or a renamed default DR site. + * ALLOW_NEW_SITE=true skips the check, for a DR failback of replicas written by + * S3C 9.5, or a renamed default DR site. * @private * @param {ObjectMD} objMD - Object metadata. * @param {string} site - Destination site name. @@ -334,26 +302,21 @@ class ReplicationStatusUpdater { } /** - * Adapts AWS SDK replication config to arsenal's ReplicationConfigurationMetadata shape. - * A bucket replication rule without StorageClass sends the objects to the S3C default - * site. That site is in the federation config (env_replication_endpoints), and we - * can't read it from here. So we use SITE_NAME as the site of these rules. Without a - * site, resolveBackends ignores the rule. + * Adapts the AWS SDK replication config to the shape arsenal expects. + * A rule without StorageClass goes to the S3C default site, set in the federation + * config (env_replication_endpoints), and we can't read it. So we give it SITE_NAME as + * site. Without a site, resolveBackends ignores the rule. * @private * @param {Object} repConfig - The full replication configuration. * @returns {Object} */ _buildArsenalConfig(repConfig) { const { Role, Rules } = repConfig; - // When a rule names SITE_NAME in its StorageClass, we assume the default site is - // another site, and the rules without StorageClass get no site. Otherwise we would - // send their objects to the wrong site. - // Example: r1 has no StorageClass (bucket dr-bucket, default site dr), r2 uses site - // sf, bucket sf-bucket. With SITE_NAME=sf, r1 is left out: we only requeue site sf, - // bucket sf-bucket. With SITE_NAME=dr, r1 gets site dr. - // If the default site is SITE_NAME too, we get it wrong: r1 is left out without a - // word. Workaround: set StorageClass sf on r1 for the run, then put the bucket - // replication config back. + // When another rule names SITE_NAME, we assume the default site is another site, and + // the rules without StorageClass get no site. If the default site is SITE_NAME too, + // they are left out (workaround in the README, SITE_NAME). + // Example: r1 has no StorageClass, r2 names sf. With SITE_NAME=sf, r1 is left out. + // With SITE_NAME=dr, r1 gets site dr. const isRuleSite = this._getConfiguredSites(Rules).includes(this.siteName); const defaultSite = isRuleSite ? undefined : this.siteName; return { @@ -435,32 +398,170 @@ class ReplicationStatusUpdater { return ['METADATA', 'DATA']; } + /** + * Rewrites the backends written by S3C 9.5 in the S3C 10.1 format, so the object has + * one format. A backend written by S3C 9.5 has no bucket and no role: backbeat takes + * them from the top-level destination and role of the object (getReplicationTargetBucket + * and getReplicationRoles in arsenal). We copy them on the backend, so backbeat uses + * the same bucket and role, and we never write the top-level destination again. + * When the object role has no destination role (a public cloud site in S3C 9.5), we + * copy only the bucket: backbeat then uses the top-level role, as before. + * A cloud site written by S3C 10.1 has no bucket, and the object has no top-level + * destination: we leave it as it is. + * Example: { destination: 'arn:aws:s3:::backup-sf', role: '...old-src,...old-dst', + * backends: [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '' }] } + * gives [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '', + * destination: 'arn:aws:s3:::backup-sf', role: '...old-dst' }] + * @private + * @param {Object} [prev] - The current replication info of the object. + * @returns {Array} The object backends, in their order. + */ + _upgradeBackends(prev) { + const backends = prev?.backends || []; + if (!prev?.destination) { + return backends; + } + const destinationRole = prev.role?.split(',')[1]; + return backends.map(b => { + if (b.destination) { + return b; + } + const upgraded = { ...b, destination: prev.destination }; + if (!b.role && destinationRole) { + upgraded.role = destinationRole; + } + return upgraded; + }); + } + + /** + * Builds the plan for one object: where the bucket replication rules send this key + * today, which object backend matches each destination, and which ones + * TARGET_REPLICATION_STATUS picks. + * Example (see _markObjectPending): for 'docs/a.pdf', we get + * [site sf, bucket backup-sf, object backend COMPLETED, not picked by NEW] + * [site dr2, bucket backup-dr2, no object backend, picked by NEW] + * @private + * @param {string} key - The object key. + * @param {Array} existingBackends - The object backends, from _upgradeBackends. + * @param {Object} repConfig - The bucket replication configuration. + * @returns {Array} One object target per destination: { candidate, existing, + * selected }. existing is the matching object backend, if any. + */ + _buildPlan(key, existingBackends, repConfig) { + + // 1. We take the destinations from the rules, like cloudserver on a PUT: site, + // bucket and role. We use them for every status, not only NEW: a requeue must + // use the current bucket and role (S3UTILS-208). resolveBackends sets them all + // PENDING: step 4 decides which ones we really send. + let candidates = ReplicationConfiguration.resolveBackends( + this._buildArsenalConfig(repConfig), key, () => false, existingBackends, + ); + + // 2. When SITE_NAME is set, we keep only that site, to requeue it without touching + // the others. When no rule sends the key to SITE_NAME anymore, we take the object + // backend for it as it is, to retry an old replication (S3UTILS-250). + // Example: SITE_NAME=dr2 keeps [site dr2, bucket backup-dr2]. + if (this.siteName) { + candidates = candidates.filter(b => b.site === this.siteName); + if (candidates.length === 0) { + candidates = existingBackends.filter(b => b.site === this.siteName); + } + } + + // 3. For each destination, we find the object backend for the same site and bucket + // (see _isSameBackend). No backend means the object was never sent there. + // 4. TARGET_REPLICATION_STATUS picks on the status of that backend. + // Example: NEW picks dr2, which has no backend. COMPLETED would pick sf. + return candidates.map(candidate => { + const existing = existingBackends.find(e => this._isSameBackend(e, candidate)); + const selected = this._statusShouldBeProcessed(existing?.status); + return { candidate, existing, selected }; + }); + } + + /** + * Builds the replication info to write from the plan. Only what the plan picks + * changes: this run must not touch the rest of the object (see the table in + * _markObjectPending). + * @private + * @param {Object} [prev] - The current replication info of the object. + * @param {Array} existingBackends - The object backends, from _upgradeBackends. + * @param {Array} plan - The object targets from _buildPlan. + * @param {Object} repConfig - The bucket replication configuration. + * @param {ObjectMD} objMD - Object metadata. + * @returns {Object} The replication info to write. + */ + _buildReplicationInfo(prev, existingBackends, plan, repConfig, objMD) { + // We keep the object backends in their order, and add the new ones at the end. + // Example: [site sf, bucket backup-sf, COMPLETED (as it was); + // site dr2, bucket backup-dr2, PENDING] + const backends = existingBackends.map(existing => { + const target = plan.find(t => t.existing === existing); + return target?.selected ? { ...target.candidate, status: 'PENDING' } : existing; + }); + for (const { candidate, existing, selected } of plan) { + if (selected && !existing) { + backends.push({ ...candidate, status: 'PENDING' }); + } + } + + // Backbeat sends a backend with ',', so we write only the + // source role, from the bucket config, like cloudserver on a PUT. + const replicationInfo = { + // Backbeat only queues objects that are PENDING. It sets PROCESSING itself. + status: 'PENDING', + role: ReplicationConfiguration.resolveSourceRole(repConfig.Role), + backends, + content: this._getReplicationContent(prev, objMD), + isNFS: prev?.isNFS ?? null, + }; + // A replica that we send further (cascading) is still a replica. + if (prev?.isReplica) { + replicationInfo.isReplica = true; + } + return replicationInfo; + } + /** * Marks an object as pending for replication. * - * We write the same format as cloudserver since S3C 10.1: each backend has its own - * destination bucket and role. Cloudserver does this for every bucket, V1 rules - * (Prefix) or V2 rules (Filter), so we do the same. + * We write the format of S3C 10.1 (and Zenko 2.16), like cloudserver on a PUT: each + * backend has its own destination bucket and role, whatever the rule format (V1 Prefix + * or V2 Filter). _markPending only calls this for keys that match a rule prefix. * - * "The rules" in the comments below means the replication rules of the bucket - * (GetBucketReplication). + * 1. _upgradeBackends: the object backends, in the S3C 10.1 format when we can. + * 2. _buildPlan: where the rules send the key, and what TARGET_REPLICATION_STATUS picks. + * 3. Nothing picked: we write nothing. + * 4. _stopOnUnknownSite: we stop the run on a SITE_NAME we don't know. + * 5. _buildReplicationInfo: what we write. * - * _markPending only calls this for keys that match a rule prefix. If a rule has no - * StorageClass, it also checks that SITE_NAME is set. + * What happens to each object backend: * - * Example used in the comments below: + * | Object backend | Matched on | Picked | Not picked | + * |------------------------------|-------------------------|----------------------|---------------------| + * | none for this destination | - | added at the end | - | + * | S3C 10.1: site, bucket, role | site + bucket (the role | rewritten in place | kept as it is | + * | | may have changed) | with the bucket and | | + * | | | role of the rule, | | + * | | | PENDING | | + * | S3C 9.5: site only | site + bucket, copied | same | rewritten with the | + * | | from the top level | | top-level bucket | + * | | (_upgradeBackends) | | and role | + * | cloud, S3C 10.1: site only | site | same | kept as it is | + * | no rule for it anymore | itself (SITE_NAME only) | PENDING in place | kept as it is | + * + * Top level: status PENDING, source role from the bucket config, content computed when + * empty (object put before the rule), isReplica kept. + * + * Example used in the comments: * bucket Role: '...:role/crr-src,...:role/crr-dst' * bucket rules (V1 or V2, same result): * r1 Prefix '' Destination { Bucket: 'arn:aws:s3:::backup-sf', StorageClass: 'sf' } * r2 Prefix 'docs/' Destination { Bucket: 'arn:aws:s3:::backup-dr2', StorageClass: 'dr2' } * TARGET_REPLICATION_STATUS=NEW, SITE_NAME not set - * object 'docs/a.pdf', already replicated to sf: - * replicationInfo: { status: 'COMPLETED', role: '...:role/crr-src', - * content: ['DATA', 'METADATA'], isNFS: null, - * backends: [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '', - * destination: 'arn:aws:s3:::backup-sf', role: '...:role/crr-dst' }] } - * result: we leave sf as it is and add dr2 as PENDING. Backbeat copies the object - * to site dr2, bucket backup-dr2, and nowhere else. + * object 'docs/a.pdf', replicated to site sf, bucket backup-sf, COMPLETED + * result: we leave sf as it is and add site dr2, bucket backup-dr2, PENDING. * * @private * @param {string} bucket - The bucket name. @@ -472,143 +573,21 @@ class ReplicationStatusUpdater { */ _markObjectPending(bucket, key, versionId, repConfig, cb) { this._updateObjectMD(bucket, key, versionId, objMD => { - // 1. We read the object backends: where the object was already sent, and how it - // went. We need them because TARGET_REPLICATION_STATUS is about the object: what - // already happened to it for each site and bucket. The bucket replication rules - // don't know that. - // Example: [site sf, bucket backup-sf, COMPLETED] const prev = objMD.getReplicationInfo(); - const existingBackends = prev?.backends || []; - - // 2. We build the candidates: where the bucket replication rules send this key - // today (site, bucket and role). We use resolveBackends, like cloudserver on a - // PUT. We need them for every status, not only NEW: a requeue must also use the - // current bucket and role, not the ones the object was sent with (S3UTILS-208). - // The role of a candidate is the destination role of the bucket Role, with the - // account of Destination.Account when the rule has one. - // resolveBackends sets every candidate to PENDING: that's not a decision yet, - // step 5 picks the ones we really send again. - // Example: two candidates, one per rule that matches the key: - // site sf, bucket backup-sf, role ...crr-dst - // site dr2, bucket backup-dr2, role ...crr-dst - const arsenalConfig = this._buildArsenalConfig(repConfig); - let candidateBackends = ReplicationConfiguration.resolveBackends( - arsenalConfig, key, () => false, existingBackends, - ); - - // 3. When SITE_NAME is set, we keep only the candidates for that site. We need it - // to requeue one site without touching the others. Example: resend everything - // to dr2 after dr2 lost its data. - // Example: SITE_NAME=dr2 keeps [site dr2, bucket backup-dr2]. - if (this.siteName) { - candidateBackends = candidateBackends.filter(b => b.site === this.siteName); - if (candidateBackends.length === 0) { - // No rule sends this key to SITE_NAME anymore, but the object may still - // have a backend for it. We need this to retry old replications to a site - // the rules moved away from: we take that backend as it is, with its own - // bucket and role. - // Example: the rules now only name dr3. With SITE_NAME=sf, we take the - // object backend for site sf. - candidateBackends = existingBackends.filter(b => b.site === this.siteName); - } - } - - // 4. For each candidate, we find the object backend for the same site and bucket - // (see _isSameBackend), and so its status. We need it to apply - // TARGET_REPLICATION_STATUS. - // No backend means the object was never sent to this site and bucket: NEW picks - // it. That's also the case when a rule moved to another bucket on the same site. - // Example: site sf, bucket backup-sf has an object backend, COMPLETED. Site dr2, - // bucket backup-dr2 has no backend. - // - // 5. TARGET_REPLICATION_STATUS picks the candidates to send again. Each object - // target holds the candidate, the object backend we found for it, and whether - // we pick it. - // Example: NEW picks dr2. COMPLETED would pick sf. - const objectTargets = candidateBackends.map(candidate => { - const existing = existingBackends.find(e => this._isSameBackend(e, candidate, prev)); - const selected = this._statusShouldBeProcessed(existing?.status); - return { candidate, existing, selected }; - }); - - // Nothing picked: we write nothing, and the object counts as skipped. - if (!objectTargets.some(t => t.selected)) { + const existingBackends = this._upgradeBackends(prev); + const plan = this._buildPlan(key, existingBackends, repConfig); + if (!plan.some(t => t.selected)) { return { skip: true }; } - - // 6. A SITE_NAME that is in neither the bucket replication rules nor the object is - // probably a typo. We stop before writing anything: a backend for a site that - // no backbeat process handles stays PENDING forever (S3UTILS-250). - // We check only the objects we write: an object we skip gets no backend, so a - // wrong SITE_NAME can't hurt it. Otherwise a FAILED retry run would stop on an - // object that has nothing to retry. - // Example: SITE_NAME is not set, so there is nothing to check. - // Example: say r1 has no StorageClass, and SITE_NAME=sff. r1 gets site sff - // (see _buildArsenalConfig), and NEW picks site sff, bucket backup-sf. The - // object only knows site sf: we stop the run. + // We check only the objects we write: an object we skip gets no backend, so a + // wrong SITE_NAME can't hurt it. if (this._stopOnUnknownSite(objMD, bucket, key, versionId, repConfig.Rules)) { return { skip: true }; } - - // 7. We build the backends to write. Only what we picked changes: this run must not - // touch the rest of the object. - // We keep the object backends in their order and add the new ones at the end. - // Backbeat finds a backend written by S3C 9.5 by its site only, so it takes the - // first backend of that site: it must stay before a new one for the same site. - // Example: [site sf, bucket backup-sf, COMPLETED (as it was); - // site dr2, bucket backup-dr2, PENDING] - const allBackends = existingBackends.map(existing => { - const target = objectTargets.find(t => t.existing === existing); - // Picked: send it again, to the site and bucket of the rule, with the role - // of the rule. Not picked, or no rule for it: leave it as it is. - return target?.selected ? { ...target.candidate, status: 'PENDING' } : existing; - }); - for (const { candidate, existing, selected } of objectTargets) { - // Picked and not on the object yet: add it at the end. - if (selected && !existing) { - allBackends.push({ ...candidate, status: 'PENDING' }); - } - } - - // 8. We build the replication info to write. - // Role: backbeat sends a backend with ','. - // So we only write the source role, from the bucket config, like cloudserver on - // a PUT. - // But a backend written by S3C 9.5 has no bucket and no role: backbeat takes - // them from the top-level destination and role. While we keep such a backend, we - // keep the destination and the destination role of the object. S3C 9.5 allowed - // one bucket and one Role per bucket config, so all these backends share them. - // The source role still comes from the bucket config: backbeat uses it for the - // new backends too, and an old one would make them fail (BadRole). - // Example: { status: 'PENDING', role: '...crr-src', backends: [sf, dr2], - // content: ['DATA', 'METADATA'], isNFS: null } - // Example: say sf is a backend written by S3C 9.5, and the object role is - // '...old-src,...old-dst'. We write role '...crr-src,...old-dst'. - const keepsOldBackend = allBackends.some(b => !b.role); - const sourceRole = ReplicationConfiguration.resolveSourceRole(repConfig.Role); - // Undefined when the object role has no destination role. Example: a public - // cloud site in S3C 9.5 has a single role. Then we write the source role alone. - const oldDestinationRole = keepsOldBackend ? prev?.role?.split(',')[1] : undefined; - const replicationInfo = { - // PENDING: backbeat only picks up objects that are PENDING. Backbeat sets - // PROCESSING itself later, when one site is done and others aren't. - status: 'PENDING', - role: oldDestinationRole ? `${sourceRole},${oldDestinationRole}` : sourceRole, - backends: allBackends, - content: this._getReplicationContent(prev, objMD), - isNFS: prev?.isNFS ?? null, - }; - if (keepsOldBackend && prev?.destination) { - replicationInfo.destination = prev.destination; - } - // A replica that we send further (cascading) is still a replica. - if (prev?.isReplica) { - replicationInfo.isReplica = true; - } - - // 9. We save it. _updateObjectMD gives the object a new microVersionId: we need it so - // the destination accepts this update as newer than what it has. - objMD.setReplicationInfo(replicationInfo); + // _updateObjectMD then gives the object a new microVersionId, so the destination + // accepts this update as newer than what it has. + objMD.setReplicationInfo( + this._buildReplicationInfo(prev, existingBackends, plan, repConfig, objMD)); return { skip: false }; }, cb); } @@ -656,9 +635,8 @@ class ReplicationStatusUpdater { (repConfig, next) => { const { Rules } = repConfig; - // A bucket replication rule without StorageClass sends the objects to the S3C - // default site, which we can't read from here: SITE_NAME must give it (see - // _buildArsenalConfig). Without it, we stop with "missing SITE_NAME". + // A rule without StorageClass needs SITE_NAME as its site (see + // _buildArsenalConfig). const hasRuleWithoutSite = Rules.some(r => r.Status === 'Enabled' && !r.Destination.StorageClass ); @@ -668,11 +646,9 @@ class ReplicationStatusUpdater { this.log.error(errMsg); return next(new Error(errMsg)); } - // We warn when no rule sends objects to SITE_NAME: no rule names it, and no rule - // uses the default site. Then we only requeue the objects that already have a - // backend for SITE_NAME, and skip the others. The warning tells the user why few - // or no objects are updated. We say it once per bucket: _markPending runs for - // each listing page. + // When no rule sends objects to SITE_NAME, we only requeue the objects that + // already have a backend for it. We warn once per bucket, so the user knows why + // few objects are updated. // Example: the rules name sf and dr2, and SITE_NAME=dr3. const ruleSites = this._getConfiguredSites(Rules); if (this.siteName && !hasRuleWithoutSite && !ruleSites.includes(this.siteName) diff --git a/README.md b/README.md index 9f7a2af4..f4a7f732 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,10 @@ Example: * s3utils 1.19.3 and later run on S3C 10.1 and later, and on Zenko 2.16 and later, only. They write the object metadata like S3C 10.1 does: each backend has its own destination bucket and role, whatever the bucket - rule format (`Prefix` or `Filter`). On S3C 9.5, use s3utils 1.18. + rule format (`Prefix` or `Filter`). When they update an object that + still has backends written by S3C 9.5, they rewrite these backends in + this format too, with the bucket and role of the object. On S3C 9.5, use + s3utils 1.18. * S3C 10.1: requeueing objects that already exist at the destination can leave them PENDING, because of the microVersionId format written by s3utils 1.18 (S3C-11591). diff --git a/tests/unit/CRR/ReplicationStatusUpdater.js b/tests/unit/CRR/ReplicationStatusUpdater.js index 7adafb96..d33f41a3 100644 --- a/tests/unit/CRR/ReplicationStatusUpdater.js +++ b/tests/unit/CRR/ReplicationStatusUpdater.js @@ -158,14 +158,17 @@ describe('ReplicationStatusUpdater', () => { content: ['METADATA', 'DATA'], isNFS: null, }; - // What we write when we keep an S3C 9.5 backend: we keep the top-level destination. + // What we write when we keep an S3C 9.5 backend: it gets the top-level destination of + // the object (see _upgradeBackends), and we don't write the top-level destination. const fromOldFormat = backends => ({ status: 'PENDING', role: 'arn:aws:iam::root:role/s3-replication-role', backends, content: ['METADATA', 'DATA'], isNFS: null, - destination: 'arn:aws:s3:::destination', + }); + const keptOldBackend = (site, status) => ({ + site, status, dataStoreVersionId: '', destination: 'arn:aws:s3:::destination', }); [ { @@ -221,7 +224,7 @@ describe('ReplicationStatusUpdater', () => { replicationStatusToProcess: ['FAILED'], // The backends keep their order: we replace aws-location in place. expectedReplicationInfo: fromOldFormat([ - { site: 'azure-location', status: 'COMPLETED', dataStoreVersionId: '' }, + keptOldBackend('azure-location', 'COMPLETED'), newBackend, ]), }, { @@ -233,8 +236,8 @@ describe('ReplicationStatusUpdater', () => { replicationStatusToProcess: ['NEW'], // The backends keep their order: we add the new one at the end. expectedReplicationInfo: fromOldFormat([ - { site: 'azure-location', status: 'COMPLETED', dataStoreVersionId: '' }, - { site: 'azure-location-2', status: 'FAILED', dataStoreVersionId: '' }, + keptOldBackend('azure-location', 'COMPLETED'), + keptOldBackend('azure-location-2', 'FAILED'), newBackend, ]), }, @@ -496,10 +499,10 @@ describe('ReplicationStatusUpdater top-level role', () => { }); }); - it('should keep the destination and destination role while a backend written by S3C 9.5 is kept', done => { - // Site sf is an S3C 9.5 backend that no rule names: we keep it, so we keep the - // top-level destination and destination role it needs. The source role comes from - // the bucket config: backbeat uses it for the new backend too. + it('should rewrite a backend written by S3C 9.5 with the top-level bucket and role', done => { + // Site sf is an S3C 9.5 backend that no rule names: we keep it, in the S3C 10.1 + // format. Its bucket and role come from the top level of the object, which we don't + // write anymore. The source role comes from the bucket config. const crr = initCrr({}, { status: 'COMPLETED', backends: [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '' }], @@ -513,20 +516,24 @@ describe('ReplicationStatusUpdater top-level role', () => { crr.run(err => { assert.ifError(err); const { replicationInfo } = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); - expect(replicationInfo.role).toBe( - 'arn:aws:iam::root:role/s3-replication-role,arn:aws:iam::123456789012:role/dest-resource'); - expect(replicationInfo.destination).toBe('arn:aws:s3:::destination2'); + expect(replicationInfo.role).toBe('arn:aws:iam::root:role/s3-replication-role'); + expect(replicationInfo.destination).toBeUndefined(); expect(replicationInfo.backends).toEqual([ - { site: 'sf', status: 'COMPLETED', dataStoreVersionId: '' }, + { + site: 'sf', status: 'COMPLETED', dataStoreVersionId: '', + destination: 'arn:aws:s3:::destination2', + role: 'arn:aws:iam::123456789012:role/dest-resource', + }, newBackend, ]); done(); }); }); - it('should write only the source role when the S3C 9.5 object role has no destination role', done => { + it('should copy only the bucket when the S3C 9.5 object role has no destination role', done => { // A public cloud site in S3C 9.5 has a single role: there is no destination role to - // keep, so we write the source role alone. + // copy. The backend gets the bucket only, and backbeat uses the top-level role, the + // source role we write. const crr = initCrr({}, { status: 'COMPLETED', backends: [{ site: 'sf', status: 'COMPLETED', dataStoreVersionId: '' }], @@ -541,7 +548,10 @@ describe('ReplicationStatusUpdater top-level role', () => { assert.ifError(err); const { replicationInfo } = JSON.parse(crr.cloudserverclient.putMetadata.mock.calls[0][0].Body); expect(replicationInfo.role).toBe('arn:aws:iam::root:role/s3-replication-role'); - expect(replicationInfo.destination).toBe('arn:aws:s3:::destination2'); + expect(replicationInfo.destination).toBeUndefined(); + expect(replicationInfo.backends[0]).toEqual({ + site: 'sf', status: 'COMPLETED', dataStoreVersionId: '', destination: 'arn:aws:s3:::destination2', + }); done(); }); }); @@ -1139,10 +1149,10 @@ describe('ReplicationStatusUpdater V2 format', () => { }); }); - it('should keep a V1-format backend unchanged when this run does not select it', done => { - // dest-A is COMPLETED in V1 format (no per-backend destination/role) — skipped by NEW filter - // dest-B is NEW and matches 'docs/' prefix — should be set to PENDING - // We leave dest-A as it is: this run doesn't pick it. + it('should keep the status of an S3C 9.5 backend that this run does not select', done => { + // dest-A is COMPLETED in the S3C 9.5 format (no bucket, no role): NEW doesn't pick it. + // dest-B has no backend and matches the 'docs/' prefix: NEW picks it. + // dest-A keeps its status: it only gets its bucket, from the top-level destination. const listVersionDocsKey = { IsTruncated: false, Versions: [{ @@ -1193,9 +1203,12 @@ describe('ReplicationStatusUpdater V2 format', () => { const destA = repInfo.backends.find(b => b.site === 'dest-A'); const destB = repInfo.backends.find(b => b.site === 'dest-B'); - // We keep dest-A, so we keep its bucket: the top-level destination. - expect(destA).toEqual({ site: 'dest-A', status: 'COMPLETED', dataStoreVersionId: 'v1-stored-id' }); - expect(repInfo.destination).toBe('arn:aws:s3:::bucket-a'); + // We keep dest-A as it was, with its bucket copied from the top-level destination. + expect(destA).toEqual({ + site: 'dest-A', status: 'COMPLETED', dataStoreVersionId: 'v1-stored-id', + destination: 'arn:aws:s3:::bucket-a', + }); + expect(repInfo.destination).toBeUndefined(); // dest-B: newly queued expect(destB.status).toBe('PENDING'); @@ -1791,18 +1804,20 @@ describe('ReplicationStatusUpdater writes the multi-destination format for every storageType: '', dataStoreVersionId: '', }; - const oldBackend = oldFormat.backends[0]; + // The same backend in the S3C 10.1 format (see _upgradeBackends). + const upgradedBackend = { ...oldFormat.backends[0], destination: 'arn:aws:s3:::d1', role: dstRole }; const sites = backends => backends.map(b => `${b.site}|${(b.destination || '-').split(':::').pop()}|${b.status}`); - it('should send the object to the new bucket with NEW, and keep the old backend first', done => { + it('should send the object to the new bucket with NEW, and keep the old backend first, upgraded', done => { const crr = initCrr({}, replication(v1Rule('d1', 'sf'), v1Rule('d2', 'sf'), v1Rule('x', 'dr2')), oldFormat); crr.run(err => { assert.ifError(err); // The old backend is the one for bucket d1. Bucket d2 and site dr2 have none, so - // NEW picks them. We write the old backend once. - expect(sites(written(crr).backends)).toEqual(['sf|-|COMPLETED', 'sf|d2|PENDING', 'dr2|x|PENDING']); - expect(written(crr).backends[0]).toEqual(oldBackend); + // NEW picks them. We write the old backend once, with its bucket and role. + expect(sites(written(crr).backends)).toEqual(['sf|d1|COMPLETED', 'sf|d2|PENDING', 'dr2|x|PENDING']); + expect(written(crr).backends[0]).toEqual(upgradedBackend); + expect(written(crr).destination).toBeUndefined(); done(); }); });