Enforce microVersionId conflicts at the metadata write level

lib/routes/routeBackbeat.js

@@ -606,6 +606,9 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
     // Note: '' is falsy so the presence check must be !== undefined, not truthy.
     const hasMicroVersionId = encodedMicroVersionId !== undefined;
     let incomingMicroVersionId = null;
+    const mongoOptions = {
+        overheadField: constants.overheadField,
+    };
     if (hasMicroVersionId && objMd) {
         // '' means source has no microVersionId, treated as older revision
         incomingMicroVersionId = encodedMicroVersionId === '' ? null : decode(encodedMicroVersionId);
@@ -659,6 +662,13 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
                 conflictErr.mvId,
             );
         }
+
+        // Atomic counterpart of the JS pre-checks above, performed at database level.
+        // Only effective when the stored revision already carries a
+        // microVersionId, need $exists and $or implementation on Arsenal for the rest
+        if (incomingMicroVersionId !== null && objMd.microVersionId) {
+            mongoOptions.conditions = { microVersionId: { $gt: incomingMicroVersionId } };
+        }
     }
 
     return _getRequestPayload(request, (err, payload) => {
@@ -810,25 +820,21 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
             omVal.replicationInfo.isNFS = !omVal.replicationInfo.isReplica;
         }
 
-        const options = {
-            overheadField: constants.overheadField,
-        };
-
         // NOTE: When 'versioning' is set to true and no 'versionId' is specified,
         // it results in the creation of a "new" version, which also updates the master.
         // NOTE: Since option fields are converted to strings when they're sent to Metadata via the query string,
         // Metadata interprets the value "false" as if it were true.
         // Therefore, to avoid this confusion, we don't pass the versioning parameter at all if its value is false.
         if (versioning) {
-            options.versioning = true;
+            mongoOptions.versioning = true;
         }
 
         // NOTE: When options fields are sent to Metadata through the query string,
         // they are converted to strings. As a result, Metadata interprets the value undefined
         // in the versionId field as an empty string ('').
         // To prevent this, the versionId field is only included in options when it is defined.
         if (versionId !== undefined) {
-            options.versionId = versionId;
+            mongoOptions.versionId = versionId;
             omVal.versionId = versionId;
 
             if (isNull) {
@@ -848,13 +854,13 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
             // The update is only done when putting a new version as updating versions that already exist
             // shouldn't affect the master.
             if (!objMd) {
-                options.repairMaster = true;
+                mongoOptions.repairMaster = true;
             }
         }
 
         // If the new null keys logic (S3C-7352) is not supported (compatibility mode), 'isNull' remains undefined.
         if (!nullVersionCompatMode) {
-            options.isNull = isNull;
+            mongoOptions.isNull = isNull;
         }
 
         const isReplicationWrite = !!headers['x-scal-replication-content'];
@@ -956,7 +962,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
                         );
 
                         if (versioningPreprocessingResult) {
-                            options.deleteNullKey = versioningPreprocessingResult.deleteNullKey;
+                            mongoOptions.deleteNullKey = versioningPreprocessingResult.deleteNullKey;
 
                             // The master references a null version only via extraMD,
                             // which is set solely in nullVersionCompatMode. In null-key
@@ -972,9 +978,9 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
                     log.trace('putting object version', {
                         objectKey: request.objectKey,
                         omVal,
-                        options,
+                        mongoOptions,
                     });
-                    return metadata.putObjectMD(bucketName, objectKey, omVal, options, log, (err, md) => {
+                    return metadata.putObjectMD(bucketName, objectKey, omVal, mongoOptions, log, (err, md) => {
                         if (err) {
                             // Handle duplicate key error during repair operation
                             // This can happen due to race conditions when multiple operations
@@ -983,7 +989,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
                             // treat this as success.
                             const errorMessage = err.message || err.toString() || '';
                             const isRepairDuplicateKeyError =
-                                options.repairMaster &&
+                                mongoOptions.repairMaster &&
                                 (errorMessage.includes('E11000') ||
                                     errorMessage.includes('duplicate key') ||
                                     errorMessage.includes('repair'));
@@ -1064,7 +1070,25 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
                     });
                 },
             ],
-            callback,
+            (err, results) => {
+                if (err?.is?.PreconditionFailed && mongoOptions.conditions) {
+                    log.debug('putMetadata: write rejected, incoming microVersionId is not newer than stored', {
+                        method: 'putMetadata',
+                        bucketName,
+                        objectKey,
+                    });
+                    request.resume();
+                    return _respondWithHeaderCrrConflict(
+                        response,
+                        log,
+                        callback,
+                        StaleMicroVersionIdException.name,
+                        'incoming revision is not newer than stored',
+                        objMd?.microVersionId,
+                    );
+                }
+                return callback(err, results);
+            },
         );
     });
 }

package.json

@@ -35,7 +35,7 @@
         "@opentelemetry/instrumentation-ioredis": "~0.64.0",
         "@opentelemetry/instrumentation-mongodb": "~0.69.0",
         "@smithy/node-http-handler": "^3.0.0",
-        "arsenal": "git+https://github.com/scality/arsenal#8.5.12",
+        "arsenal": "git+https://github.com/scality/arsenal#6675ce3895cb779b0d07756a373762c82ff5e335",
         "async": "2.6.4",
         "aws-crt": "^1.24.0",
         "bucketclient": "scality/bucketclient#8.2.7",

tests/unit/routes/routeBackbeat.js

@@ -358,6 +358,107 @@ describe('routeBackbeat', () => {
             assert.deepStrictEqual(mockResponse.body, {});
         });
 
+        it('should set an atomic microVersionId condition when updating an existing version', async () => {
+            // reverse-chronological ordering: sorted ascending, the newer revision (incoming) sorts first
+            const [incomingRaw, storedRaw] = [
+                versioning.VersionID.generateVersionId('test', 'RG001'),
+                versioning.VersionID.generateVersionId('test', 'RG001'),
+            ].sort();
+
+            mockRequest = preparePutMetadataRequest({ microVersionId: incomingRaw });
+            mockRequest.headers['x-scal-micro-version-id'] = versioning.VersionID.encode(incomingRaw);
+            mockRequest.url = '/_/backbeat/metadata/bucket0/key0' + '?versionId=aIXVkw5Tw2Pd00000000001I4j3QKsvf';
+
+            metadataUtils.standardMetadataValidateBucketAndObj.callsFake((_params, _denies, _log, callback) => {
+                callback(null, bucketInfo, { microVersionId: storedRaw });
+            });
+
+            const putObjectMDStub = sandbox
+                .stub(metadata, 'putObjectMD')
+                .callsFake((_bucketName, _objectKey, _omVal, options, _logParam, cb) => {
+                    assert.deepStrictEqual(options.conditions, { microVersionId: { $gt: incomingRaw } });
+                    cb(null, {});
+                });
+
+            routeBackbeat('127.0.0.1', mockRequest, mockResponse, log);
+            void (await endPromise);
+
+            sinon.assert.called(putObjectMDStub);
+            assert.strictEqual(mockResponse.statusCode, 200);
+        });
+
+        it('should return 409 without writing when the incoming microVersionId is already stored', async () => {
+            const incomingRaw = versioning.VersionID.generateVersionId('test', 'RG001');
+
+            mockRequest = preparePutMetadataRequest({ microVersionId: incomingRaw });
+            mockRequest.headers['x-scal-micro-version-id'] = versioning.VersionID.encode(incomingRaw);
+            mockRequest.url = '/_/backbeat/metadata/bucket0/key0' + '?versionId=aIXVkw5Tw2Pd00000000001I4j3QKsvf';
+
+            metadataUtils.standardMetadataValidateBucketAndObj.callsFake((_params, _denies, _log, callback) => {
+                callback(null, bucketInfo, { microVersionId: incomingRaw });
+            });
+
+            const putObjectMDStub = sandbox.stub(metadata, 'putObjectMD');
+
+            routeBackbeat('127.0.0.1', mockRequest, mockResponse, log);
+            void (await endPromise);
+
+            sinon.assert.notCalled(putObjectMDStub);
+            assert.strictEqual(mockResponse.statusCode, 409);
+            assert.strictEqual(mockResponse.body.code, 'MicroVersionIdAlreadyStoredException');
+        });
+
+        it('should not set a microVersionId condition when the stored revision has none', async () => {
+            const incomingRaw = versioning.VersionID.generateVersionId('test', 'RG001');
+
+            mockRequest = preparePutMetadataRequest({ microVersionId: incomingRaw });
+            mockRequest.headers['x-scal-micro-version-id'] = versioning.VersionID.encode(incomingRaw);
+            mockRequest.url = '/_/backbeat/metadata/bucket0/key0' + '?versionId=aIXVkw5Tw2Pd00000000001I4j3QKsvf';
+
+            metadataUtils.standardMetadataValidateBucketAndObj.callsFake((_params, _denies, _log, callback) => {
+                callback(null, bucketInfo, {});
+            });
+
+            const putObjectMDStub = sandbox
+                .stub(metadata, 'putObjectMD')
+                .callsFake((_bucketName, _objectKey, _omVal, options, _logParam, cb) => {
+                    assert.strictEqual(options.conditions, undefined);
+                    cb(null, {});
+                });
+
+            routeBackbeat('127.0.0.1', mockRequest, mockResponse, log);
+            void (await endPromise);
+
+            sinon.assert.called(putObjectMDStub);
+            assert.strictEqual(mockResponse.statusCode, 200);
+        });
+
+        it('should return 409 when the metadata write rejects a stale microVersionId', async () => {
+            const [incomingRaw, storedRaw] = [
+                versioning.VersionID.generateVersionId('test', 'RG001'),
+                versioning.VersionID.generateVersionId('test', 'RG001'),
+            ].sort();
+
+            mockRequest = preparePutMetadataRequest({ microVersionId: incomingRaw });
+            mockRequest.headers['x-scal-micro-version-id'] = versioning.VersionID.encode(incomingRaw);
+            mockRequest.url = '/_/backbeat/metadata/bucket0/key0' + '?versionId=aIXVkw5Tw2Pd00000000001I4j3QKsvf';
+
+            metadataUtils.standardMetadataValidateBucketAndObj.callsFake((_params, _denies, _log, callback) => {
+                callback(null, bucketInfo, { microVersionId: storedRaw });
+            });
+
+            sandbox
+                .stub(metadata, 'putObjectMD')
+                .callsFake((_bucketName, _objectKey, _omVal, _options, _logParam, cb) => {
+                    cb(errors.PreconditionFailed);
+                });
+
+            routeBackbeat('127.0.0.1', mockRequest, mockResponse, log);
+            void (await endPromise);
+
+            assert.strictEqual(mockResponse.statusCode, 409);
+        });
+
         it('should handle error when putting metadata', async () => {
             const putObjectMDStub = sandbox.stub(metadata, 'putObjectMD');
             putObjectMDStub.onCall(0).callsFake((bucketName, objectKey, omVal, options, logParam, cb) => {