diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml
index 452705cf9d..698ae1ccf0 100644
--- a/.github/docker/docker-compose.yaml
+++ b/.github/docker/docker-compose.yaml
@@ -129,7 +129,7 @@ services:
depends_on:
- redis
metadata-standalone:
- image: ghcr.io/scality/metadata:8.11.0-standalone
+ image: ghcr.io/scality/metadata:8.25.0-standalone
profiles: ['metadata-standalone']
network_mode: 'host'
volumes:
diff --git a/lib/api/listMultipartUploads.js b/lib/api/listMultipartUploads.js
index 71e428669c..4a93b0439c 100644
--- a/lib/api/listMultipartUploads.js
+++ b/lib/api/listMultipartUploads.js
@@ -24,6 +24,8 @@ const monitoring = require('../utilities/monitoringHandler');
true
my-divisor
+ CRC32
+ COMPOSITE
XMgbGlrZSBlbHZpbmcncyBub3QgaGF2aW5nIG11Y2ggbHVjaw
arn:aws:iam::111122223333:user/
diff --git a/package.json b/package.json
index ac3bdc10c2..ad1bf65f90 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,7 @@
"@azure/storage-blob": "^12.28.0",
"@hapi/joi": "^17.1.1",
"@smithy/node-http-handler": "^3.0.0",
- "arsenal": "git+https://github.com/scality/Arsenal#8.3.9",
+ "arsenal": "git+https://github.com/scality/Arsenal#8.3.10",
"async": "2.6.4",
"bucketclient": "scality/bucketclient#8.2.7",
"bufferutil": "^4.0.8",
diff --git a/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js b/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js
new file mode 100644
index 0000000000..628e810792
--- /dev/null
+++ b/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js
@@ -0,0 +1,148 @@
+const assert = require('assert');
+const {
+ CreateBucketCommand,
+ CreateMultipartUploadCommand,
+ AbortMultipartUploadCommand,
+ ListMultipartUploadsCommand,
+ DeleteBucketCommand,
+} = require('@aws-sdk/client-s3');
+
+const withV4 = require('../support/withV4');
+const BucketUtility = require('../../lib/utility/bucket-util');
+
+const bucket = `list-mpu-checksum-test-${Date.now()}`;
+const key = 'test-checksum-key';
+
+describe('ListMultipartUploads checksum fields', () =>
+ withV4(sigCfg => {
+ let bucketUtil;
+ let s3;
+
+ before(async () => {
+ bucketUtil = new BucketUtility('default', sigCfg);
+ s3 = bucketUtil.s3;
+ await s3.send(new CreateBucketCommand({ Bucket: bucket }));
+ });
+
+ after(async () => {
+ await bucketUtil.empty(bucket);
+ await s3.send(new DeleteBucketCommand({ Bucket: bucket }));
+ });
+
+ describe('MPU created without checksum algorithm', () => {
+ let uploadId;
+
+ before(async () => {
+ const res = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key }));
+ uploadId = res.UploadId;
+ });
+
+ after(async () => {
+ await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId }));
+ });
+
+ it('should not include ChecksumAlgorithm or ChecksumType in listed uploads', async () => {
+ const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+ assert.strictEqual(Uploads.length, 1);
+ assert.strictEqual(Uploads[0].UploadId, uploadId);
+ assert.strictEqual(Uploads[0].ChecksumAlgorithm, undefined);
+ assert.strictEqual(Uploads[0].ChecksumType, undefined);
+ });
+ });
+
+ describe('MPU created with checksum algorithm', () => {
+ const cases = [
+ { algo: 'CRC32', expectedType: 'COMPOSITE' },
+ { algo: 'CRC32C', expectedType: 'COMPOSITE' },
+ { algo: 'CRC64NVME', expectedType: 'FULL_OBJECT' },
+ { algo: 'SHA1', expectedType: 'COMPOSITE' },
+ { algo: 'SHA256', expectedType: 'COMPOSITE' },
+ ];
+
+ cases.forEach(({ algo, expectedType }) => {
+ describe(`${algo}`, () => {
+ let uploadId;
+
+ before(async () => {
+ const res = await s3.send(new CreateMultipartUploadCommand({
+ Bucket: bucket, Key: key, ChecksumAlgorithm: algo,
+ }));
+ uploadId = res.UploadId;
+ });
+
+ after(async () => {
+ await s3.send(new AbortMultipartUploadCommand({
+ Bucket: bucket, Key: key, UploadId: uploadId,
+ }));
+ });
+
+ it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${expectedType}`, async () => {
+ const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+ assert.strictEqual(Uploads.length, 1);
+ assert.strictEqual(Uploads[0].UploadId, uploadId);
+ assert.strictEqual(Uploads[0].ChecksumAlgorithm, algo);
+ assert.strictEqual(Uploads[0].ChecksumType, expectedType);
+ });
+ });
+ });
+ });
+
+ describe('MPU created with explicit algorithm and type', () => {
+ let uploadId;
+
+ before(async () => {
+ const res = await s3.send(new CreateMultipartUploadCommand({
+ Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC32', ChecksumType: 'FULL_OBJECT',
+ }));
+ uploadId = res.UploadId;
+ });
+
+ after(async () => {
+ await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId }));
+ });
+
+ it('should return the explicit ChecksumAlgorithm and ChecksumType', async () => {
+ const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+ assert.strictEqual(Uploads.length, 1);
+ assert.strictEqual(Uploads[0].UploadId, uploadId);
+ assert.strictEqual(Uploads[0].ChecksumAlgorithm, 'CRC32');
+ assert.strictEqual(Uploads[0].ChecksumType, 'FULL_OBJECT');
+ });
+ });
+
+ describe('multiple MPUs with mixed checksum settings', () => {
+ const uploads = [];
+
+ before(async () => {
+ const noChecksum = await s3.send(new CreateMultipartUploadCommand({
+ Bucket: bucket, Key: `${key}-no-checksum`,
+ }));
+ uploads.push({ key: `${key}-no-checksum`, uploadId: noChecksum.UploadId });
+
+ const withChecksum = await s3.send(new CreateMultipartUploadCommand({
+ Bucket: bucket, Key: `${key}-with-checksum`, ChecksumAlgorithm: 'SHA256',
+ }));
+ uploads.push({ key: `${key}-with-checksum`, uploadId: withChecksum.UploadId });
+ });
+
+ after(async () => {
+ await Promise.all(uploads.map(u =>
+ s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: u.key, UploadId: u.uploadId }))
+ ));
+ });
+
+ it('should return checksum fields only for uploads that have them', async () => {
+ const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+ assert.strictEqual(Uploads.length, 2);
+
+ const noChecksumUpload = Uploads.find(u => u.UploadId === uploads[0].uploadId);
+ assert.strictEqual(noChecksumUpload.ChecksumAlgorithm, undefined);
+ assert.strictEqual(noChecksumUpload.ChecksumType, undefined);
+
+ const withChecksumUpload = Uploads.find(u => u.UploadId === uploads[1].uploadId);
+ assert.strictEqual(withChecksumUpload.ChecksumAlgorithm, 'SHA256');
+ assert.strictEqual(withChecksumUpload.ChecksumType, 'COMPOSITE');
+ });
+ });
+ })
+);
diff --git a/tests/functional/aws-node-sdk/test/object/mpuChecksum.js b/tests/functional/aws-node-sdk/test/object/mpuChecksum.js
index cb08cda565..d68d4cc973 100644
--- a/tests/functional/aws-node-sdk/test/object/mpuChecksum.js
+++ b/tests/functional/aws-node-sdk/test/object/mpuChecksum.js
@@ -3,6 +3,7 @@ const {
CreateBucketCommand,
CreateMultipartUploadCommand,
AbortMultipartUploadCommand,
+ ListMultipartUploadsCommand,
DeleteBucketCommand,
} = require('@aws-sdk/client-s3');
@@ -32,17 +33,12 @@ describe('CreateMultipartUpload checksum headers', () =>
let res;
before(async () => {
- res = await s3.send(new CreateMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- }));
+ res = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key }));
});
after(async () => {
await s3.send(new AbortMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- UploadId: res.UploadId,
+ Bucket: bucket, Key: key, UploadId: res.UploadId,
}));
});
@@ -50,6 +46,14 @@ describe('CreateMultipartUpload checksum headers', () =>
assert.strictEqual(res.ChecksumAlgorithm, undefined);
assert.strictEqual(res.ChecksumType, undefined);
});
+
+ it('should not include ChecksumAlgorithm or ChecksumType in ListMultipartUploads', async () => {
+ const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+ const upload = Uploads.find(u => u.UploadId === res.UploadId);
+ assert(upload, 'upload not found in listing');
+ assert.strictEqual(upload.ChecksumAlgorithm, undefined);
+ assert.strictEqual(upload.ChecksumType, undefined);
+ });
});
describe('valid algorithm only', () => {
@@ -67,25 +71,28 @@ describe('CreateMultipartUpload checksum headers', () =>
before(async () => {
res = await s3.send(new CreateMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- ChecksumAlgorithm: algo,
+ Bucket: bucket, Key: key, ChecksumAlgorithm: algo,
}));
});
after(async () => {
await s3.send(new AbortMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- UploadId: res.UploadId,
+ Bucket: bucket, Key: key, UploadId: res.UploadId,
}));
});
- it(`should return ChecksumAlgorithm ${algo} and ` +
- `default ChecksumType to ${expectedType}`, () => {
+ it(`should return ChecksumAlgorithm ${algo} and default ChecksumType to ${expectedType}`, () => {
assert.strictEqual(res.ChecksumAlgorithm, algo);
assert.strictEqual(res.ChecksumType, expectedType);
});
+
+ it(`should include ${algo}/${expectedType} in ListMultipartUploads`, async () => {
+ const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+ const upload = Uploads.find(u => u.UploadId === res.UploadId);
+ assert(upload, 'upload not found in listing');
+ assert.strictEqual(upload.ChecksumAlgorithm, algo);
+ assert.strictEqual(upload.ChecksumType, expectedType);
+ });
});
});
});
@@ -107,26 +114,28 @@ describe('CreateMultipartUpload checksum headers', () =>
before(async () => {
res = await s3.send(new CreateMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- ChecksumAlgorithm: algo,
- ChecksumType: type,
+ Bucket: bucket, Key: key, ChecksumAlgorithm: algo, ChecksumType: type,
}));
});
after(async () => {
await s3.send(new AbortMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- UploadId: res.UploadId,
+ Bucket: bucket, Key: key, UploadId: res.UploadId,
}));
});
- it(`should return ChecksumAlgorithm ${algo} and ` +
- `ChecksumType ${type}`, () => {
+ it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${type}`, () => {
assert.strictEqual(res.ChecksumAlgorithm, algo);
assert.strictEqual(res.ChecksumType, type);
});
+
+ it(`should include ${algo}/${type} in ListMultipartUploads`, async () => {
+ const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+ const upload = Uploads.find(u => u.UploadId === res.UploadId);
+ assert(upload, 'upload not found in listing');
+ assert.strictEqual(upload.ChecksumAlgorithm, algo);
+ assert.strictEqual(upload.ChecksumType, type);
+ });
});
});
});
@@ -135,10 +144,7 @@ describe('CreateMultipartUpload checksum headers', () =>
it('should reject FULL_OBJECT with SHA256', async () => {
try {
await s3.send(new CreateMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- ChecksumAlgorithm: 'SHA256',
- ChecksumType: 'FULL_OBJECT',
+ Bucket: bucket, Key: key, ChecksumAlgorithm: 'SHA256', ChecksumType: 'FULL_OBJECT',
}));
assert.fail('Expected error');
} catch (err) {
@@ -149,10 +155,7 @@ describe('CreateMultipartUpload checksum headers', () =>
it('should reject FULL_OBJECT with SHA1', async () => {
try {
await s3.send(new CreateMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- ChecksumAlgorithm: 'SHA1',
- ChecksumType: 'FULL_OBJECT',
+ Bucket: bucket, Key: key, ChecksumAlgorithm: 'SHA1', ChecksumType: 'FULL_OBJECT',
}));
assert.fail('Expected error');
} catch (err) {
@@ -163,10 +166,7 @@ describe('CreateMultipartUpload checksum headers', () =>
it('should reject COMPOSITE with CRC64NVME', async () => {
try {
await s3.send(new CreateMultipartUploadCommand({
- Bucket: bucket,
- Key: key,
- ChecksumAlgorithm: 'CRC64NVME',
- ChecksumType: 'COMPOSITE',
+ Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC64NVME', ChecksumType: 'COMPOSITE',
}));
assert.fail('Expected error');
} catch (err) {
diff --git a/tests/unit/api/listMultipartUploads.js b/tests/unit/api/listMultipartUploads.js
index e24fc13f40..3141ab0108 100644
--- a/tests/unit/api/listMultipartUploads.js
+++ b/tests/unit/api/listMultipartUploads.js
@@ -4,8 +4,7 @@ const querystring = require('querystring');
const { parseString } = require('xml2js');
const { bucketPut } = require('../../../lib/api/bucketPut');
-const initiateMultipartUpload
- = require('../../../lib/api/initiateMultipartUpload');
+const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload');
const listMultipartUploads = require('../../../lib/api/listMultipartUploads');
const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers');
@@ -195,8 +194,7 @@ describe('listMultipartUploads API', () => {
});
});
- it('should return key following specified ' +
- 'key-marker', done => {
+ it('should return key following specified key-marker', done => {
const testListRequest = {
bucketName,
namespace,
@@ -208,22 +206,161 @@ describe('listMultipartUploads API', () => {
async.waterfall([
next => bucketPut(authInfo, testPutBucketRequest, log, next),
- (corsHeaders, next) => initiateMultipartUpload(authInfo,
- testInitiateMPURequest1, log, next),
- (result, corsHeaders, next) => initiateMultipartUpload(authInfo,
- testInitiateMPURequest2, log, next),
- (result, corsHeaders, next) => initiateMultipartUpload(authInfo,
- testInitiateMPURequest3, log, next),
- (result, corsHeaders, next) => listMultipartUploads(authInfo,
- testListRequest, log, next),
- (result, corsHeaders, next) =>
- parseString(result, corsHeaders, next),
+ (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next),
+ (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next),
+ (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest3, log, next),
+ (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
+ (result, corsHeaders, next) => parseString(result, corsHeaders, next),
],
(err, result) => {
- assert.strictEqual(result.ListMultipartUploadsResult
- .Upload[0].Key[0], objectName2);
- assert.strictEqual(result.ListMultipartUploadsResult
- .Upload[1], undefined);
+ assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName2);
+ assert.strictEqual(result.ListMultipartUploadsResult.Upload[1], undefined);
+ done();
+ });
+ });
+
+ it('should include ChecksumAlgorithm and ChecksumType when set on MPU', done => {
+ const checksumKey = 'checksum-object';
+ const testInitChecksumRequest = {
+ bucketName,
+ namespace,
+ objectKey: checksumKey,
+ headers: { 'x-amz-checksum-algorithm': 'CRC32' },
+ url: `/${bucketName}/${checksumKey}?uploads`,
+ actionImplicitDenies: false,
+ };
+ const testListRequest = {
+ bucketName,
+ namespace,
+ headers: { host: '/' },
+ url: `/${bucketName}?uploads`,
+ query: {},
+ actionImplicitDenies: false,
+ };
+
+ async.waterfall([
+ next => bucketPut(authInfo, testPutBucketRequest, log, next),
+ (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next),
+ (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
+ (result, corsHeaders, next) => parseString(result, corsHeaders, next),
+ ],
+ (err, result) => {
+ const upload = result.ListMultipartUploadsResult.Upload[0];
+ assert.strictEqual(upload.Key[0], checksumKey);
+ assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32');
+ assert.strictEqual(upload.ChecksumType[0], 'COMPOSITE');
+ done();
+ });
+ });
+
+ it('should not include ChecksumAlgorithm or ChecksumType when not set on MPU', done => {
+ const testListRequest = {
+ bucketName,
+ namespace,
+ headers: { host: '/' },
+ url: `/${bucketName}?uploads`,
+ query: {},
+ actionImplicitDenies: false,
+ };
+
+ async.waterfall([
+ next => bucketPut(authInfo, testPutBucketRequest, log, next),
+ (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next),
+ (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
+ (result, corsHeaders, next) => parseString(result, corsHeaders, next),
+ ],
+ (err, result) => {
+ const upload = result.ListMultipartUploadsResult.Upload[0];
+ assert.strictEqual(upload.Key[0], objectName1);
+ assert.strictEqual(upload.ChecksumAlgorithm, undefined);
+ assert.strictEqual(upload.ChecksumType, undefined);
+ done();
+ });
+ });
+
+ it('should include ChecksumAlgorithm and ChecksumType with explicit type', done => {
+ const checksumKey = 'checksum-explicit';
+ const testInitChecksumRequest = {
+ bucketName,
+ namespace,
+ objectKey: checksumKey,
+ headers: {
+ 'x-amz-checksum-algorithm': 'CRC32',
+ 'x-amz-checksum-type': 'FULL_OBJECT',
+ },
+ url: `/${bucketName}/${checksumKey}?uploads`,
+ actionImplicitDenies: false,
+ };
+ const testListRequest = {
+ bucketName,
+ namespace,
+ headers: { host: '/' },
+ url: `/${bucketName}?uploads`,
+ query: {},
+ actionImplicitDenies: false,
+ };
+
+ async.waterfall([
+ next => bucketPut(authInfo, testPutBucketRequest, log, next),
+ (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next),
+ (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
+ (result, corsHeaders, next) => parseString(result, corsHeaders, next),
+ ],
+ (err, result) => {
+ const upload = result.ListMultipartUploadsResult.Upload[0];
+ assert.strictEqual(upload.Key[0], checksumKey);
+ assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32');
+ assert.strictEqual(upload.ChecksumType[0], 'FULL_OBJECT');
+ done();
+ });
+ });
+
+ it('should list mixed uploads with and without checksum correctly', done => {
+ const checksumKey = 'aaa-checksum-object';
+ const noChecksumKey = 'zzz-no-checksum-object';
+ const testInitChecksumRequest = {
+ bucketName,
+ namespace,
+ objectKey: checksumKey,
+ headers: { 'x-amz-checksum-algorithm': 'SHA256' },
+ url: `/${bucketName}/${checksumKey}?uploads`,
+ actionImplicitDenies: false,
+ };
+ const testInitNoChecksumRequest = {
+ bucketName,
+ namespace,
+ objectKey: noChecksumKey,
+ headers: {},
+ url: `/${bucketName}/${noChecksumKey}?uploads`,
+ actionImplicitDenies: false,
+ };
+ const testListRequest = {
+ bucketName,
+ namespace,
+ headers: { host: '/' },
+ url: `/${bucketName}?uploads`,
+ query: {},
+ actionImplicitDenies: false,
+ };
+
+ async.waterfall([
+ next => bucketPut(authInfo, testPutBucketRequest, log, next),
+ (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next),
+ (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitNoChecksumRequest, log, next),
+ (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
+ (result, corsHeaders, next) => parseString(result, corsHeaders, next),
+ ],
+ (err, result) => {
+ const uploads = result.ListMultipartUploadsResult.Upload;
+ assert.strictEqual(uploads.length, 2);
+
+ const withChecksum = uploads.find(u => u.Key[0] === checksumKey);
+ assert.strictEqual(withChecksum.ChecksumAlgorithm[0], 'SHA256');
+ assert.strictEqual(withChecksum.ChecksumType[0], 'COMPOSITE');
+
+ const withoutChecksum = uploads.find(u => u.Key[0] === noChecksumKey);
+ assert.strictEqual(withoutChecksum.ChecksumAlgorithm, undefined);
+ assert.strictEqual(withoutChecksum.ChecksumType, undefined);
done();
});
});
diff --git a/yarn.lock b/yarn.lock
index c619b8e189..fc429f621a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6114,9 +6114,9 @@ arraybuffer.prototype.slice@^1.0.4:
optionalDependencies:
ioctl "^2.0.2"
-"arsenal@git+https://github.com/scality/Arsenal#8.3.9":
- version "8.3.9"
- resolved "git+https://github.com/scality/Arsenal#51e5b761f7f0612a722c828fa3d43b438c50ab7c"
+"arsenal@git+https://github.com/scality/Arsenal#8.3.10":
+ version "8.3.10"
+ resolved "git+https://github.com/scality/Arsenal#600fb069bdfb82106513a897658a027fdb96afbd"
dependencies:
"@aws-sdk/client-kms" "^3.975.0"
"@aws-sdk/client-s3" "^3.975.0"