diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml index 452705cf9d..698ae1ccf0 100644 --- a/.github/docker/docker-compose.yaml +++ b/.github/docker/docker-compose.yaml @@ -129,7 +129,7 @@ services: depends_on: - redis metadata-standalone: - image: ghcr.io/scality/metadata:8.11.0-standalone + image: ghcr.io/scality/metadata:8.25.0-standalone profiles: ['metadata-standalone'] network_mode: 'host' volumes: diff --git a/lib/api/listMultipartUploads.js b/lib/api/listMultipartUploads.js index 71e428669c..4a93b0439c 100644 --- a/lib/api/listMultipartUploads.js +++ b/lib/api/listMultipartUploads.js @@ -24,6 +24,8 @@ const monitoring = require('../utilities/monitoringHandler'); true my-divisor + CRC32 + COMPOSITE XMgbGlrZSBlbHZpbmcncyBub3QgaGF2aW5nIG11Y2ggbHVjaw arn:aws:iam::111122223333:user/ diff --git a/package.json b/package.json index ac3bdc10c2..ad1bf65f90 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.3.9", + "arsenal": "git+https://github.com/scality/Arsenal#8.3.10", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js b/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js new file mode 100644 index 0000000000..628e810792 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js @@ -0,0 +1,148 @@ +const assert = require('assert'); +const { + CreateBucketCommand, + CreateMultipartUploadCommand, + AbortMultipartUploadCommand, + ListMultipartUploadsCommand, + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); + +const withV4 = require('../support/withV4'); +const BucketUtility = require('../../lib/utility/bucket-util'); + +const bucket = `list-mpu-checksum-test-${Date.now()}`; +const key = 'test-checksum-key'; + +describe('ListMultipartUploads checksum fields', () => + withV4(sigCfg => { + let bucketUtil; + let s3; + + before(async () => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + }); + + after(async () => { + await bucketUtil.empty(bucket); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + describe('MPU created without checksum algorithm', () => { + let uploadId; + + before(async () => { + const res = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key })); + uploadId = res.UploadId; + }); + + after(async () => { + await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId })); + }); + + it('should not include ChecksumAlgorithm or ChecksumType in listed uploads', async () => { + const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket })); + assert.strictEqual(Uploads.length, 1); + assert.strictEqual(Uploads[0].UploadId, uploadId); + assert.strictEqual(Uploads[0].ChecksumAlgorithm, undefined); + assert.strictEqual(Uploads[0].ChecksumType, undefined); + }); + }); + + describe('MPU created with checksum algorithm', () => { + const cases = [ + { algo: 'CRC32', expectedType: 'COMPOSITE' }, + { algo: 'CRC32C', expectedType: 'COMPOSITE' }, + { algo: 'CRC64NVME', expectedType: 'FULL_OBJECT' }, + { algo: 'SHA1', expectedType: 'COMPOSITE' }, + { algo: 'SHA256', expectedType: 'COMPOSITE' }, + ]; + + cases.forEach(({ algo, expectedType }) => { + describe(`${algo}`, () => { + let uploadId; + + before(async () => { + const res = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, Key: key, ChecksumAlgorithm: algo, + })); + uploadId = res.UploadId; + }); + + after(async () => { + await s3.send(new AbortMultipartUploadCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + })); + }); + + it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${expectedType}`, async () => { + const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket })); + assert.strictEqual(Uploads.length, 1); + assert.strictEqual(Uploads[0].UploadId, uploadId); + assert.strictEqual(Uploads[0].ChecksumAlgorithm, algo); + assert.strictEqual(Uploads[0].ChecksumType, expectedType); + }); + }); + }); + }); + + describe('MPU created with explicit algorithm and type', () => { + let uploadId; + + before(async () => { + const res = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC32', ChecksumType: 'FULL_OBJECT', + })); + uploadId = res.UploadId; + }); + + after(async () => { + await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId })); + }); + + it('should return the explicit ChecksumAlgorithm and ChecksumType', async () => { + const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket })); + assert.strictEqual(Uploads.length, 1); + assert.strictEqual(Uploads[0].UploadId, uploadId); + assert.strictEqual(Uploads[0].ChecksumAlgorithm, 'CRC32'); + assert.strictEqual(Uploads[0].ChecksumType, 'FULL_OBJECT'); + }); + }); + + describe('multiple MPUs with mixed checksum settings', () => { + const uploads = []; + + before(async () => { + const noChecksum = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, Key: `${key}-no-checksum`, + })); + uploads.push({ key: `${key}-no-checksum`, uploadId: noChecksum.UploadId }); + + const withChecksum = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, Key: `${key}-with-checksum`, ChecksumAlgorithm: 'SHA256', + })); + uploads.push({ key: `${key}-with-checksum`, uploadId: withChecksum.UploadId }); + }); + + after(async () => { + await Promise.all(uploads.map(u => + s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: u.key, UploadId: u.uploadId })) + )); + }); + + it('should return checksum fields only for uploads that have them', async () => { + const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket })); + assert.strictEqual(Uploads.length, 2); + + const noChecksumUpload = Uploads.find(u => u.UploadId === uploads[0].uploadId); + assert.strictEqual(noChecksumUpload.ChecksumAlgorithm, undefined); + assert.strictEqual(noChecksumUpload.ChecksumType, undefined); + + const withChecksumUpload = Uploads.find(u => u.UploadId === uploads[1].uploadId); + assert.strictEqual(withChecksumUpload.ChecksumAlgorithm, 'SHA256'); + assert.strictEqual(withChecksumUpload.ChecksumType, 'COMPOSITE'); + }); + }); + }) +); diff --git a/tests/functional/aws-node-sdk/test/object/mpuChecksum.js b/tests/functional/aws-node-sdk/test/object/mpuChecksum.js index cb08cda565..d68d4cc973 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/mpuChecksum.js @@ -3,6 +3,7 @@ const { CreateBucketCommand, CreateMultipartUploadCommand, AbortMultipartUploadCommand, + ListMultipartUploadsCommand, DeleteBucketCommand, } = require('@aws-sdk/client-s3'); @@ -32,17 +33,12 @@ describe('CreateMultipartUpload checksum headers', () => let res; before(async () => { - res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })); + res = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key })); }); after(async () => { await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: res.UploadId, + Bucket: bucket, Key: key, UploadId: res.UploadId, })); }); @@ -50,6 +46,14 @@ describe('CreateMultipartUpload checksum headers', () => assert.strictEqual(res.ChecksumAlgorithm, undefined); assert.strictEqual(res.ChecksumType, undefined); }); + + it('should not include ChecksumAlgorithm or ChecksumType in ListMultipartUploads', async () => { + const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket })); + const upload = Uploads.find(u => u.UploadId === res.UploadId); + assert(upload, 'upload not found in listing'); + assert.strictEqual(upload.ChecksumAlgorithm, undefined); + assert.strictEqual(upload.ChecksumType, undefined); + }); }); describe('valid algorithm only', () => { @@ -67,25 +71,28 @@ describe('CreateMultipartUpload checksum headers', () => before(async () => { res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - ChecksumAlgorithm: algo, + Bucket: bucket, Key: key, ChecksumAlgorithm: algo, })); }); after(async () => { await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: res.UploadId, + Bucket: bucket, Key: key, UploadId: res.UploadId, })); }); - it(`should return ChecksumAlgorithm ${algo} and ` + - `default ChecksumType to ${expectedType}`, () => { + it(`should return ChecksumAlgorithm ${algo} and default ChecksumType to ${expectedType}`, () => { assert.strictEqual(res.ChecksumAlgorithm, algo); assert.strictEqual(res.ChecksumType, expectedType); }); + + it(`should include ${algo}/${expectedType} in ListMultipartUploads`, async () => { + const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket })); + const upload = Uploads.find(u => u.UploadId === res.UploadId); + assert(upload, 'upload not found in listing'); + assert.strictEqual(upload.ChecksumAlgorithm, algo); + assert.strictEqual(upload.ChecksumType, expectedType); + }); }); }); }); @@ -107,26 +114,28 @@ describe('CreateMultipartUpload checksum headers', () => before(async () => { res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - ChecksumAlgorithm: algo, - ChecksumType: type, + Bucket: bucket, Key: key, ChecksumAlgorithm: algo, ChecksumType: type, })); }); after(async () => { await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: res.UploadId, + Bucket: bucket, Key: key, UploadId: res.UploadId, })); }); - it(`should return ChecksumAlgorithm ${algo} and ` + - `ChecksumType ${type}`, () => { + it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${type}`, () => { assert.strictEqual(res.ChecksumAlgorithm, algo); assert.strictEqual(res.ChecksumType, type); }); + + it(`should include ${algo}/${type} in ListMultipartUploads`, async () => { + const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket })); + const upload = Uploads.find(u => u.UploadId === res.UploadId); + assert(upload, 'upload not found in listing'); + assert.strictEqual(upload.ChecksumAlgorithm, algo); + assert.strictEqual(upload.ChecksumType, type); + }); }); }); }); @@ -135,10 +144,7 @@ describe('CreateMultipartUpload checksum headers', () => it('should reject FULL_OBJECT with SHA256', async () => { try { await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - ChecksumAlgorithm: 'SHA256', - ChecksumType: 'FULL_OBJECT', + Bucket: bucket, Key: key, ChecksumAlgorithm: 'SHA256', ChecksumType: 'FULL_OBJECT', })); assert.fail('Expected error'); } catch (err) { @@ -149,10 +155,7 @@ describe('CreateMultipartUpload checksum headers', () => it('should reject FULL_OBJECT with SHA1', async () => { try { await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - ChecksumAlgorithm: 'SHA1', - ChecksumType: 'FULL_OBJECT', + Bucket: bucket, Key: key, ChecksumAlgorithm: 'SHA1', ChecksumType: 'FULL_OBJECT', })); assert.fail('Expected error'); } catch (err) { @@ -163,10 +166,7 @@ describe('CreateMultipartUpload checksum headers', () => it('should reject COMPOSITE with CRC64NVME', async () => { try { await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - ChecksumAlgorithm: 'CRC64NVME', - ChecksumType: 'COMPOSITE', + Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC64NVME', ChecksumType: 'COMPOSITE', })); assert.fail('Expected error'); } catch (err) { diff --git a/tests/unit/api/listMultipartUploads.js b/tests/unit/api/listMultipartUploads.js index e24fc13f40..3141ab0108 100644 --- a/tests/unit/api/listMultipartUploads.js +++ b/tests/unit/api/listMultipartUploads.js @@ -4,8 +4,7 @@ const querystring = require('querystring'); const { parseString } = require('xml2js'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const listMultipartUploads = require('../../../lib/api/listMultipartUploads'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); @@ -195,8 +194,7 @@ describe('listMultipartUploads API', () => { }); }); - it('should return key following specified ' + - 'key-marker', done => { + it('should return key following specified key-marker', done => { const testListRequest = { bucketName, namespace, @@ -208,22 +206,161 @@ describe('listMultipartUploads API', () => { async.waterfall([ next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest3, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, - testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest3, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), ], (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[0].Key[0], objectName2); - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[1], undefined); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName2); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1], undefined); + done(); + }); + }); + + it('should include ChecksumAlgorithm and ChecksumType when set on MPU', done => { + const checksumKey = 'checksum-object'; + const testInitChecksumRequest = { + bucketName, + namespace, + objectKey: checksumKey, + headers: { 'x-amz-checksum-algorithm': 'CRC32' }, + url: `/${bucketName}/${checksumKey}?uploads`, + actionImplicitDenies: false, + }; + const testListRequest = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads`, + query: {}, + actionImplicitDenies: false, + }; + + async.waterfall([ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + const upload = result.ListMultipartUploadsResult.Upload[0]; + assert.strictEqual(upload.Key[0], checksumKey); + assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32'); + assert.strictEqual(upload.ChecksumType[0], 'COMPOSITE'); + done(); + }); + }); + + it('should not include ChecksumAlgorithm or ChecksumType when not set on MPU', done => { + const testListRequest = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads`, + query: {}, + actionImplicitDenies: false, + }; + + async.waterfall([ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + const upload = result.ListMultipartUploadsResult.Upload[0]; + assert.strictEqual(upload.Key[0], objectName1); + assert.strictEqual(upload.ChecksumAlgorithm, undefined); + assert.strictEqual(upload.ChecksumType, undefined); + done(); + }); + }); + + it('should include ChecksumAlgorithm and ChecksumType with explicit type', done => { + const checksumKey = 'checksum-explicit'; + const testInitChecksumRequest = { + bucketName, + namespace, + objectKey: checksumKey, + headers: { + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'FULL_OBJECT', + }, + url: `/${bucketName}/${checksumKey}?uploads`, + actionImplicitDenies: false, + }; + const testListRequest = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads`, + query: {}, + actionImplicitDenies: false, + }; + + async.waterfall([ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + const upload = result.ListMultipartUploadsResult.Upload[0]; + assert.strictEqual(upload.Key[0], checksumKey); + assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32'); + assert.strictEqual(upload.ChecksumType[0], 'FULL_OBJECT'); + done(); + }); + }); + + it('should list mixed uploads with and without checksum correctly', done => { + const checksumKey = 'aaa-checksum-object'; + const noChecksumKey = 'zzz-no-checksum-object'; + const testInitChecksumRequest = { + bucketName, + namespace, + objectKey: checksumKey, + headers: { 'x-amz-checksum-algorithm': 'SHA256' }, + url: `/${bucketName}/${checksumKey}?uploads`, + actionImplicitDenies: false, + }; + const testInitNoChecksumRequest = { + bucketName, + namespace, + objectKey: noChecksumKey, + headers: {}, + url: `/${bucketName}/${noChecksumKey}?uploads`, + actionImplicitDenies: false, + }; + const testListRequest = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads`, + query: {}, + actionImplicitDenies: false, + }; + + async.waterfall([ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitNoChecksumRequest, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + const uploads = result.ListMultipartUploadsResult.Upload; + assert.strictEqual(uploads.length, 2); + + const withChecksum = uploads.find(u => u.Key[0] === checksumKey); + assert.strictEqual(withChecksum.ChecksumAlgorithm[0], 'SHA256'); + assert.strictEqual(withChecksum.ChecksumType[0], 'COMPOSITE'); + + const withoutChecksum = uploads.find(u => u.Key[0] === noChecksumKey); + assert.strictEqual(withoutChecksum.ChecksumAlgorithm, undefined); + assert.strictEqual(withoutChecksum.ChecksumType, undefined); done(); }); }); diff --git a/yarn.lock b/yarn.lock index c619b8e189..fc429f621a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6114,9 +6114,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.3.9": - version "8.3.9" - resolved "git+https://github.com/scality/Arsenal#51e5b761f7f0612a722c828fa3d43b438c50ab7c" +"arsenal@git+https://github.com/scality/Arsenal#8.3.10": + version "8.3.10" + resolved "git+https://github.com/scality/Arsenal#600fb069bdfb82106513a897658a027fdb96afbd" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0"