CLDSRV-887: ListMultipartUploads return checksum algo and type

.github/docker/docker-compose.yaml

@@ -129,7 +129,7 @@ services:
     depends_on:
       - redis
   metadata-standalone:
-    image: ghcr.io/scality/metadata:8.11.0-standalone
+    image: ghcr.io/scality/metadata:8.25.0-standalone
     profiles: ['metadata-standalone']
     network_mode: 'host'
     volumes:

lib/api/listMultipartUploads.js

@@ -24,6 +24,8 @@ const monitoring = require('../utilities/monitoringHandler');
   <IsTruncated>true</IsTruncated>
   <Upload>
     <Key>my-divisor</Key>
+    <ChecksumAlgorithm>CRC32</ChecksumAlgorithm>
+    <ChecksumType>COMPOSITE</ChecksumType>
     <UploadId>XMgbGlrZSBlbHZpbmcncyBub3QgaGF2aW5nIG11Y2ggbHVjaw</UploadId>
     <Initiator>
       <ID>arn:aws:iam::111122223333:user/

package.json

@@ -33,7 +33,7 @@
     "@azure/storage-blob": "^12.28.0",
     "@hapi/joi": "^17.1.1",
     "@smithy/node-http-handler": "^3.0.0",
-    "arsenal": "git+https://github.com/scality/Arsenal#8.3.9",
+    "arsenal": "git+https://github.com/scality/Arsenal#8.3.10",
     "async": "2.6.4",
     "bucketclient": "scality/bucketclient#8.2.7",
     "bufferutil": "^4.0.8",

tests/functional/aws-node-sdk/test/object/listMpuChecksum.js

@@ -0,0 +1,148 @@
+const assert = require('assert');
+const {
+    CreateBucketCommand,
+    CreateMultipartUploadCommand,
+    AbortMultipartUploadCommand,
+    ListMultipartUploadsCommand,
+    DeleteBucketCommand,
+} = require('@aws-sdk/client-s3');
+
+const withV4 = require('../support/withV4');
+const BucketUtility = require('../../lib/utility/bucket-util');
+
+const bucket = `list-mpu-checksum-test-${Date.now()}`;
+const key = 'test-checksum-key';
+
+describe('ListMultipartUploads checksum fields', () =>
+    withV4(sigCfg => {
+        let bucketUtil;
+        let s3;
+
+        before(async () => {
+            bucketUtil = new BucketUtility('default', sigCfg);
+            s3 = bucketUtil.s3;
+            await s3.send(new CreateBucketCommand({ Bucket: bucket }));
+        });
+
+        after(async () => {
+            await bucketUtil.empty(bucket);
+            await s3.send(new DeleteBucketCommand({ Bucket: bucket }));
+        });
+
+        describe('MPU created without checksum algorithm', () => {
+            let uploadId;
+
+            before(async () => {
+                const res = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key }));
+                uploadId = res.UploadId;
+            });
+
+            after(async () => {
+                await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId }));
+            });
+
+            it('should not include ChecksumAlgorithm or ChecksumType in listed uploads', async () => {
+                const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+                assert.strictEqual(Uploads.length, 1);
+                assert.strictEqual(Uploads[0].UploadId, uploadId);
+                assert.strictEqual(Uploads[0].ChecksumAlgorithm, undefined);
+                assert.strictEqual(Uploads[0].ChecksumType, undefined);
+            });
+        });
+
+        describe('MPU created with checksum algorithm', () => {
+            const cases = [
+                { algo: 'CRC32', expectedType: 'COMPOSITE' },
+                { algo: 'CRC32C', expectedType: 'COMPOSITE' },
+                { algo: 'CRC64NVME', expectedType: 'FULL_OBJECT' },
+                { algo: 'SHA1', expectedType: 'COMPOSITE' },
+                { algo: 'SHA256', expectedType: 'COMPOSITE' },
+            ];
+
+            cases.forEach(({ algo, expectedType }) => {
+                describe(`${algo}`, () => {
+                    let uploadId;
+
+                    before(async () => {
+                        const res = await s3.send(new CreateMultipartUploadCommand({
+                            Bucket: bucket, Key: key, ChecksumAlgorithm: algo,
+                        }));
+                        uploadId = res.UploadId;
+                    });
+
+                    after(async () => {
+                        await s3.send(new AbortMultipartUploadCommand({
+                            Bucket: bucket, Key: key, UploadId: uploadId,
+                        }));
+                    });
+
+                    it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${expectedType}`, async () => {
+                        const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+                        assert.strictEqual(Uploads.length, 1);
+                        assert.strictEqual(Uploads[0].UploadId, uploadId);
+                        assert.strictEqual(Uploads[0].ChecksumAlgorithm, algo);
+                        assert.strictEqual(Uploads[0].ChecksumType, expectedType);
+                    });
+                });
+            });
+        });
+
+        describe('MPU created with explicit algorithm and type', () => {
+            let uploadId;
+
+            before(async () => {
+                const res = await s3.send(new CreateMultipartUploadCommand({
+                    Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC32', ChecksumType: 'FULL_OBJECT',
+                }));
+                uploadId = res.UploadId;
+            });
+
+            after(async () => {
+                await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId }));
+            });
+
+            it('should return the explicit ChecksumAlgorithm and ChecksumType', async () => {
+                const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+                assert.strictEqual(Uploads.length, 1);
+                assert.strictEqual(Uploads[0].UploadId, uploadId);
+                assert.strictEqual(Uploads[0].ChecksumAlgorithm, 'CRC32');
+                assert.strictEqual(Uploads[0].ChecksumType, 'FULL_OBJECT');
+            });
+        });
+
+        describe('multiple MPUs with mixed checksum settings', () => {
+            const uploads = [];
+
+            before(async () => {
+                const noChecksum = await s3.send(new CreateMultipartUploadCommand({
+                    Bucket: bucket, Key: `${key}-no-checksum`,
+                }));
+                uploads.push({ key: `${key}-no-checksum`, uploadId: noChecksum.UploadId });
+
+                const withChecksum = await s3.send(new CreateMultipartUploadCommand({
+                    Bucket: bucket, Key: `${key}-with-checksum`, ChecksumAlgorithm: 'SHA256',
+                }));
+                uploads.push({ key: `${key}-with-checksum`, uploadId: withChecksum.UploadId });
+            });
+
+            after(async () => {
+                await Promise.all(uploads.map(u =>
+                    s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: u.key, UploadId: u.uploadId }))
+                ));
+            });
+
+            it('should return checksum fields only for uploads that have them', async () => {
+                const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+                assert.strictEqual(Uploads.length, 2);
+
+                const noChecksumUpload = Uploads.find(u => u.UploadId === uploads[0].uploadId);
+                assert.strictEqual(noChecksumUpload.ChecksumAlgorithm, undefined);
+                assert.strictEqual(noChecksumUpload.ChecksumType, undefined);
+
+                const withChecksumUpload = Uploads.find(u => u.UploadId === uploads[1].uploadId);
+                assert.strictEqual(withChecksumUpload.ChecksumAlgorithm, 'SHA256');
+                assert.strictEqual(withChecksumUpload.ChecksumType, 'COMPOSITE');
+            });
+        });
+    })
+);

tests/functional/aws-node-sdk/test/object/mpuChecksum.js

@@ -3,6 +3,7 @@ const {
     CreateBucketCommand,
     CreateMultipartUploadCommand,
     AbortMultipartUploadCommand,
+    ListMultipartUploadsCommand,
     DeleteBucketCommand,
 } = require('@aws-sdk/client-s3');
 
@@ -32,24 +33,27 @@ describe('CreateMultipartUpload checksum headers', () =>
             let res;
 
             before(async () => {
-                res = await s3.send(new CreateMultipartUploadCommand({
-                    Bucket: bucket,
-                    Key: key,
-                }));
+                res = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key }));
             });
 
             after(async () => {
                 await s3.send(new AbortMultipartUploadCommand({
-                    Bucket: bucket,
-                    Key: key,
-                    UploadId: res.UploadId,
+                    Bucket: bucket, Key: key, UploadId: res.UploadId,
                 }));
             });
 
             it('should not return ChecksumAlgorithm and ChecksumType', () => {
                 assert.strictEqual(res.ChecksumAlgorithm, undefined);
                 assert.strictEqual(res.ChecksumType, undefined);
             });
+
+            it('should not include ChecksumAlgorithm or ChecksumType in ListMultipartUploads', async () => {
+                const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+                const upload = Uploads.find(u => u.UploadId === res.UploadId);
+                assert(upload, 'upload not found in listing');
+                assert.strictEqual(upload.ChecksumAlgorithm, undefined);
+                assert.strictEqual(upload.ChecksumType, undefined);
+            });
         });
 
         describe('valid algorithm only', () => {
@@ -67,25 +71,28 @@ describe('CreateMultipartUpload checksum headers', () =>
 
                     before(async () => {
                         res = await s3.send(new CreateMultipartUploadCommand({
-                            Bucket: bucket,
-                            Key: key,
-                            ChecksumAlgorithm: algo,
+                            Bucket: bucket, Key: key, ChecksumAlgorithm: algo,
                         }));
                     });
 
                     after(async () => {
                         await s3.send(new AbortMultipartUploadCommand({
-                            Bucket: bucket,
-                            Key: key,
-                            UploadId: res.UploadId,
+                            Bucket: bucket, Key: key, UploadId: res.UploadId,
                         }));
                     });
 
-                    it(`should return ChecksumAlgorithm ${algo} and ` +
-                       `default ChecksumType to ${expectedType}`, () => {
+                    it(`should return ChecksumAlgorithm ${algo} and default ChecksumType to ${expectedType}`, () => {
                         assert.strictEqual(res.ChecksumAlgorithm, algo);
                         assert.strictEqual(res.ChecksumType, expectedType);
                     });
+
+                    it(`should include ${algo}/${expectedType} in ListMultipartUploads`, async () => {
+                        const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+                        const upload = Uploads.find(u => u.UploadId === res.UploadId);
+                        assert(upload, 'upload not found in listing');
+                        assert.strictEqual(upload.ChecksumAlgorithm, algo);
+                        assert.strictEqual(upload.ChecksumType, expectedType);
+                    });
                 });
             });
         });
@@ -107,26 +114,28 @@ describe('CreateMultipartUpload checksum headers', () =>
 
                     before(async () => {
                         res = await s3.send(new CreateMultipartUploadCommand({
-                            Bucket: bucket,
-                            Key: key,
-                            ChecksumAlgorithm: algo,
-                            ChecksumType: type,
+                            Bucket: bucket, Key: key, ChecksumAlgorithm: algo, ChecksumType: type,
                         }));
                     });
 
                     after(async () => {
                         await s3.send(new AbortMultipartUploadCommand({
-                            Bucket: bucket,
-                            Key: key,
-                            UploadId: res.UploadId,
+                            Bucket: bucket, Key: key, UploadId: res.UploadId,
                         }));
                     });
 
-                    it(`should return ChecksumAlgorithm ${algo} and ` +
-                       `ChecksumType ${type}`, () => {
+                    it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${type}`, () => {
                         assert.strictEqual(res.ChecksumAlgorithm, algo);
                         assert.strictEqual(res.ChecksumType, type);
                     });
+
+                    it(`should include ${algo}/${type} in ListMultipartUploads`, async () => {
+                        const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
+                        const upload = Uploads.find(u => u.UploadId === res.UploadId);
+                        assert(upload, 'upload not found in listing');
+                        assert.strictEqual(upload.ChecksumAlgorithm, algo);
+                        assert.strictEqual(upload.ChecksumType, type);
+                    });
                 });
             });
         });
@@ -135,10 +144,7 @@ describe('CreateMultipartUpload checksum headers', () =>
             it('should reject FULL_OBJECT with SHA256', async () => {
                 try {
                     await s3.send(new CreateMultipartUploadCommand({
-                        Bucket: bucket,
-                        Key: key,
-                        ChecksumAlgorithm: 'SHA256',
-                        ChecksumType: 'FULL_OBJECT',
+                        Bucket: bucket, Key: key, ChecksumAlgorithm: 'SHA256', ChecksumType: 'FULL_OBJECT',
                     }));
                     assert.fail('Expected error');
                 } catch (err) {
@@ -149,10 +155,7 @@ describe('CreateMultipartUpload checksum headers', () =>
             it('should reject FULL_OBJECT with SHA1', async () => {
                 try {
                     await s3.send(new CreateMultipartUploadCommand({
-                        Bucket: bucket,
-                        Key: key,
-                        ChecksumAlgorithm: 'SHA1',
-                        ChecksumType: 'FULL_OBJECT',
+                        Bucket: bucket, Key: key, ChecksumAlgorithm: 'SHA1', ChecksumType: 'FULL_OBJECT',
                     }));
                     assert.fail('Expected error');
                 } catch (err) {
@@ -163,10 +166,7 @@ describe('CreateMultipartUpload checksum headers', () =>
             it('should reject COMPOSITE with CRC64NVME', async () => {
                 try {
                     await s3.send(new CreateMultipartUploadCommand({
-                        Bucket: bucket,
-                        Key: key,
-                        ChecksumAlgorithm: 'CRC64NVME',
-                        ChecksumType: 'COMPOSITE',
+                        Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC64NVME', ChecksumType: 'COMPOSITE',
                     }));
                     assert.fail('Expected error');
                 } catch (err) {