diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml index ffd1172b94..ec860ba00f 100644 --- a/.github/docker/docker-compose.yaml +++ b/.github/docker/docker-compose.yaml @@ -47,6 +47,7 @@ services: - S3QUOTA - QUOTA_ENABLE_INFLIGHTS - S3_VERSION_ID_ENCODING_TYPE + - S3_INTEGRITY_CHECKS_ENABLED - S3_SERVER_ACCESS_LOGS_MODE=ENABLED - S3_ENABLE_SERVER_ACCESS_LOGS=true - RATE_LIMIT_SERVICE_USER_ARN=arn:aws:iam::123456789013:root diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f96a040dc7..6fe10a3fe4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -558,6 +558,55 @@ jobs: source: /tmp/artifacts if: always() + # Runs the server with x-amz-checksum-* computation turned off and asserts the + # resulting behaviour. Needs its own job because every other functional suite + # runs against a server with checksums enabled. + checksums-disabled-tests: + runs-on: ubuntu-24.04 + needs: build + env: + S3BACKEND: mem + S3VAULT: mem + CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage + MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} + MPU_TESTING: 'yes' + S3_INTEGRITY_CHECKS_ENABLED: 'false' + JOB_NAME: ${{ github.job }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup CI services + run: docker compose up -d + working-directory: .github/docker + - name: Run checksums-disabled tests + run: |- + set -o pipefail; + bash wait_for_local_port.bash 8000 40 + yarn run ft_checksums_disabled | tee /tmp/artifacts/${{ github.job }}/tests.log + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: checksums-disabled-tests + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() + # Configure and run as Integration run S3C tests s3c-ft-tests: strategy: diff --git a/config.json b/config.json index 7d0f7d9efe..6846ead615 100644 --- a/config.json +++ b/config.json @@ -152,6 +152,9 @@ "multiObjectDelete": 2097152, "bucketPutPolicy": 20480 }, + "integrityChecks": { + "enabled": true + }, "serverAccessLogs": { "mode": "DISABLED", "outputFile": "/logs/server-access.log", diff --git a/lib/Config.js b/lib/Config.js index 3c9887fed1..af21a103be 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -584,6 +584,58 @@ function parseServerAccessLogs(config) { return res; } +/** + * Parse the `integrityChecks` config section. + * + * `enabled` is a kill switch for `x-amz-checksum-*` digests, defaulting to true. + * Setting it to false stops CloudServer computing them at all — the point is to + * reclaim the CPU spent hashing every payload byte, so nothing is compared and + * no checksum is stored. Objects and MPU parts then carry no checksum metadata, + * the same state as objects predating checksum support. + * + * The headers are then ignored outright rather than merely unenforced: a + * malformed or unsupported `x-amz-checksum-*` value is accepted instead of + * rejected, and neither CreateMultipartUpload nor CompleteMultipartUpload + * validates `x-amz-checksum-algorithm`/`-type`. + * + * Content-MD5 and x-amz-content-sha256 are unaffected and remain enforced. + * + * Safe to disable part-way through a multipart upload, but not to re-enable: + * parts uploaded while disabled carry no digest, so a later CompleteMPU cannot + * compose the final checksum and fails. + * + * @param {object} config - raw parsed config file contents + * @return {{enabled: boolean}} the parsed integrityChecks section + */ +function parseIntegrityChecks(config) { + const res = { enabled: true }; + + if (config && config.integrityChecks) { + assert( + typeof config.integrityChecks === 'object' && !Array.isArray(config.integrityChecks), + 'bad config: integrityChecks must be an object', + ); + + if ('enabled' in config.integrityChecks) { + assert( + typeof config.integrityChecks.enabled === 'boolean', + 'bad config: integrityChecks.enabled must be a boolean', + ); + res.enabled = config.integrityChecks.enabled; + } + } + + if (process.env.S3_INTEGRITY_CHECKS_ENABLED !== undefined) { + assert( + ['true', 'false'].includes(process.env.S3_INTEGRITY_CHECKS_ENABLED), + "bad config: S3_INTEGRITY_CHECKS_ENABLED must be 'true' or 'false'", + ); + res.enabled = process.env.S3_INTEGRITY_CHECKS_ENABLED === 'true'; + } + + return res; +} + /** * Reads from a config file and returns the content as a config object */ @@ -1829,6 +1881,7 @@ class Config extends EventEmitter { this.apiBodySizeLimits[apiKey] = limit; } } + this.integrityChecks = parseIntegrityChecks(config); this.serverAccessLogs = parseServerAccessLogs(config); /** * S3C-10336: PutObject max size of 5GB is new in 9.5.1 @@ -2201,4 +2254,5 @@ module.exports = { azureGetStorageAccountName, azureGetLocationCredentials, parseSupportedLifecycleRules, + parseIntegrityChecks, }; diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 99f083aeb1..9d84c9152c 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -3,6 +3,7 @@ const crypto = require('crypto'); const { crc32: crtCrc32, crc32c: crtCrc32c } = require('aws-crt').checksums; const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); const { errors: ArsenalErrors, errorInstances } = require('arsenal'); +const { config } = require('../../../Config'); const { combinePartCrcs } = require('./crcCombine'); const { supportedSignatureChecksums, unsupportedSignatureChecksums } = require('../../../../constants'); @@ -37,6 +38,10 @@ const errMPUTypeInvalid = errorInstances.InvalidRequest.customizeDescription( const errMPUTypeWithoutAlgo = errorInstances.InvalidRequest.customizeDescription( 'The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.', ); +const errMPUTypeNotConfigured = errorInstances.InvalidRequest.customizeDescription( + 'The upload was not created with a checksum mode. ' + + 'The complete request must not include a x-amz-checksum-type header.', +); // TODO(S3C-11278): Update with 'MD5', 'SHA512', 'XXHASH128', 'XXHASH3', 'XXHASH64' when they are introduced. // https://scality.atlassian.net/browse/S3C-11278 @@ -111,6 +116,8 @@ const ChecksumError = Object.freeze({ MPUAlgoNotSupported: 'MPUAlgoNotSupported', MPUTypeInvalid: 'MPUTypeInvalid', MPUTypeWithoutAlgo: 'MPUTypeWithoutAlgo', + MPUTypeNotConfigured: 'MPUTypeNotConfigured', + MPUTypeModeMismatch: 'MPUTypeModeMismatch', MPUInvalidCombination: 'MPUInvalidCombination', CopyChecksumAlgoNotSupported: 'CopyChecksumAlgoNotSupported', ContentSHA256Missing: 'ContentSHA256Missing', @@ -120,6 +127,18 @@ const ChecksumError = Object.freeze({ const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/; +/** + * When false, x-amz-checksum-* headers are ignored: no digest is calculated, + * compared, or stored. + * + * Content-MD5 and x-amz-content-sha256 are not impacted. + * + * @return {boolean} true when checksums are enabled. + */ +function areChecksumsEnabled() { + return config.integrityChecks?.enabled !== false; +} + function uint32ToBase64(num) { const buf = Buffer.alloc(4); buf.writeUInt32BE(num, 0); @@ -558,6 +577,13 @@ function arsenalErrorFromChecksumError(err) { return errMPUTypeInvalid; case ChecksumError.MPUTypeWithoutAlgo: return errMPUTypeWithoutAlgo; + case ChecksumError.MPUTypeNotConfigured: + return errMPUTypeNotConfigured; + case ChecksumError.MPUTypeModeMismatch: + return errorInstances.InvalidRequest.customizeDescription( + `The upload was created using the ${err.details.type} checksum mode. ` + + 'The complete request must use the same checksum mode.', + ); case ChecksumError.MPUInvalidCombination: return errorInstances.InvalidRequest.customizeDescription( `The ${err.details.type} checksum type cannot be used ` + @@ -680,6 +706,9 @@ async function validateMethodChecksumNoChunking(request, body, log) { } if (request.apiMethod in checksumedMethods) { + if (!areChecksumsEnabled()) { + return md5OnlyValidationFunc(request, body, log); + } return await defaultValidationFunc(request, body, log); } @@ -758,6 +787,46 @@ function getChecksumDataFromMPUHeaders(headers) { return { algorithm: algo, type: defaultChecksumType[algo], isDefault: false }; } +/** + * Validate the x-amz-checksum-type header on a CompleteMultipartUpload request + * against the checksum type the MPU was created with. + * + * x-amz-checksum-algorithm is deliberately not validated: AWS ignores a mismatch + * on that header for CompleteMultipartUpload. + * + * @param {object} headers - request headers (lowercased keys) + * @param {string|undefined} mpuChecksumType - the checksum type recorded on the + * MPU at CreateMPU time; falsy for a legacy MPU predating type tracking + * @param {boolean} isExternal - external-backend MPU. Those record no checksum + * config (CLDSRV-964), so an absent type means "not tracked here" rather than + * a legacy upload, and the header is ignored instead of rejected. + * @returns {{error: string, details: object}|null} null when valid + */ +function validateCompleteMPUChecksumType(headers, mpuChecksumType, isExternal) { + const headerType = headers['x-amz-checksum-type']; + if (!headerType) { + return null; + } + + const headerTypeUpper = headerType.toUpperCase(); + if (!validMPUTypes.has(headerTypeUpper)) { + return { error: ChecksumError.MPUTypeInvalid, details: { type: headerType } }; + } + + if (!mpuChecksumType) { + if (isExternal) { + return null; + } + return { error: ChecksumError.MPUTypeNotConfigured, details: { type: headerType } }; + } + + if (headerTypeUpper !== mpuChecksumType.toUpperCase()) { + return { error: ChecksumError.MPUTypeModeMismatch, details: { type: mpuChecksumType } }; + } + + return null; +} + // ============================================================================= // MPU final-object checksum computation // ============================================================================= @@ -876,8 +945,10 @@ module.exports = { algorithms, checksumedMethods, getChecksumDataFromMPUHeaders, + validateCompleteMPUChecksumType, computeCompositeMPUChecksum, computeFullObjectMPUChecksum, validateCompleteMultipartUploadChecksum, getCopyObjectChecksumAlgorithm, + areChecksumsEnabled, }; diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 24d915acd0..82f26ccfc5 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -19,6 +19,7 @@ const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError, validateXAmzContentSHA256, + areChecksumsEnabled, } = require('../integrity/validateChecksums'); const { externalBackends, versioningNotImplBackends } = constants; @@ -43,6 +44,11 @@ function zeroSizeBodyChecksumCheck(headers, metadataStoreParams, callback) { if (contentSHA256Err) { return callback(arsenalErrorFromChecksumError(contentSHA256Err)); } + // Nothing is computed or stored when checksums are disabled, so a zero-byte + // object ends up with no checksum metadata like every other object. + if (!areChecksumsEnabled()) { + return callback(null); + } const checksumData = getChecksumDataFromHeaders(headers) || defaultChecksumData; if (checksumData.error) { return callback(arsenalErrorFromChecksumError(checksumData)); @@ -316,7 +322,7 @@ function createAndStoreObject( } } - const headerChecksum = getChecksumDataFromHeaders(request.headers); + const headerChecksum = areChecksumsEnabled() ? getChecksumDataFromHeaders(request.headers) : null; if (headerChecksum && headerChecksum.error) { return next(arsenalErrorFromChecksumError(headerChecksum)); } diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index 38f1a07836..e0a70c990a 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -2,7 +2,7 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform'); const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform'); const ContentSHA256Transform = require('../../../auth/streamingV4/ContentSHA256Transform'); -const { parseContentSHA256, ContentSHA256Type } = require('../integrity/validateChecksums'); +const { parseContentSHA256, ContentSHA256Type, areChecksumsEnabled } = require('../integrity/validateChecksums'); const { errors, errorInstances, jsutil } = require('arsenal'); const { unsupportedSignatureChecksums } = require('../../../../constants'); @@ -80,7 +80,7 @@ function pipeChecksumStreams(inputStream, primary, secondary, onStreamError, log */ function prepareStream(request, streamingV4Params, checksums, log, errCb) { const xAmzContentSHA256 = request.headers['x-amz-content-sha256']; - const { primary = null, secondary = null } = checksums || {}; + const { primary = null, secondary = null } = (areChecksumsEnabled() && checksums) || {}; switch (xAmzContentSHA256) { case 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': { diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index feb1a09bc1..be85005f2f 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -32,6 +32,8 @@ const { computeCompositeMPUChecksum, computeFullObjectMPUChecksum, validateCompleteMultipartUploadChecksum, + validateCompleteMPUChecksumType, + areChecksumsEnabled, } = require('./apiUtils/integrity/validateChecksums'); const versionIdUtils = versioning.VersionID; @@ -336,41 +338,21 @@ function completeMultipartUpload(authInfo, request, log, callback) { log.error('error validating request', { error: err }); return next(err, destBucket); } - // Validate x-amz-checksum-type header (if present) matches - // the checksum type the MPU was created with. - // x-amz-checksum-algorithm is not validated: AWS ignores - // a mismatch on this header for CompleteMultipartUpload. - const headerType = request.headers['x-amz-checksum-type']; - if (headerType) { - const headerTypeUpper = headerType.toUpperCase(); - if (headerTypeUpper !== 'COMPOSITE' && headerTypeUpper !== 'FULL_OBJECT') { - const typeErr = errorInstances.InvalidRequest.customizeDescription( - 'Value for x-amz-checksum-type header is invalid.', - ); - return next(typeErr, destBucket); - } - const mpuType = storedMetadata.checksumType; - if (!mpuType) { - // External-backend MPUs record no checksum config - // (CLDSRV-964): ignore the header, like every other - // checksum input on CompleteMPU for external backends. - const mpuLocation = storedMetadata.controllingLocationConstraint; - const isExternalMpu = - !!constants.externalBackends[config.getLocationConstraintType(mpuLocation)]; - if (!isExternalMpu) { - // Legacy MPU created before checksumType was tracked. - const typeErr = errorInstances.InvalidRequest.customizeDescription( - 'The upload was not created with a checksum mode. ' + - 'The complete request must not include a x-amz-checksum-type header.', - ); - return next(typeErr, destBucket); - } - } else if (headerTypeUpper !== mpuType.toUpperCase()) { - const typeErr = errorInstances.InvalidRequest.customizeDescription( - `The upload was created using the ${mpuType} checksum mode. ` + - 'The complete request must use the same checksum mode.', - ); - return next(typeErr, destBucket); + // External-backend MPUs record no checksum config (CLDSRV-964), + // so an absent stored type means "not tracked" rather than legacy. + // MPUs created while checksums were disabled record none either, + // so the header is ignored rather than validated. + if (areChecksumsEnabled()) { + const mpuLocation = storedMetadata.controllingLocationConstraint; + const isExternalMpu = + !!constants.externalBackends[config.getLocationConstraintType(mpuLocation)]; + const typeErr = validateCompleteMPUChecksumType( + request.headers, + storedMetadata.checksumType, + isExternalMpu, + ); + if (typeErr) { + return next(arsenalErrorFromChecksumError(typeErr), destBucket); } } return next(null, destBucket, objMD, mpuBucket, storedMetadata); @@ -470,7 +452,10 @@ function completeMultipartUpload(authInfo, request, log, callback) { type: storedMetadata.checksumType, isDefault: storedMetadata.checksumIsDefault, }; - const checksumErr = validatePerPartChecksums(jsonList, storedParts, splitter, mpuChecksum); + let checksumErr = null; + if (areChecksumsEnabled()) { + checksumErr = validatePerPartChecksums(jsonList, storedParts, splitter, mpuChecksum); + } if (checksumErr) { log.debug('per-part checksum validation failed', { error: checksumErr, @@ -609,7 +594,8 @@ function completeMultipartUpload(authInfo, request, log, callback) { // - if no filteredPartsObj then there is no per-part info to compute from (aws_s3/gcp/ingestion // return no filteredPartsObj; azure returns filteredPartsObj, but its parts store no checksum) // - if completeObjData is present it means the MPU was completed by an external backend - if (!filteredPartsObj || completeObjData) { + // - if checksums are disabled the parts carry no digest to compose from + if (!filteredPartsObj || completeObjData || !areChecksumsEnabled()) { return continueProcessParts(null); } computeFinalChecksum( diff --git a/lib/api/initiateMultipartUpload.js b/lib/api/initiateMultipartUpload.js index a5331f890e..a216290096 100644 --- a/lib/api/initiateMultipartUpload.js +++ b/lib/api/initiateMultipartUpload.js @@ -24,6 +24,7 @@ const { updateEncryption } = require('./apiUtils/bucket/updateEncryption'); const { getChecksumDataFromMPUHeaders, arsenalErrorFromChecksumError, + areChecksumsEnabled, } = require('./apiUtils/integrity/validateChecksums'); const { config } = require('../Config'); const kms = require('../kms/wrapper'); @@ -87,8 +88,8 @@ function initiateMultipartUpload(authInfo, request, log, callback) { log.debug('invalid x-amz-website-redirect-location' + `value ${websiteRedirectHeader}`, { error: err }); return callback(err); } - const checksumConfig = getChecksumDataFromMPUHeaders(request.headers); - if (checksumConfig.error) { + const checksumConfig = areChecksumsEnabled() ? getChecksumDataFromMPUHeaders(request.headers) : null; + if (checksumConfig && checksumConfig.error) { const checksumErr = arsenalErrorFromChecksumError(checksumConfig); log.debug('checksum header validation failed', { error: checksumErr, method: 'initiateMultipartUpload' }); monitoring.promMetrics('PUT', bucketName, checksumErr.code, 'initiateMultipartUpload'); @@ -150,9 +151,11 @@ function initiateMultipartUpload(authInfo, request, log, callback) { initiatorDisplayName, splitter: constants.splitter, }; - metadataStoreParams.checksumAlgorithm = checksumConfig.algorithm; - metadataStoreParams.checksumType = checksumConfig.type; - metadataStoreParams.checksumIsDefault = checksumConfig.isDefault; + if (checksumConfig) { + metadataStoreParams.checksumAlgorithm = checksumConfig.algorithm; + metadataStoreParams.checksumType = checksumConfig.type; + metadataStoreParams.checksumIsDefault = checksumConfig.isDefault; + } const tagging = request.headers['x-amz-tagging']; if (tagging) { metadataStoreParams.tagging = tagging; @@ -224,7 +227,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) { // Only respond the headers if the user sent them and // the MPU can honor them (not an external backend). - if (!checksumConfig.isDefault && !isExternalLocation) { + if (checksumConfig && !checksumConfig.isDefault && !isExternalLocation) { // eslint-disable-next-line no-param-reassign corsHeaders['x-amz-checksum-algorithm'] = checksumConfig.algorithm.toUpperCase(); // eslint-disable-next-line no-param-reassign diff --git a/lib/api/objectCopy.js b/lib/api/objectCopy.js index 19d043181e..153aa21f03 100644 --- a/lib/api/objectCopy.js +++ b/lib/api/objectCopy.js @@ -29,6 +29,7 @@ const { algorithms, arsenalErrorFromChecksumError, getCopyObjectChecksumAlgorithm, + areChecksumsEnabled, } = require('./apiUtils/integrity/validateChecksums'); const ChecksumTransform = require('../auth/streamingV4/ChecksumTransform'); const ChecksumWritable = require('../auth/streamingV4/ChecksumWritable'); @@ -75,6 +76,9 @@ function _orphanedDataLocations(dataToDelete, newDataGetInfo) { * @returns {boolean} */ function _shouldRecomputeChecksum(headers, sourceObjMD) { + if (!areChecksumsEnabled()) { + return false; + } const requestedAlgo = headers['x-amz-checksum-algorithm']?.toLowerCase(); if ( sourceObjMD.checksum?.checksumType === 'FULL_OBJECT' && @@ -410,7 +414,7 @@ function _prepMetadata( storeMetadataParams.defaultRetention = defaultRetentionConfig; } - if (sourceObjMD.checksum && !_shouldRecomputeChecksum(headers, sourceObjMD)) { + if (areChecksumsEnabled() && sourceObjMD.checksum && !_shouldRecomputeChecksum(headers, sourceObjMD)) { storeMetadataParams.checksum = { algorithm: sourceObjMD.checksum.checksumAlgorithm, value: sourceObjMD.checksum.checksumValue, @@ -503,7 +507,9 @@ function objectCopy(authInfo, request, sourceBucket, sourceObject, sourceVersion monitoring.promMetrics('PUT', destBucketName, err.code, 'copyObject'); return callback(err); } - const { error: checksumAlgoErr, algorithm: requestedAlgo } = getCopyObjectChecksumAlgorithm(request.headers); + const { error: checksumAlgoErr, algorithm: requestedAlgo } = areChecksumsEnabled() + ? getCopyObjectChecksumAlgorithm(request.headers) + : { error: null, algorithm: null }; if (checksumAlgoErr) { const err = arsenalErrorFromChecksumError(checksumAlgoErr); log.debug('invalid x-amz-checksum-algorithm', { error: checksumAlgoErr }); diff --git a/lib/api/objectPutCopyPart.js b/lib/api/objectPutCopyPart.js index ccba422c71..5d7fc53691 100644 --- a/lib/api/objectPutCopyPart.js +++ b/lib/api/objectPutCopyPart.js @@ -16,7 +16,7 @@ const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { initializeInternalLogRequestQueue, queueInternalLogRequest } = require('../utilities/serverAccessLogger'); -const { algorithms } = require('./apiUtils/integrity/validateChecksums'); +const { algorithms, areChecksumsEnabled } = require('./apiUtils/integrity/validateChecksums'); const { buildSourcePartsStream, computeChecksumFromDataLocator } = require('./apiUtils/object/sourceChecksum'); const { config } = require('../Config'); const kms = require('../kms/wrapper'); @@ -28,6 +28,9 @@ const { BackendInfo } = models; const skipError = new Error('skip'); function _shouldRecomputeChecksum(request, sourceChecksum, algo) { + if (!areChecksumsEnabled()) { + return false; + } if (request.headers['x-amz-copy-source-range']) { return true; } @@ -564,7 +567,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket, sourceObject, reqVer // Reuse the source's stored checksum, or none for a legacy or // external-backend MPU. const partChecksum = - algo && !destIsExternal + algo && !destIsExternal && areChecksumsEnabled() ? { algorithm: algo, value: sourceObjMD.checksum.checksumValue } : undefined; if (isSkip) { diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index 780c1e9b8e..c314c3bd2b 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -19,7 +19,11 @@ const { BackendInfo } = models; const writeContinue = require('../utilities/writeContinue'); const { parseObjectEncryptionHeaders } = require('./apiUtils/bucket/bucketEncryption'); const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol'); -const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError } = require('./apiUtils/integrity/validateChecksums'); +const { + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, + areChecksumsEnabled, +} = require('./apiUtils/integrity/validateChecksums'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { storeServerAccessLogInfo } = require('../metadata/metadataUtils'); @@ -373,7 +377,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log, cb) { }; const backendInfo = new BackendInfo(config, objectLocationConstraint); - const headerChecksum = getChecksumDataFromHeaders(request.headers); + const checksumsEnabled = areChecksumsEnabled(); + const headerChecksum = checksumsEnabled ? getChecksumDataFromHeaders(request.headers) : null; if (headerChecksum && headerChecksum.error) { return next(arsenalErrorFromChecksumError(headerChecksum), destinationBucket); } @@ -392,9 +397,7 @@ function objectPutPart(authInfo, request, streamingV4Params, log, cb) { return next(checksumTypeMismatchErr(mpuChecksumAlgo, headerChecksum.algorithm), destinationBucket); } - // A COMPOSITE MPU's final checksum is composed from the per-part - // checksums, so every part must carry one. - if (!headerChecksum && mpuChecksumType === 'COMPOSITE') { + if (checksumsEnabled && !headerChecksum && mpuChecksumType === 'COMPOSITE') { return next(checksumTypeMismatchErr(mpuChecksumAlgo, 'null'), destinationBucket); } diff --git a/lib/routes/veeam/utils.js b/lib/routes/veeam/utils.js index cf3fa0f7a3..3634f18a34 100644 --- a/lib/routes/veeam/utils.js +++ b/lib/routes/veeam/utils.js @@ -10,6 +10,7 @@ const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError, defaultChecksumData, + areChecksumsEnabled, } = require('../../api/apiUtils/integrity/validateChecksums'); const UtilizationService = require('../../utilization/instance'); const metadata = require('../../metadata/wrapper'); @@ -49,7 +50,7 @@ async function receiveData(request, log) { `maximum allowed content-length is ${ContentLengthThreshold} bytes`, ); } - const headerChecksum = getChecksumDataFromHeaders(request.headers); + const headerChecksum = areChecksumsEnabled() ? getChecksumDataFromHeaders(request.headers) : null; if (headerChecksum && headerChecksum.error) { throw arsenalErrorFromChecksumError(headerChecksum); } @@ -89,12 +90,12 @@ async function receiveData(request, log) { // Checksum transforms only compute digests while streaming: validation // against the expected values (header or trailer) must be done once the // stream is fully consumed. - // `checksums.primary` is always set above, so primaryChecksumStream is the - // end of the pipeline here; validate it explicitly rather than relying on - // `prepared.stream` happening to be that transform. + // Validate the primary stream explicitly rather than relying on + // `prepared.stream` happening to be that transform. It is absent when + // checksums are disabled, in which case there is nothing to validate. const checksumErr = (prepared.contentSHA256Stream && prepared.contentSHA256Stream.validateChecksum()) || - prepared.primaryChecksumStream.validateChecksum(); + (prepared.primaryChecksumStream && prepared.primaryChecksumStream.validateChecksum()); if (checksumErr) { log.debug('failed checksum validation', { error: checksumErr }); throw arsenalErrorFromChecksumError(checksumErr); diff --git a/lib/server.js b/lib/server.js index a3ccdc29fa..bd45191371 100644 --- a/lib/server.js +++ b/lib/server.js @@ -419,6 +419,11 @@ class S3Server { } } + logger.info('integrityChecks config', { config: _config.integrityChecks }); + if (!_config.integrityChecks.enabled) { + logger.warn('x-amz-checksum-* digests are disabled: not computed, not validated, and not stored'); + } + try { logger.info('ServerAccessLogger config', { config: _config.serverAccessLogs }); if ( diff --git a/package.json b/package.json index 8b99acb926..939de713a8 100644 --- a/package.json +++ b/package.json @@ -121,6 +121,7 @@ "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support --exit", "ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/versioning/ --exit", "ft_awssdk_external_backends": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/multipleBackend --exit", + "ft_checksums_disabled": "cd tests/functional/checksumsDisabled && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 120000 *.js --exit", "ft_mixed_bucket_format_version": "cd tests/functional/metadata && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json MixedVersionFormat.js --exit", "ft_management": "cd tests/functional/report && yarn test", "ft_backbeat": "cd tests/functional/backbeat && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", diff --git a/tests/functional/checksumsDisabled/checksumsDisabled.js b/tests/functional/checksumsDisabled/checksumsDisabled.js new file mode 100644 index 0000000000..b70d92e18b --- /dev/null +++ b/tests/functional/checksumsDisabled/checksumsDisabled.js @@ -0,0 +1,365 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const { + S3Client, + CreateBucketCommand, + DeleteBucketCommand, + PutObjectCommand, + GetObjectCommand, + HeadObjectCommand, + CopyObjectCommand, + GetObjectAttributesCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + UploadPartCopyCommand, + CompleteMultipartUploadCommand, + AbortMultipartUploadCommand, + ListPartsCommand, + PutBucketTaggingCommand, + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); + +const getConfig = require('../aws-node-sdk/test/support/config'); +const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); + +/* + * These tests require CloudServer to be running with checksums DISABLED: + * + * S3_INTEGRITY_CHECKS_ENABLED=false + * + * or `integrityChecks: { enabled: false }` in config.json. They assert the + * opposite of the normal checksum suites, so they are deliberately kept out of + * tests/functional/aws-node-sdk/test/ — `yarn ft_awssdk` runs that tree against + * a server with checksums on, where every assertion here would fail. + * + * Run with: yarn ft_checksums_disabled + */ + +const bucket = `checksums-disabled-${Date.now()}`; +const body = Buffer.from('I am the body of an object', 'utf8'); +const bodyMd5 = crypto.createHash('md5').update(body).digest('base64'); +// A syntactically valid CRC32 that does not match `body`. +const wrongCrc32 = 'AAAAAA=='; +// 5MB, the minimum size for a non-final MPU part. +const partBody = Buffer.alloc(5 * 1024 * 1024, 'a'); + +// Every checksum field the SDK may surface on a response. +const CHECKSUM_FIELDS = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + 'ChecksumType', +]; + +function assertNoChecksum(res, context) { + CHECKSUM_FIELDS.forEach(field => { + assert.strictEqual(res[field], undefined, `${context}: expected no ${field}, got ${res[field]}`); + }); +} + +describe('with checksums disabled', () => { + let s3; + let bucketUtil; + + before(async () => { + bucketUtil = new BucketUtility('default', {}); + s3 = new S3Client({ ...getConfig('default', {}), maxAttempts: 0 }); + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + + // Fail fast and loudly rather than emitting a wall of confusing + // assertion errors if the server was started with checksums enabled. + const probe = await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'probe', Body: body })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'probe' })); + if (CHECKSUM_FIELDS.some(f => probe[f] !== undefined)) { + throw new Error( + 'This suite requires CloudServer running with checksums disabled ' + + '(S3_INTEGRITY_CHECKS_ENABLED=false); the server returned a checksum.', + ); + } + }); + + after(async () => { + await bucketUtil.empty(bucket); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + describe('PutObject', () => { + it('should not return a checksum when none is requested', async () => { + const res = await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'plain', Body: body })); + assertNoChecksum(res, 'PutObject'); + }); + + it('should accept a checksum that does not match the body', async () => { + // Enabled, this is BadDigest. + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'wrong-checksum', + Body: body, + ChecksumCRC32: wrongCrc32, + }), + ); + assertNoChecksum(res, 'PutObject with a wrong checksum'); + }); + + it('should not return a checksum on GET, HEAD or GetObjectAttributes', async () => { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'readback', Body: body })); + + const get = await s3.send( + new GetObjectCommand({ Bucket: bucket, Key: 'readback', ChecksumMode: 'ENABLED' }), + ); + assertNoChecksum(get, 'GetObject'); + await get.Body.transformToByteArray(); + + const head = await s3.send( + new HeadObjectCommand({ Bucket: bucket, Key: 'readback', ChecksumMode: 'ENABLED' }), + ); + assertNoChecksum(head, 'HeadObject'); + + const attrs = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: 'readback', + ObjectAttributes: ['Checksum', 'ETag'], + }), + ); + assert.strictEqual(attrs.Checksum, undefined, 'GetObjectAttributes should report no Checksum'); + assert(attrs.ETag, 'GetObjectAttributes should still report an ETag'); + }); + + it('should still enforce Content-MD5', async () => { + const wrongMd5 = crypto.createHash('md5').update('not the body').digest('base64'); + await assert.rejects( + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'bad-md5', + Body: body, + ContentMD5: wrongMd5, + }), + ), + err => err.name === 'BadDigest' || err.Code === 'BadDigest', + ); + }); + + it('should accept a correct Content-MD5', async () => { + const res = await s3.send( + new PutObjectCommand({ Bucket: bucket, Key: 'good-md5', Body: body, ContentMD5: bodyMd5 }), + ); + assertNoChecksum(res, 'PutObject with a valid Content-MD5'); + }); + + it('should store no checksum for a zero-byte object', async () => { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'empty', Body: Buffer.alloc(0) })); + const head = await s3.send( + new HeadObjectCommand({ Bucket: bucket, Key: 'empty', ChecksumMode: 'ENABLED' }), + ); + assertNoChecksum(head, 'HeadObject on a zero-byte object'); + }); + }); + + describe('CopyObject', () => { + it('should not carry a checksum to the destination', async () => { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'copy-src', Body: body })); + const res = await s3.send( + new CopyObjectCommand({ + Bucket: bucket, + Key: 'copy-dst', + CopySource: `/${bucket}/copy-src`, + }), + ); + assertNoChecksum(res.CopyObjectResult || {}, 'CopyObject'); + + const head = await s3.send( + new HeadObjectCommand({ Bucket: bucket, Key: 'copy-dst', ChecksumMode: 'ENABLED' }), + ); + assertNoChecksum(head, 'HeadObject on the copy'); + }); + + it('should ignore a requested checksum algorithm', async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: bucket, + Key: 'copy-dst-sha256', + CopySource: `/${bucket}/copy-src`, + ChecksumAlgorithm: 'SHA256', + }), + ); + assertNoChecksum(res.CopyObjectResult || {}, 'CopyObject with ChecksumAlgorithm'); + }); + }); + + describe('multipart upload', () => { + async function runMpu(key, createParams, partParams) { + const create = await s3.send( + new CreateMultipartUploadCommand({ Bucket: bucket, Key: key, ...createParams }), + ); + const uploadId = create.UploadId; + try { + const part = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 1, + Body: partBody, + ...partParams, + }), + ); + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { Parts: [{ ETag: part.ETag, PartNumber: 1 }] }, + }), + ); + return { create, part, complete, uploadId }; + } catch (err) { + await s3 + .send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId })) + .catch(() => {}); + throw err; + } + } + + it('should not echo a checksum algorithm from CreateMultipartUpload', async () => { + const { create, part, complete } = await runMpu('mpu-explicit', { + ChecksumAlgorithm: 'CRC32', + }); + assert.strictEqual(create.ChecksumAlgorithm, undefined); + assert.strictEqual(create.ChecksumType, undefined); + assertNoChecksum(part, 'UploadPart'); + assertNoChecksum(complete, 'CompleteMultipartUpload'); + }); + + it('should complete an MPU created with an explicit algorithm and no per-part checksums', async () => { + // Enabled, a CRC32 MPU is COMPOSITE and UploadPart would reject a + // part carrying no x-amz-checksum-crc32. + const { complete } = await runMpu('mpu-no-part-checksums', { ChecksumAlgorithm: 'CRC32' }); + assert(complete.ETag, 'CompleteMultipartUpload should succeed'); + assertNoChecksum(complete, 'CompleteMultipartUpload'); + }); + + it('should accept a per-part checksum that does not match the part', async () => { + const { complete } = await runMpu( + 'mpu-wrong-part-checksum', + { ChecksumAlgorithm: 'CRC32' }, + { ChecksumCRC32: wrongCrc32 }, + ); + assert(complete.ETag); + }); + + it('should report no checksum in ListParts', async () => { + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-listparts', + ChecksumAlgorithm: 'CRC32', + }), + ); + const part = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: 'mpu-listparts', + UploadId: create.UploadId, + PartNumber: 1, + Body: partBody, + }), + ); + assert(part.ETag); + + const list = await s3.send( + new ListPartsCommand({ Bucket: bucket, Key: 'mpu-listparts', UploadId: create.UploadId }), + ); + assert.strictEqual(list.ChecksumAlgorithm, undefined); + (list.Parts || []).forEach(p => assertNoChecksum(p, 'ListParts part')); + + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-listparts', + UploadId: create.UploadId, + }), + ); + }); + + it('should not store a checksum on UploadPartCopy', async () => { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'copypart-src', Body: partBody })); + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-copypart', + ChecksumAlgorithm: 'CRC32', + }), + ); + const copied = await s3.send( + new UploadPartCopyCommand({ + Bucket: bucket, + Key: 'mpu-copypart', + UploadId: create.UploadId, + PartNumber: 1, + CopySource: `/${bucket}/copypart-src`, + }), + ); + assertNoChecksum(copied.CopyPartResult || {}, 'UploadPartCopy'); + + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-copypart', + UploadId: create.UploadId, + MultipartUpload: { + Parts: [{ ETag: copied.CopyPartResult.ETag, PartNumber: 1 }], + }, + }), + ); + assertNoChecksum(complete, 'CompleteMultipartUpload after UploadPartCopy'); + }); + + it('should not store a checksum on a ranged UploadPartCopy', async () => { + // A copy-source range always forces a recompute when enabled. + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-copypart-range', + ChecksumAlgorithm: 'CRC32', + }), + ); + const copied = await s3.send( + new UploadPartCopyCommand({ + Bucket: bucket, + Key: 'mpu-copypart-range', + UploadId: create.UploadId, + PartNumber: 1, + CopySource: `/${bucket}/copypart-src`, + CopySourceRange: `bytes=0-${partBody.length - 1}`, + }), + ); + assertNoChecksum(copied.CopyPartResult || {}, 'ranged UploadPartCopy'); + + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-copypart-range', + UploadId: create.UploadId, + }), + ); + }); + }); + + describe('buffered-body endpoints', () => { + it('should accept a wrong x-amz-checksum on PutBucketTagging', async () => { + // Enabled, this is BadDigest. + await s3.send( + new PutBucketTaggingCommand({ + Bucket: bucket, + Tagging: { TagSet: [{ Key: 'k', Value: 'v' }] }, + ChecksumCRC32: wrongCrc32, + }), + ); + }); + }); +}); diff --git a/tests/unit/Config.js b/tests/unit/Config.js index 4ecdbe4bbb..a7d801e0a9 100644 --- a/tests/unit/Config.js +++ b/tests/unit/Config.js @@ -7,6 +7,7 @@ const { azureGetLocationCredentials, locationConstraintAssert, parseSupportedLifecycleRules, + parseIntegrityChecks, ConfigObject, } = require('../../lib/Config'); @@ -908,6 +909,49 @@ describe('Config', () => { }); }); + describe('parse integrity checks', () => { + afterEach(() => { + delete process.env.S3_INTEGRITY_CHECKS_ENABLED; + }); + + it('should default to enabled when not configured', () => { + assert.deepStrictEqual(parseIntegrityChecks(null), { enabled: true }); + assert.deepStrictEqual(parseIntegrityChecks({}), { enabled: true }); + }); + + it('should read the configured value', () => { + assert.deepStrictEqual(parseIntegrityChecks({ integrityChecks: { enabled: false } }), { enabled: false }); + assert.deepStrictEqual(parseIntegrityChecks({ integrityChecks: { enabled: true } }), { enabled: true }); + }); + + it('should throw if integrityChecks is not an object', () => { + assert.throws(() => parseIntegrityChecks({ integrityChecks: 'yes' }), /must be an object/); + assert.throws(() => parseIntegrityChecks({ integrityChecks: [true] }), /must be an object/); + }); + + it('should throw if enabled is not a boolean', () => { + assert.throws(() => parseIntegrityChecks({ integrityChecks: { enabled: 'false' } }), /must be a boolean/); + assert.throws(() => parseIntegrityChecks({ integrityChecks: { enabled: 0 } }), /must be a boolean/); + }); + + it('should let S3_INTEGRITY_CHECKS_ENABLED override the config file', () => { + process.env.S3_INTEGRITY_CHECKS_ENABLED = 'false'; + assert.deepStrictEqual(parseIntegrityChecks({ integrityChecks: { enabled: true } }), { enabled: false }); + process.env.S3_INTEGRITY_CHECKS_ENABLED = 'true'; + assert.deepStrictEqual(parseIntegrityChecks({ integrityChecks: { enabled: false } }), { enabled: true }); + }); + + it('should throw on a non-boolean S3_INTEGRITY_CHECKS_ENABLED', () => { + process.env.S3_INTEGRITY_CHECKS_ENABLED = 'nope'; + assert.throws(() => parseIntegrityChecks(null), /S3_INTEGRITY_CHECKS_ENABLED/); + }); + + it('should expose integrityChecks on the config object', () => { + const config = new ConfigObject(); + assert.deepStrictEqual(config.integrityChecks, { enabled: true }); + }); + }); + describe('serverHeader', () => { let sandbox; let readFileStub; diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index f456739152..b2fec1353e 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -14,8 +14,11 @@ const { arsenalErrorFromChecksumError, getChecksumDataFromMPUHeaders, validateCompleteMultipartUploadChecksum, + validateCompleteMPUChecksumType, getCopyObjectChecksumAlgorithm, + areChecksumsEnabled, } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { config } = require('../../../../../lib/Config'); const { errors: ArsenalErrors } = require('arsenal'); describe('validateChecksumsNoChunking MD5', () => { @@ -1465,3 +1468,207 @@ describe('validateMethodChecksumNoChunking x-amz-content-sha256', () => { assert.ifError(result); }); }); + +describe('validateCompleteMPUChecksumType', () => { + describe('when the header is absent', () => { + it('should return null whatever the MPU checksum type is', () => { + assert.strictEqual(validateCompleteMPUChecksumType({}, 'COMPOSITE'), null); + assert.strictEqual(validateCompleteMPUChecksumType({}, 'FULL_OBJECT'), null); + assert.strictEqual(validateCompleteMPUChecksumType({}, undefined), null); + }); + + it('should return null for an empty header value', () => { + assert.strictEqual(validateCompleteMPUChecksumType({ 'x-amz-checksum-type': '' }, 'COMPOSITE'), null); + }); + }); + + describe('when the header matches the MPU checksum type', () => { + ['COMPOSITE', 'FULL_OBJECT'].forEach(type => { + it(`should return null for ${type}`, () => { + assert.strictEqual(validateCompleteMPUChecksumType({ 'x-amz-checksum-type': type }, type), null); + }); + }); + + it('should compare case-insensitively on both sides', () => { + const lowerHeader = { 'x-amz-checksum-type': 'composite' }; + assert.strictEqual(validateCompleteMPUChecksumType(lowerHeader, 'COMPOSITE'), null); + const upperHeader = { 'x-amz-checksum-type': 'FULL_OBJECT' }; + assert.strictEqual(validateCompleteMPUChecksumType(upperHeader, 'full_object'), null); + }); + }); + + describe('when the header value is not a valid checksum type', () => { + it('should return MPUTypeInvalid', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'BOGUS' }, 'COMPOSITE'); + assert.strictEqual(result.error, ChecksumError.MPUTypeInvalid); + assert.strictEqual(result.details.type, 'BOGUS'); + }); + + it('should take precedence over an unset MPU checksum type', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'BOGUS' }, undefined); + assert.strictEqual(result.error, ChecksumError.MPUTypeInvalid); + }); + + it('should map to InvalidRequest (400)', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'BOGUS' }, 'COMPOSITE'); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidRequest'); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.description, 'Value for x-amz-checksum-type header is invalid.'); + }); + }); + + describe('when the MPU was created without a checksum type', () => { + it('should return MPUTypeNotConfigured', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'COMPOSITE' }, undefined); + assert.strictEqual(result.error, ChecksumError.MPUTypeNotConfigured); + }); + + it('should map to InvalidRequest (400) describing the legacy upload', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'COMPOSITE' }, undefined); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidRequest'); + assert.strictEqual(err.code, 400); + assert.strictEqual( + err.description, + 'The upload was not created with a checksum mode. ' + + 'The complete request must not include a x-amz-checksum-type header.', + ); + }); + + it('should ignore the header for an external-backend MPU', () => { + // External backends record no checksum config (CLDSRV-964), so an + // absent type means "not tracked here", not a legacy upload. + const headers = { 'x-amz-checksum-type': 'COMPOSITE' }; + assert.strictEqual(validateCompleteMPUChecksumType(headers, undefined, true), null); + }); + + it('should still reject a mismatch on an external-backend MPU that has a type', () => { + const headers = { 'x-amz-checksum-type': 'COMPOSITE' }; + const result = validateCompleteMPUChecksumType(headers, 'FULL_OBJECT', true); + assert.strictEqual(result.error, ChecksumError.MPUTypeModeMismatch); + }); + }); + + describe('when the header does not match the MPU checksum type', () => { + it('should return MPUTypeModeMismatch carrying the MPU type', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'COMPOSITE' }, 'FULL_OBJECT'); + assert.strictEqual(result.error, ChecksumError.MPUTypeModeMismatch); + assert.strictEqual(result.details.type, 'FULL_OBJECT'); + }); + + it('should map to InvalidRequest (400) naming the mode the MPU was created with', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'COMPOSITE' }, 'FULL_OBJECT'); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidRequest'); + assert.strictEqual(err.code, 400); + assert.strictEqual( + err.description, + 'The upload was created using the FULL_OBJECT checksum mode. ' + + 'The complete request must use the same checksum mode.', + ); + }); + }); +}); + +describe('areChecksumsEnabled', () => { + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + }); + + it('should be enabled with the shipped config', () => { + assert.strictEqual(areChecksumsEnabled(), true); + }); + + it('should be disabled only when enabled is exactly false', () => { + config.integrityChecks = { enabled: false }; + assert.strictEqual(areChecksumsEnabled(), false); + }); + + it('should be enabled when enabled is true', () => { + config.integrityChecks = { enabled: true }; + assert.strictEqual(areChecksumsEnabled(), true); + }); + + it('should default to enabled when the section or the key is missing', () => { + config.integrityChecks = undefined; + assert.strictEqual(areChecksumsEnabled(), true); + config.integrityChecks = {}; + assert.strictEqual(areChecksumsEnabled(), true); + }); + + it('should not treat a falsy non-false value as disabled', () => { + // Guards the `!== false` comparison: only an explicit boolean false + // turns checksums off, so a mis-typed config cannot silently disable them. + [0, '', null, 'false'].forEach(value => { + config.integrityChecks = { enabled: value }; + assert.strictEqual(areChecksumsEnabled(), true, `enabled: ${JSON.stringify(value)}`); + }); + }); +}); + +describe('validateMethodChecksumNoChunking with checksums disabled', () => { + const body = 'Hello, World!'; + const sigV4Header = 'AWS4-HMAC-SHA256 Credential=x'; + const correctMd5 = crypto.createHash('md5').update(body).digest('base64'); + const wrongMd5 = crypto.createHash('md5').update('other').digest('base64'); + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + config.integrityChecks = { enabled: false }; + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + }); + + const run = headers => + validateMethodChecksumNoChunking({ apiMethod: 'bucketPutCors', headers }, body, new DummyRequestLogger()); + + it('should ignore a wrong x-amz-checksum- for every checksumed method', async () => { + for (const method of Object.keys(checksumedMethods)) { + const request = { apiMethod: method, headers: { 'x-amz-checksum-crc32': 'AAAAAA==' } }; + const result = await validateMethodChecksumNoChunking(request, body, new DummyRequestLogger()); + assert.ifError(result, `${method} should not reject`); + } + }); + + it('should ignore x-amz-checksum-* headers that would otherwise be rejected', async () => { + // Malformed, unsupported, multiple, and a mismatched sdk-algorithm all + // become no-ops: the header is not looked at once checksums are off. + assert.ifError(await run({ 'x-amz-checksum-crc32': 'not-base64!!' })); + assert.ifError(await run({ 'x-amz-checksum-md5': 'AAAAAA==' })); + assert.ifError(await run({ 'x-amz-checksum-crc32': 'AAAAAA==', 'x-amz-checksum-crc32c': 'AAAAAA==' })); + assert.ifError(await run({ 'x-amz-checksum-crc32': 'AAAAAA==', 'x-amz-sdk-checksum-algorithm': 'SHA256' })); + }); + + it('should still enforce Content-MD5', async () => { + const result = await run({ 'content-md5': wrongMd5 }); + assert.strictEqual(result.message, 'BadDigest'); + assert.ifError(await run({ 'content-md5': correctMd5 })); + }); + + it('should still enforce Content-MD5 alongside an ignored x-amz-checksum', async () => { + const result = await run({ 'content-md5': wrongMd5, 'x-amz-checksum-crc32': 'AAAAAA==' }); + assert.strictEqual(result.message, 'BadDigest'); + }); + + it('should still enforce x-amz-content-sha256', async () => { + const wrongHex = crypto.createHash('sha256').update('other').digest('hex'); + const result = await run({ authorization: sigV4Header, 'x-amz-content-sha256': wrongHex }); + assert.strictEqual(result.message, 'XAmzContentSHA256Mismatch'); + }); + + it('should reject a wrong x-amz-checksum- again once re-enabled', async () => { + config.integrityChecks = { enabled: true }; + const result = await run({ 'x-amz-checksum-crc32': 'AAAAAA==' }); + assert.strictEqual(result.message, 'BadDigest'); + }); +}); diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js index 5a582f03ef..efabb3d0b2 100644 --- a/tests/unit/api/apiUtils/object/prepareStream.js +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -10,6 +10,7 @@ const TrailingChecksumTransform = require('../../../../../lib/auth/streamingV4/t const { DummyRequestLogger } = require('../../../helpers'); const DummyRequest = require('../../../DummyRequest'); const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { config } = require('../../../../../lib/Config'); const log = new DummyRequestLogger(); const defaultChecksums = { primary: defaultChecksumData, secondary: null }; @@ -419,3 +420,67 @@ describe('prepareStream', () => { }); }); }); + +describe('prepareStream with checksums disabled', () => { + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + config.integrityChecks = { enabled: false }; + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + }); + + // Disabling must land in exactly the state a caller requesting no checksum + // already produces, which the 'no checksum requested' suite above pins down. + it('should build no ChecksumTransform even when checksums are requested', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const checksums = { primary: defaultChecksumData, secondary: { algorithm: 'crc32', isTrailer: false } }; + const result = prepareStream(request, null, checksums, log, () => {}); + assert.strictEqual(result.error, null); + assert.strictEqual(result.stream, request, 'the request should pass through untouched'); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + }); + + it('should drop the checksum transforms on the chunked upload path', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, defaultChecksums, log, () => {}); + assert(result.stream instanceof V4Transform, 'v4 chunk decoding must still happen'); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + }); + + it('should drop the checksum transforms on the trailer path', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert(result.stream instanceof TrailingChecksumTransform, 'trailer framing must still be parsed'); + assert.strictEqual(result.primaryChecksumStream, null); + }); + + it('should still validate a literal x-amz-content-sha256 payload hash', done => { + // x-amz-content-sha256 is SigV4 load-bearing and out of scope for the flag. + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }, bodyData); + const result = prepareStream(request, null, defaultChecksums, log, done); + assert.strictEqual(result.primaryChecksumStream, null); + assert(result.contentSHA256Stream instanceof ContentSHA256Transform); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.contentSHA256Stream.validateChecksum(), null); + done(); + }); + result.stream.on('error', done); + }); + + it('should build the transforms again once re-enabled', () => { + config.integrityChecks = { enabled: true }; + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert(result.primaryChecksumStream instanceof ChecksumTransform); + }); +}); diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index 1128cd4bb7..4ea1e69c26 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -4792,3 +4792,293 @@ describe('CompleteMultipartUpload final checksum on azure-style external backend await _assertNoChecksumInResult(xml); }); }); + +describe('CompleteMultipartUpload with checksums disabled', () => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // Starts a CRC64NVME (FULL_OBJECT) MPU and uploads one part, returning what CompleteMPU needs. + // Runs with checksums enabled so the part carries a stored digest — this + // commit only gates CompleteMPU, UploadPart still computes as usual. + async function initiateAndUploadPart() { + config.integrityChecks = { enabled: true }; + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc64nvme' }; + const initRes = await util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + const uploadId = (await parseStringPromise(initRes)).InitiateMultipartUploadResult.UploadId[0]; + + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + + let storedChecksum; + for (const [key, value] of metadata.keyMaps.get(mpuBucket)) { + if (key.startsWith(uploadId) && !key.startsWith('overview')) { + storedChecksum = value.checksumValue; + } + } + assert(storedChecksum, 'part should have a stored checksum to validate against'); + + return { uploadId, eTag, storedChecksum }; + } + + function completeRequestWithPartChecksum(uploadId, eTag, checksumValue) { + const post = [ + '', + '', + '1', + `"${eTag}"`, + `${checksumValue}`, + '', + '', + ]; + return { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${uploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId }, + post, + actionImplicitDenies: false, + }; + } + + const complete = request => + util.promisify(cb => completeMultipartUpload(authInfo, request, log, (err, xml) => cb(err, xml)))(); + + it('should reject a mismatched per-part checksum while enabled', async () => { + const { uploadId, eTag } = await initiateAndUploadPart(); + const request = completeRequestWithPartChecksum(uploadId, eTag, 'AQIDBAUGBwg='); + await assert.rejects(complete(request), { message: 'InvalidPart' }); + }); + + it('should accept a mismatched per-part checksum once disabled', async () => { + const { uploadId, eTag } = await initiateAndUploadPart(); + config.integrityChecks = { enabled: false }; + const request = completeRequestWithPartChecksum(uploadId, eTag, 'AQIDBAUGBwg='); + const xml = await complete(request); + assert.match(xml, / { + const { uploadId, eTag, storedChecksum } = await initiateAndUploadPart(); + + const enabledXml = await complete(completeRequestWithPartChecksum(uploadId, eTag, storedChecksum)); + assert.match(enabledXml, //, 'enabled should return a final checksum'); + + const second = await initiateAndUploadPart(); + config.integrityChecks = { enabled: false }; + const disabledXml = await complete( + completeRequestWithPartChecksum(second.uploadId, second.eTag, second.storedChecksum), + ); + assert.doesNotMatch(disabledXml, //, 'disabled should omit the final checksum'); + }); +}); + +describe('initiateMultipartUpload with checksums disabled', () => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + config.integrityChecks = { enabled: false }; + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + const initiate = headers => util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + + async function uploadIdFrom(xml) { + return (await parseStringPromise(xml)).InitiateMultipartUploadResult.UploadId[0]; + } + + function storedOverview(uploadId) { + for (const [key, value] of metadata.keyMaps.get(mpuBucket)) { + if (key.startsWith('overview') && value.uploadId === uploadId) { + return value; + } + } + return null; + } + + it('should record no checksum config on the MPU', async () => { + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc32' }; + const overview = storedOverview(await uploadIdFrom(await initiate(headers))); + assert.strictEqual(overview.checksumAlgorithm, undefined); + assert.strictEqual(overview.checksumType, undefined); + assert.strictEqual(overview.checksumIsDefault, undefined); + }); + + it('should not reject an unsupported x-amz-checksum-algorithm', async () => { + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'NOT-AN-ALGO' }; + await assert.doesNotReject(initiate(headers)); + }); + + it('should not reject x-amz-checksum-type without an algorithm', async () => { + const headers = { ...initiateRequest.headers, 'x-amz-checksum-type': 'FULL_OBJECT' }; + await assert.doesNotReject(initiate(headers)); + }); + + it('should complete an MPU whose parts carry no checksum', async () => { + // The end-to-end case CreateMPU previously broke: an explicit algorithm + // was recorded, so CompleteMPU demanded per-part digests that never existed. + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc32' }; + const uploadId = await uploadIdFrom(await initiate(headers)); + + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + + const completeRequest = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + const xml = await util.promisify(cb => + completeMultipartUpload(authInfo, completeRequest, log, (err, res) => cb(err, res)), + )(); + assert.match(xml, / { + const uploadId = await uploadIdFrom(await initiate(initiateRequest.headers)); + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + + const completeRequest = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + completeRequest.headers = { ...completeRequest.headers, 'x-amz-checksum-type': 'FULL_OBJECT' }; + await assert.doesNotReject( + util.promisify(cb => completeMultipartUpload(authInfo, completeRequest, log, (err, res) => cb(err, res)))(), + ); + }); + + it('should record the checksum config again once re-enabled', async () => { + config.integrityChecks = { enabled: true }; + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc32' }; + const overview = storedOverview(await uploadIdFrom(await initiate(headers))); + assert.strictEqual(overview.checksumAlgorithm, 'crc32'); + assert.strictEqual(overview.checksumType, 'COMPOSITE'); + }); +}); + +describe('CompleteMPU x-amz-checksum-type validation vs the checksum flag', () => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // Creates an MPU under `createEnabled`, uploads one part, and returns a + // CompleteMPU request carrying `headerType` as x-amz-checksum-type. + async function buildComplete(createEnabled, createHeaders, headerType) { + config.integrityChecks = { enabled: createEnabled }; + const headers = { ...initiateRequest.headers, ...createHeaders }; + const initRes = await util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + const uploadId = (await parseStringPromise(initRes)).InitiateMultipartUploadResult.UploadId[0]; + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + const request = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + request.headers = { ...request.headers, 'x-amz-checksum-type': headerType }; + return request; + } + + const complete = request => + util.promisify(cb => completeMultipartUpload(authInfo, request, log, (err, xml) => cb(err, xml)))(); + + describe('when disabled', () => { + it('should ignore a type header on an MPU that recorded none', async () => { + const request = await buildComplete(false, { 'x-amz-checksum-algorithm': 'crc32' }, 'COMPOSITE'); + config.integrityChecks = { enabled: false }; + await assert.doesNotReject(complete(request)); + }); + + it('should ignore an invalid type value', async () => { + const request = await buildComplete(false, {}, 'NOT-A-TYPE'); + config.integrityChecks = { enabled: false }; + await assert.doesNotReject(complete(request)); + }); + + it('should ignore a type that mismatches one recorded while enabled', async () => { + // MPU created with checksums on, completed after the flag was flipped. + const request = await buildComplete(true, { 'x-amz-checksum-algorithm': 'crc64nvme' }, 'COMPOSITE'); + config.integrityChecks = { enabled: false }; + await assert.doesNotReject(complete(request)); + }); + }); + + describe('when enabled', () => { + it('should still reject a type header on an MPU that recorded none', async () => { + const request = await buildComplete(false, {}, 'FULL_OBJECT'); + config.integrityChecks = { enabled: true }; + await assert.rejects(complete(request), { message: 'InvalidRequest' }); + }); + + it('should still reject an invalid type value', async () => { + const request = await buildComplete(true, { 'x-amz-checksum-algorithm': 'crc64nvme' }, 'NOT-A-TYPE'); + config.integrityChecks = { enabled: true }; + await assert.rejects(complete(request), { message: 'InvalidRequest' }); + }); + + it('should still reject a type that mismatches the recorded one', async () => { + const request = await buildComplete(true, { 'x-amz-checksum-algorithm': 'crc64nvme' }, 'COMPOSITE'); + config.integrityChecks = { enabled: true }; + await assert.rejects(complete(request), { message: 'InvalidRequest' }); + }); + }); +}); + +describe('MPU created with checksums, then disabled mid-upload', () => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // crc32 defaults to COMPOSITE, the combination that previously wedged: + // UploadPart demanded a per-part checksum that CompleteMPU would never use. + it('should complete an explicit COMPOSITE MPU flipped to disabled', async () => { + config.integrityChecks = { enabled: true }; + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc32' }; + const initRes = await util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + const uploadId = (await parseStringPromise(initRes)).InitiateMultipartUploadResult.UploadId[0]; + + config.integrityChecks = { enabled: false }; + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + + const completeRequest = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + const xml = await util.promisify(cb => + completeMultipartUpload(authInfo, completeRequest, log, (err, res) => cb(err, res)), + )(); + assert.match(xml, / { }); }); }); + +describe('objectCopy with checksums disabled', () => { + const sourceChecksum = { + checksumAlgorithm: 'crc32', + checksumValue: 'AAAAAA==', + checksumType: 'FULL_OBJECT', + }; + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + async.series( + [ + next => bucketPut(authInfo, putDestBucketRequest, log, next), + next => bucketPut(authInfo, putSourceBucketRequest, log, next), + next => + objectPut( + authInfo, + versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[0]), + undefined, + log, + next, + ), + ], + done, + ); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + sinon.restore(); + cleanup(); + }); + + // Runs a copy with checksums disabled and hands the destination metadata back. + function copyDisabled(headers, cb) { + config.integrityChecks = { enabled: false }; + const req = _createObjectCopyRequest(destBucketName, headers); + return objectCopy(authInfo, req, sourceBucketName, objectKey, undefined, log, (err, xml) => { + assert.ifError(err); + return metadata.getObjectMD(destBucketName, objectKey, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + cb(xml, md); + }); + }); + } + + it('should not store a checksum when the source has none', done => { + setSourceChecksum(null, err => { + assert.ifError(err); + copyDisabled(undefined, (xml, md) => { + assert.strictEqual(md.checksum, undefined); + assert.doesNotMatch(xml, / { + setSourceChecksum(sourceChecksum, err => { + assert.ifError(err); + copyDisabled(undefined, (xml, md) => { + assert.strictEqual(md.checksum, undefined, 'destination must not inherit the source checksum'); + done(); + }); + }); + }); + + it('should ignore x-amz-checksum-algorithm instead of recomputing', done => { + setSourceChecksum(sourceChecksum, err => { + assert.ifError(err); + // sha256 differs from the source's crc32, which would normally force a + // recompute — the expensive path this flag exists to avoid. + copyDisabled({ 'x-amz-checksum-algorithm': 'SHA256' }, (xml, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + }); + + it('should not reject an invalid x-amz-checksum-algorithm', done => { + setSourceChecksum(null, err => { + assert.ifError(err); + copyDisabled({ 'x-amz-checksum-algorithm': 'NOT-AN-ALGO' }, (xml, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + }); + + it('should still reject an invalid x-amz-checksum-algorithm when enabled', done => { + config.integrityChecks = { enabled: true }; + const req = _createObjectCopyRequest(destBucketName, { 'x-amz-checksum-algorithm': 'NOT-AN-ALGO' }); + objectCopy(authInfo, req, sourceBucketName, objectKey, undefined, log, err => { + assert(err, 'should reject'); + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + + it('should recompute again once re-enabled', done => { + config.integrityChecks = { enabled: true }; + setSourceChecksum(null, err => { + assert.ifError(err); + const req = _createObjectCopyRequest(destBucketName); + objectCopy(authInfo, req, sourceBucketName, objectKey, undefined, log, (err, xml) => { + assert.ifError(err); + assert.match(xml, //, 'enabled should recompute the default checksum'); + done(); + }); + }); + }); +}); diff --git a/tests/unit/api/objectCopyPart.js b/tests/unit/api/objectCopyPart.js index cbcf713d40..80dd8b3169 100644 --- a/tests/unit/api/objectCopyPart.js +++ b/tests/unit/api/objectCopyPart.js @@ -457,3 +457,110 @@ describe('objectPutCopyPart._copyPartStreamingWithChecksum', () => { }); }); }); + +describe('objectPutCopyPart with checksums disabled', () => { + const { _shouldRecomputeChecksum } = objectPutCopyPart; + const objData = Buffer.from('foo', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + sinon.spy(metadataswitch, 'putObjectMD'); + async.waterfall( + [ + cb => bucketPut(authInfo, putDestBucketRequest, log, e => cb(e)), + cb => bucketPut(authInfo, putSourceBucketRequest, log, e => cb(e)), + cb => + objectPut( + authInfo, + versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData), + undefined, + log, + e => cb(e), + ), + ], + done, + ); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + sinon.restore(); + cleanup(); + }); + + describe('_shouldRecomputeChecksum', () => { + it('should never recompute, whatever the source or range', () => { + config.integrityChecks = { enabled: false }; + const withRange = { headers: { 'x-amz-copy-source-range': 'bytes=0-1' } }; + const noRange = { headers: {} }; + const composite = { checksumType: 'COMPOSITE', checksumAlgorithm: 'crc32' }; + // Each of these returns true when enabled. + assert.strictEqual(_shouldRecomputeChecksum(withRange, composite, 'crc32'), false); + assert.strictEqual(_shouldRecomputeChecksum(noRange, undefined, 'crc32'), false); + assert.strictEqual(_shouldRecomputeChecksum(noRange, composite, 'crc32'), false); + }); + + it('should recompute again once re-enabled', () => { + config.integrityChecks = { enabled: true }; + assert.strictEqual(_shouldRecomputeChecksum({ headers: {} }, undefined, 'crc32'), true); + }); + }); + + describe('part metadata', () => { + function copyPartDisabled({ sourceChecksum, headers } = {}) { + return new Promise((resolve, reject) => { + const initReq = _createInitiateRequest(destBucketName, { + 'x-amz-checksum-algorithm': 'CRC32', + }); + // Create the MPU with checksums on so an algorithm is recorded, + // then disable: the flag must be honoured at copy time. + config.integrityChecks = { enabled: true }; + return initiateMultipartUpload(authInfo, initReq, log, (err, res) => { + if (err) { + return reject(err); + } + return parseString(res, (parseErr, json) => { + if (parseErr) { + return reject(parseErr); + } + const uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + if (sourceChecksum) { + metadata.keyMaps.get(sourceBucketName).get(objectKey).checksum = sourceChecksum; + } else { + delete metadata.keyMaps.get(sourceBucketName).get(objectKey).checksum; + } + config.integrityChecks = { enabled: false }; + const req = _createObjectCopyPartRequest(destBucketName, uploadId, headers); + return objectPutCopyPart(authInfo, req, sourceBucketName, objectKey, undefined, log, copyErr => + copyErr ? reject(copyErr) : resolve(metadataswitch.putObjectMD.lastCall.args[2]), + ); + }); + }); + }); + } + + it('should store no checksum when the source has one to reuse', async () => { + const omVal = await copyPartDisabled({ + sourceChecksum: { checksumType: 'FULL_OBJECT', checksumAlgorithm: 'crc32', checksumValue: 'AAAAAA==' }, + }); + assert.strictEqual(omVal.checksumAlgorithm, undefined); + assert.strictEqual(omVal.checksumValue, undefined); + }); + + it('should store no checksum when the source has none', async () => { + // Would previously recompute; must not dereference the absent source checksum. + const omVal = await copyPartDisabled(); + assert.strictEqual(omVal.checksumAlgorithm, undefined); + assert.strictEqual(omVal.checksumValue, undefined); + }); + + it('should store no checksum for a ranged copy', async () => { + // A range always forces a recompute when enabled. + const omVal = await copyPartDisabled({ headers: { 'x-amz-copy-source-range': 'bytes=0-1' } }); + assert.strictEqual(omVal.checksumAlgorithm, undefined); + assert.strictEqual(omVal.checksumValue, undefined); + }); + }); +}); diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js index 3d3f29bc8f..29b81d965c 100644 --- a/tests/unit/api/objectPut.js +++ b/tests/unit/api/objectPut.js @@ -1411,3 +1411,186 @@ describe('objectPut with objectKeyByteLimit', () => { }); }); }); + +describe('objectPut with checksums disabled', () => { + const sha256Value = crypto.createHash('sha256').update(postBody).digest('base64'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, testPutBucketRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + const putRequest = headers => + new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com`, ...headers }, + url: '/', + }, + postBody, + ); + + function putDisabled(headers, cb) { + config.integrityChecks = { enabled: false }; + return objectPut(authInfo, putRequest(headers), undefined, log, (err, resHeaders) => { + assert.ifError(err); + return metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + cb(resHeaders, md); + }); + }); + } + + it('should store no checksum metadata when none is requested', done => { + putDisabled(undefined, (resHeaders, md) => { + // Enabled, this stores the implicit crc64nvme default. + assert.strictEqual(md.checksum, undefined); + assert.strictEqual(resHeaders['x-amz-checksum-crc64nvme'], undefined); + done(); + }); + }); + + it('should ignore a client-supplied checksum rather than storing it', done => { + putDisabled({ 'x-amz-checksum-sha256': sha256Value }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + assert.strictEqual(resHeaders['x-amz-checksum-sha256'], undefined); + done(); + }); + }); + + it('should accept a client-supplied checksum that does not match the body', done => { + const wrong = crypto.createHash('sha256').update('not the body').digest('base64'); + putDisabled({ 'x-amz-checksum-sha256': wrong }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + + it('should accept a malformed checksum value', done => { + // Enabled, this is InvalidRequest. + putDisabled({ 'x-amz-checksum-crc32': 'not-base64!' }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + + it('should accept an unsupported checksum algorithm', done => { + // Enabled, this is InvalidRequest. + putDisabled({ 'x-amz-checksum-md5': 'AAAAAA==' }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + + it('should accept multiple checksum headers', done => { + // Enabled, this is InvalidRequest. + putDisabled({ 'x-amz-checksum-sha256': sha256Value, 'x-amz-checksum-crc32': 'AAAAAA==' }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + + it('should still reject a mismatched Content-MD5', done => { + config.integrityChecks = { enabled: false }; + // checkHashMatchMD5 reads request.contentMD5, not the header. + const request = putRequest(); + request.contentMD5 = crypto.createHash('md5').update('not the body').digest('base64'); + objectPut(authInfo, request, undefined, log, err => { + assert(err, 'should reject'); + assert.strictEqual(err.message, 'BadDigest'); + done(); + }); + }); + + it('should store no checksum for a zero-byte object', done => { + config.integrityChecks = { enabled: false }; + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, + Buffer.alloc(0), + ); + objectPut(authInfo, request, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + }); + + it('should ignore a wrong checksum on a zero-byte object', done => { + config.integrityChecks = { enabled: false }; + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'host': `${bucketName}.s3.amazonaws.com`, + 'x-amz-checksum-crc32': 'AAAAAA==', + }, + url: '/', + }, + Buffer.alloc(0), + ); + objectPut(authInfo, request, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + }); + + it('should store the zero-byte checksum again once re-enabled', done => { + config.integrityChecks = { enabled: true }; + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, + Buffer.alloc(0), + ); + objectPut(authInfo, request, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + assert(md.checksum, 'enabled should store the empty-body checksum'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + done(); + }); + }); + }); + + it('should store the checksum again once re-enabled', done => { + config.integrityChecks = { enabled: true }; + objectPut(authInfo, putRequest(), undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + assert(md.checksum, 'enabled should store the default checksum'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + done(); + }); + }); + }); +}); diff --git a/tests/unit/api/objectPutPartChecksum.js b/tests/unit/api/objectPutPartChecksum.js index f933e56dc1..a020529965 100644 --- a/tests/unit/api/objectPutPartChecksum.js +++ b/tests/unit/api/objectPutPartChecksum.js @@ -11,6 +11,7 @@ const constants = require('../../../constants'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const DummyRequest = require('../DummyRequest'); const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const { config } = require('../../../lib/Config'); const { metadata } = storage.metadata.inMemory.metadata; @@ -392,3 +393,102 @@ describe('objectPutPart checksum validation', () => { }); }); }); + +describe('objectPutPart with checksums disabled', () => { + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // Creates the MPU with checksums enabled so an algorithm is recorded, then + // disables before uploading: the mid-flight case where the flag is flipped + // between CreateMPU and UploadPart. + function uploadPartAfterDisabling(initiateHeaders, partHeaders, cb) { + config.integrityChecks = { enabled: true }; + return initiateMPU(initiateHeaders, (err, uploadId) => { + assert.ifError(err); + config.integrityChecks = { enabled: false }; + const request = makePutPartRequest(uploadId, 1, partBody, partHeaders); + return objectPutPart(authInfo, request, undefined, log, (putErr, resHeaders) => + cb(putErr, uploadId, resHeaders), + ); + }); + } + + it('should accept a part with no checksum on a COMPOSITE MPU', done => { + // Enabled, this is rejected: a COMPOSITE MPU requires a per-part checksum. + uploadPartAfterDisabling({ 'x-amz-checksum-algorithm': 'crc32' }, {}, (err, uploadId) => { + assert.ifError(err); + const part = getPartMetadata(uploadId); + assert.strictEqual(part.checksumValue, undefined); + done(); + }); + }); + + it('should accept a part whose algorithm differs from the MPU', done => { + uploadPartAfterDisabling( + { 'x-amz-checksum-algorithm': 'crc32' }, + { 'x-amz-checksum-sha256': 'YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=' }, + (err, uploadId) => { + assert.ifError(err); + assert.strictEqual(getPartMetadata(uploadId).checksumValue, undefined); + done(); + }, + ); + }); + + it('should accept a part whose checksum does not match the body', done => { + uploadPartAfterDisabling( + { 'x-amz-checksum-algorithm': 'crc32' }, + { 'x-amz-checksum-crc32': 'AAAAAA==' }, + (err, uploadId) => { + assert.ifError(err); + assert.strictEqual(getPartMetadata(uploadId).checksumValue, undefined); + done(); + }, + ); + }); + + it('should not echo a checksum back in the response', done => { + uploadPartAfterDisabling( + { 'x-amz-checksum-algorithm': 'crc32' }, + { 'x-amz-checksum-crc32': 'AAAAAA==' }, + (err, uploadId, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders['x-amz-checksum-crc32'], undefined); + done(); + }, + ); + }); + + it('should not reject a malformed per-part checksum header', done => { + uploadPartAfterDisabling( + { 'x-amz-checksum-algorithm': 'crc32' }, + { 'x-amz-checksum-crc32': 'not-base64!!' }, + err => { + assert.ifError(err); + done(); + }, + ); + }); + + it('should reject a part with no checksum on a COMPOSITE MPU once re-enabled', done => { + config.integrityChecks = { enabled: true }; + initiateMPU({ 'x-amz-checksum-algorithm': 'crc32' }, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody, {}); + objectPutPart(authInfo, request, undefined, log, putErr => { + assert(putErr, 'should reject'); + assert.strictEqual(putErr.message, 'InvalidRequest'); + done(); + }); + }); + }); +}); diff --git a/tests/unit/routes/veeam-utils.js b/tests/unit/routes/veeam-utils.js index 6e31dd5035..0960f96be6 100644 --- a/tests/unit/routes/veeam-utils.js +++ b/tests/unit/routes/veeam-utils.js @@ -5,6 +5,7 @@ const { Readable } = require('stream'); const UtilizationService = require('../../../lib/utilization/instance'); const metadata = require('../../../lib/metadata/wrapper'); const { fetchCapacityMetrics, buildVeeamFileData, receiveData } = require('../../../lib/routes/veeam/utils'); +const { config } = require('../../../lib/Config'); const { DummyRequestLogger } = require('../helpers'); describe('fetchCapacityMetrics', () => { @@ -385,4 +386,62 @@ describe('receiveData', () => { ); await assert.rejects(receiveData(request, log), err => err.is.InvalidArgument); }); + + describe('with checksums disabled', () => { + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + config.integrityChecks = { enabled: false }; + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + }); + + it('should accept a malformed x-amz-checksum header', async () => { + // Enabled, this is InvalidRequest. + const request = makeRequest( + payload, + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc64nvme': 'not-base64!', + }, + payload.length, + ); + const data = await receiveData(request, log); + assert.strictEqual(data, payload); + }); + + it('should ignore a mismatched x-amz-checksum header', async () => { + // Enabled, this is BadDigest. + const request = makeRequest( + payload, + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-sha256': crypto.createHash('sha256').update('other').digest('base64'), + }, + payload.length, + ); + const data = await receiveData(request, log); + assert.strictEqual(data, payload); + }); + + it('should still strip an unvalidated trailing checksum from the body', async () => { + // Enabled, this digest is a BadDigest mismatch. + const body = chunkedBody('AAAAAAAAAAA='); + const request = makeRequest( + body, + { + 'content-length': `${body.length}`, + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc64nvme', + 'x-amz-decoded-content-length': `${payload.length}`, + }, + payload.length, + ); + const data = await receiveData(request, log); + assert.strictEqual(data, payload); + }); + }); });