Storage - Content Validation Decoder Stress Tests

sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml

@@ -108,6 +108,150 @@ matrix:
       durationMin: 60
       imageBuildDir: "../../.."
 
+    # content validation downloads using BlobDownloadStreamOptions with CRC64 validation
+    crc64downloadstreamsm:
+      testScenario: downloadstreamwithcrc64
+      sync: true
+      sizeBytes: 1024
+      downloadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and async client
+    crc64downloadstreamasyncsm:
+      testScenario: downloadstreamwithcrc64
+      sync: false
+      sizeBytes: 1024
+      downloadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and large payload
+    crc64downloadstreamlg:
+      testScenario: downloadstreamwithcrc64
+      sync: true
+      sizeBytes: "52428800"
+      downloadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadStreamOptions with CRC64 validation, async client, and large payload
+    crc64downloadstreamasynclg:
+      testScenario: downloadstreamwithcrc64
+      sync: false
+      sizeBytes: "52428800"
+      downloadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadContentOptions with CRC64 validation
+    crc64downloadcontentsm:
+      testScenario: downloadcontentwithcrc64
+      sync: true
+      sizeBytes: 1024
+      downloadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadContentOptions with CRC64 validation and async client
+    crc64downloadcontentasyncsm:
+      testScenario: downloadcontentwithcrc64
+      sync: false
+      sizeBytes: 1024
+      downloadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadContentOptions with CRC64 validation and large payload
+    crc64downloadcontentlg:
+      testScenario: downloadcontentwithcrc64
+      sync: true
+      sizeBytes: "52428800"
+      downloadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadContentOptions with CRC64 validation, async client, and large payload
+    crc64downloadcontentasynclg:
+      testScenario: downloadcontentwithcrc64
+      sync: false
+      sizeBytes: "52428800"
+      downloadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadToFileOptions with CRC64 validation
+    crc64downloadfilesm:
+      testScenario: downloadtofilewithcrc64
+      sync: true
+      sizeBytes: 1024
+      downloadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and async client
+    crc64downloadfileasyncsm:
+      testScenario: downloadtofilewithcrc64
+      sync: false
+      sizeBytes: 1024
+      downloadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and multi-block payload
+    crc64downloadfilemd:
+      testScenario: downloadtofilewithcrc64
+      sync: true
+      sizeBytes: "16777216"
+      downloadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobDownloadToFileOptions with CRC64 validation, async client, and multi-block payload
+    crc64downloadfileasyncmd:
+      testScenario: downloadtofilewithcrc64
+      sync: false
+      sizeBytes: "16777216"
+      downloadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobInputStreamOptions with CRC64 validation
+    crc64inputstreamsm:
+      testScenario: openinputstreamwithcrc64
+      sync: true
+      sizeBytes: 1024
+      downloadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobInputStreamOptions with CRC64 validation and large payload
+    crc64inputstreamlg:
+      testScenario: openinputstreamwithcrc64
+      sync: true
+      sizeBytes: "52428800"
+      downloadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation
+    crc64bytechannelreadsm:
+      testScenario: openseekablebytechannelreadwithcrc64
+      sync: true
+      sizeBytes: 1024
+      downloadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation and large payload
+    crc64bytechannelreadlg:
+      testScenario: openseekablebytechannelreadwithcrc64
+      sync: true
+      sizeBytes: "52428800"
+      downloadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
     # this test uploads 1KB (1024 bytes) to append blob, no chunking
     appendblocksmall:
       testScenario: appendblock

sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java

@@ -15,6 +15,11 @@ public static void main(String[] args) {
             BlockBlobOutputStream.class,
             BlockBlobUpload.class,
             CommitBlockList.class,
+            DownloadContentWithCRC64.class,
+            DownloadStreamWithCRC64.class,
+            DownloadToFileWithCRC64.class,
+            OpenInputStreamWithCRC64.class,
+            OpenSeekableByteChannelReadWithCRC64.class,
             DownloadToFile.class,
             DownloadStream.class,
             DownloadContent.class,

sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java

@@ -0,0 +1,70 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.storage.blob.stress;
+
+import com.azure.core.http.HttpHeaderName;
+import com.azure.core.util.BinaryData;
+import com.azure.core.util.Context;
+import com.azure.storage.blob.BlobAsyncClient;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.options.BlobDownloadContentOptions;
+import com.azure.storage.blob.options.BlobDownloadStreamOptions;
+import com.azure.storage.blob.stress.utils.OriginalContent;
+import com.azure.storage.common.ContentValidationAlgorithm;
+import com.azure.storage.stress.StorageStressOptions;
+import reactor.core.publisher.Mono;
+
+/**
+ * Download content with CRC64 Algorithm enabled.
+ * Verifies the correctness of the download response content via CRC.
+ */
+public class DownloadContentWithCRC64 extends BlobScenarioBase<StorageStressOptions> {
+    private final OriginalContent originalContent = new OriginalContent();
+    private final BlobClient syncClient;
+    private final BlobAsyncClient asyncClient;
+    private final BlobAsyncClient asyncNoFaultClient;
+
+    public DownloadContentWithCRC64(StorageStressOptions options) {
+        super(options);
+        String blobName = generateBlobName();
+        this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName);
+        this.syncClient = getSyncContainerClient().getBlobClient(blobName);
+        this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName);
+    }
+
+    @Override
+    protected void runInternal(Context span) {
+        originalContent.checkMatch(
+            syncClient.downloadContentWithResponse(
+                new BlobDownloadContentOptions()
+                    .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64),
+                null, span).getValue(),
+            span).block();
+    }
+
+    @Override
+    protected Mono<Void> runInternalAsync(Context span) {
+        // TODO return downloadContent once it stops buffering.
+        return asyncClient.downloadStreamWithResponse(
+            new BlobDownloadStreamOptions()
+                .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))
+            .flatMap(response -> {
+                long contentLength = Long.valueOf(response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH));
+                return BinaryData.fromFlux(response.getValue(), contentLength, false);
+            })
+            .flatMap(bd -> originalContent.checkMatch(bd, span));
+    }
+
+    @Override
+    public Mono<Void> setupAsync() {
+        return super.setupAsync()
+            .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize()));
+    }
+
+    @Override
+    public Mono<Void> cleanupAsync() {
+        return asyncNoFaultClient.deleteIfExists()
+            .then(super.cleanupAsync());
+    }
+}

sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.storage.blob.stress;
+
+import com.azure.core.util.Context;
+import com.azure.storage.blob.BlobAsyncClient;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.options.BlobDownloadStreamOptions;
+import com.azure.storage.blob.stress.utils.OriginalContent;
+import com.azure.storage.common.ContentValidationAlgorithm;
+import com.azure.storage.stress.CrcOutputStream;
+import com.azure.storage.stress.StorageStressOptions;
+import reactor.core.publisher.Mono;
+
+import java.io.IOException;
+
+/**
+ * Streaming blob download with CRC64 Algorithm enabled.
+ * Verifies the correctness of the download response content via CRC.
+ */
+public class DownloadStreamWithCRC64 extends BlobScenarioBase<StorageStressOptions> {
+    private final OriginalContent originalContent = new OriginalContent();
+    private final BlobClient syncClient;
+    private final BlobAsyncClient asyncClient;
+    private final BlobAsyncClient asyncNoFaultClient;
+
+    public DownloadStreamWithCRC64(StorageStressOptions options) {
+        super(options);
+        String blobName = generateBlobName();
+        this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName);
+        this.syncClient = getSyncContainerClient().getBlobClient(blobName);
+        this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName);
+    }
+
+    @Override
+    protected void runInternal(Context span) throws IOException {
+        try (CrcOutputStream outputStream = new CrcOutputStream()) {
+            syncClient.downloadStreamWithResponse(outputStream,
+                new BlobDownloadStreamOptions()
+                    .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64),
+                null, span);
+            outputStream.close();
+            originalContent.checkMatch(outputStream.getContentInfo(), span).block();
+        }
+    }
+
+    @Override
+    protected Mono<Void> runInternalAsync(Context span) {
+        return asyncClient.downloadStreamWithResponse(
+            new BlobDownloadStreamOptions()
+                .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))
+            .flatMap(response -> originalContent.checkMatch(response.getValue(), span));
+    }
+
+    @Override
+    public Mono<Void> setupAsync() {
+        return super.setupAsync()
+            .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize()));
+    }
+
+    @Override
+    public Mono<Void> cleanupAsync() {
+        return asyncNoFaultClient.deleteIfExists()
+            .then(super.cleanupAsync());
+    }
+}