diff --git a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml index 3a8920a67b35..79abd617f5b6 100644 --- a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml +++ b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml @@ -108,6 +108,150 @@ matrix: durationMin: 60 imageBuildDir: "../../.." + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation + crc64downloadstreamsm: + testScenario: downloadstreamwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and async client + crc64downloadstreamasyncsm: + testScenario: downloadstreamwithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and large payload + crc64downloadstreamlg: + testScenario: downloadstreamwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation, async client, and large payload + crc64downloadstreamasynclg: + testScenario: downloadstreamwithcrc64 + sync: false + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation + crc64downloadcontentsm: + testScenario: downloadcontentwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation and async client + crc64downloadcontentasyncsm: + testScenario: downloadcontentwithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation and large payload + crc64downloadcontentlg: + testScenario: downloadcontentwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation, async client, and large payload + crc64downloadcontentasynclg: + testScenario: downloadcontentwithcrc64 + sync: false + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation + crc64downloadfilesm: + testScenario: downloadtofilewithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and async client + crc64downloadfileasyncsm: + testScenario: downloadtofilewithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and multi-block payload + crc64downloadfilemd: + testScenario: downloadtofilewithcrc64 + sync: true + sizeBytes: "16777216" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation, async client, and multi-block payload + crc64downloadfileasyncmd: + testScenario: downloadtofilewithcrc64 + sync: false + sizeBytes: "16777216" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobInputStreamOptions with CRC64 validation + crc64inputstreamsm: + testScenario: openinputstreamwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobInputStreamOptions with CRC64 validation and large payload + crc64inputstreamlg: + testScenario: openinputstreamwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation + crc64bytechannelreadsm: + testScenario: openseekablebytechannelreadwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation and large payload + crc64bytechannelreadlg: + testScenario: openseekablebytechannelreadwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + # this test uploads 1KB (1024 bytes) to append blob, no chunking appendblocksmall: testScenario: appendblock diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java index e38bd16791ca..194a290d7a3c 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java @@ -15,6 +15,11 @@ public static void main(String[] args) { BlockBlobOutputStream.class, BlockBlobUpload.class, CommitBlockList.class, + DownloadContentWithCRC64.class, + DownloadStreamWithCRC64.class, + DownloadToFileWithCRC64.class, + OpenInputStreamWithCRC64.class, + OpenSeekableByteChannelReadWithCRC64.class, DownloadToFile.class, DownloadStream.class, DownloadContent.class, diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java new file mode 100644 index 000000000000..6c92e1563c60 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +/** + * Download content with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadContentWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public DownloadContentWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + originalContent.checkMatch( + syncClient.downloadContentWithResponse( + new BlobDownloadContentOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span).getValue(), + span).block(); + } + + @Override + protected Mono runInternalAsync(Context span) { + // TODO return downloadContent once it stops buffering. + return asyncClient.downloadStreamWithResponse( + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(response -> { + long contentLength = Long.valueOf(response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + return BinaryData.fromFlux(response.getValue(), contentLength, false); + }) + .flatMap(bd -> originalContent.checkMatch(bd, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java new file mode 100644 index 000000000000..64cccb2c7dd4 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcOutputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +/** + * Streaming blob download with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadStreamWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public DownloadStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + try (CrcOutputStream outputStream = new CrcOutputStream()) { + syncClient.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + outputStream.close(); + originalContent.checkMatch(outputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return asyncClient.downloadStreamWithResponse( + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(response -> originalContent.checkMatch(response.getValue(), span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java new file mode 100644 index 000000000000..ceb2faa8b153 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.UUID; + +/** + * Download to file with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadToFileWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(DownloadToFileWithCRC64.class); + private final Path directoryPath; + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public DownloadToFileWithCRC64(StorageStressOptions options) { + super(options); + this.directoryPath = getTempPath("test"); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()); + } + + @Override + protected void runInternal(Context span) { + Path downloadPath = directoryPath.resolve(UUID.randomUUID() + ".txt"); + BlobDownloadToFileOptions blobOptions = new BlobDownloadToFileOptions(downloadPath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try { + syncClient.downloadToFileWithResponse(blobOptions, Duration.ofSeconds(options.getDuration()), span); + originalContent.checkMatch(BinaryData.fromFile(downloadPath), span).block(); + } finally { + deleteFile(downloadPath); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return Mono.using( + () -> directoryPath.resolve(UUID.randomUUID() + ".txt"), + path -> asyncClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(path.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(path), span)), + DownloadToFileWithCRC64::deleteFile); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } + + private Path getTempPath(String prefix) { + try { + return Files.createTempDirectory(prefix); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } + + private static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (Throwable e) { + LOGGER.atError() + .addKeyValue("path", path) + .log("failed to delete file", e); + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java new file mode 100644 index 000000000000..eaae1570560b --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.InputStream; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Open input stream with CRC64 Algorithm enabled (sync only). + * Verifies the correctness of the download response content via CRC. + */ +public class OpenInputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(OpenInputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public OpenInputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + try (InputStream stream = syncClient.openInputStream( + new BlobInputStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + span)) { + try (CrcInputStream crcStream = new CrcInputStream(stream)) { + byte[] buffer = new byte[8192]; + while (crcStream.read(buffer) != -1) { + // do nothing + } + originalContent.checkMatch(crcStream.getContentInfo(), span).block(); + } + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("openInputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java new file mode 100644 index 000000000000..e6376238fe92 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.BlobSeekableByteChannelReadResult; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.channels.Channels; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Seekable byte channel read with CRC64 Algorithm enabled (sync only). + * Verifies the correctness of the download response content via CRC. + */ +public class OpenSeekableByteChannelReadWithCRC64 + extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(OpenSeekableByteChannelReadWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public OpenSeekableByteChannelReadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlobSeekableByteChannelReadResult result = syncClient.openSeekableByteChannelRead( + new BlobSeekableByteChannelReadOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + span); + try (CrcInputStream crcStream = new CrcInputStream(Channels.newInputStream(result.getChannel()))) { + byte[] buffer = new byte[8192]; + while (crcStream.read(buffer) != -1) { + // do nothing + } + originalContent.checkMatch(crcStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, + new RuntimeException("openSeekableByteChannelRead() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/templates/stress-test-job.yaml b/sdk/storage/azure-storage-blob-stress/templates/stress-test-job.yaml index 8cc1c2d71b5b..03a0609beebd 100644 --- a/sdk/storage/azure-storage-blob-stress/templates/stress-test-job.yaml +++ b/sdk/storage/azure-storage-blob-stress/templates/stress-test-job.yaml @@ -77,4 +77,4 @@ spec: cpu: "0.7" {{- include "stress-test-addons.container-env" . | nindent 6 }} -{{- end -}} \ No newline at end of file +{{- end -}}