diff --git a/src/Client/Core/LargePayloadPurgeDisposition.cs b/src/Client/Core/LargePayloadPurgeDisposition.cs new file mode 100644 index 00000000..bfdedca2 --- /dev/null +++ b/src/Client/Core/LargePayloadPurgeDisposition.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// The outcome of a single large-payload blob deletion attempt. The split is by whether a failure can +/// self-heal. Mirrors the LargePayloadPurgeDisposition protobuf enum. +/// +public enum LargePayloadPurgeDisposition +{ + /// + /// No disposition was specified. + /// + Unspecified = 0, + + /// + /// Terminal success. The blob was deleted, was already absent, or was deliberately left in place because + /// it is not owned by the payload store. The backend deletes the tombstone in all three cases. + /// + Deleted = 1, + + /// + /// The failure may self-heal, so the row stays pending and the backend sets the next attempt. + /// + Retry = 2, + + /// + /// A deterministic failure or protocol violation that retrying can never fix. The backend preserves the + /// evidence, alerts, and stops automatic retries. + /// + Quarantined = 3, +} diff --git a/src/Client/Core/LargePayloadPurgeResult.cs b/src/Client/Core/LargePayloadPurgeResult.cs new file mode 100644 index 00000000..e834e62a --- /dev/null +++ b/src/Client/Core/LargePayloadPurgeResult.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable outcome of exactly one attempted large-payload blob deletion. Mirrors the +/// LargePayloadPurgeResult protobuf message but is safe to pass through the orchestration/activity +/// boundary. The backend owns retry scheduling and branches solely on +/// : it deletes rows reported as +/// , reschedules +/// on its own backoff, and moves +/// rows out of the active fetch. The worker never +/// computes a retry delay. +/// +/// +/// The disposition is deliberately the only outcome field: anything finer would be write-only on the backend. +/// Why an attempt failed stays in the worker's own telemetry, which holds the cause at full fidelity rather +/// than as a lossy classification. +/// +/// +/// The opaque correlation token echoed unchanged from the fetched . It is +/// what identifies the row being reported on, so it must be passed back exactly as received: callers must not +/// parse it, derive from it, or construct one. +/// +/// Opaqueness here is encapsulation, not security. The token is not an authentication credential and carries +/// no integrity guarantee, so treating a well-formed token as proof that the caller was entitled to report on +/// that row would be wrong. Authentication and task-hub scope are the security boundary. +/// +/// +/// The disposition of the deletion attempt. +public sealed record LargePayloadPurgeResult( + string TombstoneToken, + LargePayloadPurgeDisposition Disposition); diff --git a/src/Client/Core/LargePayloadTombstone.cs b/src/Client/Core/LargePayloadTombstone.cs new file mode 100644 index 00000000..7f82557b --- /dev/null +++ b/src/Client/Core/LargePayloadTombstone.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable representation of a tombstoned large-payload row whose external blob a credentialed caller +/// must delete. Mirrors the LargePayloadTombstone protobuf message but is safe to pass through the +/// orchestration/activity boundary. +/// +/// +/// The opaque, backend-issued correlation token for this exact tombstone version. Its format is deliberately +/// not part of the contract: callers must not parse it, and must echo it back unchanged in the corresponding +/// so the backend can resolve the row it came from. +/// +/// +/// The self-describing blob:v2:{fullBlobUrl} payload token whose backing blob should be deleted. +/// +public sealed record LargePayloadTombstone(string TombstoneToken, string PayloadToken) +{ + /// + /// The maximum number of tombstones a single GetLargePayloadTombstones request may ask for. The + /// service clamps a larger request down to its own maximum, so this is the largest value that is worth + /// asking for rather than a value that will be rejected. Validators that bound a caller-supplied limit + /// compare against this shared value rather than a hard-coded literal so the bound cannot drift between + /// the client and the auto-purge extension. + /// + public const int MaxRequestLimit = 1000; +} diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 865a83b3..f240d722 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -14,6 +14,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; +using LP = Microsoft.DurableTask.Protobuf.LargePayloads; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Client.Grpc; @@ -21,10 +22,11 @@ namespace Microsoft.DurableTask.Client.Grpc; /// /// Durable Task client implementation that uses gRPC to connect to a remote "sidecar" process. /// -public sealed class GrpcDurableTaskClient : DurableTaskClient +public sealed class GrpcDurableTaskClient : DurableTaskClient, Internal.ILargePayloadAutoPurgeClient { readonly ILogger logger; readonly TaskHubSidecarServiceClient sidecarClient; + readonly LP.LargePayloadPurge.LargePayloadPurgeClient largePayloadPurgeClient; readonly GrpcDurableTaskClientOptions options; readonly DurableEntityClient? entityClient; AsyncDisposable asyncDisposable; @@ -56,6 +58,13 @@ public GrpcDurableTaskClient(string name, GrpcDurableTaskClientOptions options, this.asyncDisposable = GetCallInvoker(options, logger, out CallInvoker callInvoker); this.sidecarClient = new TaskHubSidecarServiceClient(callInvoker); + // Built on the SAME effective invoker as the sidecar client, which is what keeps the DTS-only purge + // service on this client's existing channel and inside its configured interceptors. That invoker may be + // a ChannelRecreatingCallInvoker, so the generated client follows the client's channel swaps rather than + // pinning the channel it happened to see here. Constructed unconditionally: it opens no connection of + // its own, so an app that never touches auto-purge pays only the allocation. + this.largePayloadPurgeClient = new LP.LargePayloadPurge.LargePayloadPurgeClient(callInvoker); + if (this.options.EnableEntitySupport) { this.entityClient = new GrpcDurableEntityClient(this.Name, this.DataConverter, this.sidecarClient, logger); @@ -625,6 +634,26 @@ public override async Task> GetOrchestrationHistoryAsync( } } + /// + async Task Internal.ILargePayloadAutoPurgeClient.SetLargePayloadAutoPurgeAsync(bool enabled, CancellationToken cancellation) + { + try + { + await this.largePayloadPurgeClient.SetLargePayloadAutoPurgeAsync( + new LP.SetLargePayloadAutoPurgeRequest { Enabled = enabled }, + cancellationToken: cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + "The SetLargePayloadAutoPurge operation was canceled.", e, cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) + { + throw new NotImplementedException(e.Status.Detail); + } + } + static AsyncDisposable GetCallInvoker(GrpcDurableTaskClientOptions options, ILogger logger, out CallInvoker callInvoker) { AsyncDisposable disposable = GetCallInvokerCore(options, logger, out CallInvoker core); diff --git a/src/Client/Grpc/Internal/ILargePayloadAutoPurgeClient.cs b/src/Client/Grpc/Internal/ILargePayloadAutoPurgeClient.cs new file mode 100644 index 00000000..fcfb52a6 --- /dev/null +++ b/src/Client/Grpc/Internal/ILargePayloadAutoPurgeClient.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client.Grpc.Internal; + +/// +/// Exposes the backend's large-payload auto-purge setting to the Azure Blob payloads extension, which owns the +/// public API that turns the feature on and off. +/// +/// +/// +/// This is an internal API that supports the DurableTask infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should not implement it, and should not use it directly in your code. Doing so +/// can result in application failures when updating to a new DurableTask release. +/// +/// +/// Deliberately narrow. The extension needs exactly one backend operation, and it must reach it over the +/// client's own transport - the same post-interceptor CallInvoker the client uses for every other RPC, +/// so the call carries the configured auth chain and follows channel recreation. Handing out the raw +/// CallInvoker instead would let a caller build arbitrary clients on the transport. The other two purge +/// RPCs are not here: they are executed by the worker's activities on the worker's transport, and no client +/// ever calls them. +/// +/// +/// Implemented explicitly by , so a client that is not the gRPC client - or +/// a gRPC client from an SDK version that predates this - fails the cast rather than silently doing nothing. +/// +/// +public interface ILargePayloadAutoPurgeClient +{ + /// + /// Sets the large-payload blob auto-purge setting for the caller's authenticated task hub. + /// + /// The setting to persist. + /// The cancellation token. + /// A task that completes once the backend has acknowledged the setting. + Task SetLargePayloadAutoPurgeAsync(bool enabled, CancellationToken cancellation = default); +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs new file mode 100644 index 00000000..7085e1ad --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -0,0 +1,238 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Globalization; +using System.Net; +using Azure; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that deletes a chunk of externalized payload blobs given their tokens, classifying each attempt as +/// , , or +/// and returning one outcome per token, positionally +/// aligned to the input. Deleting a whole chunk in a single activity call is what keeps orchestration history +/// small. Deletion is idempotent, so re-delivered tokens, a retried chunk, and concurrent workers are all safe. +/// +/// +/// The split between retry and quarantine is whether the failure can self-heal, verified against the +/// Azure.Storage.Blobs / Azure.Core exception model (not assumed): +/// +/// +/// The Azure SDK already retries transient failures internally (connection errors plus HTTP +/// 408/429/500/502/503/504, with exponential backoff), so any exception that escapes +/// means those built-in retries were already exhausted. It is still +/// classified as retryable, because the backend - not this activity - owns retry scheduling and can defer the +/// row past a storage outage. +/// +/// +/// Quarantine is reserved for deterministic failures and protocol violations that retrying can never fix: a +/// known version prefix whose body does not parse, a request storage rejected as permanently invalid, and a +/// legacy v1 token. Quarantine preserves the row and its token as durable evidence, so a permanent failure +/// neither blocks the queue nor destroys the only record of the blob. +/// +/// +/// A blob is never deleted on an uncertain error, and a single bad token never fails the whole batch: a +/// failure is returned as a disposition rather than thrown. +/// +/// +/// The reported result carries the disposition alone, so every branch below logs its cause where the cause is +/// still exact, rather than deriving it afterwards from a value that crossed the wire. That log is the only +/// record of why an attempt failed. Per design ยง7 it still carries neither the token nor raw exception text - +/// a token exposes the storage account, container, and blob path - so the cause is a bounded classification +/// string plus a bounded, sanitized storage error code. The token itself is preserved on the backend's +/// quarantined row. +/// +/// The payload store used to delete blobs. +/// The logger instance. +[DurableTask] +public class DeleteExternalBlobActivity( + PayloadStore store, + ILogger logger) + : TaskActivity, List> +{ + // Concurrency cap for the deletes WITHIN one chunk. The orchestrator runs at most + // BlobPurgeJobOrchestrator.MaxParallelChunkActivities (4) of these activities at once, so the total number + // of storage deletes in flight across the worker is 4 x 8 = 32 - identical to the flat cap this chunked + // design replaced. That product is the real budget: if either factor changes the other must move to keep it + // at 32, or the worker will either starve throughput or multiply into hundreds of concurrent storage calls + // (e.g. 20 chunks x 32 = 640). + const int MaxParallelDeletesPerChunk = 8; + + readonly PayloadStore store = Check.NotNull(store); + readonly ILogger logger = Check.NotNull(logger); + + /// + /// Gets or sets the wall-clock ceiling for a single blob delete. The store's own retry policy allows up to + /// 8 attempts against a 2-minute network timeout (~18 minutes worst case) for one blob, and a chunk awaits + /// its slowest delete, so without a bound one hung blob would hold a concurrency slot for many minutes and + /// stall the whole wave. Capping it well under the activity's own 15s/30s/60s retry cadence means a stuck + /// delete gives up, surfaces as , and is classified + /// for the backend to defer - rather than pinning the slot. + /// It is settable only so a test can shrink it; it is never reconfigured at runtime. + /// + internal TimeSpan SingleDeleteTimeout { get; set; } = TimeSpan.FromSeconds(60); + + /// + public override async Task> RunAsync(TaskActivityContext context, List input) + { + Check.NotNull(input, nameof(input)); + + // Write each outcome at its token's INDEX, never in completion order: the deletes below run + // concurrently and the orchestrator zips these back onto tombstones positionally, so a delete finishing + // out of order must not shift a disposition onto the wrong row. + BlobPurgeOutcome[] outcomes = new BlobPurgeOutcome[input.Count]; + + // A single bad token never fails its peers: DeleteAsync returns a disposition on every branch instead + // of throwing (its catch-all absorbs everything but OutOfMemory/StackOverflow), so Task.WhenAll never + // observes a fault from a classified failure and the other tokens in the chunk still complete and report. + using SemaphoreSlim gate = new(MaxParallelDeletesPerChunk, MaxParallelDeletesPerChunk); + Task[] deletes = new Task[input.Count]; + for (int i = 0; i < input.Count; i++) + { + deletes[i] = DeleteAtAsync(i); + } + + await Task.WhenAll(deletes); + + return new List(outcomes); + + async Task DeleteAtAsync(int index) + { + await gate.WaitAsync().ConfigureAwait(false); + try + { + outcomes[index] = await this.DeleteAsync(input[index]).ConfigureAwait(false); + } + finally + { + gate.Release(); + } + } + } + + /// + /// Extracts a bounded, sanitized storage error code for diagnostics. The service's own error code (for + /// example BlobNotFound) is a fixed vocabulary and the numeric status is the fallback, so neither + /// can carry a token or raw exception text. + /// + static string SanitizeErrorCode(RequestFailedException exception) + { + // Pattern-matched rather than string.IsNullOrEmpty: on netstandard2.0 that method carries no + // [NotNullWhen(false)] annotation, so flow analysis cannot prove the else branch is non-null and warns. + // A constant pattern is analyzed by the compiler itself and so behaves the same on every target. + string? errorCode = exception.ErrorCode; + return errorCode is null or "" + ? exception.Status.ToString(CultureInfo.InvariantCulture) + : errorCode; + } + + async Task DeleteAsync(string token) + { + // Classify the token's version prefix before consulting the store. The store reports every token it + // cannot decode as the same ArgumentException, but the three cases have opposite dispositions, so they + // are separated here, where the prefix is still visible. + if (token.StartsWith(BlobPayloadStore.TokenPrefixV1, StringComparison.Ordinal)) + { + // A v1 token carries a container *name* but not the storage account, so a delete against the + // currently-configured account cannot be verified: if the store has since been repointed, + // DeleteIfExists returns false and the purge would falsely report success while the real blob + // survives in the old account. Retrying cannot fix that, and a success-shaped discard would destroy + // the only durable record of the blob, so the row is quarantined instead - the backend preserves + // its token as evidence and stops polling it. The backend excludes v1 at insertion time, so + // reaching this branch is an invariant violation rather than an expected path. + this.logger.BlobPurgeDeleteQuarantined("LegacyV1Token", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); + } + + if (!token.StartsWith(BlobPayloadStore.TokenPrefixV2, StringComparison.Ordinal)) + { + // An unrecognized prefix is most likely a token written by a newer SDK than this worker runs. That + // recovers after an upgrade, so it earns a deferral rather than quarantine. Quarantine is + // permanent and requires an operator to unwind; a deferral only leaves the row idle and visible, + // so an unrecognized token is deliberately kept on the recoverable side of that asymmetry. + this.logger.BlobPurgeDeleteRetryable("UnsupportedTokenVersion", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); + } + + try + { + // Bound the delete's wall-clock time (see SingleDeleteTimeout). If it elapses, the token cancels the + // store call, which surfaces as OperationCanceledException and is classified retryable by the + // catch-all below - so a hung blob yields to the backend's deferral instead of pinning its slot. + // TaskActivityContext exposes no ambient cancellation token to link, so this timeout is the only + // cancellation source; host shutdown is handled by the worker tearing the activity down. + using CancellationTokenSource timeout = new(this.SingleDeleteTimeout); + PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, timeout.Token); + + // The blob exists but this store never wrote it, so it was left untouched. That is an expected + // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it would + // fill the quarantine set with non-defects. The tombstone is still resolved, because a blob the + // store does not own is not the store's to delete. + if (outcome == PayloadDeleteOutcome.NotStoreOwned) + { + this.logger.BlobPurgeBlobNotStoreOwned(); + } + + // Deleted, AlreadyAbsent, and NotStoreOwned are all terminal successes: none can be improved by + // trying again. + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Deleted); + } + catch (ArgumentException) + { + // The prefix gate above proves this is a v2 token, so the only remaining decode failure is a v2 + // body that does not parse. The SDK and backend control both sides of the protocol, so that + // indicates a producer, corruption, or compatibility bug; retrying can never fix it. + this.logger.BlobPurgeDeleteQuarantined("MalformedToken", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); + } + catch (NotSupportedException) + { + // The registered store does not implement deletion. Every payload would fail the same way, so the + // work is kept recoverable until an operator registers a store that can delete. + this.logger.BlobPurgeDeleteRetryable("StoreCannotDelete", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); + } + catch (PayloadStorageException) + { + // The token is well formed but points at a storage account this worker's credential cannot reach + // (account-key auth is account-specific). Recoverable after a configuration or credential change, + // so it is deferred rather than discarded. + this.logger.BlobPurgeDeleteRetryable("StorageAccountUnreachable", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.BadRequest) + { + // Storage rejected a request generated from a well-formed token as permanently invalid (for + // example InvalidUri / InvalidResourceName). Retrying can never succeed. + this.logger.BlobPurgeDeleteQuarantined("InvalidStorageRequest", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); + } + catch (RequestFailedException ex) when ( + ex.Status == (int)HttpStatusCode.Unauthorized || ex.Status == (int)HttpStatusCode.Forbidden) + { + // Authorization can be transient or fixed by reconfiguration, so it stays recoverable rather than + // dropping data an operator can still reclaim. + this.logger.BlobPurgeDeleteRetryable("StorageAuthorizationFailed", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); + } + catch (RequestFailedException ex) + { + // Throttling, 5xx, and anything else the service reported, including a failed If-Match on the + // ownership check: transient by default. + this.logger.BlobPurgeDeleteRetryable("TransientStorageFailure", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + // Timeouts, cancellation, and network failures. A blob is never dropped on an uncertain error. + // Storage reported no code here, so the exception's type name is appended to the cause: it is a + // bounded value that cannot carry a token, and it is the only thing separating a timeout from a + // cancellation or a DNS failure now that no classification crosses the wire. + this.logger.BlobPurgeDeleteRetryable($"UnexpectedFailure:{ex.GetType().Name}", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); + } + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs new file mode 100644 index 00000000..7f021250 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; +using static Microsoft.DurableTask.Protobuf.LargePayloads.LargePayloadPurge; +using LP = Microsoft.DurableTask.Protobuf.LargePayloads; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that fetches a bounded batch of due large-payload tombstones from the backend for the auto-purge +/// job to delete. +/// +/// The large-payload purge service client used to query the backend for tombstones. +/// The logger instance. +[DurableTask] +internal sealed class GetLargePayloadTombstonesActivity( + LargePayloadPurgeClient client, + ILogger logger) + : TaskActivity> +{ + readonly LargePayloadPurgeClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task> RunAsync(TaskActivityContext context, int input) + { + if (input <= 0 || input > LargePayloadTombstone.MaxRequestLimit) + { + throw new ArgumentOutOfRangeException( + nameof(input), input, $"Limit must be greater than 0 and less than or equal to {LargePayloadTombstone.MaxRequestLimit}."); + } + + LP.GetLargePayloadTombstonesResponse response; + try + { + response = await this.client.GetLargePayloadTombstonesAsync( + new LP.GetLargePayloadTombstonesRequest { Limit = input }); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + "The GetLargePayloadTombstonesAsync operation was canceled.", e); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) + { + // Mixed-rollout guard: an older backend build (or a stale local emulator image) does not implement + // this RPC. Surfacing NotImplementedException lets the orchestrator disable the job instead of + // retrying an operation that can never succeed - see BlobPurgeJobOrchestrator's handling of it. + throw new NotImplementedException( + "The Durable Task backend does not implement the GetLargePayloadTombstones RPC required for " + + "large-payload auto-purge. Auto-purge is now disabled. Upgrade the backend (or re-pull " + + "'mcr.microsoft.com/dts/dts-emulator'), then call SetLargePayloadAutoPurgeAsync(true, ...) " + + $"again to re-enable it. Backend detail: {e.Status.Detail}", + e); + } + + List tombstones = new(response.Tombstones.Count); + foreach (LP.LargePayloadTombstone tombstone in response.Tombstones) + { + tombstones.Add(new LargePayloadTombstone(tombstone.TombstoneToken, tombstone.PayloadToken)); + } + + this.logger.BlobPurgeFetchedTombstones(tombstones.Count); + return tombstones; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs new file mode 100644 index 00000000..f0e6001d --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; +using static Microsoft.DurableTask.Protobuf.LargePayloads.LargePayloadPurge; +using LP = Microsoft.DurableTask.Protobuf.LargePayloads; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that reports the outcome of every attempted blob deletion to the backend, so it can delete the +/// resolved tombstones, reschedule the retryable ones, and quarantine the rest. Every attempted row is +/// reported, not only the successful ones: the backend owns retry scheduling, so a row it hears nothing about +/// would simply be re-served unchanged on the next cycle. +/// +/// The large-payload purge service client used to report purge results to the backend. +/// The logger instance. +[DurableTask] +internal sealed class ReportLargePayloadPurgeResultsActivity( + LargePayloadPurgeClient client, + ILogger logger) + : TaskActivity, object?> +{ + readonly LargePayloadPurgeClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task RunAsync( + TaskActivityContext context, List input) + { + if (input is null || input.Count == 0) + { + return null; + } + + LP.ReportLargePayloadPurgeResultsRequest request = new(); + foreach (LargePayloadPurgeResult result in input) + { + request.Results.Add(new LP.LargePayloadPurgeResult + { + // Echoed back exactly as it was received. The SDK never parses or rebuilds this token, so a + // change to what the backend puts in it needs no change here. + TombstoneToken = result.TombstoneToken, + + // The managed disposition enum declares the same numeric values as its protobuf counterpart, + // so it maps across by value. This is the only enum on the message and it only travels + // outbound, so the SDK can never receive a value it does not know. + Disposition = (LP.LargePayloadPurgeDisposition)result.Disposition, + }); + } + + if (request.Results.Count == 0) + { + return null; + } + + try + { + await this.client.ReportLargePayloadPurgeResultsAsync(request); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + "The ReportLargePayloadPurgeResultsAsync operation was canceled.", e); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) + { + // Mixed-rollout guard: an older backend build (or a stale local emulator image) does not implement + // this RPC. Surfacing NotImplementedException lets the orchestrator disable the job instead of + // retrying an operation that can never succeed - see BlobPurgeJobOrchestrator's handling of it. + throw new NotImplementedException( + "The Durable Task backend does not implement the ReportLargePayloadPurgeResults RPC required " + + "for large-payload auto-purge. Auto-purge is now disabled. Upgrade the backend (or re-pull " + + "'mcr.microsoft.com/dts/dts-emulator'), then call SetLargePayloadAutoPurgeAsync(true, ...) " + + $"again to re-enable it. Backend detail: {e.Status.Detail}", + e); + } + + this.logger.BlobPurgeReportedResults(input.Count); + return null; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/DurableTaskClientExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/DurableTaskClientExtensions.AzureBlobPayloads.cs new file mode 100644 index 00000000..c8f2ab4c --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/DurableTaskClientExtensions.AzureBlobPayloads.cs @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Client.Grpc.Internal; +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.Client; + +/// +/// Extension methods that turn Azure Blob large-payload auto-purge on and off for a task hub. +/// +public static class DurableTaskClientExtensionsAzureBlobPayloads +{ + /// + /// Turns large-payload blob auto-purge on or off for the task hub this client is authenticated against, and + /// starts or stops the singleton purge job that does the deleting. + /// + /// The Durable Task client. Must be the gRPC client, with entity support enabled. + /// + /// true to tombstone externalized payloads on instance purge and run the job that deletes their + /// blobs; false to stop tombstoning and stop the job. + /// + /// + /// The maximum number of tombstones the job requests from the backend per cycle. Must be between 1 and + /// inclusive. Ignored entirely when + /// is false, including when out of range, because a job that is being + /// stopped has no cycle to size. + /// + /// The cancellation token. + /// A task that completes once both steps below have been performed. + /// + /// + /// One call owns one transition, and the caller owns when it happens. There is no host that applies this at + /// startup and no loop that reasserts it: the setting and the job persist in the task hub, so a value set + /// once outlives every process that was running when it was set. Repeating the same call is safe - the + /// backend setting is last-writer-wins and the entity operations are idempotent - so a caller that is unsure + /// of the current state can simply call again. + /// + /// + /// This performs TWO separate operations against the task hub, in this order: the backend setting is written + /// first and awaited, then the singleton job entity is signalled. The order is deliberate on both paths. + /// Enabling the setting before starting the job means the job cannot poll for tombstones the backend is not + /// yet writing; disabling it before stopping the job means no new tombstones are created while the job winds + /// down. There is no transaction across the two, and none is possible: they are different subsystems. + /// + /// + /// Because they are separate, completion means the backend acknowledged the setting and the entity signal + /// was reliably enqueued - not that the job has actually begun or ended. Starting is asynchronous: the + /// entity schedules the perpetual orchestrator, which begins its first cycle shortly after. Stopping is + /// cooperative: the orchestrator reads the job state at the top of each cycle and exits on its own, so + /// tombstones already fetched and deletes already in flight run to completion first. + /// + /// + /// If the setting succeeds and the entity signal then fails or is cancelled, the setting is NOT rolled back + /// and this throws. Rolling back would be its own operation that can fail in turn, and it would be wrong as + /// often as it was right - a concurrent caller may have set the value the rollback would undo. Retry the + /// same call instead; it converges from any partial state. + /// + /// + /// There is no coordination between callers: no lease, no owner, no fencing. Two clients calling with + /// different values race, and the last write wins for the backend setting and, independently, for the job + /// entity - so a sufficiently unlucky interleaving can leave the setting from one caller with the job state + /// from the other. Deciding who calls this, and when, is the caller's responsibility. + /// + /// + /// Auto-purge reclaims only blobs referenced by self-describing blob:v2: tokens. Payloads written by + /// SDK versions that emitted legacy blob:v1: tokens are not reclaimed, because a v1 token identifies + /// the container by name only and not the storage account, so the delete cannot be verified. Their backing + /// blobs remain in storage exactly as they did before auto-purge existed, and the backend removes their rows + /// normally. Note also that when the storage account has blob versioning or blob soft delete enabled, + /// auto-purge deletes the current base blob, but retained versions and soft-deleted blobs keep consuming + /// storage until a lifecycle-management policy or the retention period reclaims them. + /// + /// + /// is null. + /// + /// is true and is out of range. + /// + /// + /// The client is not the gRPC client, or was built with entity support disabled. + /// + /// The backend does not implement large-payload purge. + public static async Task SetLargePayloadAutoPurgeAsync( + this DurableTaskClient client, + bool enabled, + int batchSize = BlobPurgeConstants.DefaultBatchSize, + CancellationToken cancellationToken = default) + { + Check.NotNull(client); + + // Validated only on the enabling path. On the disabling path the value is not used at all - there is no + // cycle left to size - so rejecting it would fail a call that would otherwise have done exactly what the + // caller wanted, which matters most for the caller who is passing a batch size through from + // configuration and flipping only the flag. + if (enabled && (batchSize < 1 || batchSize > BlobPurgeConstants.MaxBatchSize)) + { + throw new ArgumentOutOfRangeException( + nameof(batchSize), + batchSize, + $"{nameof(batchSize)} must be between 1 and {BlobPurgeConstants.MaxBatchSize} (inclusive)."); + } + + // Both local prerequisites are resolved BEFORE the RPC, so a client that cannot complete the call fails + // without having changed anything in the backend. Discovering the missing entity client afterwards would + // leave the setting written and the job unreachable - the enabling path's worst outcome, since the + // backend would start tombstoning payloads that nothing is running to delete. + if (client is not ILargePayloadAutoPurgeClient autoPurgeClient) + { + throw new NotSupportedException( + $"Large-payload auto-purge requires the gRPC Durable Task client, but this client is " + + $"'{client.GetType().FullName}'."); + } + + DurableEntityClient entities = client.Entities; + + await autoPurgeClient.SetLargePayloadAutoPurgeAsync(enabled, cancellationToken); + + EntityInstanceId entityId = new(nameof(BlobPurgeJob), BlobPurgeConstants.JobId); + + // Signalled directly rather than driven through an orchestration. The entity is the job's authority and + // its operations are idempotent, so there is nothing a bridge orchestration would add here: the caller + // is explicit and single, not a fleet of hosts racing to converge. + if (enabled) + { + await entities.SignalEntityAsync( + entityId, nameof(BlobPurgeJob.Create), batchSize, cancellation: cancellationToken); + } + else + { + // A stop aimed at an entity that was never created materializes it with default (Pending) state, + // because the framework persists entity state after every operation. That is accepted rather than + // avoided with a read-before-signal: the read would be a second round trip that can disagree with + // the signal that follows it, and a Pending job is exactly the state an explicit disable wants. + await entities.SignalEntityAsync( + entityId, nameof(BlobPurgeJob.Stop), cancellation: cancellationToken); + } + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs new file mode 100644 index 00000000..c0d323b9 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Constants used throughout the blob payload auto-purge functionality. +/// +static class BlobPurgeConstants +{ + /// + /// The fixed, process-global job ID for the singleton blob payload auto-purge job. A single job drains + /// tombstoned payloads for the whole scheduler, so the ID is hard-coded rather than caller-supplied. + /// + public const string JobId = "__dt_blob_payload_autopurge__"; + + /// + /// The default number of tombstoned payloads the auto-purge job requests from the backend per cycle, + /// used whenever a batch size is not passed explicitly. + /// + public const int DefaultBatchSize = 500; + + /// + /// The maximum batch size the auto-purge job may request per cycle. Delegates to + /// , the single authority for the gRPC + /// GetLargePayloadTombstones contract bound, so the two cannot drift. + /// + public const int MaxBatchSize = LargePayloadTombstone.MaxRequestLimit; + + /// + /// The prefix used for generating blob purge job orchestrator instance IDs. Format: "BlobPurgeJob-{jobId}". + /// + public const string OrchestratorInstanceIdPrefix = "BlobPurgeJob-"; + + /// + /// Generates an orchestrator instance ID for a given blob purge job ID. + /// + /// The blob purge job ID. + /// The orchestrator instance ID. + public static string GetOrchestratorInstanceId(string jobId) => $"{OrchestratorInstanceIdPrefix}{jobId}"; +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs new file mode 100644 index 00000000..1e5e7957 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Durable entity that manages the lifecycle of the singleton blob payload auto-purge job. +/// +/// The logger instance. +class BlobPurgeJob(ILogger logger) : TaskEntity +{ + /// + /// Creates the auto-purge job, and starts its orchestrator if one is not already running. Because the job is + /// a per-task-hub singleton, callers racing to create it converge on the same result rather than disturbing + /// a running job. It also takes the batch size when the job is already + /// , which is what lets a later call resize a running job. + /// + /// + /// There is deliberately no guard against reviving an job. + /// Create is only ever reached from an explicit client call, so reviving is precisely what the caller asked + /// for - and it is the documented recovery once the backend has been upgraded to implement the purge RPCs. + /// A guard here would make that recovery impossible without a state reset the API does not expose. If the + /// backend is still unsupported, the orchestrator discovers it on the next cycle and marks the job again. + /// + /// The entity context. + /// + /// The maximum number of tombstoned payloads to request from the backend per cycle. + /// + public void Create(TaskEntityContext context, int purgeBatchSize) + { + if (this.State.Status == BlobPurgeJobStatus.Active) + { + // The batch size is taken because this is the only path by which a changed size reaches an active + // job - the orchestrator re-reads it from here every cycle. Without this the value written by the + // very first Create would be the only one the job ever used, and a batch size the backend rejects + // would wedge it permanently. + // + // Written only when the value actually differs, which is what keeps LastModifiedAt tracking real + // changes rather than the time of the last call. An entity written by a build that predates this + // field carries zero, which differs from any real size, so the first Create after an upgrade still + // repairs it. + if (this.State.PurgeBatchSize != purgeBatchSize) + { + this.State.PurgeBatchSize = purgeBatchSize; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } + + logger.BlobPurgeJobAlreadyRunning(context.Id.Key); + + // Run is re-signalled even though the job is already active, and this is what lets a repeated call + // heal a job whose orchestrator has died. The signal is deliberately blind. An entity-initiated + // start carries no reuse policy, so the backend decides its fate: it discards the start while the + // target instance exists in any non-completed status, and purges and replaces it once the instance + // has completed, terminated, failed or been canceled. A healthy orchestrator is therefore left + // strictly alone and only a dead one is replaced. + // + // Being blind is the point, not a shortcut. The backend reaches that decision atomically, so + // delegating it removes the race entirely. Checking the orchestrator's status here and signalling + // only when it looked dead would reintroduce the window in which it dies - or recovers - between + // the read and the signal, which is strictly worse than not asking. + context.SignalEntity(context.Id, nameof(this.Run)); + return; + } + + this.State.Status = BlobPurgeJobStatus.Active; + this.State.PurgeBatchSize = purgeBatchSize; + this.State.CreatedAt ??= DateTimeOffset.UtcNow; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + this.State.LastError = null; + + logger.BlobPurgeJobCreated(context.Id.Key); + + // Signal Run to start the perpetual purge orchestrator. + context.SignalEntity(context.Id, nameof(this.Run)); + } + + /// + /// Starts the purge orchestrator if the job is active. + /// + /// + /// + /// The orchestrator runs under a fixed instance ID, which is what keeps the singleton a singleton. No reuse + /// policy is passed, and none can be: the entity's start action has no field to carry one, so anything set + /// here would be dropped before it reached the wire. That default is the behaviour the job relies on rather + /// than an omission - the backend discards a start aimed at an instance that already exists in a + /// non-completed status, and replaces the instance only once it has completed, terminated, failed or been + /// canceled. Signalling this operation is therefore always safe, whatever the orchestrator is doing. + /// + /// + /// This operation deliberately writes no state. It schedules an orchestrator and nothing more, so touching + /// here would overwrite a real change with the time of a + /// call that changed nothing. + /// + /// + /// The entity context. + public void Run(TaskEntityContext context) + { + if (this.State.Status != BlobPurgeJobStatus.Active) + { + return; + } + + string instanceId = BlobPurgeConstants.GetOrchestratorInstanceId(context.Id.Key); + StartOrchestrationOptions startOrchestrationOptions = new(instanceId); + + context.ScheduleNewOrchestration( + new TaskName(nameof(BlobPurgeJobOrchestrator)), + new BlobPurgeJobRunRequest(context.Id, this.State.PurgeBatchSize), + startOrchestrationOptions); + } + + /// + /// Stops the auto-purge job. + /// + /// + /// + /// The perpetual orchestrator is deliberately not terminated here, and this operation does not touch its + /// instance ID at all. The orchestrator reads this entity at the top of every cycle and exits on its own + /// once the job is no longer , so shutdown is cooperative: there is + /// no window in which one party terminates an orchestrator that the other believes is healthy, and the + /// in-flight cycle finishes rather than being cut off part-way through a batch of deletes. + /// + /// + /// , and + /// are preserved. They are the job's history and its + /// configuration, both of which are wanted if it is started again, and keeping CreatedAt is also what + /// distinguishes a stopped job from one that was never started. + /// + /// + /// The entity context. + public void Stop(TaskEntityContext context) + { + if (this.State.Status != BlobPurgeJobStatus.Active) + { + // Load-bearing. Stop is signalled explicitly and blind - the caller does not read the job's state + // first - so a disable aimed at a job that is already stopped, or that was never created, lands + // here. Concurrent callers signalling at once land here for the same reason. Making the no-op + // harmless in the entity is what allows the public API to be a plain signal rather than a + // read-then-signal pair whose two round trips could disagree. + // + // Returning here also leaves the state exactly as it was found, which is what keeps + // LastModifiedAt meaning "when this job stopped" rather than "when a stop was last signalled at + // it". That only holds because no other operation rewrites the field on a no-op either: Run never + // writes it, and Create rewrites it only when the batch size actually differs. Breaking either of + // those breaks this too. It does not avoid materializing the entity - the framework persists + // entity state after every operation, so a stop signal to an entity that does not exist yet + // creates it with default state, which an explicit disable accepts. + logger.BlobPurgeJobAlreadyStopped(context.Id.Key); + return; + } + + this.State.Status = BlobPurgeJobStatus.Pending; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.BlobPurgeJobStopped(context.Id.Key); + } + + /// + /// Marks the job because the backend does not implement the + /// large-payload purge RPCs. + /// + /// + /// This is a real stop, not a pause. The orchestrator reaches it after the fetch or report activity + /// surfaces a gRPC Unimplemented as a , and then exits; nothing + /// restarts the job on its own. Recovery is an explicit call to the public enable API once the backend + /// implements the RPCs - revives an unsupported job deliberately. + /// + /// The entity context. + /// A human-readable description of why the backend is unsupported. + public void MarkUnsupported(TaskEntityContext context, string detail) + { + if (this.State.Status == BlobPurgeJobStatus.Unsupported) + { + // Idempotent no-op. The orchestrator awaits this call and then exits, but concurrent orchestrators + // (one per replica, all hitting the same unsupported backend) can each report before the others + // exit. Leaving the state untouched on the repeat keeps LastModifiedAt meaning "when the job was + // disabled" rather than "when the last replica noticed", mirroring the guard discipline on Stop. + return; + } + + this.State.Status = BlobPurgeJobStatus.Unsupported; + this.State.LastError = detail; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.BlobPurgeJobMarkedUnsupported(context.Id.Key, detail); + } + + /// + /// Records progress after a purge cycle completes. + /// + /// The entity context. + /// The number of blobs purged in the cycle. + public void RecordPurged(TaskEntityContext context, long purgedCount) + { + this.State.PurgedCount += purgedCount; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } + + /// + /// Gets the current state of the auto-purge job. + /// + /// The entity context. + /// The current job state. + public BlobPurgeJobState Get(TaskEntityContext context) => this.State; +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs new file mode 100644 index 00000000..b4ec602f --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Log messages for the Azure Blob externalized-payload auto-purge job. +/// +static partial class Logs +{ + [LoggerMessage(EventId = 810, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' created.")] + public static partial void BlobPurgeJobCreated(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 811, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is already active. Its batch size was updated to the requested one if it differed, and its orchestrator was re-signalled, which starts one only if none is running.")] + public static partial void BlobPurgeJobAlreadyRunning(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] + public static partial void BlobPurgeJobOrchestratorStopping(this ILogger logger, string? jobId, string status); + + [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Blob payload auto-purge quarantined a payload; cause '{cause}', storage code '{storageCode}'. The failure is deterministic and cannot succeed on a retry. The backend preserves the tombstone row and its token as evidence and stops polling it. The reported result carries the disposition alone, so this log is the only record of the cause.")] + public static partial void BlobPurgeDeleteQuarantined(this ILogger logger, string cause, string? storageCode); + + [LoggerMessage(EventId = 814, Level = LogLevel.Debug, Message = "Blob payload auto-purge fetched {count} tombstoned payload(s) from the backend.")] + public static partial void BlobPurgeFetchedTombstones(this ILogger logger, int count); + + [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge reported {count} purge result(s) to the backend.")] + public static partial void BlobPurgeReportedResults(this ILogger logger, int count); + + [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' stopped. The perpetual orchestrator is not terminated; it reads the job state at the start of its next cycle and exits on its own.")] + public static partial void BlobPurgeJobStopped(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; cause '{cause}', storage code '{storageCode}'. The backend reschedules the tombstone for a later attempt. The reported result carries the disposition alone, so this log is the only record of the cause.")] + public static partial void BlobPurgeDeleteRetryable(this ILogger logger, string cause, string? storageCode); + + [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] + public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); + + [LoggerMessage(EventId = 821, Level = LogLevel.Warning, Message = "An externalized payload blob does not carry this store's ownership marker, so it was left untouched; the tombstone is still resolved. This is expected for payloads written before the marker shipped, and for blobs the store never created whose token text matches the payload token grammar.")] + public static partial void BlobPurgeBlobNotStoreOwned(this ILogger logger); + + [LoggerMessage(EventId = 822, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is already stopped; ignoring the stop request.")] + public static partial void BlobPurgeJobAlreadyStopped(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 827, Level = LogLevel.Warning, Message = "Blob payload auto-purge job '{jobId}' was disabled because the backend does not implement the large-payload purge RPCs: {detail}. This is expected against an older backend build or a stale local emulator image. Upgrade the Durable Task backend (or re-pull 'mcr.microsoft.com/dts/dts-emulator'), then enable auto-purge again; the job stays disabled until something enables it.")] + public static partial void BlobPurgeJobMarkedUnsupported(this ILogger logger, string? jobId, string detail); + + [LoggerMessage(EventId = 830, Level = LogLevel.Error, Message = "Blob payload auto-purge for job '{jobId}' stopped because the backend does not implement the large-payload purge RPCs: {detail}. The job is now disabled and will not delete blobs. Upgrade the Durable Task backend (or re-pull 'mcr.microsoft.com/dts/dts-emulator'), then enable auto-purge again to resume.")] + public static partial void BlobPurgeBackendUnsupported(this ILogger logger, string? jobId, string detail); +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs new file mode 100644 index 00000000..edf88b8f --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// State for the singleton blob payload auto-purge job, stored in the entity. +/// +public sealed class BlobPurgeJobState +{ + /// + /// Gets or sets the current status of the auto-purge job. + /// + public BlobPurgeJobStatus Status { get; set; } + + /// + /// Gets or sets the time when the job was first created. + /// + public DateTimeOffset? CreatedAt { get; set; } + + /// + /// Gets or sets the time of the last meaningful change to the job: when it was started, when it was + /// stopped, when it was given a different from the one it already had, or + /// when it last recorded a non-zero number of purged blobs. + /// + /// + /// This is not a liveness or heartbeat signal, and it must not be read as one. A repeated enable call that + /// changes nothing does not move it, and an active job whose cycles keep finding nothing to purge leaves it + /// untouched indefinitely, so a value far in the past is equally consistent with a healthy idle job and a + /// dead one. + /// + public DateTimeOffset? LastModifiedAt { get; set; } + + /// + /// Gets or sets the total number of payload blobs the job has purged. + /// + public long PurgedCount { get; set; } + + /// + /// Gets or sets the last error message, if any. + /// + public string? LastError { get; set; } + + /// + /// Gets or sets the maximum number of tombstoned payloads requested from the backend per cycle. + /// + public int PurgeBatchSize { get; set; } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs new file mode 100644 index 00000000..0d3a5f92 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Represents the current status of the singleton blob payload auto-purge job. +/// +public enum BlobPurgeJobStatus +{ + /// + /// The job is not running. This is both the state of a job that has never been started and the resting + /// state of one that has been stopped, which distinguishes: it is + /// null only for a job that was never created. It is kept as the zero value so a brand-new entity does not + /// accidentally appear active. + /// + Pending, + + /// + /// The job is active and draining tombstoned payloads from the backend. + /// + Active, + + /// + /// The backend does not implement the large-payload purge RPCs, so the job cannot run. This happens against + /// an older backend build or a stale local emulator image that predates the feature. It is kept distinct + /// from so that a job which is down because the backend cannot support it is not + /// mistaken for one that was deliberately stopped. Nothing revives it on its own: recovery is an explicit + /// re-enable once the backend implements the RPCs, and that call's + /// reactivates the job. + /// + Unsupported = 2, +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs new file mode 100644 index 00000000..d4c6291f --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// The outcome of attempting to delete a single externalized payload blob during an auto-purge cycle. The +/// orchestrator pairs it with the tombstone token it kept for that row to build the reported +/// . +/// +/// +/// Carries the disposition alone, because that is the only outcome field the contract reports. Why an attempt +/// reached its disposition is logged by at the point it is +/// classified, at higher fidelity than any value that could be carried here. +/// +/// Whether the row is resolved, should be retried, or must be quarantined. +public sealed record BlobPurgeOutcome(LargePayloadPurgeDisposition Disposition); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs new file mode 100644 index 00000000..f48ebcc2 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -0,0 +1,262 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Orchestrator input describing the purge job to run. +/// +/// The entity ID of the owning . +/// The maximum number of tombstoned payloads to request per cycle. +/// The number of cycles processed since the last continue-as-new. +public sealed record BlobPurgeJobRunRequest( + EntityInstanceId JobEntityId, int PurgeBatchSize, int ProcessedCycles = 0); + +/// +/// Perpetual orchestrator that drains due large-payload tombstones from the backend, deletes their blobs with +/// capped parallelism, and reports every outcome so the backend can resolve, reschedule, or quarantine each +/// row. It idles on a timer when there is nothing to purge and continues-as-new periodically to keep its +/// history small. +/// +[DurableTask] +public class BlobPurgeJobOrchestrator : TaskOrchestrator +{ + const int ContinueAsNewFrequency = 5; + + // Tombstones are deleted a chunk at a time: one activity call handles DeleteChunkSize tokens instead of a + // single token, which is what keeps orchestration history small. A full 1000-row cycle becomes 1000 / 50 = + // 20 activity calls (~40 history events) instead of 1000 calls (~2000 events) - a 50x reduction - and drains + // in ceil(20 / 4) = 5 waves instead of 32. + const int DeleteChunkSize = 50; + + // How many chunk-delete activities run concurrently. TOTAL concurrent storage deletes are this times the + // activity's own MaxParallelDeletesPerChunk (8): 4 x 8 = 32, exactly the flat cap the per-token design used. + // The product is the budget - moving one factor without the other either starves throughput or multiplies + // into hundreds of concurrent storage calls. + const int MaxParallelChunkActivities = 4; + static readonly TimeSpan IdleDelay = TimeSpan.FromMinutes(1); + static readonly TimeSpan ErrorBackoff = TimeSpan.FromMinutes(1); + + // Retry policy for the purge activities: 3 attempts with exponential backoff (15s, 30s, capped at 60s). + static readonly RetryPolicy PurgeActivityRetryPolicy = new( + maxNumberOfAttempts: 3, + firstRetryInterval: TimeSpan.FromSeconds(15), + backoffCoefficient: 2.0, + maxRetryInterval: TimeSpan.FromSeconds(60)) + { + // A NotImplementedException means the backend does not implement the purge RPCs (mixed rollout / stale + // emulator). That cannot be fixed by retrying, so short-circuit the ~45s retry budget and let the + // failure surface on the first attempt. RunAsync catches it and disables the job instead of looping. + HandleFailure = details => !details.IsCausedBy(), + }; + + /// + public override async Task RunAsync(TaskOrchestrationContext context, BlobPurgeJobRunRequest input) + { + ILogger logger = context.CreateReplaySafeLogger(); + string jobId = input.JobEntityId.Key; + + int batchSize = input.PurgeBatchSize; + int processedCycles = input.ProcessedCycles; + + while (true) + { + processedCycles++; + if (processedCycles > ContinueAsNewFrequency) + { + context.ContinueAsNew(new BlobPurgeJobRunRequest(input.JobEntityId, batchSize, ProcessedCycles: 0)); + return null!; + } + + try + { + // Stop cleanly if the job has been stopped or removed. BlobPurgeJob.Stop is what makes this + // reachable: it moves the entity off Active without touching this orchestrator at all, so + // shutdown is cooperative - the in-flight cycle finishes and the loop exits on its own terms + // rather than being terminated part-way through a batch of deletes. + // input: null is named deliberately. A bare positional null binds to the (id, name, options) + // overload instead, which reads as if an input were being passed when it is not. + BlobPurgeJobState? state = await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.Get), input: null); + + if (state is null || state.Status != BlobPurgeJobStatus.Active) + { + logger.BlobPurgeJobOrchestratorStopping(jobId, state?.Status.ToString() ?? "null"); + return null; + } + + // Take the batch size from the entity rather than from this orchestrator's input. A perpetual + // orchestrator outlives configuration changes: its input is fixed when it is created and is + // carried verbatim through every continue-as-new, so using input.PurgeBatchSize would pin the + // value written by the very first Create for the entire life of the job. Re-reading it from the + // state fetch this cycle already performs costs no extra call and is what lets a changed batch + // size actually take effect. That matters because a batch size the backend rejects fails every + // fetch: without this the job would be wedged with no recovery short of deleting the entity. + // + // Fall back to the input when the stored value is not positive. An entity written by an older + // build carries no batch size at all, and asking the backend for zero rows every cycle would be + // a silent, permanent stall. + int cycleBatchSize = state.PurgeBatchSize > 0 ? state.PurgeBatchSize : batchSize; + + List tombstones = await context.CallActivityAsync>( + nameof(GetLargePayloadTombstonesActivity), + cycleBatchSize, + new TaskOptions(PurgeActivityRetryPolicy)); + + if (tombstones is null || tombstones.Count == 0) + { + // Nothing to purge right now: block on a timer (push-free idle) then check again. + await context.CreateTimer(IdleDelay, default); + continue; + } + + List results = await this.DeleteBatchAsync(context, tombstones); + + // Every attempted row produces a result, including the retryable ones: the backend owns retry + // scheduling, so it needs to hear about a failure to defer the row. Reporting unconditionally + // is what keeps a failing row from being re-served unchanged on the very next cycle. + await context.CallActivityAsync( + nameof(ReportLargePayloadPurgeResultsActivity), + results, + new TaskOptions(PurgeActivityRetryPolicy)); + + // Two different questions, deliberately not conflated. Progress counts only payloads that were + // actually purged; the backoff decision asks whether ANY row left the retry queue, because a + // quarantined row also stops being re-served even though nothing was reclaimed. + int purged = CountDisposition(results, LargePayloadPurgeDisposition.Deleted); + int resolved = results.Count - CountDisposition(results, LargePayloadPurgeDisposition.Retry); + + if (purged > 0) + { + await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)purged); + } + + if (resolved == 0) + { + // The whole batch came back retryable (e.g. a storage outage or throttling). Deletes report + // failure as a return value rather than an exception, so no activity retry policy applies on + // that path. Continuing immediately would refetch and re-attempt in a tight loop for as long + // as the outage lasts, so back off before the next cycle. + await context.CreateTimer(ErrorBackoff, default); + } + } + catch (TaskFailedException ex) when (ex.FailureDetails.IsCausedBy()) + { + // The backend does not implement the large-payload purge RPCs (an older backend build or a + // stale local emulator image). Retrying cannot help, so disable the job durably and exit the + // perpetual loop cleanly instead of logging a generic failure and backing off forever. + // MarkUnsupported is AWAITED, not signalled, so the disable is committed to the entity before we + // return. Nothing restarts the job automatically: it stays Unsupported until a caller + // explicitly enables auto-purge again, and that call's Create revives it. This is deliberately + // a distinct diagnostic from BlobPurgeCycleFailed below. + logger.BlobPurgeBackendUnsupported(jobId, ex.FailureDetails.ErrorMessage); + await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.MarkUnsupported), ex.FailureDetails.ErrorMessage); + return null; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + // A single bad cycle (transient backend/entity/activity failure) must not kill the perpetual + // loop. Log, back off, then continue so the job self-heals and keeps draining. + logger.BlobPurgeCycleFailed(ex, jobId); + await context.CreateTimer(ErrorBackoff, default); + continue; + } + } + } + + static int CountDisposition( + List results, LargePayloadPurgeDisposition disposition) + { + int count = 0; + foreach (LargePayloadPurgeResult result in results) + { + if (result.Disposition == disposition) + { + count++; + } + } + + return count; + } + + static async Task DrainAsync( + List>> tasks, List results) + { + List[] completed = await Task.WhenAll(tasks); + foreach (List chunkResults in completed) + { + results.AddRange(chunkResults); + } + } + + async Task> DeleteBatchAsync( + TaskOrchestrationContext context, List tombstones) + { + List results = new(tombstones.Count); + List>> inFlight = new(); + + for (int start = 0; start < tombstones.Count; start += DeleteChunkSize) + { + int count = Math.Min(DeleteChunkSize, tombstones.Count - start); + List chunk = tombstones.GetRange(start, count); + inFlight.Add(this.DeleteChunkAsync(context, chunk)); + + if (inFlight.Count >= MaxParallelChunkActivities) + { + await DrainAsync(inFlight, results); + inFlight.Clear(); + } + } + + if (inFlight.Count > 0) + { + await DrainAsync(inFlight, results); + } + + return results; + } + + async Task> DeleteChunkAsync( + TaskOrchestrationContext context, List chunk) + { + List tokens = new(chunk.Count); + foreach (LargePayloadTombstone tombstone in chunk) + { + tokens.Add(tombstone.PayloadToken); + } + + List outcomes = await context.CallActivityAsync>( + nameof(DeleteExternalBlobActivity), + tokens, + new TaskOptions(PurgeActivityRetryPolicy)); + + // The activity contract is one outcome per input token, positionally aligned. Assert that before zipping + // so a broken contract fails loudly here instead of silently pinning each disposition onto the wrong row + // - which would tell the backend to delete, retry, or quarantine the wrong payloads. + if (outcomes is null || outcomes.Count != chunk.Count) + { + throw new InvalidOperationException( + $"The blob delete activity returned {outcomes?.Count ?? 0} outcomes for a chunk of {chunk.Count} " + + "tokens; expected exactly one per token. Refusing to attribute dispositions to the wrong rows."); + } + + List results = new(chunk.Count); + for (int i = 0; i < chunk.Count; i++) + { + LargePayloadTombstone tombstone = chunk[i]; + + // The tombstone token is echoed back unchanged: it is opaque to the SDK and is the only thing that + // tells the backend which row this disposition belongs to. Retry scheduling is the backend's job, + // so no next-attempt time is computed here. + results.Add(new LargePayloadPurgeResult(tombstone.TombstoneToken, outcomes[i].Disposition)); + } + + return results; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/RebindableCallInvoker.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/RebindableCallInvoker.cs new file mode 100644 index 00000000..7623ba74 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/RebindableCallInvoker.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// A that forwards every call to whichever invoker is currently bound, and can be +/// re-bound to a replacement at any time. +/// +/// +/// The purge activities reach the backend over the worker's own transport rather than a second +/// connection. That transport is not stable for the life of the process: the worker recreates its gRPC channel +/// when the existing one is wedged, and the invoker it builds is wrapped with the configured interceptors +/// (authentication among them). A client constructed once against a captured invoker would therefore keep +/// using a disposed channel after a recreate. This indirection is what lets a singleton +/// LargePayloadPurgeClient - already constructed, already injected into an activity - route its next +/// call through the worker's replacement invoker. +/// The worker publishes into this type through the internal call-invoker publisher hook, always with the +/// effective post-interceptor invoker, so callers here never bypass the configured chain. +/// Calls in flight when a rebind happens complete against the invoker they already captured; only +/// subsequent calls observe the replacement. The worker defers disposal of the previous channel for exactly +/// this reason. +/// +sealed class RebindableCallInvoker : CallInvoker +{ + // Written by the worker (startup and each successful channel recreate) and read by activity call sites on + // other threads. Volatile access publishes the reference without a lock; each call reads it exactly once so + // a rebind cannot tear a single call across two invokers. + CallInvoker? current; + + /// + /// Binds (or re-binds) the invoker that subsequent calls are forwarded to. + /// + /// The invoker to forward to. + public void Rebind(CallInvoker invoker) + { + Volatile.Write(ref this.current, Check.NotNull(invoker)); + } + + /// + public override TResponse BlockingUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => this.Current().BlockingUnaryCall(method, host, options, request); + + /// + public override AsyncUnaryCall AsyncUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => this.Current().AsyncUnaryCall(method, host, options, request); + + /// + public override AsyncServerStreamingCall AsyncServerStreamingCall( + Method method, string? host, CallOptions options, TRequest request) + => this.Current().AsyncServerStreamingCall(method, host, options, request); + + /// + public override AsyncClientStreamingCall AsyncClientStreamingCall( + Method method, string? host, CallOptions options) + => this.Current().AsyncClientStreamingCall(method, host, options); + + /// + public override AsyncDuplexStreamingCall AsyncDuplexStreamingCall( + Method method, string? host, CallOptions options) + => this.Current().AsyncDuplexStreamingCall(method, host, options); + + CallInvoker Current() + { + // Fail loudly rather than inventing a transport. Reaching here means an activity ran before the worker + // published its invoker, which cannot happen through the normal path (work items only arrive after the + // worker has connected) - so a null here is a wiring defect worth surfacing, not a race to paper over. + return Volatile.Read(ref this.current) ?? throw new InvalidOperationException( + "The Durable Task worker has not published a gRPC transport yet, so externalized payloads cannot be " + + "purged. This indicates the purge activities were invoked outside a running gRPC worker."); + } +} diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index d8607ec6..9f859b55 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Client.Grpc; using Microsoft.DurableTask.Converters; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; namespace Microsoft.DurableTask; @@ -12,8 +14,34 @@ namespace Microsoft.DurableTask; /// /// Extension methods to enable externalized payloads using Azure Blob Storage for Durable Task Client. /// +/// +/// Externalized payloads are configured per host, not per named builder. The PayloadStore is registered +/// as a container-wide singleton (shared with the worker builder in the same host), so the first builder that +/// calls UseExternalizedPayloads supplies the configuration the whole host uses. Configuring multiple +/// named clients in the same host with different storage accounts or different backends is therefore not +/// supported: later builders silently share the first builder's registration. A single named client, or +/// several named clients that share one configuration, is fully supported. +/// public static class DurableTaskClientBuilderExtensionsAzureBlobPayloads { + /// + /// Enables externalized payload storage using Azure Blob Storage for the specified client builder. + /// + /// The builder to configure. + /// The callback to configure the storage options. + /// The original builder, for call chaining. + public static IDurableTaskClientBuilder UseExternalizedPayloads( + this IDurableTaskClientBuilder builder, + Action configure) + { + Check.NotNull(builder); + Check.NotNull(configure); + + builder.Services.Configure(builder.Name, configure); + + return UseExternalizedPayloadsCore(builder); + } + /// /// Enables externalized payload storage using a pre-configured shared payload store. /// This overload helps ensure client and worker use the same configuration. @@ -29,6 +57,15 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientBuilder builder) { + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the worker builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + // Wrap the gRPC CallInvoker with our interceptor when using the gRPC client builder.Services .AddOptions(builder.Name) @@ -42,6 +79,16 @@ static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientB opt.Interceptors.Add(new AzureBlobPayloadsSideCarInterceptor(store, opts)); }); + // The explicit auto-purge API (SetLargePayloadAutoPurgeAsync) reaches the singleton job through + // client.Entities on BOTH paths - enabling signals Create, disabling signals Stop - so entity support + // must be on whenever externalized payloads are configured. Set it on the base options so an explicit + // UseGrpc client that disables entity support still wins (DurableTaskClientOptions.ApplyTo copies this + // value only when the derived options did not set it explicitly). + builder.Configure(options => + { + options.EnableEntitySupport = true; + }); + return builder; } } diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index e1f8387d..c27a1360 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -1,11 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Microsoft.DurableTask.Converters; +using Grpc.Core; +using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; +using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; +using static Microsoft.DurableTask.Protobuf.LargePayloads.LargePayloadPurge; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask; @@ -13,6 +17,14 @@ namespace Microsoft.DurableTask; /// /// Extension methods to enable externalized payloads using Azure Blob Storage for Durable Task Worker. /// +/// +/// Externalized payloads are configured per host, not per named builder. The PayloadStore and the +/// purge LargePayloadPurgeClient are registered as container-wide singletons, so the first builder +/// in the host that calls UseExternalizedPayloads supplies the configuration that both of them use. +/// Configuring multiple named workers in the same host with different storage accounts or different backends +/// is therefore not supported: later builders silently share the first builder's registration. A single named +/// worker, or several named workers that share one configuration, is fully supported. +/// public static class DurableTaskWorkerBuilderExtensionsAzureBlobPayloads { /// @@ -29,11 +41,6 @@ public static IDurableTaskWorkerBuilder UseExternalizedPayloads( Check.NotNull(configure); builder.Services.Configure(builder.Name, configure); - builder.Services.AddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); return UseExternalizedPayloadsCore(builder); } @@ -53,10 +60,20 @@ public static IDurableTaskWorkerBuilder UseExternalizedPayloads( static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerBuilder builder) { + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the client builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + // Wrap the gRPC CallInvoker with our interceptor when using the gRPC worker builder.Services .AddOptions(builder.Name) - .PostConfigure>((opt, store, monitor) => + .PostConfigure, RebindableCallInvoker>( + (opt, store, monitor, purgeInvoker) => { LargePayloadStorageOptions opts = monitor.Get(builder.Name); @@ -66,8 +83,55 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB opt.Interceptors.Add(new AzureBlobPayloadsSideCarInterceptor(store, opts)); opt.Capabilities.Add(P.WorkerCapability.LargePayloads); + + // Follow the worker's transport instead of capturing one. The worker publishes its effective + // post-interceptor invoker here at startup and after every channel recreate, which is what + // keeps the purge activities on the live channel and inside the configured auth chain. + opt.SetCallInvokerPublisher(purgeInvoker.Rebind); + }); + + // The auto-purge job is entity-driven: its orchestrator drives the BlobPurgeJob entity, and an + // orchestrator that touches entities with support off throws (TaskOrchestrationContextWrapper). Enable + // it whenever externalized payloads are configured - mirroring the client side - so the job can run + // whenever a client has turned the feature on, whichever host that client lives in. + builder.Services + .AddOptions(builder.Name) + .Configure(options => + { + options.EnableEntitySupport = true; }); + // The purge activities talk to the backend over the worker's OWN transport, so a worker-only host + // (which never registers a DurableTaskClient) can still run the job. The worker's transport is not + // fixed for the life of the process - it recreates its channel when the current one is wedged, and the + // invoker it hands out is the one left after the configured interceptors (auth included) have been + // applied. Capturing options.CallInvoker/Channel here would therefore take a RAW invoker, skip the + // interceptors, reject the Address-only configuration outright, and keep pointing at channel A after + // the worker had moved to channel B. Instead, register an indirection the worker publishes into and + // build the client on that. + // TryAddSingleton (rather than a keyed/named registration) is deliberate: the consumers - + // GetLargePayloadTombstonesActivity and ReportLargePayloadPurgeResultsActivity - are constructed from + // the plain IServiceProvider at dispatch with no worker name in scope, so a keyed registration would + // have no resolvable consumer. This is the per-host single-configuration constraint documented on the + // class remarks: in a multi-named-worker host the first builder's options win here. Do not "fix" this + // into keyed DI - without a worker-name-aware consumer there is nothing to resolve the keyed client. + builder.Services.TryAddSingleton(); + builder.Services.TryAddSingleton( + sp => new LargePayloadPurgeClient(sp.GetRequiredService())); + + // Register the entity/orchestrator/activities that run the singleton auto-purge job. These are ALWAYS + // registered (never gated on configuration) so that a job a client has turned on always has something + // to execute here. Workers never call SetLargePayloadAutoPurge - the setting is owned by the explicit + // client API - but they do fetch and report via the worker's LargePayloadPurgeClient above. + builder.AddTasks(r => + { + r.AddEntity(); + r.AddOrchestrator(); + r.AddActivity(); + r.AddActivity(); + r.AddActivity(); + }); + return builder; } } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index f519b196..0601751d 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -23,8 +23,33 @@ namespace Microsoft.DurableTask; Justification = "SemaphoreSlim does not allocate a disposable resource unless AvailableWaitHandle is accessed.")] public sealed class BlobPayloadStore : PayloadStore { - const string TokenPrefixV1 = "blob:v1:"; - const string TokenPrefixV2 = "blob:v2:"; + /// + /// The prefix of legacy v1 payload tokens, which identify the container by name only and not the storage + /// account. Auto-purge uses this to detect and skip v1 tokens. + /// + internal const string TokenPrefixV1 = "blob:v1:"; + + /// + /// The prefix of self-describing v2 payload tokens, which carry the blob's absolute URI including the + /// storage account. Auto-purge uses this to tell a malformed v2 token (a protocol defect) apart from an + /// unrecognized version prefix (a token written by a newer SDK). + /// + internal const string TokenPrefixV2 = "blob:v2:"; + + /// + /// The metadata name of the ownership marker written on every blob this store creates. Recognizing a + /// token proves only that its text matches the store's grammar; the marker is what proves the store + /// actually wrote the blob, so a customer's own blob is never deleted just because an orchestration + /// referenced it. Azure requires metadata names to follow the naming rules for C# identifiers, so the + /// marker is spelled with an underscore rather than a hyphen. + /// + internal const string OwnershipMarkerName = "managed_by"; + + /// + /// The fixed value of the ownership marker written on every blob this store creates. + /// + internal const string OwnershipMarkerValue = "dts"; + const string ContentEncodingGzip = "gzip"; const int MaxRetryAttempts = 8; const int BaseDelayMs = 250; @@ -120,6 +145,7 @@ public override async Task UploadAsync(string payLoad, CancellationToken BlobOpenWriteOptions writeOptions = new() { HttpHeaders = new BlobHttpHeaders { ContentEncoding = ContentEncodingGzip }, + Metadata = CreateOwnershipMetadata(), }; using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, cancellationToken); using GZipStream compressedBlobStream = new(blobStream, System.IO.Compression.CompressionLevel.Optimal, leaveOpen: true); @@ -133,7 +159,15 @@ public override async Task UploadAsync(string payLoad, CancellationToken } else { - using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); + // The uncompressed path still needs write options purely to carry the ownership marker: + // the marker must be written by every path that creates a blob, or auto-purge would later + // decline to delete the store's own uncompressed payloads. It rides along in the PUT the + // upload already issues, so it costs no extra request. + BlobOpenWriteOptions writeOptions = new() + { + Metadata = CreateOwnershipMetadata(), + }; + using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, cancellationToken); // using MemoryStream payloadStream = new(payloadBuffer, writable: false); // await payloadStream.CopyToAsync(blobStream, bufferSize: DefaultCopyBufferSize, cancellationToken); @@ -198,6 +232,90 @@ public override async Task DownloadAsync(string token, CancellationToken return await DownloadFromBlobAsync(blob, cancellationToken); } + /// + public override async Task DeleteAsync(string token, CancellationToken cancellationToken) + { + DecodeTokenResult decoded = DecodeToken(token); + + BlobClient blob; + if (!decoded.IsV2) + { + // v1 tokens do not carry the account, so the payload is assumed to live in the configured container. + if (!string.Equals(decoded.Container, this.containerClient.Name, StringComparison.Ordinal)) + { + throw new ArgumentException("Token container does not match configured container.", nameof(token)); + } + + blob = this.containerClient.GetBlobClient(decoded.Name); + } + else if (this.IsConfiguredContainer(decoded.ContainerUri!)) + { + // Same account and container as the configured store: reuse it (works with any auth mode). + blob = this.containerClient.GetBlobClient(decoded.Name); + } + else if (this.options.Credential != null) + { + // The payload lives in a different account (e.g. the store was repointed). Identity auth can still + // delete it as long as the credential has RBAC access to that account. + blob = new BlobClient(decoded.BlobUri, this.options.Credential, this.clientOptions); + } + else + { + throw new PayloadStorageException( + $"The externalized payload lives in a different storage account ('{decoded.ContainerUri}') than the " + + $"currently-configured payload store ('{this.containerClient.Uri}'). Cross-account payload deletes " + + "require identity (AAD) authentication with access to both accounts; connection-string / " + + "account-key credentials are account-specific and cannot delete in another account."); + } + + // Recognizing the token proves only that its text matches this store's grammar - not that this store + // wrote the blob. A customer may keep an expensive dataset in Blob Storage and have orchestrations + // reference it by URL; deleting that would destroy data the store never created. So ownership is read + // from the object itself before anything is deleted. + BlobProperties properties; + try + { + Response response = await blob.GetPropertiesAsync( + conditions: null, cancellationToken: cancellationToken); + properties = response.Value; + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) + { + // Already gone. Deletion is idempotent, so a re-delivered tombstone or a concurrent worker + // replica that won the race is a success, not an error. + return PayloadDeleteOutcome.AlreadyAbsent; + } + + if (!HasOwnershipMarker(properties.Metadata)) + { + // Positive evidence that the blob is customer-owned: leave it untouched. The caller still resolves + // the payload reference, because a blob this store never wrote is not this store's to delete. + return PayloadDeleteOutcome.NotStoreOwned; + } + + // Pair the ownership read with the delete using the ETag from that same read. If anything rewrites the + // blob in between - including a customer overwriting it with content that no longer carries the marker + // - the If-Match condition fails the delete instead of removing the newer content, so the read-then- + // delete behaves as a single check-and-delete without taking a lease. + // Idempotent by design: DeleteIfExistsAsync returns false (rather than throwing) when the blob is + // already gone, so re-delivered tombstones and concurrent purges from multiple worker replicas are safe. + // IncludeSnapshots removes the blob's snapshots along with the base blob, but it does NOT delete blob + // *versions*: with versioning enabled, deleting the base blob turns the current version into a retained + // previous version. Versions are deliberately not enumerated and deleted here - doing so still would not + // guarantee the bytes are reclaimed, because blob soft delete is a storage-account-level policy that + // retains deleted content for its retention period regardless of how the delete was issued and regardless + // of which container the blob lives in. Immediate reclamation is therefore unobtainable client-side and + // belongs to an account lifecycle-management policy (whose rule can be scoped to the payload container's + // blob prefix) or the retention expiry, so Deleted means "accepted by storage", not "bytes reclaimed" + // (see PayloadDeleteOutcome.Deleted). + Response deleted = await blob.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + conditions: new BlobRequestConditions { IfMatch = properties.ETag }, + cancellationToken: cancellationToken); + + return deleted.Value ? PayloadDeleteOutcome.Deleted : PayloadDeleteOutcome.AlreadyAbsent; + } + /// public override bool IsKnownPayloadToken(string value) { @@ -261,6 +379,38 @@ internal static DecodeTokenResult DecodeToken(string token) throw new ArgumentException("Invalid external payload token.", nameof(token)); } + /// + /// Creates the ownership metadata stamped on every blob this store writes, so a later purge can prove the + /// store created the blob before deleting it. + /// + static Dictionary CreateOwnershipMetadata() => + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [OwnershipMarkerName] = OwnershipMarkerValue, + }; + + /// + /// Returns whether the supplied blob metadata carries this store's ownership marker. Azure treats metadata + /// names as case-insensitive, so the lookup is too. + /// + static bool HasOwnershipMarker(IDictionary? metadata) + { + if (metadata is null) + { + return false; + } + + foreach (KeyValuePair entry in metadata) + { + if (string.Equals(entry.Key, OwnershipMarkerName, StringComparison.OrdinalIgnoreCase)) + { + return string.Equals(entry.Value, OwnershipMarkerValue, StringComparison.Ordinal); + } + } + + return false; + } + static async Task WritePayloadAsync(byte[] payloadBuffer, Stream target, CancellationToken cancellationToken) { #if NETSTANDARD2_0 diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs new file mode 100644 index 00000000..afc95115 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask; + +/// +/// The outcome of deleting a payload through . All three values are +/// successful terminal outcomes; failures are surfaced as exceptions instead. +/// +public enum PayloadDeleteOutcome +{ + /// + /// The delete was accepted by storage on this call - the backing object's current version was removed. + /// This means the delete was accepted, not that the underlying bytes were reclaimed: if the storage account + /// has blob versioning or blob soft delete enabled, the prior version or the soft-deleted blob is retained + /// until a lifecycle-management policy or the configured retention period removes it. + /// + Deleted, + + /// + /// The payload's backing object was already absent. Deletion is idempotent, so this is a success. + /// + AlreadyAbsent, + + /// + /// The backing object exists but does not carry the store's ownership marker, so the store did not + /// create it and left it untouched. The payload reference is still resolved, because an object the + /// store never wrote is not the store's to delete. + /// + NotStoreOwned, +} diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs index b0fe6f80..92636789 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs @@ -24,6 +24,32 @@ public abstract class PayloadStore /// Payload string. public abstract Task DownloadAsync(string token, CancellationToken cancellationToken); + /// + /// Deletes the payload referenced by the token. Implementations that support deletion must be + /// idempotent: deleting a payload that no longer exists is a no-op and must not throw. + /// + /// + /// The default implementation throws . Stores that externalize + /// payloads to deletable storage (for example Azure Blob Storage) should override it. It is declared + /// virtual rather than abstract so that adding it does not break existing external subclasses. + /// Implementations must delete only objects they created; an object that carries no proof of the + /// store's ownership must be left untouched and reported as + /// . + /// A result means the store no longer references the object + /// and the underlying storage accepted the delete; it does not guarantee that the bytes have been + /// reclaimed. Storage-level retention features such as versioning, soft delete, or retention policies + /// may keep the content for a policy-defined period, and implementations are not expected to defeat them. + /// + /// The opaque reference token. + /// Cancellation token. + /// + /// The outcome of the deletion: whether the object was deleted, was already absent, or was left in + /// place because the store does not own it. + /// + public virtual Task DeleteAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException( + $"This {nameof(PayloadStore)} implementation does not support deleting payloads."); + /// /// Returns true if the specified value appears to be a token understood by this store. /// Implementations should not throw for unknown tokens. diff --git a/src/Grpc/durable-task-scheduler/large_payload_purge.proto b/src/Grpc/durable-task-scheduler/large_payload_purge.proto new file mode 100644 index 00000000..9d9e222a --- /dev/null +++ b/src/Grpc/durable-task-scheduler/large_payload_purge.proto @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +syntax = "proto3"; + +package microsoft.durabletask.largepayloads; + +option csharp_namespace = "Microsoft.DurableTask.Protobuf.LargePayloads"; + +// Blob auto-purge for externalized large payloads. +// +// A payload too large to store inline is written to blob storage and the backend keeps only a token. +// Purging the instance cannot reclaim that blob on its own: the blob lives in the customer's storage +// account and only the worker holds credentials for it. This service is how the backend hands that +// deletion to the worker and learns the outcome. +// +// The contract is a ledger, not a stream. Purge writes one tombstone per externalized payload, the +// worker fetches the tombstones that are due, deletes the blobs, and reports each outcome. The +// backend owns retry scheduling and row lifetime; the worker owns the delete and the classification +// of its result, and never computes a retry delay. +service LargePayloadPurge { + // Explicitly sets large-payload blob auto-purge for the caller's authenticated task hub. This + // operation is invoked by a Durable Task client instance; it is not a worker-connection handshake. + // + // This is a setting, not a WorkerCapability: a worker being able to resolve externalized payloads + // is not the same as the customer opting into deleting them, and a capability list is presence-only + // so it cannot carry an explicit false. + // + // The last successful explicit call wins. Conflicting values from multiple client instances are + // the callers' responsibility; the backend performs no coordination. Not calling this operation + // leaves the stored value untouched. + // + // Enabling allows instance purge to preserve externalized payload tokens as tombstones and allows + // due tombstones to be fetched. Disabling stops creation of new tombstones and makes + // GetLargePayloadTombstones return no new work, while preserving existing tombstones for a later + // re-enable. Calls and fetches already in flight may complete. + // + // An optional purge batch size exposed by a client SDK configures its singleton purge job + // separately. It is not part of this backend setting and is intentionally absent from this RPC. + rpc SetLargePayloadAutoPurge(SetLargePayloadAutoPurgeRequest) returns (SetLargePayloadAutoPurgeResponse); + + // Returns a bounded, deterministically ordered batch of due large-payload tombstones whose + // external blobs the worker must delete. Scoped to the caller's authenticated task hub. + // Only rows that are pending and whose next attempt time has arrived are returned; a row stays + // pending until its outcome is reported, so this is safe under retries and duplicate callers. + // Returns no work while large-payload auto-purge is disabled. + rpc GetLargePayloadTombstones(GetLargePayloadTombstonesRequest) returns (GetLargePayloadTombstonesResponse); + + // Reports the outcome of each attempted blob deletion. The backend owns retry scheduling and + // branches solely on `disposition`: it deletes rows reported as DELETED, reschedules RETRY on + // its own backoff, and moves QUARANTINED rows out of the active fetch while preserving their + // evidence. The worker never computes a retry delay. + rpc ReportLargePayloadPurgeResults(ReportLargePayloadPurgeResultsRequest) returns (ReportLargePayloadPurgeResultsResponse); +} + +// client -> server: an explicit task-hub auto-purge setting from a Durable Task client instance. +message SetLargePayloadAutoPurgeRequest { + // The customer's explicit choice. Omitting the RPC leaves the stored value untouched, so there is + // no third state to encode on the wire. + bool enabled = 1; +} + +// server -> client: acknowledgement that the setting was recorded. +message SetLargePayloadAutoPurgeResponse { +} + +// server -> client: one tombstoned large-payload row whose external blob the worker must delete. +message LargePayloadTombstone { + // Opaque, backend-issued correlation token for this exact tombstone version. The worker must not + // interpret its format and must echo it unchanged in the purge result. + string tombstone_token = 1; + + // A self-describing SDK v2 token: "blob:v2:{fullBlobUrl}". + // Legacy v1 tokens are never tombstoned: v1 carries a container name but not the storage + // account, so a delete against the configured account cannot be verified. The backend + // hard-deletes v1 payload rows instead. + string payload_token = 2; +} + +// The outcome of a single blob deletion attempt. The split is by whether a failure can self-heal. +enum LargePayloadPurgeDisposition { + // Required: proto3 reserves 0 as the first value, and scalars have no field presence, so an + // unset field arrives as 0. Keeping 0 meaningless is load-bearing here: if 0 meant DELETED, a + // client that failed to set this field would make the backend delete tombstones and orphan the + // blobs permanently. The backend must reject a result carrying this value. + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; + + // Terminal success: the tombstone is resolved and the backend deletes it. Covers the blob being + // deleted, the blob already being absent, and the blob being deliberately left in place because + // the payload store does not own it. All three are terminal because none of them can be + // improved by trying again. + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; + + // The failure may self-heal, so the row stays pending and the backend sets the next attempt. + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2; + + // A deterministic failure or protocol violation that retrying can never fix. The row leaves the + // polling set but is never deleted or expired: it keeps the token, which after the payload row is + // gone is the only durable record of the blob, so discarding it would orphan the blob silently. + // Resolving a quarantined row is a deliberate operator action. Why it failed is not recorded here + // and is not meant to be; that detail lives in the worker's telemetry at full fidelity. + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; +} + +// client -> server: the outcome of exactly one tombstoned row. +message LargePayloadPurgeResult { + // Echoed unchanged from the corresponding LargePayloadTombstone. + string tombstone_token = 1; + + // The only field the backend acts on. Deliberately the only outcome field on this message: + // anything finer would be write-only. Failure detail stays in the worker's own telemetry, which + // holds the full exception rather than a lossy classification. + LargePayloadPurgeDisposition disposition = 2; +} + +// client -> server: request up to `limit` due tombstones for the caller's task hub. +message GetLargePayloadTombstonesRequest { + // The maximum number of rows to return. The service clamps this to its own maximum. + int32 limit = 1; +} + +// server -> client: the due tombstones whose blobs the worker must delete. +message GetLargePayloadTombstonesResponse { + repeated LargePayloadTombstone tombstones = 1; +} + +// client -> server: a bounded batch of purge outcomes. +message ReportLargePayloadPurgeResultsRequest { + repeated LargePayloadPurgeResult results = 1; +} + +// server -> client: acknowledgement that the reported outcomes were recorded. +message ReportLargePayloadPurgeResultsResponse { +} diff --git a/src/Grpc/refresh-protos.ps1 b/src/Grpc/refresh-protos.ps1 index ffa0d529..e5d4dc39 100644 --- a/src/Grpc/refresh-protos.ps1 +++ b/src/Grpc/refresh-protos.ps1 @@ -23,6 +23,9 @@ $protoFiles = @( }, @{ SourcePath = "durable-task-scheduler/sandbox_service.proto" + }, + @{ + SourcePath = "durable-task-scheduler/large_payload_purge.proto" } ) diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 5dd18d52..439f07a0 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -318,27 +318,15 @@ async ValueTask BuildRuntimeStateAsync( async Task> ConnectAsync(CancellationToken cancellation) { - TimeSpan helloDeadline = this.internalOptions.HelloDeadline; - DateTime? deadline = null; - - if (helloDeadline > TimeSpan.Zero) - { - // Clamp to a UTC DateTime.MaxValue so a misconfigured (very large) HelloDeadline cannot - // throw ArgumentOutOfRangeException out of DateTime.Add and so the gRPC deadline remains - // unambiguous during internal normalization. - DateTime now = DateTime.UtcNow; - DateTime maxDeadlineUtc = DateTime.SpecifyKind(DateTime.MaxValue, DateTimeKind.Utc); - TimeSpan maxOffset = maxDeadlineUtc - now; - deadline = helloDeadline >= maxOffset ? maxDeadlineUtc : now.Add(helloDeadline); - } - - await this.client!.HelloAsync(EmptyMessage, deadline: deadline, cancellationToken: cancellation); - this.Logger.EstablishedWorkItemConnection(); + await this.client!.HelloAsync( + EmptyMessage, + deadline: this.NextHelloDeadline(), + cancellationToken: cancellation); DurableTaskWorkerOptions workerOptions = this.worker.workerOptions; // Get the stream for receiving work-items - return this.client!.GetWorkItems( + AsyncServerStreamingCall stream = this.client!.GetWorkItems( new P.GetWorkItemsRequest { MaxConcurrentActivityWorkItems = @@ -351,6 +339,34 @@ async ValueTask BuildRuntimeStateAsync( WorkItemFilters = this.worker.workItemFilters?.ToGrpcWorkItemFilters(), }, cancellationToken: cancellation); + + // Logged last, not straight after Hello: the message claims a work-item streaming connection, so it + // must not be emitted until the call that opens one has actually been created. A synchronous throw + // out of GetWorkItems would otherwise leave a retry loop reporting connections that never existed. + this.Logger.EstablishedWorkItemConnection(); + return stream; + } + + /// + /// Computes a fresh absolute deadline for the Hello handshake from the configured + /// HelloDeadline interval. + /// + /// The absolute UTC deadline, or null when the deadline is disabled. + DateTime? NextHelloDeadline() + { + TimeSpan interval = this.internalOptions.HelloDeadline; + if (interval <= TimeSpan.Zero) + { + return null; + } + + // Clamp to a UTC DateTime.MaxValue so a misconfigured (very large) HelloDeadline cannot + // throw ArgumentOutOfRangeException out of DateTime.Add and so the gRPC deadline remains + // unambiguous during internal normalization. + DateTime now = DateTime.UtcNow; + DateTime maxDeadlineUtc = DateTime.SpecifyKind(DateTime.MaxValue, DateTimeKind.Utc); + TimeSpan maxOffset = maxDeadlineUtc - now; + return interval >= maxOffset ? maxDeadlineUtc : now.Add(interval); } async Task ProcessWorkItemsAsync( diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.cs index 1ebd6294..b034a465 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.cs @@ -6,6 +6,7 @@ using Microsoft.DurableTask.Worker.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; namespace Microsoft.DurableTask.Worker.Grpc; @@ -69,6 +70,7 @@ public GrpcDurableTaskWorker( protected override async Task ExecuteAsync(CancellationToken stoppingToken) { AsyncDisposable workerOwnedChannelDisposable = this.GetCallInvoker(out CallInvoker callInvoker, out string address); + this.PublishCallInvoker(callInvoker); // Seed the tracker from the configured channel once, then update latestObservedChannel after // each successful recreate. Do not re-read this.grpcOptions.Channel inside the loop: the options @@ -81,7 +83,15 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) while (!stoppingToken.IsCancellationRequested) { - Processor processor = new(this, new(callInvoker), this.orchestrationFilter, this.ExceptionPropertiesProvider); + // Built on the SAME CallInvoker as the sidecar client so the DTS-only purge service the + // activities use rides the worker's existing channel and interceptors rather than opening a + // second connection. The worker itself never calls SetLargePayloadAutoPurge; that setting is + // owned by the explicit client API. + Processor processor = new( + this, + new TaskHubSidecarServiceClient(callInvoker), + this.orchestrationFilter, + this.ExceptionPropertiesProvider); ProcessorExitReason reason = await processor.ExecuteAsync(stoppingToken); if (reason == ProcessorExitReason.Shutdown || stoppingToken.IsCancellationRequested) @@ -272,6 +282,7 @@ void ApplySuccessfulRecreate( TimeSpan deferredDisposeGracePeriod) { callInvoker = result.NewCallInvoker!; + this.PublishCallInvoker(callInvoker); address = result.NewAddress!; latestObservedChannel = result.NewChannel; AsyncDisposable previousDisposable = workerOwnedChannelDisposable; @@ -317,6 +328,16 @@ AsyncDisposable GetCallInvoker(out CallInvoker callInvoker, out string address) return disposable; } + // Called with the post-interceptor invoker only, at startup and after each successful recreate, so a + // subscriber shares the worker's transport (and its auth chain) instead of holding an invoker that goes + // stale the moment the channel is replaced. Deliberately not wrapped in a catch: the hook is documented as + // non-throwing, and swallowing a failure here would leave subscribers silently pinned to a dead channel + // with nothing to indicate it. + void PublishCallInvoker(CallInvoker callInvoker) + { + this.grpcOptions.Internal.CallInvokerPublisher?.Invoke(callInvoker); + } + AsyncDisposable GetCallInvokerCore(out CallInvoker callInvoker, out string address) { if (this.grpcOptions.Channel is GrpcChannel c) diff --git a/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs b/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs index 930ee99d..9ed5a342 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs @@ -126,9 +126,11 @@ internal class InternalOptions public bool InsertEntityUnlocksOnCompletion { get; set; } /// - /// Gets or sets the maximum amount of time to wait for the initial Hello handshake against the - /// backend before treating the connect attempt as failed and retrying. A non-positive value disables - /// the deadline. Defaults to 30 seconds. This guards against half-open HTTP/2 connections that can + /// Gets or sets the maximum amount of time to wait for a single connection-setup RPC against the + /// backend before treating the connect attempt as failed and retrying. Each pre-stream RPC - the + /// Hello handshake, and the large-payload auto-purge announcement when the worker has one to make - + /// gets this much time on its own rather than sharing one budget. A non-positive value disables the + /// deadline. Defaults to 30 seconds. This guards against half-open HTTP/2 connections that can /// otherwise cause reconnect to hang indefinitely. /// public TimeSpan HelloDeadline { get; set; } = TimeSpan.FromSeconds(30); @@ -143,9 +145,9 @@ internal class InternalOptions public TimeSpan SilentDisconnectTimeout { get; set; } = TimeSpan.FromSeconds(120); /// - /// Gets or sets the number of consecutive connect failures (Hello timeouts, Unavailable responses, or - /// silent stream disconnects) after which the underlying gRPC channel will be recreated to clear - /// stale DNS, sub-channel state, or routing-affinity bindings. Setting to 0 or a negative value + /// Gets or sets the number of consecutive connect failures (connection-setup timeouts, Unavailable + /// responses, or silent stream disconnects) after which the underlying gRPC channel will be recreated + /// to clear stale DNS, sub-channel state, or routing-affinity bindings. Setting to 0 or a negative value /// disables channel recreation. Defaults to 5. /// public int ChannelRecreateFailureThreshold { get; set; } = 5; @@ -197,5 +199,14 @@ internal class InternalOptions /// Gets or sets a callback that is invoked when activity work items are received or finished. /// public Action? NotifyActivity { get; set; } + + /// + /// Gets or sets a callback invoked with the worker's current effective - the + /// one produced after have been applied. It is invoked once when the worker + /// starts and again after every successful channel recreate, so a component that shares the worker's + /// transport can follow it instead of capturing an invoker that later points at a disposed channel. + /// Implementations must not throw and must not block. + /// + public Action? CallInvokerPublisher { get; set; } } } diff --git a/src/Worker/Grpc/Internal/InternalOptionsExtensions.cs b/src/Worker/Grpc/Internal/InternalOptionsExtensions.cs index 81ad09d5..cec95c2c 100644 --- a/src/Worker/Grpc/Internal/InternalOptionsExtensions.cs +++ b/src/Worker/Grpc/Internal/InternalOptionsExtensions.cs @@ -115,4 +115,24 @@ public static void SetSilentDisconnectTimeout(this GrpcDurableTaskWorkerOptions { options.Internal.SilentDisconnectTimeout = timeout; } + + /// + /// Sets a callback invoked with the worker's current effective - the one that + /// remains after the configured interceptors have been applied. The worker invokes it once at startup and + /// again after every successful channel recreate, so a component that shares the worker's transport can + /// follow it rather than capturing an invoker that is later left pointing at a disposed channel. + /// + /// The gRPC worker options. + /// The publish callback. It must not throw and must not block. + /// + /// This is an internal API that supports the DurableTask infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. + /// + public static void SetCallInvokerPublisher( + this GrpcDurableTaskWorkerOptions options, + Action publisher) + { + options.Internal.CallInvokerPublisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); + } } diff --git a/src/Worker/Grpc/Worker.Grpc.csproj b/src/Worker/Grpc/Worker.Grpc.csproj index 039a2efa..2b5d4113 100644 --- a/src/Worker/Grpc/Worker.Grpc.csproj +++ b/src/Worker/Grpc/Worker.Grpc.csproj @@ -20,4 +20,11 @@ + + + + + diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs new file mode 100644 index 00000000..9c121eab --- /dev/null +++ b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Reflection; +using Microsoft.DurableTask.Client; +using LP = Microsoft.DurableTask.Protobuf.LargePayloads; + +namespace Microsoft.DurableTask.Client.Grpc.Tests; + +/// +/// ReportLargePayloadPurgeResultsActivity maps the managed purge disposition +/// onto its protobuf counterpart by numeric value rather than by name, which is only correct while the two +/// sides agree on every value. A silent drift would not fail to compile; it would send the backend a different +/// disposition than the worker decided and delete or quarantine the wrong rows. These tests pin the mapping. +/// +public class LargePayloadPurgeEnumParityTests +{ + [Fact] + public void Disposition_ManagedAndProtobufValues_AreIdentical() + { + // Arrange & Act + Dictionary managed = Enum.GetValues(typeof(LargePayloadPurgeDisposition)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + Dictionary proto = Enum.GetValues(typeof(LP.LargePayloadPurgeDisposition)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + + // Assert - same numeric values AND the same names at each value, so neither side can gain, lose, or + // renumber a member unnoticed. + managed.Should().Equal(proto); + } + + /// + /// The numeric cast is safe only because no enum crosses the wire inbound on this feature: the SDK + /// casts a value it defined itself, so it can never receive an unknown value and silently reinterpret it. + /// That invariant holds today by the shape of the contract, not by construction, and nothing in the code + /// states it. Adding an enum to an inbound type would create exactly that path - a newer backend sending a + /// value this SDK does not know, mapped by raw numeric cast onto a valid-but-wrong member - and it would + /// compile silently. This pins the invariant so it fails here instead. + /// + /// A type carrying server-to-client data for the purge feature. + [Theory] + [InlineData(typeof(LargePayloadTombstone))] + [InlineData(typeof(LP.LargePayloadTombstone))] + [InlineData(typeof(LP.GetLargePayloadTombstonesResponse))] + [InlineData(typeof(LP.ReportLargePayloadPurgeResultsResponse))] + public void InboundTypes_ExposeNoEnumMembers(Type inboundType) + { + // Arrange & Act + List enumMembers = inboundType + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => (Nullable.GetUnderlyingType(p.PropertyType) ?? p.PropertyType).IsEnum) + .Select(p => $"{inboundType.Name}.{p.Name}") + .ToList(); + + // Assert + enumMembers.Should().BeEmpty( + "an enum on an inbound type invalidates the numeric enum cast in " + + "ReportLargePayloadPurgeResultsActivity. The SDK would map a value chosen by the " + + "backend - including one a newer backend added that this SDK does not know - onto a managed member " + + "by raw numeric value, silently mis-dispositioning rows. Map inbound enums explicitly instead, with " + + "a switch that handles unknown values"); + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs new file mode 100644 index 00000000..d145feb2 --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs @@ -0,0 +1,443 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class BlobPurgeJobOrchestratorTests +{ + static readonly EntityInstanceId JobEntityId = new(nameof(BlobPurgeJob), BlobPurgeConstants.JobId); + + readonly List requested = []; + readonly TestLogger logger = new(); + + [Fact] + public async Task RunAsync_UsesBatchSizeFromEntity_NotFromInput() + { + // Arrange - the orchestrator is perpetual, so its input is fixed at creation and carried verbatim + // through every continue-as-new. Reading the batch size from the input would pin the value written by + // the very first Create for the life of the job, which is what made a configuration change impossible + // to apply. The entity is the authority. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 777 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert + this.AssertNoCycleFailed(); + this.requested.Should().Equal(777); + } + + [Fact] + public async Task RunAsync_WhenEntityBatchSizeIsUnset_FallsBackToInput() + { + // Arrange - an entity written by an older build carries no batch size at all. Passing that zero to the + // fetch activity would ask the backend for nothing on every cycle: a silent, permanent stall. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 0 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert + this.AssertNoCycleFailed(); + this.requested.Should().Equal(100); + } + + [Fact] + public async Task RunAsync_WhenJobIsStopped_ExitsWithoutFetching() + { + // Arrange - Stop moves the entity off Active without touching this orchestrator, so the exit is the + // orchestrator's own decision on its own schedule. Nothing must be fetched on a stopped job. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending, PurgeBatchSize = 777 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert - an empty fetch list is also what a broken mock produces, so the stopping log is asserted + // too: it is the only evidence that the orchestrator read the state and chose to exit. + this.AssertNoCycleFailed(); + this.requested.Should().BeEmpty(); + this.logger.Logs.Should().Contain(entry => entry.Message.Contains("stopping")); + } + + [Fact] + public async Task RunAsync_WhenBackendDoesNotImplementPurgeRpcs_DisablesJobAndExits() + { + // Arrange - the fetch activity surfaced a gRPC Unimplemented as NotImplementedException (mixed rollout / + // stale emulator). The orchestrator must disable the job durably and exit its perpetual loop, rather + // than logging a generic cycle failure and retrying on every backoff forever. + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 250 }); + + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + + context + .Setup(c => c.CallActivityAsync>( + It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new TaskFailedException( + nameof(GetLargePayloadTombstonesActivity), + 1, + new NotImplementedException("backend does not implement GetLargePayloadTombstones"))); + + // Act + object? result = await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert - disabled through an AWAITED MarkUnsupported so the write is durable before the loop exits, a + // dedicated diagnostic is logged (not the generic cycle-failed one), and RunAsync returns rather than + // continuing. The awaited call, not a signal, is what guarantees the disable is committed here. + result.Should().BeNull(); + entities.Verify( + e => e.CallEntityAsync( + JobEntityId, + nameof(BlobPurgeJob.MarkUnsupported), + It.IsAny(), + It.IsAny()), + Times.Once); + this.logger.Logs.Should().Contain( + entry => entry.Message.Contains("does not implement the large-payload purge RPCs")); + this.AssertNoCycleFailed(); + } + + [Fact] + public async Task RunAsync_DeletesLargeBatch_InChunksNotOneActivityPerToken() + { + // Arrange - a full 1000-row batch. The whole point of chunking is that a batch this size fans out to a + // HANDFUL of delete-activity calls (ceil(1000 / 50) = 20), not one per token (1000), which is what + // bloated the orchestration history. The activity contract is one outcome per token, so the mock returns + // exactly that; the orchestrator asserts the count before zipping. + const int batchSize = 1000; + List tombstones = []; + for (int i = 0; i < batchSize; i++) + { + tombstones.Add(new LargePayloadTombstone( + TombstoneToken: $"tombstone-{i}", + PayloadToken: $"blob:v2:https://acct.blob.core.windows.net/c/{i}")); + } + + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // Active on the first read, stopped on the second, so the perpetual loop runs exactly one cycle. + entities + .SetupSequence(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = batchSize }) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending }); + + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + + context + .Setup(c => c.CallActivityAsync>( + It.Is(n => n.Name == nameof(GetLargePayloadTombstonesActivity)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(tombstones); + + int chunkActivityCalls = 0; + int tokensDeleted = 0; + context + .Setup(c => c.CallActivityAsync>( + It.Is(n => n.Name == nameof(DeleteExternalBlobActivity)), + It.IsAny(), + It.IsAny())) + .Returns((_, input, _) => + { + List chunk = (List)input!; + Interlocked.Increment(ref chunkActivityCalls); + Interlocked.Add(ref tokensDeleted, chunk.Count); + + // One outcome per token, positionally aligned - the shape the orchestrator asserts before zipping. + List outcomes = new(chunk.Count); + foreach (string _ in chunk) + { + outcomes.Add(new BlobPurgeOutcome(LargePayloadPurgeDisposition.Deleted)); + } + + return Task.FromResult(outcomes); + }); + + List? reported = null; + context + .Setup(c => c.CallActivityAsync( + It.Is(n => n.Name == nameof(ReportLargePayloadPurgeResultsActivity)), + It.IsAny(), + It.IsAny())) + .Callback((_, input, _) => reported = (List)input!) + .Returns(Task.CompletedTask); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: batchSize)); + + // Assert - 20 chunk activities for the whole batch, not 1000; every token was still handled exactly once; + // and one result is reported per row so the backend hears about all 1000. + this.AssertNoCycleFailed(); + chunkActivityCalls.Should().Be(20); + tokensDeleted.Should().Be(batchSize); + reported.Should().NotBeNull(); + reported!.Should().HaveCount(batchSize); + } + + [Fact] + public async Task RunAsync_SendsPayloadTokenToStorage_AndEchoesTombstoneTokenToBackend() + { + // Arrange - a tombstone now carries two opaque strings: PayloadToken addresses the blob, TombstoneToken + // identifies the ledger row. Both are plain strings, so swapping them compiles silently and would fail + // in the worst possible way - deleting nothing while telling the backend rows were purged. Distinct, + // non-overlapping values make a swap impossible to miss. + List tombstones = + [ + new("tombstone-a", "blob:v2:https://acct.blob.core.windows.net/c/a"), + new("tombstone-b", "blob:v2:https://acct.blob.core.windows.net/c/b"), + ]; + + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + entities + .SetupSequence(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 100 }) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending }); + + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + + context + .Setup(c => c.CallActivityAsync>( + It.Is(n => n.Name == nameof(GetLargePayloadTombstonesActivity)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(tombstones); + + List sentToDelete = []; + context + .Setup(c => c.CallActivityAsync>( + It.Is(n => n.Name == nameof(DeleteExternalBlobActivity)), + It.IsAny(), + It.IsAny())) + .Returns((_, input, _) => + { + List chunk = (List)input!; + sentToDelete.AddRange(chunk); + + // Distinct dispositions so the zip cannot be proven by a constant. + return Task.FromResult>( + [ + new(LargePayloadPurgeDisposition.Deleted), + new(LargePayloadPurgeDisposition.Quarantined), + ]); + }); + + List? reported = null; + context + .Setup(c => c.CallActivityAsync( + It.Is(n => n.Name == nameof(ReportLargePayloadPurgeResultsActivity)), + It.IsAny(), + It.IsAny())) + .Callback((_, input, _) => reported = (List)input!) + .Returns(Task.CompletedTask); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert - storage saw only payload tokens, the backend heard only tombstone tokens, and each row kept + // its own disposition rather than the batch collapsing onto one. + this.AssertNoCycleFailed(); + sentToDelete.Should().Equal( + "blob:v2:https://acct.blob.core.windows.net/c/a", + "blob:v2:https://acct.blob.core.windows.net/c/b"); + reported.Should().NotBeNull(); + List results = reported!; + results.Select(r => r.TombstoneToken).Should().Equal("tombstone-a", "tombstone-b"); + results.Select(r => r.Disposition).Should().Equal( + LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeDisposition.Quarantined); + } + + [Fact] + public async Task RunAsync_WhenFetchReturnsNoTombstones_IdlesWithoutDeletingOrReporting() + { + // Arrange - an Active job whose fetch comes back empty. Nothing is due, so the cycle must short-circuit + // to the idle timer and touch neither the delete activity nor the report activity: an empty backend + // response has to cost a wait and nothing else, or every quiet worker would hammer storage and the + // backend on every tick. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 250 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert - the fetch ran (recording the batch size) and the idle timer was created, but neither the + // delete nor the report activity was ever invoked on the empty batch. + this.AssertNoCycleFailed(); + this.requested.Should().Equal(250); + context.Verify( + c => c.CreateTimer(It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + context.Verify( + c => c.CallActivityAsync>( + It.Is(n => n.Name == nameof(DeleteExternalBlobActivity)), + It.IsAny(), + It.IsAny()), + Times.Never); + context.Verify( + c => c.CallActivityAsync( + It.Is(n => n.Name == nameof(ReportLargePayloadPurgeResultsActivity)), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task RunAsync_ContinuesAsNewOnlyAfterConfiguredCycles() + { + // Arrange - a job that stays Active with nothing to purge, so the continue-as-new guard is the ONLY + // thing that can end the perpetual loop. That guard is the sole bound on history growth here: a fresh + // run must process exactly ContinueAsNewFrequency cycles and then continue-as-new with a reset count. + // Firing early would reset the job's progress window too often; never firing would let the orchestration + // history grow without bound until the instance died days later. + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // Always Active: the job never stops, so continue-as-new is the only exit from the loop. + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 250 }); + + context + .Setup(c => c.CallActivityAsync>( + It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, input, _) => this.requested.Add((int)input!)) + .ReturnsAsync([]); + + BlobPurgeJobRunRequest? restarted = null; + context + .Setup(c => c.ContinueAsNew(It.IsAny(), It.IsAny())) + .Callback((input, _) => restarted = (BlobPurgeJobRunRequest)input!); + + // Act - start fresh (zero processed cycles). + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 250)); + + // Assert - five cycles ran (one empty fetch each) before a single continue-as-new that reset the cycle + // count while carrying the same job identity and batch size forward. The five recorded fetches are what + // prove it did not continue-as-new early; the single ContinueAsNew is what proves it does not grow the + // history forever. + this.AssertNoCycleFailed(); + this.requested.Should().HaveCount(5); + context.Verify(c => c.ContinueAsNew(It.IsAny(), It.IsAny()), Times.Once); + restarted.Should().NotBeNull(); + restarted!.JobEntityId.Should().Be(JobEntityId); + restarted.PurgeBatchSize.Should().Be(250); + restarted.ProcessedCycles.Should().Be(0); + } + + /// + /// Guards against the whole test passing through the orchestrator's catch-all cycle handler, which would + /// leave every observation empty and make the assertions vacuous. + /// + void AssertNoCycleFailed() => + this.logger.Logs.Should().NotContain(entry => entry.Message.Contains("cycle for job")); + + /// + /// Builds a context whose first entity read returns and whose second returns a + /// stopped job, so the perpetual loop runs at most one cycle and then exits. Batch sizes passed to the + /// fetch activity are recorded. + /// + Mock ContextFor(BlobPurgeJobState first) + { + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + entities + .SetupSequence(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(first) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending }); + + context + .Setup(c => c.CallActivityAsync>( + It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, input, _) => this.requested.Add((int)input!)) + .ReturnsAsync([]); + + return context; + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs new file mode 100644 index 00000000..8a12d55b --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -0,0 +1,473 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Entities; +using Microsoft.DurableTask.Entities.Tests; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class BlobPurgeJobTests +{ + readonly BlobPurgeJob job = new(new TestLogger()); + + [Fact] + public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() + { + // Arrange + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(null), + 250); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(250); + state.CreatedAt.Should().NotBeNull(); + state.LastModifiedAt.Should().NotBeNull(); + + // Starting the job means signalling Run, and this path has always done so. Asserted explicitly because + // the already-active path now signals Run too, which makes this the case that would silently stop being + // covered if the two branches were ever collapsed. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Create_WhenAlreadyActive_UpdatesBatchSizeAndReSignalsRun() + { + // Arrange - the job is already running and the configured batch size has changed. Create is the only + // path by which a new batch size can reach an active job, so it must be taken. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 100, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 999); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(999); + + // Run is re-signalled even though the job is already active. That is what lets a job whose orchestrator + // has died be rebuilt by a later explicit enable call, and it is safe because the backend discards the + // resulting start while the orchestrator is alive rather than replacing it. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Create_WhenAlreadyActive_SignalsNothingOtherThanRun() + { + // Arrange - pins that re-signalling Run is the only signal the already-active path emits. Verifying the + // Run signal alone would still pass if a second, different signal were added beside it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 100, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 999); + + // Act + await this.job.RunAsync(operation); + + // Assert - the Times.Once above is the positive control for this Times.Never: both target the same + // four-argument overload on the same mock, so this cannot be passing because nothing was recorded. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + It.Is(name => name != nameof(BlobPurgeJob.Run)), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Create_WhenAlreadyActive_AndBatchSizeUnchanged_DoesNotMoveLastModifiedAt() + { + // Arrange - the steady state. Create runs on every explicit enable call, and a repeated enable almost + // always carries the same batch size the job already has. If that rewrote LastModifiedAt, the field + // would degrade to "time of the last call" and say nothing about the job. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 250); + + // Act + await this.job.RunAsync(operation); + + // Assert - the sibling test below is the positive control: identical wiring, differing only in the + // batch size passed in, and it proves this same path does move the field when something changes. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.LastModifiedAt.Should().Be(configuredAt); + state.PurgeBatchSize.Should().Be(250); + } + + [Fact] + public async Task Create_WhenAlreadyActive_AndBatchSizeChanged_MovesLastModifiedAt() + { + // Arrange - a real configuration change reaching an active job, which is the one thing this path + // exists to deliver and the one case that must be recorded. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 500); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.PurgeBatchSize.Should().Be(500); + state.LastModifiedAt.Should().BeAfter(configuredAt); + } + + [Fact] + public async Task Run_DoesNotMoveLastModifiedAt() + { + // Arrange - Run schedules an orchestrator and changes nothing about the job. It is signalled by every + // Create, so writing here would move the field on every enable call and undo the conditional + // write above. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert - scheduling is asserted first as the positive control. Without it a misdispatched operation + // would write nothing and pass this test for entirely the wrong reason. + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.LastModifiedAt.Should().Be(configuredAt); + } + + [Fact] + public async Task Run_WhenActive_SchedulesOrchestratorAtTheFixedInstanceId() + { + // Arrange - the fixed instance ID is the mechanism the whole restart story rests on. It is what lets the + // backend recognize a start as targeting the existing orchestrator, and therefore discard it while that + // orchestrator is alive instead of running a second one alongside it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + Mock.Get(operation.Context) + .Setup(c => c.Id) + .Returns(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); + + // Act + await this.job.RunAsync(operation); + + // Assert + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.Is(o => + o.InstanceId == BlobPurgeConstants.GetOrchestratorInstanceId(BlobPurgeConstants.JobId))), + Times.Once); + } + + [Fact] + public async Task Run_WhenNotActive_SchedulesNothing() + { + // Arrange - a Run signal arriving after the job was stopped. Create signals Run rather than starting the + // orchestrator itself, so a Stop landing between the two leaves this signal in flight against a job that + // must no longer purge. This guard is what makes the stop win instead of the stale signal restarting it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Pending, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert - the Times.Once above is the positive control: same mocked type, same three-argument overload, + // so this cannot be passing merely because the mock records nothing. + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Stop_WhenActive_MovesToPendingAndKeepsHistory() + { + // Arrange - a running job. Stopping must not discard the configuration or the progress counters: they + // are wanted if the job is started again, and CreatedAt is what distinguishes a stopped job from one + // that was never started. + DateTimeOffset createdAt = DateTimeOffset.UtcNow.AddDays(-3); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + CreatedAt = createdAt, + PurgedCount = 17, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Stop), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Pending); + state.CreatedAt.Should().Be(createdAt); + state.PurgedCount.Should().Be(17); + state.PurgeBatchSize.Should().Be(250); + state.LastModifiedAt.Should().NotBeNull(); + } + + [Fact] + public async Task Stop_WhenNotActive_LeavesStateUntouched() + { + // Arrange - a job that is already stopped. A repeated explicit disable lands here, as does one that + // races another caller's disable: the job stopped between when this caller decided to disable it and + // this signal landing. Rewriting LastModifiedAt here would report the losing side of that race as if + // it were the moment the job stopped. + DateTimeOffset stoppedAt = DateTimeOffset.UtcNow.AddHours(-6); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Pending, + CreatedAt = DateTimeOffset.UtcNow.AddDays(-3), + LastModifiedAt = stoppedAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Stop), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Pending); + state.LastModifiedAt.Should().Be(stoppedAt); + state.PurgeBatchSize.Should().Be(250); + } + + [Fact] + public async Task Create_StoresBatchSizeVerbatim_WithoutCoercion() + { + // Arrange - the batch size is validated once at specification (LargePayloadStorageOptions), so the + // entity trusts its input and performs no coercion of its own. A zero here is stored as-is, proving + // the previous non-positive-to-default fallback was removed. + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(null), + 0); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.PurgeBatchSize.Should().Be(0); + } + + [Fact] + public async Task Get_ReturnsCurrentState() + { + // Arrange + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 42, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Get), + new TestEntityState(existing), + null); + + // Act + object? result = await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType(result); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(42); + } + + [Fact] + public async Task Create_WhenUnsupported_RevivesJob() + { + // Arrange - the backend was found not to implement the purge RPCs, so the job was disabled. Create is + // now reached only from an explicit client call, so reviving is exactly what the caller asked for, and + // it is the documented recovery once the backend has been upgraded. Refusing here would strand the job + // permanently, because nothing else clears the status. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Unsupported, + LastError = "backend does not implement GetLargePayloadTombstones", + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 250); + + // Act + await this.job.RunAsync(operation); + + // Assert - reviving means both halves: the status goes Active and the orchestrator is scheduled. The + // stale error is dropped so a later failure is not read as this one. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.LastError.Should().BeNull(); + + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task MarkUnsupported_WhenActive_DisablesJobAndRecordsDetail() + { + // Arrange - a running job whose fetch/report activity just surfaced a gRPC Unimplemented. The job must + // move to a status Create refuses to revive, and the detail is retained so an operator can see why. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.MarkUnsupported), + new TestEntityState(existing), + "backend does not implement GetLargePayloadTombstones"); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Unsupported); + state.LastError.Should().Be("backend does not implement GetLargePayloadTombstones"); + state.LastModifiedAt.Should().NotBeNull(); + + // Unlike Create, this must not (re)start the orchestrator - the job is being disabled, not run. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task MarkUnsupported_WhenAlreadyUnsupported_LeavesStateUntouched() + { + // Arrange - a second replica reporting the same unsupported backend before the first orchestrator has + // exited. The repeat must be a no-op so LastModifiedAt keeps meaning "when the job was disabled" and the + // original detail is not overwritten by a later, possibly less specific, one. + DateTimeOffset disabledAt = DateTimeOffset.UtcNow.AddMinutes(-5); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Unsupported, + LastError = "original detail", + LastModifiedAt = disabledAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.MarkUnsupported), + new TestEntityState(existing), + "a different detail"); + + // Act + await this.job.RunAsync(operation); + + // Assert - the sibling test above is the positive control: identical wiring off an Active job does move + // both fields, proving this no-op is the guard's doing and not a dead path. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Unsupported); + state.LastError.Should().Be("original detail"); + state.LastModifiedAt.Should().Be(disabledAt); + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs new file mode 100644 index 00000000..893ef5fe --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -0,0 +1,377 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure; +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +/// +/// The reported outcome carries the disposition alone, so each case pins two things: the disposition, which is +/// what the backend acts on, and the logged cause, which is now the only record of why an attempt reached that +/// disposition. Several branches share a disposition and are told apart only by their cause, so asserting the +/// disposition alone would let two branches collapse into one unnoticed. +/// +public class DeleteExternalBlobActivityTests +{ + const string V2Token = "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"; + + [Fact] + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_Quarantines() + { + // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. + StubPayloadStore store = new(new RequestFailedException(400, "bad", "InvalidResourceName", null)); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; + + // Assert - quarantined (evidence preserved), never a success-shaped discard. The sanitized storage + // error code is logged alongside the cause and is what distinguishes this from the parse failures. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + logger.Logs.Should().ContainSingle( + l => l.Message.Contains("InvalidStorageRequest") && l.Message.Contains("InvalidResourceName")); + } + + [Fact] + public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_RetriesAsTransient() + { + // Arrange - a Status 503 that escaped the SDK's internal retries is still treated as transient. + StubPayloadStore store = new(new RequestFailedException(503, "busy", "ServerBusy", null)); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + logger.Logs.Should().ContainSingle( + l => l.Message.Contains("TransientStorageFailure") && l.Message.Contains("ServerBusy")); + } + + [Theory] + [InlineData(401)] + [InlineData(403)] + public async Task RunAsync_WhenDeleteThrowsAuthorizationFailure_Retries(int status) + { + // Arrange - authorization can be fixed by reconfiguration, so it stays recoverable. + StubPayloadStore store = new(new RequestFailedException(status, "denied", "AuthorizationFailure", null)); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StorageAuthorizationFailed")); + } + + [Fact] + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_Retries() + { + // Arrange - the payload lives in a storage account the configured credential cannot reach. That is + // recoverable after a configuration or credential change, so it is deferred rather than discarded. + StubPayloadStore store = new(new PayloadStorageException("cross-account delete requires identity auth")); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync( + null!, ["blob:v2:https://other.blob.core.windows.net/c/abc123"]))[0]; + + // Assert - the cause is what separates this from the other retryable storage failures, which the + // contract no longer distinguishes. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StorageAccountUnreachable")); + } + + [Fact] + public async Task RunAsync_V1Token_QuarantinesWithoutCallingStore() + { + // Arrange - a v1 token names a container but not the storage account, so a delete against the + // configured account cannot be verified and would falsely report success if the store was repointed. + Mock store = new(); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, ["blob:v1:payloads:abc123"]))[0]; + + // Assert - quarantined by the gate, and the store's DeleteAsync was never invoked. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("LegacyV1Token")); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsync_UnknownTokenVersion_RetriesWithoutCallingStore() + { + // Arrange - an unrecognized prefix most likely came from a newer SDK, which recovers after an upgrade. + Mock store = new(); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, ["blob:v9:https://acct.blob.core.windows.net/c/x"]))[0]; + + // Assert - retried, NOT quarantined: quarantine is terminal and requires an operator to unwind, while a + // deferral only leaves the row idle and visible. This branch and the token branches that quarantine are + // told apart only by cause, so the disposition is asserted deliberately: folding them together would + // strand rows that an SDK upgrade would have resolved. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnsupportedTokenVersion")); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsync_MalformedV2Token_Quarantines() + { + // Arrange - a recognized v2 prefix whose body does not parse; the store signals that with + // ArgumentException. Retrying can never fix a body the SDK itself produced malformed. + StubPayloadStore store = new(new ArgumentException("Invalid token")); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, ["blob:v2:not-a-uri"]))[0]; + + // Assert - contrast with the unknown-prefix case above, which is retried. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("MalformedToken")); + } + + [Fact] + public async Task RunAsync_V2Token_CallsStoreAndReportsDeleted() + { + // Arrange - a self-describing v2 token is not gated and must reach the store. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.Deleted); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; + + // Assert - an ordinary success is silent; any log here would mean a failure branch was taken. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + logger.Logs.Should().BeEmpty(); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public async Task RunAsync_WhenBlobAlreadyAbsent_ReportsDeleted() + { + // Arrange - deletion is idempotent, so a blob a previous attempt already removed is not a failure. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.AlreadyAbsent); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + logger.Logs.Should().BeEmpty(); + } + + [Fact] + public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeleting() + { + // Arrange - the blob exists but carries no ownership marker, so the store left it untouched. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.NotStoreOwned); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; + + // Assert - the tombstone is still resolved: a blob the store does not own is not the store's to delete, + // and re-serving the row forever would never make it deletable. The reported result is now identical to + // an ordinary delete, so the log is the only thing that records the difference. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("ownership marker")); + } + + [Fact] + public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombstone() + { + // Arrange - a store that cannot delete (the base PayloadStore.DeleteAsync throws NotSupportedException). + StubPayloadStore store = new(new NotSupportedException()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; + + // Assert - retried (tombstone preserved): resolving it would destroy the backend's cleanup ledger while + // the blob survives. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StoreCannotDelete")); + } + + [Fact] + public async Task RunAsync_WhenDeleteTimesOut_RetriesAsTransient() + { + // Arrange - a non-Azure exception (timeout / network failure) must not drop a blob on doubt. + StubPayloadStore store = new(new TimeoutException()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; + + // Assert - storage reported no code here, so the exception's type name is what identifies the failure. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnexpectedFailure:TimeoutException")); + } + + [Fact] + public async Task RunAsync_MixedChunk_AttributesEachOutcomeToItsOwnTokenByIndex() + { + // Arrange - one chunk whose five tokens resolve to deliberately different dispositions and complete in + // the REVERSE of their input order. Adjacent indices differ, so a driver that recorded results in + // completion order (or shifted by one) would misattribute at least one row. Index 3 throws a raw + // exception from the store to prove that a single failing token neither faults its peers nor escapes the + // activity. + string[] tokens = + [ + "blob:v2:https://acct.blob.core.windows.net/payloads/0", + "blob:v2:https://acct.blob.core.windows.net/payloads/1", + "blob:v2:https://acct.blob.core.windows.net/payloads/2", + "blob:v2:https://acct.blob.core.windows.net/payloads/3", + "blob:v2:https://acct.blob.core.windows.net/payloads/4", + ]; + + Dictionary> gates = new(); + foreach (string token in tokens) + { + // Synchronous continuations (the default) make completion strictly ordered: each Set* below resolves + // exactly one delete inline, in call order, so completion order is deterministically the reverse of + // input order. The index-based driver ignores completion order, but a regression to completion-order + // recording would then attribute deterministically wrong - which is what makes this test bite. + gates[token] = new TaskCompletionSource(); + } + + ControlledPayloadStore store = new(gates); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act - start the chunk (every delete is now parked on its gate), then release the tokens back-to-front + // so completion order is the exact reverse of input order. + Task> run = activity.RunAsync(null!, [.. tokens]); + + gates[tokens[4]].SetResult(PayloadDeleteOutcome.AlreadyAbsent); // -> Deleted + gates[tokens[3]].SetException(new TimeoutException()); // -> Retry + gates[tokens[2]].SetResult(PayloadDeleteOutcome.Deleted); // -> Deleted + gates[tokens[1]].SetException(new RequestFailedException(503, "busy", "ServerBusy", null)); // -> Retry + gates[tokens[0]].SetException( + new RequestFailedException(400, "bad", "InvalidResourceName", null)); // -> Quarantined + + List outcomes = await run; + + // Assert - each disposition sits at its own token's index regardless of completion order, and the raw + // failure at index 3 left indices 0, 1, 2, and 4 untouched. + outcomes.Should().HaveCount(tokens.Length); + outcomes[0].Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + outcomes[1].Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcomes[2].Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + outcomes[3].Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcomes[4].Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + } + + [Fact] + public async Task RunAsync_WhenSingleDeleteExceedsTimeout_ReturnsRetryInsteadOfHanging() + { + // Arrange - a store whose delete never completes on its own and only observes cancellation. With the + // per-delete timeout the activity must abandon it and classify it retryable, rather than pinning the + // concurrency slot for the store's full multi-minute retry budget. + BlockingPayloadStore store = new(); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger) + { + SingleDeleteTimeout = TimeSpan.FromMilliseconds(50), + }; + + // Act - guard the await so a regression (an unbounded delete) surfaces as a bounded assertion failure + // instead of hanging the whole test run. + Task> run = activity.RunAsync(null!, [V2Token]); + Task first = await Task.WhenAny(run, Task.Delay(TimeSpan.FromSeconds(10))); + first.Should().BeSameAs(run, "the delete must yield to its timeout rather than run unbounded"); + + List outcomes = await run; + + // Assert - the timeout surfaced as a cancellation and was classified retryable by the catch-all, exactly + // as a network timeout would be; no exception escaped RunAsync. + outcomes.Should().ContainSingle(); + outcomes[0].Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnexpectedFailure:TaskCanceledException")); + } + + sealed class StubPayloadStore : PayloadStore + { + readonly Exception? deleteError; + + public StubPayloadStore(Exception? deleteError) => this.deleteError = deleteError; + + public override Task DeleteAsync(string token, CancellationToken cancellationToken) => + this.deleteError is null + ? Task.FromResult(PayloadDeleteOutcome.Deleted) + : throw this.deleteError; + + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => true; + } + + sealed class ControlledPayloadStore : PayloadStore + { + readonly IReadOnlyDictionary> gates; + + public ControlledPayloadStore( + IReadOnlyDictionary> gates) => this.gates = gates; + + public override Task DeleteAsync(string token, CancellationToken cancellationToken) => + this.gates[token].Task; + + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => true; + } + + sealed class BlockingPayloadStore : PayloadStore + { + public override async Task DeleteAsync(string token, CancellationToken cancellationToken) + { + // Never completes on its own; only cancellation (the activity's per-delete timeout) ends the wait. + await Task.Delay(Timeout.Infinite, cancellationToken); + return PayloadDeleteOutcome.Deleted; + } + + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => true; + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeActivityUnimplementedTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeActivityUnimplementedTests.cs new file mode 100644 index 00000000..8006f80f --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeActivityUnimplementedTests.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using static Microsoft.DurableTask.Protobuf.LargePayloads.LargePayloadPurge; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +/// +/// The two purge activities are the real runtime path to the backend - they call the generated purge service client +/// directly. During a mixed rollout an older backend returns gRPC Unimplemented for the new purge RPCs; +/// each activity must translate that into a so the orchestrator can +/// disable the job instead of retrying an operation that can never succeed. +/// +public class PurgeActivityUnimplementedTests +{ + [Fact] + public async Task GetLargePayloadTombstones_WhenBackendUnimplemented_ThrowsNotImplemented() + { + // Arrange - the backend rejects the fetch RPC because it does not implement it. + LargePayloadPurgeClient client = new( + new ThrowingCallInvoker(new RpcException(new Status(StatusCode.Unimplemented, "unknown method")))); + GetLargePayloadTombstonesActivity activity = new(client, new TestLogger()); + + // Act + Func act = () => activity.RunAsync(null!, 100); + + // Assert - a bare RpcException would let the retry policy spin forever; NotImplementedException is the + // signal the orchestrator matches on to stop. + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task ReportLargePayloadPurgeResults_WhenBackendUnimplemented_ThrowsNotImplemented() + { + // Arrange - a non-empty result list is required to get past the activity's empty-input short-circuit and + // actually reach the report RPC, which the backend does not implement. + LargePayloadPurgeClient client = new( + new ThrowingCallInvoker(new RpcException(new Status(StatusCode.Unimplemented, "unknown method")))); + ReportLargePayloadPurgeResultsActivity activity = + new(client, new TestLogger()); + List results = new() + { + new LargePayloadPurgeResult("tombstone-token-1", LargePayloadPurgeDisposition.Deleted), + }; + + // Act + Func act = () => activity.RunAsync(null!, results); + + // Assert + await act.Should().ThrowAsync(); + } + + /// + /// A minimal whose unary calls fault with a configured . + /// Building a real over it exercises the activity's own catch + /// clause exactly as a live channel would, without mocking the generated client. The faulted + /// surfaces the exception on await, matching real gRPC. + /// + sealed class ThrowingCallInvoker(RpcException error) : CallInvoker + { + public override AsyncUnaryCall AsyncUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => new( + Task.FromException(error), + Task.FromResult(new Metadata()), + () => error.Status, + () => new Metadata(), + () => { }); + + public override TResponse BlockingUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => throw error; + + public override AsyncServerStreamingCall AsyncServerStreamingCall( + Method method, string? host, CallOptions options, TRequest request) + => throw new NotSupportedException(); + + public override AsyncClientStreamingCall AsyncClientStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + + public override AsyncDuplexStreamingCall AsyncDuplexStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeTransportTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeTransportTests.cs new file mode 100644 index 00000000..d65977ce --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeTransportTests.cs @@ -0,0 +1,473 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Concurrent; +using System.Globalization; +using System.Net; +using System.Net.Http.Headers; +using FluentAssertions; +using Google.Protobuf.WellKnownTypes; +using Grpc.Core; +using Grpc.Core.Interceptors; +using Grpc.Net.Client; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.Grpc; +using Microsoft.DurableTask.Worker.Grpc.Internal; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; +using static Microsoft.DurableTask.Protobuf.LargePayloads.LargePayloadPurge; +using LP = Microsoft.DurableTask.Protobuf.LargePayloads; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +/// +/// Covers how the purge activities reach the backend. They share the worker's transport rather than opening a +/// second connection, and that transport moves: the worker replaces its channel when the current one is wedged, +/// and the invoker it hands out is the one left after the configured interceptors have run. A client that +/// captured an invoker at registration time would miss both. +/// +public class PurgeTransportTests +{ + [Fact] + public void AddressOnlyWorker_ResolvesThePurgeClient() + { + // Arrange - Address-only is a supported worker configuration and the one that leaves both Channel and + // CallInvoker null. Resolving the purge client used to read those two properties and throw here. + ServiceCollection services = new(); + services.AddSingleton(NullLoggerFactory.Instance); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(options => options.Address = "http://localhost:4001"); + builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + }); + + using ServiceProvider provider = services.BuildServiceProvider(); + + // Act + Func resolve = () => provider.GetRequiredService(); + + // Assert + resolve.Should().NotThrow(); + } + + [Fact] + public async Task RunningWorker_RoutesPurgeCallsThroughItsOwnInterceptedTransport() + { + // Arrange - one worker, one transport. The interceptor stands in for the configured cross-cutting chain + // (authentication is the one that matters in production): if the purge client were built on a raw + // invoker instead of the worker's effective one, the interceptor would see the sidecar RPCs and none of + // the purge RPCs. + RecordingInterceptor interceptor = new(); + FakeCallInvoker transport = new(); + ServiceCollection services = new(); + services.AddSingleton(NullLoggerFactory.Instance); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(options => + { + options.CallInvoker = transport; + options.Interceptors.Add(interceptor); + }); + builder.UseExternalizedPayloads(options => + { + options.ConnectionString = "UseDevelopmentStorage=true"; + }); + }); + + await using ServiceProvider provider = services.BuildServiceProvider(); + LargePayloadPurgeClient client = provider.GetRequiredService(); + IHostedService worker = provider.GetServices().Single(); + + // Act - start the worker and wait until it has actually opened the work-item stream, which is the point + // at which activities could start running. Then issue the two purge RPCs an activity would issue. + await worker.StartAsync(CancellationToken.None); + try + { + await transport.WorkItemsRequested.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + await client.GetLargePayloadTombstonesAsync(new LP.GetLargePayloadTombstonesRequest { Limit = 1 }); + await client.ReportLargePayloadPurgeResultsAsync(new LP.ReportLargePayloadPurgeResultsRequest()); + } + finally + { + await worker.StopAsync(CancellationToken.None); + } + + // Assert - the configured interceptor observed the worker's own handshake AND both purge RPCs, so all + // three rode the same intercepted transport. The worker itself never calls SetLargePayloadAutoPurge: + // that setting is written by an explicit client call, not by a worker connecting. + interceptor.Calls.Should().Contain("Hello"); + interceptor.Calls.Should().NotContain("SetLargePayloadAutoPurge"); + interceptor.Calls.Should().Contain("GetLargePayloadTombstones"); + interceptor.Calls.Should().Contain("ReportLargePayloadPurgeResults"); + } + + [Fact] + public async Task WhenTheWorkerRecreatesItsChannel_AnExistingPurgeClientFollowsIt() + { + // Arrange - two channels backed by handlers that record which one received a call. The worker starts on + // A, is forced to recreate onto B, and the purge client is resolved ONCE up front: the property under + // test is that an already-constructed client routes its later calls through the replacement. + RecordingHandler handlerA = new(); + RecordingHandler handlerB = new(); + using GrpcChannel channelA = CreateChannel("http://localhost:14001", handlerA); + using GrpcChannel channelB = CreateChannel("http://localhost:14002", handlerB); + + // The recreator hands over B exactly once, and only when the test releases the gate, so the window in + // which A is the published transport is bounded by the test rather than by timing. Later requests park + // until shutdown so the connect loop cannot spin against a channel that can never succeed. + TaskCompletionSource releaseB = new(TaskCreationOptions.RunContinuationsAsynchronously); + int recreateCalls = 0; + + ServiceCollection services = new(); + services.AddSingleton(NullLoggerFactory.Instance); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(options => + { + options.Channel = channelA; + options.Internal.ChannelRecreateFailureThreshold = 1; + options.SetChannelRecreator(async (current, cancellation) => + { + if (Interlocked.Increment(ref recreateCalls) == 1) + { + await releaseB.Task.WaitAsync(cancellation); + return channelB; + } + + await Task.Delay(Timeout.Infinite, cancellation); + return current; + }); + }); + builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + }); + + await using ServiceProvider provider = services.BuildServiceProvider(); + LargePayloadPurgeClient client = provider.GetRequiredService(); + IHostedService worker = provider.GetServices().Single(); + + // Act + await worker.StartAsync(CancellationToken.None); + try + { + // The worker's Hello landing on A proves A is the published transport, with no polling or sleeping. + await handlerA.FirstHello.WaitAsync(TimeSpan.FromSeconds(30)); + await FetchTombstonesIgnoringTransportFailureAsync(client); + + int fetchesOnAWhileAWasCurrent = handlerA.PurgeFetchCount; + int fetchesOnBWhileAWasCurrent = handlerB.PurgeFetchCount; + + // Let the recreate complete; Hello landing on B proves the replacement has been published. + releaseB.SetResult(true); + await handlerB.FirstHello.WaitAsync(TimeSpan.FromSeconds(30)); + await FetchTombstonesIgnoringTransportFailureAsync(client); + + // Assert - the same client instance moved from A to B, and did not keep a second call on A. + fetchesOnAWhileAWasCurrent.Should().Be(1); + fetchesOnBWhileAWasCurrent.Should().Be(0); + handlerB.PurgeFetchCount.Should().Be(1); + handlerA.PurgeFetchCount.Should().Be(1); + } + finally + { + await worker.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task RebindableCallInvoker_MovesAnAlreadyConstructedClientToTheReplacement() + { + // Arrange - the generated client caches the invoker it was constructed with, so this is the mechanism + // that lets a singleton client follow the worker without being rebuilt. + RebindableCallInvoker invoker = new(); + FakeCallInvoker first = new(); + FakeCallInvoker second = new(); + LargePayloadPurgeClient client = new(invoker); + + // Act + invoker.Rebind(first); + await client.GetLargePayloadTombstonesAsync(new LP.GetLargePayloadTombstonesRequest { Limit = 1 }); + invoker.Rebind(second); + await client.GetLargePayloadTombstonesAsync(new LP.GetLargePayloadTombstonesRequest { Limit = 1 }); + + // Assert + first.Calls.Should().ContainSingle().Which.Should().Be("GetLargePayloadTombstones"); + second.Calls.Should().ContainSingle().Which.Should().Be("GetLargePayloadTombstones"); + } + + [Fact] + public void RebindableCallInvoker_BeforeAnythingIsPublished_Throws() + { + // Arrange - reaching a purge call with no worker transport is a wiring defect. Surfacing it beats + // inventing a connection or returning a null the caller would dereference somewhere less obvious. + RebindableCallInvoker invoker = new(); + LargePayloadPurgeClient client = new(invoker); + + // Act + Func call = () => + client.GetLargePayloadTombstonesAsync(new LP.GetLargePayloadTombstonesRequest { Limit = 1 }).ResponseAsync; + + // Assert + call.Should().ThrowAsync(); + } + + [Fact] + public void RebindableCallInvoker_ForwardsEveryCallShape() + { + // Arrange - CallInvoker has five abstract members and the generated clients only exercise two of them + // today. Cover the rest so a future streaming RPC on the purge service cannot silently bypass the + // indirection. + RebindableCallInvoker invoker = new(); + ShapeRecordingCallInvoker target = new(); + invoker.Rebind(target); + Method method = new( + MethodType.Unary, + "svc", + "m", + Marshallers.Create(_ => Array.Empty(), _ => new Empty()), + Marshallers.Create(_ => Array.Empty(), _ => new Empty())); + + // Act + invoker.BlockingUnaryCall(method, null, default, new Empty()); + invoker.AsyncUnaryCall(method, null, default, new Empty()); + invoker.AsyncServerStreamingCall(method, null, default, new Empty()); + invoker.AsyncClientStreamingCall(method, null, default); + invoker.AsyncDuplexStreamingCall(method, null, default); + + // Assert + target.Shapes.Should().Equal("blocking", "unary", "server", "client", "duplex"); + } + + static GrpcChannel CreateChannel(string address, HttpMessageHandler handler) + => GrpcChannel.ForAddress(address, new GrpcChannelOptions { HttpHandler = handler }); + + static async Task FetchTombstonesIgnoringTransportFailureAsync(LargePayloadPurgeClient client) + { + // The fake endpoints answer every call with a trailers-only Unavailable, so the call always fails. + // Which handler recorded it is the signal, not whether it succeeded - but the status is asserted rather + // than swallowed, so a call that failed for some unrelated reason (a wrong method, a broken marshaller) + // cannot masquerade as the expected transport failure. + RpcException failure = await Assert.ThrowsAsync( + () => client.GetLargePayloadTombstonesAsync(new LP.GetLargePayloadTombstonesRequest { Limit = 1 }) + .ResponseAsync); + failure.StatusCode.Should().Be(StatusCode.Unavailable); + } + + /// + /// Records the gRPC method names that pass through the configured interceptor chain. + /// + sealed class RecordingInterceptor : Interceptor + { + readonly ConcurrentQueue calls = new(); + + public IReadOnlyCollection Calls => this.calls.ToArray(); + + public override AsyncUnaryCall AsyncUnaryCall( + TRequest request, + ClientInterceptorContext context, + AsyncUnaryCallContinuation continuation) + { + this.calls.Enqueue(context.Method.Name); + return continuation(request, context); + } + + public override AsyncServerStreamingCall AsyncServerStreamingCall( + TRequest request, + ClientInterceptorContext context, + AsyncServerStreamingCallContinuation continuation) + { + this.calls.Enqueue(context.Method.Name); + return continuation(request, context); + } + } + + /// + /// A transport that answers the worker's connection-setup RPCs and the purge RPCs, and parks the work-item + /// stream so the worker stays connected for the duration of a test. + /// + sealed class FakeCallInvoker : CallInvoker + { + readonly ConcurrentQueue calls = new(); + + public IReadOnlyCollection Calls => this.calls.ToArray(); + + public TaskCompletionSource WorkItemsRequested { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public override TResponse BlockingUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => (TResponse)this.Respond(method.Name); + + public override AsyncUnaryCall AsyncUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + { + return new AsyncUnaryCall( + Task.FromResult((TResponse)this.Respond(method.Name)), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { }); + } + + public override AsyncServerStreamingCall AsyncServerStreamingCall( + Method method, string? host, CallOptions options, TRequest request) + { + this.calls.Enqueue(method.Name); + this.WorkItemsRequested.TrySetResult(true); + return new AsyncServerStreamingCall( + new ParkedStreamReader(), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { }); + } + + public override AsyncClientStreamingCall AsyncClientStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + + public override AsyncDuplexStreamingCall AsyncDuplexStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + + object Respond(string methodName) + { + this.calls.Enqueue(methodName); + return methodName switch + { + "Hello" => new Empty(), + "SetLargePayloadAutoPurge" => new LP.SetLargePayloadAutoPurgeResponse(), + "GetLargePayloadTombstones" => new LP.GetLargePayloadTombstonesResponse(), + "ReportLargePayloadPurgeResults" => new LP.ReportLargePayloadPurgeResultsResponse(), + _ => throw new RpcException(new Status(StatusCode.Unimplemented, methodName)), + }; + } + } + + /// + /// Records which member was forwarded, so every call shape can be covered. + /// + sealed class ShapeRecordingCallInvoker : CallInvoker + { + public List Shapes { get; } = new(); + + public override TResponse BlockingUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + { + this.Shapes.Add("blocking"); + return default!; + } + + public override AsyncUnaryCall AsyncUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + { + this.Shapes.Add("unary"); + return new AsyncUnaryCall( + Task.FromResult(default(TResponse)!), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { }); + } + + public override AsyncServerStreamingCall AsyncServerStreamingCall( + Method method, string? host, CallOptions options, TRequest request) + { + this.Shapes.Add("server"); + return new AsyncServerStreamingCall( + new ParkedStreamReader(), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { }); + } + + public override AsyncClientStreamingCall AsyncClientStreamingCall( + Method method, string? host, CallOptions options) + { + this.Shapes.Add("client"); + return new AsyncClientStreamingCall( + null!, + Task.FromResult(default(TResponse)!), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { }); + } + + public override AsyncDuplexStreamingCall AsyncDuplexStreamingCall( + Method method, string? host, CallOptions options) + { + this.Shapes.Add("duplex"); + return new AsyncDuplexStreamingCall( + null!, + new ParkedStreamReader(), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { }); + } + } + + /// + /// A work-item stream that never yields a message and completes only when the reader is cancelled. + /// + sealed class ParkedStreamReader : IAsyncStreamReader + { + public T Current => default!; + + public async Task MoveNext(CancellationToken cancellationToken) + { + await Task.Delay(Timeout.Infinite, cancellationToken); + return false; + } + } + + /// + /// A channel-backing handler that records the gRPC method path of every request and answers every call with + /// a trailers-only , so a call can be attributed to one channel without + /// any network I/O and the worker's connect loop classifies the failure the same way every run. + /// + sealed class RecordingHandler : HttpMessageHandler + { + readonly ConcurrentQueue paths = new(); + readonly TaskCompletionSource firstHello = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task FirstHello => this.firstHello.Task; + + public int PurgeFetchCount => + this.paths.Count(p => p.EndsWith("/GetLargePayloadTombstones", StringComparison.Ordinal)); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + string path = request.RequestUri!.AbsolutePath; + this.paths.Enqueue(path); + + // Returning a real gRPC status rather than throwing keeps the classification out of the transport + // library's exception-mapping rules: the worker must see Unavailable for the connect loop to count + // the failure toward a channel recreate. + HttpResponseMessage response = new(HttpStatusCode.OK) + { + Version = new Version(2, 0), + Content = new ByteArrayContent(Array.Empty()), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc"); + response.Headers.TryAddWithoutValidation("grpc-status", ((int)StatusCode.Unavailable).ToString(CultureInfo.InvariantCulture)); + response.Headers.TryAddWithoutValidation("grpc-message", "This endpoint intentionally never answers."); + + if (path.EndsWith("/Hello", StringComparison.Ordinal)) + { + this.firstHello.TrySetResult(true); + } + + return Task.FromResult(response); + } + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/SetLargePayloadAutoPurgeAsyncTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/SetLargePayloadAutoPurgeAsyncTests.cs new file mode 100644 index 00000000..07e5c51c --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/SetLargePayloadAutoPurgeAsyncTests.cs @@ -0,0 +1,545 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Headers; +using FluentAssertions; +using Grpc.Core; +using Grpc.Core.Interceptors; +using Grpc.Net.Client; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Client.Grpc; +using Microsoft.DurableTask.Client.Grpc.Internal; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; +using LP = Microsoft.DurableTask.Protobuf.LargePayloads; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +/// +/// Covers the public, explicitly-invoked auto-purge control. One call owns one task-hub transition, and it is +/// two operations rather than one: the backend setting is written first and awaited, then the singleton job +/// entity is signalled. Almost everything worth asserting here is about that boundary - what runs before the +/// backend is touched, what does not run when the first operation fails, and what is deliberately NOT undone +/// when the second one does. +/// +public class SetLargePayloadAutoPurgeAsyncTests +{ + const string ExpectedEntityId = "@blobpurgejob@__dt_blob_payload_autopurge__"; + + /// + /// The gRPC client's default number of consecutive transport failures before it recreates its channel. + /// Mirrored here rather than configured because the internal option is not reachable from this assembly. + /// + const int ConsecutiveFailuresBeforeRecreate = 5; + + [Fact] + public async Task Enable_WritesTheBackendSettingBeforeSignallingCreate() + { + // Arrange - both operations ride the same invoker, so their relative order is directly observable. + RecordingCallInvoker invoker = new(); + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + await client.SetLargePayloadAutoPurgeAsync(enabled: true); + + // Assert - the order is the point: enabling the setting before starting the job means the job cannot + // poll for tombstones the backend is not yet writing. + invoker.Methods.Should().Equal("SetLargePayloadAutoPurge", "SignalEntity"); + invoker.Sets.Should().ContainSingle().Which.Enabled.Should().BeTrue(); + + P.SignalEntityRequest signal = invoker.Signals.Should().ContainSingle().Subject; + signal.InstanceId.Should().Be(ExpectedEntityId); + signal.Name.Should().Be("Create"); + signal.Input.Should().Be("500"); + } + + [Fact] + public async Task Enable_PassesTheRequestedBatchSizeToTheJob() + { + // Arrange + RecordingCallInvoker invoker = new(); + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + await client.SetLargePayloadAutoPurgeAsync(enabled: true, batchSize: 42); + + // Assert - the size is the operation's input, so a caller resizing a running job does it with the same + // call that started it. + invoker.Signals.Should().ContainSingle().Which.Input.Should().Be("42"); + } + + [Fact] + public async Task Disable_WritesTheBackendSettingBeforeSignallingStop() + { + // Arrange + RecordingCallInvoker invoker = new(); + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + await client.SetLargePayloadAutoPurgeAsync(enabled: false); + + // Assert - the mirror-image order: turning tombstoning off first means no new tombstones are created + // while the job winds down. + invoker.Methods.Should().Equal("SetLargePayloadAutoPurge", "SignalEntity"); + invoker.Sets.Should().ContainSingle().Which.Enabled.Should().BeFalse(); + + P.SignalEntityRequest signal = invoker.Signals.Should().ContainSingle().Subject; + signal.InstanceId.Should().Be(ExpectedEntityId); + signal.Name.Should().Be("Stop"); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1001)] + public async Task Disable_IgnoresTheBatchSizeEvenWhenItIsOutOfRange(int batchSize) + { + // Arrange - the caller who threads a batch size through from configuration and flips only the flag. + // Rejecting the size on the disable path would fail a call that would otherwise have done exactly what + // was asked, since a job being stopped has no cycle to size. + RecordingCallInvoker invoker = new(); + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + await client.SetLargePayloadAutoPurgeAsync(enabled: false, batchSize: batchSize); + + // Assert + invoker.Methods.Should().Equal("SetLargePayloadAutoPurge", "SignalEntity"); + invoker.Signals.Should().ContainSingle().Which.Name.Should().Be("Stop"); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1001)] + public async Task Enable_WithAnOutOfRangeBatchSize_TouchesNothing(int batchSize) + { + // Arrange + RecordingCallInvoker invoker = new(); + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + Func act = () => client.SetLargePayloadAutoPurgeAsync(enabled: true, batchSize: batchSize); + + // Assert - validation runs before anything leaves the process, so a rejected call cannot leave the + // backend tombstoning payloads that no job was started to delete. + await act.Should().ThrowAsync(); + invoker.Methods.Should().BeEmpty(); + } + + [Fact] + public async Task NonGrpcClient_FailsWithoutAnyRpc() + { + // Arrange - the RPC lives on the gRPC client, so any other implementation must be rejected rather than + // silently no-op'd. + StubDurableTaskClient client = new(); + + // Act + Func act = () => client.SetLargePayloadAutoPurgeAsync(enabled: true); + + // Assert + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task EntitiesDisabledClient_FailsBeforeTheBackendIsTouched() + { + // Arrange - the worst partial failure this ordering prevents. Discovering the missing entity client + // after the setting was written would leave the backend tombstoning payloads with nothing running to + // delete them, so the local prerequisite is resolved first. + RecordingCallInvoker invoker = new(); + await using GrpcDurableTaskClient client = CreateClient(invoker, enableEntitySupport: false); + + // Act + Func act = () => client.SetLargePayloadAutoPurgeAsync(enabled: true); + + // Assert + await act.Should().ThrowAsync(); + invoker.Methods.Should().BeEmpty(); + } + + [Fact] + public async Task WhenTheBackendSettingFails_TheJobIsNotSignalled() + { + // Arrange + RecordingCallInvoker invoker = new() + { + SetFailure = new RpcException(new Status(StatusCode.Unavailable, "backend is down")), + }; + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + Func act = () => client.SetLargePayloadAutoPurgeAsync(enabled: true); + + // Assert - the failure is propagated verbatim, not translated into a partial success, and the second + // operation never runs. + (await act.Should().ThrowAsync()).Which.StatusCode.Should().Be(StatusCode.Unavailable); + invoker.Methods.Should().Equal("SetLargePayloadAutoPurge"); + } + + [Fact] + public async Task WhenTheBackendDoesNotImplementTheRpc_TheJobIsNotSignalled() + { + // Arrange - an older backend, or one that is not the Durable Task Scheduler at all. + RecordingCallInvoker invoker = new() + { + SetFailure = new RpcException(new Status(StatusCode.Unimplemented, "no such service")), + }; + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + Func act = () => client.SetLargePayloadAutoPurgeAsync(enabled: true); + + // Assert - mapped exactly as every other public method on the gRPC client maps Unimplemented. + (await act.Should().ThrowAsync()).Which.Message.Should().Be("no such service"); + invoker.Methods.Should().Equal("SetLargePayloadAutoPurge"); + } + + [Fact] + public async Task WhenTheBackendSettingIsCancelled_TheJobIsNotSignalled() + { + // Arrange + RecordingCallInvoker invoker = new() + { + SetFailure = new RpcException(new Status(StatusCode.Cancelled, "canceled")), + }; + await using GrpcDurableTaskClient client = CreateClient(invoker); + using CancellationTokenSource cts = new(); + cts.Cancel(); + + // Act + Func act = () => client.SetLargePayloadAutoPurgeAsync(enabled: true, cancellationToken: cts.Token); + + // Assert + await act.Should().ThrowAsync(); + invoker.Methods.Should().Equal("SetLargePayloadAutoPurge"); + } + + [Fact] + public async Task WhenTheEntitySignalFails_TheSettingIsSurfacedButNotRolledBack() + { + // Arrange - the one genuinely partial outcome. Rolling the setting back would be its own operation that + // can fail in turn, and it would be wrong as often as it was right, because a concurrent caller may + // have written the value the rollback would undo. + RecordingCallInvoker invoker = new() + { + SignalFailure = new RpcException(new Status(StatusCode.Internal, "entity unreachable")), + }; + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + Func act = () => client.SetLargePayloadAutoPurgeAsync(enabled: true); + + // Assert - the failure reaches the caller, and no compensating Set follows it. A second Set here would + // be the rollback the design deliberately does not perform. + await act.Should().ThrowAsync(); + invoker.Methods.Should().Equal("SetLargePayloadAutoPurge", "SignalEntity"); + invoker.Sets.Should().ContainSingle(); + } + + [Fact] + public async Task RepeatedIdenticalCalls_IssueTheSameTwoOperationsAgain() + { + // Arrange - the retry story. Both operations are idempotent, so a caller that is unsure of the current + // state, or that is retrying after a partial failure, can simply call again. + RecordingCallInvoker invoker = new(); + await using GrpcDurableTaskClient client = CreateClient(invoker); + + // Act + await client.SetLargePayloadAutoPurgeAsync(enabled: true, batchSize: 250); + await client.SetLargePayloadAutoPurgeAsync(enabled: true, batchSize: 250); + + // Assert + invoker.Methods.Should().Equal( + "SetLargePayloadAutoPurge", "SignalEntity", "SetLargePayloadAutoPurge", "SignalEntity"); + invoker.Sets.Should().HaveCount(2).And.OnlyContain(s => s.Enabled); + invoker.Signals.Should().HaveCount(2).And.OnlyContain(s => s.Name == "Create" && s.Input == "250"); + } + + [Fact] + public async Task BothOperationsRideTheClientsConfiguredInterceptorChain() + { + // Arrange - the interceptor stands in for the configured cross-cutting chain (authentication is the one + // that matters in production). If the purge client were built on a raw invoker instead of the client's + // effective one, the interceptor would see the entity signal and not the setting. + RecordingCallInvoker invoker = new(); + RecordingInterceptor interceptor = new(); + GrpcDurableTaskClientOptions options = new() + { + CallInvoker = invoker, + EnableEntitySupport = true, + }; + options.Interceptors.Add(interceptor); + await using GrpcDurableTaskClient client = new("test", options, NullLogger.Instance); + + // Act + await client.SetLargePayloadAutoPurgeAsync(enabled: true); + + // Assert + interceptor.Calls.Should().Contain("SetLargePayloadAutoPurge"); + interceptor.Calls.Should().Contain("SignalEntity"); + } + + [Fact] + public async Task WhenTheClientRecreatesItsChannel_APreconstructedClientFollowsIt() + { + // Arrange - two channels backed by handlers that record which one received a call. The client is + // constructed ONCE up front, before any recreate, so the property under test is that a client which has + // already built its purge client routes later calls through the replacement channel rather than the one + // it happened to see at construction. + RecordingHandler handlerA = new(); + RecordingHandler handlerB = new(); + using GrpcChannel channelA = GrpcChannel.ForAddress( + "http://localhost:14101", new GrpcChannelOptions { HttpHandler = handlerA }); + using GrpcChannel channelB = GrpcChannel.ForAddress( + "http://localhost:14102", new GrpcChannelOptions { HttpHandler = handlerB }); + + GrpcDurableTaskClientOptions options = new() + { + Channel = channelA, + EnableEntitySupport = true, + }; + + options.SetChannelRecreator((_, _) => Task.FromResult(channelB)); + + await using GrpcDurableTaskClient client = new("test", options, NullLogger.Instance); + + // Act - the client recreates its channel only after a run of consecutive transport failures, so the + // arrange step has to actually produce that run. Every one of these lands on A, which is what arms the + // recreate; the final call afterwards then has to land on B. + for (int i = 0; i < ConsecutiveFailuresBeforeRecreate; i++) + { + await SetIgnoringTransportFailureAsync(client); + } + + await WaitForAsync(() => handlerB.Calls > 0 || handlerA.Calls > ConsecutiveFailuresBeforeRecreate); + int callsOnABeforeTheLastAttempt = handlerA.Calls; + await SetIgnoringTransportFailureAsync(client); + + // Assert - B saw the later call, so the already-constructed purge client followed the swap. No second + // transport was created: both calls went through the client's own channel-recreating invoker. + handlerA.SetCalls.Should().BeGreaterThan(0); + handlerB.SetCalls.Should().BeGreaterThan(0); + handlerA.Calls.Should().Be(callsOnABeforeTheLastAttempt); + } + + static GrpcDurableTaskClient CreateClient(CallInvoker invoker, bool enableEntitySupport = true) + => new( + "test", + new GrpcDurableTaskClientOptions + { + CallInvoker = invoker, + EnableEntitySupport = enableEntitySupport, + }, + NullLogger.Instance); + + static async Task SetIgnoringTransportFailureAsync(GrpcDurableTaskClient client) + { + // The fake endpoints answer every call with a trailers-only Unavailable, so the call always fails. + // Which handler recorded it is the signal, not whether it succeeded - but the status is asserted rather + // than swallowed so an unrelated failure cannot masquerade as the expected transport failure. + RpcException failure = await Assert.ThrowsAsync( + () => client.SetLargePayloadAutoPurgeAsync(enabled: true)); + failure.StatusCode.Should().Be(StatusCode.Unavailable); + } + + static async Task WaitForAsync(Func condition) + { + DateTime deadline = DateTime.UtcNow.AddSeconds(10); + while (!condition() && DateTime.UtcNow < deadline) + { + await Task.Delay(20); + } + } + + /// + /// Records the gRPC methods the client issues, in order, and answers each one. Both the purge service and + /// the sidecar service ride the same invoker, which is exactly what makes their relative order observable. + /// + sealed class RecordingCallInvoker : CallInvoker + { + readonly ConcurrentQueue methods = new(); + readonly ConcurrentQueue sets = new(); + readonly ConcurrentQueue signals = new(); + + public IReadOnlyList Methods => this.methods.ToArray(); + + public IReadOnlyList Sets => this.sets.ToArray(); + + public IReadOnlyList Signals => this.signals.ToArray(); + + public RpcException? SetFailure { get; init; } + + public RpcException? SignalFailure { get; init; } + + public override AsyncUnaryCall AsyncUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + { + this.methods.Enqueue(method.Name); + + Task response; + switch (request) + { + case LP.SetLargePayloadAutoPurgeRequest set: + this.sets.Enqueue(set); + response = this.SetFailure is null + ? Task.FromResult((TResponse)(object)new LP.SetLargePayloadAutoPurgeResponse()) + : Task.FromException(this.SetFailure); + break; + case P.SignalEntityRequest signal: + this.signals.Enqueue(signal); + response = this.SignalFailure is null + ? Task.FromResult((TResponse)(object)new P.SignalEntityResponse()) + : Task.FromException(this.SignalFailure); + break; + default: + response = Task.FromException( + new RpcException(new Status(StatusCode.Unimplemented, method.Name))); + break; + } + + return new AsyncUnaryCall( + response, + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { }); + } + + public override TResponse BlockingUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => throw new NotSupportedException(); + + public override AsyncServerStreamingCall AsyncServerStreamingCall( + Method method, string? host, CallOptions options, TRequest request) + => throw new NotSupportedException(); + + public override AsyncClientStreamingCall AsyncClientStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + + public override AsyncDuplexStreamingCall AsyncDuplexStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + } + + /// + /// Records the gRPC method names that pass through the client's configured interceptor chain. + /// + sealed class RecordingInterceptor : Interceptor + { + readonly ConcurrentQueue calls = new(); + + public IReadOnlyCollection Calls => this.calls.ToArray(); + + public override AsyncUnaryCall AsyncUnaryCall( + TRequest request, + ClientInterceptorContext context, + AsyncUnaryCallContinuation continuation) + { + this.calls.Enqueue(context.Method.Name); + return continuation(request, context); + } + } + + /// + /// A channel-backing handler that answers every call with a trailers-only + /// , so a call can be attributed to one channel without network I/O and + /// the client's recreate logic classifies the failure the same way every run. + /// + sealed class RecordingHandler : HttpMessageHandler + { + int calls; + int setCalls; + + public int Calls => Volatile.Read(ref this.calls); + + public int SetCalls => Volatile.Read(ref this.setCalls); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref this.calls); + if (request.RequestUri!.AbsolutePath.EndsWith("/SetLargePayloadAutoPurge", StringComparison.Ordinal)) + { + Interlocked.Increment(ref this.setCalls); + } + + // Returning a real gRPC status rather than throwing keeps the classification out of the transport + // library's exception-mapping rules. Ownership of the response transfers to the caller, so it is + // deliberately not disposed here. + HttpResponseMessage response = new(HttpStatusCode.OK) + { + Version = new Version(2, 0), + Content = new ByteArrayContent(Array.Empty()), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc"); + response.Headers.TryAddWithoutValidation("grpc-status", ((int)StatusCode.Unavailable).ToString()); + response.Headers.TryAddWithoutValidation("grpc-message", "This endpoint intentionally never answers."); + return Task.FromResult(response); + } + } + + /// + /// A non-gRPC client, which the extension must reject before it changes anything. + /// + sealed class StubDurableTaskClient : DurableTaskClient + { + public StubDurableTaskClient() + : base("stub") + { + } + + public override ValueTask DisposeAsync() => default; + + public override Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, + object? input = null, + StartOrchestrationOptions? options = null, + CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override Task RaiseEventAsync( + string instanceId, string eventName, object? eventPayload = null, CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override Task TerminateInstanceAsync( + string instanceId, TerminateInstanceOptions? options = null, CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override Task SuspendInstanceAsync( + string instanceId, string? reason = null, CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override Task ResumeInstanceAsync( + string instanceId, string? reason = null, CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override Task GetInstancesAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override AsyncPageable GetAllInstancesAsync(OrchestrationQuery? filter = null) + => throw new NotSupportedException(); + + public override Task WaitForInstanceStartAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override Task WaitForInstanceCompletionAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override Task PurgeInstanceAsync( + string instanceId, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) + => throw new NotSupportedException(); + + public override Task PurgeAllInstancesAsync( + PurgeInstancesFilter filter, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) + => throw new NotSupportedException(); + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj b/test/Extensions/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj index 40851465..6accf793 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj +++ b/test/Extensions/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj @@ -13,6 +13,7 @@ + diff --git a/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs b/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs index c03ee76b..4a2111b5 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs @@ -373,6 +373,48 @@ public async Task UploadAsync_StaleContainerNotFoundFailure_DoesNotOverwriteNewe createCalls.Should().Be(2); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task UploadAsync_WritesOwnershipMarkerOnBothWritePaths(bool compressionEnabled) + { + // Arrange - the marker is what lets auto-purge prove the store wrote a blob before deleting it, so it + // must be written on the compressed and the uncompressed path alike. It rides the existing + // BlobOpenWriteOptions, so it costs no extra request. + BlobOpenWriteOptions? capturedOptions = null; + Mock containerClientMock = new(); + containerClientMock.Setup(c => c.Name).Returns("test-container"); + containerClientMock + .Setup(c => c.GetBlobClient(It.IsAny())) + .Returns(() => + { + Mock blobClientMock = new(); + blobClientMock.SetupGet(b => b.Uri).Returns( + new Uri("https://testaccount.blob.core.windows.net/test-container/payload")); + blobClientMock + .Setup(b => b.OpenWriteAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((bool _, BlobOpenWriteOptions options, CancellationToken _) => + { + capturedOptions = options; + return new MemoryStream(); + }); + return blobClientMock.Object; + }); + + LargePayloadStorageOptions options = new() { CompressionEnabled = compressionEnabled }; + BlobPayloadStore store = new(options, containerClientMock.Object); + + // Act + await store.UploadAsync("payload", CancellationToken.None); + + // Assert + capturedOptions.Should().NotBeNull(); + capturedOptions!.Metadata.Should().NotBeNull(); + capturedOptions.Metadata.Should().Contain( + BlobPayloadStore.OwnershipMarkerName, BlobPayloadStore.OwnershipMarkerValue); + } + static Mock CreateContainerClientMock() { Mock containerClientMock = new(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs new file mode 100644 index 00000000..0aa3ec23 --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -0,0 +1,339 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Grpc.Core; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Grpc; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.Grpc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.DependencyInjection; + +public class UseExternalizedPayloadsTests +{ + [Fact] + public void UseExternalizedPayloads_Client_RegistersNoHostedService() + { + // Arrange - auto-purge is now an explicit client call, not a declarative option, so nothing about this + // extension may start background work. A hosted service here would reintroduce the behaviour the + // redesign removed: a host that reasserts a task-hub-wide setting simply because it booted. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act + builder.Object.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + + // Assert + services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + } + + [Fact] + public void UseExternalizedPayloads_Worker_RegistersNoHostedService() + { + // Arrange - the worker mirror. A worker never announces the auto-purge setting either; it only runs the + // job's orchestrator and activities when something else has started the job. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + services.AddOptions(string.Empty) + .Configure(o => o.CallInvoker = Mock.Of()); + + // Act + builder.Object.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + + // Assert + services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + } + + [Fact] + public void UseExternalizedPayloads_ConfigureDelegate_InvokedExactlyOnce() + { + // Arrange - a delegate that counts its invocations. The old probe-at-registration ran configure a second + // time against a throwaway options instance; this locks in that user code runs exactly once, when the + // named options are first materialized. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + int invocations = 0; + + // Act + builder.Object.UseExternalizedPayloads(options => invocations++); + using ServiceProvider provider = services.BuildServiceProvider(); + provider.GetRequiredService>().Get(string.Empty); + + // Assert - configure ran once (at options materialization), not a second time at registration. + invocations.Should().Be(1); + } + + [Fact] + public void UseExternalizedPayloads_ClientOnly_RegistersResolvablePayloadStore() + { + // Arrange - a client-only host with no worker and no explicit AddExternalizedPayloadStore. This is the + // exact shape that previously failed: the core method declared a PostConfigure dependency on + // PayloadStore without ever registering it, so options resolution threw at runtime. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act - UseDevelopmentStorage=true is a valid connection string that BlobServiceClient accepts with no + // network I/O, so the store constructs offline. Build the provider and actually resolve PayloadStore. + builder.Object.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + using ServiceProvider clientProvider = services.BuildServiceProvider(); + + // Assert - the store resolves without throwing and is the blob-backed implementation. + PayloadStore store = clientProvider.GetRequiredService(); + store.Should().BeOfType(); + } + + [Fact] + public void UseExternalizedPayloads_Worker_LeavesTheGrpcWorkerOptionsAlone() + { + // Arrange - the worker used to carry a resolved auto-purge flag that it announced once per connection. + // That is gone: the setting belongs to the task hub and is written by an explicit client call, so this + // extension must not add a worker-scoped copy of it. What the worker DOES configure is the transport the + // purge activities ride, which PurgeTransportTests covers. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + CallInvoker configured = Mock.Of(); + services.AddOptions(string.Empty) + .Configure(o => o.CallInvoker = configured); + + // Act + builder.Object.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + + using ServiceProvider provider = services.BuildServiceProvider(); + GrpcDurableTaskWorkerOptions grpcOptions = + provider.GetRequiredService>().Get(string.Empty); + + // Assert - the configured transport survives PostConfigure untouched, and the large-payload capability + // is the only thing this extension adds to the worker's gRPC options. + grpcOptions.CallInvoker.Should().BeSameAs(configured); + grpcOptions.Channel.Should().BeNull(); + } + + [Fact] + public void UseExternalizedPayloads_Client_EnablesEntitySupport() + { + // Arrange - the public SetLargePayloadAutoPurgeAsync extension reaches the singleton job through + // client.Entities on BOTH the enable and disable paths, so the client must turn entity support on + // whenever externalized payloads are configured. Nothing gates this on whether auto-purge is on: the + // client cannot know that, and reading client.Entities is what the API does before it touches the + // backend. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act + builder.Object.UseExternalizedPayloads(options => { }); + using ServiceProvider provider = services.BuildServiceProvider(); + DurableTaskClientOptions options = + provider.GetRequiredService>().Get(string.Empty); + + // Assert + options.EnableEntitySupport.Should().BeTrue(); + } + + [Fact] + public void UseExternalizedPayloads_Worker_EnablesEntitySupport() + { + // Arrange - the purge orchestrator drives the BlobPurgeJob entity, and an orchestrator that touches + // entities with support off throws, so the worker must turn entity support on whenever externalized + // payloads are configured. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act + builder.Object.UseExternalizedPayloads(options => { }); + using ServiceProvider provider = services.BuildServiceProvider(); + DurableTaskWorkerOptions options = + provider.GetRequiredService>().Get(string.Empty); + + // Assert + options.EnableEntitySupport.Should().BeTrue(); + } + + [Fact] + public void UseExternalizedPayloads_WorkerOnly_PurgeActivitiesAreConstructible() + { + // Arrange - a WORKER-ONLY host: AddDurableTaskWorker + UseExternalizedPayloads, with no AddDurableTaskClient + // anywhere. This is the split-deployment shape ("client triggers, worker executes"). The purge activities + // used to inject a concrete DurableTaskClient, which a worker-only host never registers, so they threw at + // dispatch time - and only at dispatch, because DurableTaskRegistry stores a lazy + // ActivatorUtilities.GetServiceOrCreateInstance factory - leaving auto-purge silently broken. They now + // inject the worker's own LargePayloadPurge client, which rides the transport the worker publishes at + // runtime, so they construct with no DurableTaskClient present. UseGrpc is here because a real worker + // configures a transport, not because resolving the purge client needs one - see + // PurgeTransportTests.AddressOnlyWorker_ResolvesThePurgeClient for the configuration that has neither + // Channel nor CallInvoker. + ServiceCollection services = new(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(options => options.CallInvoker = Mock.Of()); + builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + }); + + using ServiceProvider provider = services.BuildServiceProvider(); + + // Act - construct both activities through the exact reflection path DurableTaskRegistry uses at dispatch. + Action constructGet = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(GetLargePayloadTombstonesActivity)); + Action constructReport = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(ReportLargePayloadPurgeResultsActivity)); + + // Assert - both must resolve without a client in the container. This throws on the pre-fix code. + constructGet.Should().NotThrow(); + constructReport.Should().NotThrow(); + } + + [Fact] + public void UseExternalizedPayloads_NamedClientBuilder_ResolvesEverythingUnderThatName() + { + // Arrange - every other test in this file drives builder.Name == string.Empty, so the entire named code + // path had no coverage: the storage options, the client's entity-support flip and the intercepted gRPC + // client options are ALL keyed on builder.Name. Drive a non-empty name end to end. + const string name = "client-hub"; + ServiceCollection services = new(); + services.AddSingleton(Mock.Of()); + services.AddOptions(name) + .Configure(o => o.CallInvoker = Mock.Of()); + + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(name); + + // Act + builder.Object.UseExternalizedPayloads( + options => options.ConnectionString = "UseDevelopmentStorage=true"); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - the storage options, the entity-support flip and the intercepted gRPC client options all + // materialize under the builder name, and the store resolves from the built provider. + provider.GetRequiredService>().Get(name) + .ConnectionString.Should().Be("UseDevelopmentStorage=true"); + provider.GetRequiredService>().Get(name) + .EnableEntitySupport.Should().BeTrue(); + provider.GetRequiredService>().Get(name) + .CallInvoker.Should().NotBeNull(); + provider.GetRequiredService().Should().BeOfType(); + } + + [Fact] + public void UseExternalizedPayloads_NamedWorkerBuilder_ResolvesEverythingUnderThatName() + { + // Arrange - the worker mirror of the named-client gap: the storage options, the worker's entity-support + // flip and the purge transport the activities inject are all keyed on builder.Name, and every existing + // worker test uses the empty name. Drive a non-empty name. + const string name = "worker-hub"; + ServiceCollection services = new(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddOptions(name) + .Configure(o => o.CallInvoker = Mock.Of()); + + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(name); + + // Act + builder.Object.UseExternalizedPayloads( + options => options.ConnectionString = "UseDevelopmentStorage=true"); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - both option sets resolve under the name, the store is blob-backed, and both purge activities + // construct through the exact reflection path DurableTaskRegistry uses at dispatch. + provider.GetRequiredService>().Get(name) + .ConnectionString.Should().Be("UseDevelopmentStorage=true"); + provider.GetRequiredService>().Get(name) + .EnableEntitySupport.Should().BeTrue(); + provider.GetRequiredService().Should().BeOfType(); + + Action constructGet = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(GetLargePayloadTombstonesActivity)); + Action constructReport = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(ReportLargePayloadPurgeResultsActivity)); + constructGet.Should().NotThrow(); + constructReport.Should().NotThrow(); + } + + [Fact] + public void UseExternalizedPayloads_NamedClientBuilder_DoesNotLeakOptionsToOtherNames() + { + // Arrange - configure ONLY the "client-hub" name. This is the assertion with teeth: a happy-path named + // test still passes even if every .Get(builder.Name) were hard-coded to Options.DefaultName, because the + // requested name and the default would resolve to the same populated instance. Asserting that OTHER names + // stay at their defaults is what actually pins the configuration to builder.Name. + const string name = "client-hub"; + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(name); + + // Act + builder.Object.UseExternalizedPayloads( + options => options.ConnectionString = "UseDevelopmentStorage=true"); + using ServiceProvider provider = services.BuildServiceProvider(); + IOptionsMonitor monitor = + provider.GetRequiredService>(); + + // Assert - neither a different name nor the default name observes "client-hub"'s configuration. + monitor.Get("other-hub").ConnectionString.Should().NotBe("UseDevelopmentStorage=true"); + monitor.Get(string.Empty).ConnectionString.Should().NotBe("UseDevelopmentStorage=true"); + } + + [Fact] + public async Task UseExternalizedPayloads_ClientAndWorkerInOneHost_ShareOneStore() + { + // Arrange - the combined "client triggers, worker executes" host: both AddDurableTaskClient and + // AddDurableTaskWorker call UseExternalizedPayloads in one ServiceCollection. PayloadStore is registered + // with TryAdd on both sides, so the container must hold exactly one. + ServiceCollection services = new(); + services.AddSingleton(NullLoggerFactory.Instance); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddDurableTaskClient(builder => + { + builder.UseGrpc(options => options.CallInvoker = Mock.Of()); + builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + }); + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(options => options.CallInvoker = Mock.Of()); + builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + }); + + // AddDurableTaskClient registers a ClientContainer that is IAsyncDisposable-only, so the provider must be + // disposed asynchronously. + await using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - one shared store, and the only hosted service is the worker itself: neither builder adds + // background work of its own now that auto-purge is an explicit call. The worker's purge activity still + // constructs in the combined container. + provider.GetServices().Should().ContainSingle() + .Which.Should().BeOfType(); + provider.GetServices().Should().ContainSingle() + .Which.Should().BeOfType(); + + Action constructActivity = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(GetLargePayloadTombstonesActivity)); + constructActivity.Should().NotThrow(); + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs new file mode 100644 index 00000000..30c7cee3 --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure; +using Azure.Core; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests; + +/// +/// Unit tests for , covering legacy v1 back-compatibility, the +/// self-describing v2 token resolution (same account, cross-account with identity, cross-account without), and +/// the ownership marker that gates every delete. +/// +public class BlobPayloadStoreDeleteTests +{ + const string ContainerName = "payloads"; + const string ConfiguredAccountUrl = "https://myaccount.blob.core.windows.net"; + + static readonly ETag KnownETag = new("\"0x8DTEST\""); + + static Mock CreateContainer(Mock blob, string expectedBlobName) + { + Mock container = new(); + container.Setup(c => c.Name).Returns(ContainerName); + container.Setup(c => c.Uri).Returns(new Uri($"{ConfiguredAccountUrl}/{ContainerName}")); + container.Setup(c => c.GetBlobClient(expectedBlobName)).Returns(blob.Object); + return container; + } + + /// + /// Creates a blob that exists and carries this store's ownership marker, which is the ordinary case for a + /// payload the store itself uploaded. + /// + static Mock CreateBlob(bool existed) => CreateBlob(existed, owned: true); + + static Mock CreateBlob(bool existed, bool owned) + { + Mock blob = new(); + + if (existed) + { + Dictionary metadata = owned + ? new() { [BlobPayloadStore.OwnershipMarkerName] = BlobPayloadStore.OwnershipMarkerValue } + : new() { ["customer-tag"] = "not-ours" }; + + blob + .Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue( + BlobsModelFactory.BlobProperties(metadata: metadata, eTag: KnownETag), Mock.Of())); + } + else + { + blob + .Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(404, "not found", "BlobNotFound", null)); + } + + blob + .Setup(b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(existed, Mock.Of())); + return blob; + } + + [Fact] + public async Task DeleteAsync_V1Token_DeletesBackingBlobIncludingSnapshots() + { + // Arrange + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); + + // Act + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v1:{ContainerName}:abc123", CancellationToken.None); + + // Assert + outcome.Should().Be(PayloadDeleteOutcome.Deleted); + container.Verify(c => c.GetBlobClient("abc123"), Times.Once); + blob.Verify( + b => b.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + It.Is(c => c.IfMatch == KnownETag), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DeleteAsync_MissingBlob_IsIdempotentAndReportsAlreadyAbsent() + { + // Arrange + Mock blob = CreateBlob(existed: false); + Mock container = CreateContainer(blob, "missing"); + BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); + + // Act (a missing blob must be a no-op, not an error) + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v1:{ContainerName}:missing", CancellationToken.None); + + // Assert - the ownership probe already proved absence, so no delete request is needed. + outcome.Should().Be(PayloadDeleteOutcome.AlreadyAbsent); + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DeleteAsync_BlobWithoutOwnershipMarker_LeavesBlobUntouched() + { + // Arrange - a blob whose token text matches the v2 grammar but which this store never wrote (for + // example a customer dataset referenced by URL, or a payload written before the marker shipped). + Mock blob = CreateBlob(existed: true, owned: false); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); + + // Act + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v2:{ConfiguredAccountUrl}/{ContainerName}/abc123", CancellationToken.None); + + // Assert - reported distinctly and, critically, never deleted. + outcome.Should().Be(PayloadDeleteOutcome.NotStoreOwned); + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DeleteAsync_V1TokenContainerMismatch_ThrowsAndDoesNotDelete() + { + // Arrange - a v1 token does not carry the account, so its container must match the configured store. + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); + + // Act & Assert + await Assert.ThrowsAsync( + () => store.DeleteAsync("blob:v1:other-container:abc123", CancellationToken.None)); + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Theory] + [InlineData("not-a-token")] + [InlineData("blob:v1:only-container")] + [InlineData("blob:v1::blobname")] + public async Task DeleteAsync_InvalidToken_ThrowsArgumentException(string token) + { + // Arrange + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => store.DeleteAsync(token, CancellationToken.None)); + } + + [Fact] + public async Task DeleteAsync_V2TokenSameContainer_DeletesViaConfiguredClient() + { + // Arrange - a self-describing v2 token whose account+container match the configured store. The store + // recognizes it via IsConfiguredContainer and deletes through the existing container client (which works + // with any auth mode), never building a cross-account client. + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); + + // Act + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v2:{ConfiguredAccountUrl}/{ContainerName}/abc123", CancellationToken.None); + + // Assert + outcome.Should().Be(PayloadDeleteOutcome.Deleted); + container.Verify(c => c.GetBlobClient("abc123"), Times.Once); + blob.Verify( + b => b.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + It.Is(c => c.IfMatch == KnownETag), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DeleteAsync_V2TokenDifferentAccountWithCredential_DoesNotUseConfiguredContainer() + { + // Arrange - a v2 token pointing at a DIFFERENT account than the configured store, with identity auth + // available. The store must build a BlobClient bound to the token's own account using the credential and + // must not touch the configured container client. A credential that throws on token acquisition proves + // the cross-account path is taken without any network call (the throw short-circuits before the send). + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + SentinelCredential credential = new(); + BlobPayloadStore store = new( + new LargePayloadStorageOptions(new Uri(ConfiguredAccountUrl), credential), container.Object); + string token = "blob:v2:https://otheraccount.blob.core.windows.net/othercontainer/abc123"; + + // Act + Exception error = await Assert.ThrowsAnyAsync( + () => store.DeleteAsync(token, CancellationToken.None)); + + // Assert - the cross-account BlobClient invoked our sentinel credential (directly or wrapped), proving + // that branch ran; the configured container client is never used for a different account. + Assert.True( + error is SentinelCredential.InvokedException || error.InnerException is SentinelCredential.InvokedException, + $"Expected the cross-account BlobClient to invoke the credential, but got: {error}"); + container.Verify(c => c.GetBlobClient(It.IsAny()), Times.Never); + } + + [Fact] + public async Task DeleteAsync_V2TokenDifferentAccountWithoutCredential_ThrowsPayloadStorageExceptionAndDoesNotDelete() + { + // Arrange - the configured store uses a connection string (account-key auth, no TokenCredential) and the + // token points at a different account. Account keys are account-specific, so the delete cannot cross + // accounts and must fail fast with a clear PayloadStorageException before any network call. + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true"), container.Object); + string token = "blob:v2:https://otheraccount.blob.core.windows.net/othercontainer/abc123"; + + // Act + PayloadStorageException error = await Assert.ThrowsAsync( + () => store.DeleteAsync(token, CancellationToken.None)); + + // Assert - fails before touching the network or the configured container. + Assert.Contains("different storage account", error.Message, StringComparison.Ordinal); + container.Verify(c => c.GetBlobClient(It.IsAny()), Times.Never); + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + // A TokenCredential that throws as soon as a token is requested, proving the cross-account BlobClient path + // was taken without performing any network I/O. + sealed class SentinelCredential : TokenCredential + { + public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken) => + throw new InvokedException(); + + public override ValueTask GetTokenAsync( + TokenRequestContext requestContext, CancellationToken cancellationToken) => + throw new InvokedException(); + + public sealed class InvokedException : Exception + { + } + } +} diff --git a/test/Worker/Grpc.Tests/RunBackgroundTaskLoggingTests.cs b/test/Worker/Grpc.Tests/RunBackgroundTaskLoggingTests.cs index 5786d749..2574367b 100644 --- a/test/Worker/Grpc.Tests/RunBackgroundTaskLoggingTests.cs +++ b/test/Worker/Grpc.Tests/RunBackgroundTaskLoggingTests.cs @@ -11,7 +11,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq; -using P = Microsoft.DurableTask.Protobuf; +using P = Microsoft.DurableTask.Protobuf; using Xunit; using Grpc.Core; using Xunit.Abstractions;