diff --git a/Directory.Packages.props b/Directory.Packages.props index 3b4d223bc03..cf88d7638cc 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -38,6 +38,7 @@ + @@ -70,7 +71,6 @@ - diff --git a/README.md b/README.md index 822f535bcda..e9826e29144 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,7 @@ ModularPipelines takes a different approach: each unit of work is a self-contain * **Secret obfuscation** - Automatic masking in logs * **Hooks** - Run code before/after any module * **Skip conditions** - Dynamically skip modules based on custom logic -* **Retry policies** - Configurable retry with Polly integration +* **Resilience shields** - Configurable retries and advanced Kevlar integration * **Requirements validation** - Check prerequisites before running * **Progress reporting** - Real-time console output with parallel execution visualization * **Source controlled** - Your pipeline is code, version it like code diff --git a/README_Template.md b/README_Template.md index 5783c24b21f..16f6662077e 100644 --- a/README_Template.md +++ b/README_Template.md @@ -186,7 +186,7 @@ ModularPipelines takes a different approach: each unit of work is a self-contain * **Secret obfuscation** - Automatic masking in logs * **Hooks** - Run code before/after any module * **Skip conditions** - Dynamically skip modules based on custom logic -* **Retry policies** - Configurable retry with Polly integration +* **Resilience shields** - Configurable retries and advanced Kevlar integration * **Requirements validation** - Check prerequisites before running * **Progress reporting** - Real-time console output with parallel execution visualization * **Source controlled** - Your pipeline is code, version it like code diff --git a/docs/docs/architecture/module-execution-lifecycle.md b/docs/docs/architecture/module-execution-lifecycle.md index da4c78a7423..8a1ed09862b 100644 --- a/docs/docs/architecture/module-execution-lifecycle.md +++ b/docs/docs/architecture/module-execution-lifecycle.md @@ -24,7 +24,7 @@ For a module that runs successfully, the phases are: 11. Attribute `IModuleEndHandler` handlers run sequentially by priority. 12. The module result is published and dependants become eligible. -`OnBeforeExecuteAsync` and `OnAfterExecuteAsync` wrap the complete retry policy, not each +`OnBeforeExecuteAsync` and `OnAfterExecuteAsync` wrap the complete resilience shield, not each individual attempt. ## Skipped modules diff --git a/docs/docs/how-to/defining-modules.md b/docs/docs/how-to/defining-modules.md index ce2d8590672..2dad6fbe284 100644 --- a/docs/docs/how-to/defining-modules.md +++ b/docs/docs/how-to/defining-modules.md @@ -92,7 +92,7 @@ public class MyModule : Module |--------|-------------| | `.WithTimeout(TimeSpan)` | Maximum execution time before module is cancelled | | `.WithRetry(int, TimeSpan?, Func?)` | Retry attempts, jittered base delay, and optional exception filter | -| `.Advanced.WithRetryPolicy(IAsyncPolicy)` | Custom Polly policy for advanced scenarios | +| `.Advanced.WithShield(Shield)` | Custom Kevlar resilience shield for advanced scenarios | | `.WithSkipWhen(...)` | Condition to skip the module | | `.WithIgnoreFailures()` | Don't fail the pipeline if this module fails | | `.WithIgnoreFailuresWhen(...)` | Conditionally ignore failures | diff --git a/docs/docs/how-to/hooks.md b/docs/docs/how-to/hooks.md index ef28da992b7..3b89f7c6d4c 100644 --- a/docs/docs/how-to/hooks.md +++ b/docs/docs/how-to/hooks.md @@ -139,7 +139,7 @@ The order for a successful module is: 3. Global `OnModuleStartAsync` 4. Attribute `IModuleStartHandler` 5. Module `OnBeforeExecuteAsync` -6. Module `ExecuteAsync` with its retry policy +6. Module `ExecuteAsync` through its configured resilience shield 7. Module `OnAfterExecuteAsync` 8. Global `OnModuleEndAsync` 9. Attribute `IModuleEndHandler` diff --git a/docs/docs/how-to/retry-policy.md b/docs/docs/how-to/retry-policy.md index 63452901f7c..5785a383edd 100644 --- a/docs/docs/how-to/retry-policy.md +++ b/docs/docs/how-to/retry-policy.md @@ -1,9 +1,9 @@ --- -title: Retry Policies +title: Retries and Resilience Shields sidebar_position: 6 --- -# Retry Policies +# Retries and Resilience Shields When creating modules, you can configure retries per module using the `Configure()` method. The standard API supports exponential backoff, jitter, and exception filtering without exposing @@ -42,18 +42,18 @@ protected override ModuleConfiguration Configure() => ModuleConfiguration.Create .Build(); ``` -### Advanced Polly Policy +### Advanced Resilience Shield -For policy features outside the standard API, use the explicit `.Advanced` surface: +For resilience features outside the standard API, use a Kevlar `Shield` through the explicit `.Advanced` surface: ```csharp public class MyModule : Module { protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() .Advanced - .WithRetryPolicy( - Policy.Handle() - .WaitAndRetryAsync(5, i => TimeSpan.FromSeconds(i * i))) + .WithShield( + Shield.When() + .Retry(5, Backoff.Custom(i => TimeSpan.FromSeconds(i * i)))) .Build(); protected override async Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) @@ -63,20 +63,20 @@ public class MyModule : Module } ``` -### Context-Aware Retry Policy +### Context-Aware Resilience Shield -If you need access to the pipeline context when building your policy: +If you need access to the pipeline context when building your shield: ```csharp public class MyModule : Module { protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() .Advanced - .WithRetryPolicy(ctx => + .WithShield(ctx => { var retryCount = ctx.Environment.IsCI ? 5 : 2; - return Policy.Handle() - .WaitAndRetryAsync(retryCount, i => TimeSpan.FromSeconds(i)); + return Shield.When() + .Retry(retryCount, Backoff.Custom(i => TimeSpan.FromSeconds(i))); }) .Build(); } @@ -84,7 +84,7 @@ public class MyModule : Module ## Combining with Other Behaviors -Retry policies can be combined with other module behaviors: +Retry configuration can be combined with other module behaviors: ```csharp public class ResilientModule : Module @@ -97,9 +97,9 @@ public class ResilientModule : Module } ``` -## Default Retry Policy +## Default Retry Configuration -Retry policies are off by default. You can set a default retry count on the `PipelineOptions`: +Retries are off by default. You can set a default retry count on the `PipelineOptions`: ```csharp var builder = Pipeline.CreateBuilder(args); @@ -117,4 +117,4 @@ builder.ConfigurePipelineOptions(options => options with await builder.ExecutePipelineAsync(); ``` -This applies to all modules that don't override their retry policy. Modules can override this default by configuring their own retry policy in `Configure()`. +This applies to all modules that don't override their retry configuration. Modules can override this default by configuring retries in `Configure()`. diff --git a/docs/docs/migrating-to-v3.md b/docs/docs/migrating-to-v3.md index 90fcda91763..f73d448bb7c 100644 --- a/docs/docs/migrating-to-v3.md +++ b/docs/docs/migrating-to-v3.md @@ -240,7 +240,7 @@ public class MyModule : Module | V2 Override | V3 Configure() Method | |-------------|----------------------| | `TimeSpan Timeout` property | `.WithTimeout(TimeSpan)` | -| `AsyncRetryPolicy RetryPolicy` property | `.WithRetry(int, ...)` or `.Advanced.WithRetryPolicy(IAsyncPolicy)` | +| `AsyncRetryPolicy RetryPolicy` property | `.WithRetry(int, ...)` or `.Advanced.WithShield(Shield)` | | `Task ShouldSkip()` method | `.WithSkipWhen(...)` | | `Task ShouldIgnoreFailures()` method | `.WithIgnoreFailures()` or `.WithIgnoreFailuresWhen(...)` | | `ModuleRunType.AlwaysRun` | `.WithAlwaysRun()` | @@ -867,7 +867,7 @@ The following have been removed in V3: | `ShouldIgnoreFailures()` method | `Configure().WithIgnoreFailures()` | | `ModuleRunType` property | `Configure().WithAlwaysRun()` | | `Timeout` property | `Configure().WithTimeout()` | -| `RetryPolicy` property | `Configure().WithRetry()` or `.Advanced.WithRetryPolicy()` | +| `RetryPolicy` property | `Configure().WithRetry()` or `.Advanced.WithShield()` | | `GetModule()` on module | `context.GetModule()` | ## New Features in V3 diff --git a/src/ModularPipelines.Build/Modules/UnitTests/RunUnitTestModule.cs b/src/ModularPipelines.Build/Modules/UnitTests/RunUnitTestModule.cs index ecc9f8eeb85..d785325c2bf 100644 --- a/src/ModularPipelines.Build/Modules/UnitTests/RunUnitTestModule.cs +++ b/src/ModularPipelines.Build/Modules/UnitTests/RunUnitTestModule.cs @@ -1,4 +1,5 @@ using System.Text.RegularExpressions; +using Kevlar; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ModularPipelines; @@ -14,7 +15,6 @@ using ModularPipelines.Models; using ModularPipelines.Modules; using ModularPipelines.Options; -using Polly; using Spectre.Console; using File = ModularPipelines.FileSystem.File; @@ -40,7 +40,7 @@ public abstract partial class RunUnitTestModule(IOptions pipel protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() .WithSkipWhen(GetSkipDecision) .Advanced - .WithRetryPolicy(Policy.Handle().RetryAsync(0)) + .WithShield(Shield.Retry(0)) .Build(); protected virtual SkipDecision GetSkipDecision(IModuleContext context) => SkipDecision.DoNotSkip; diff --git a/src/ModularPipelines/Configuration/AdvancedModuleConfigurationBuilder.cs b/src/ModularPipelines/Configuration/AdvancedModuleConfigurationBuilder.cs index efc89ba9361..114324205ad 100644 --- a/src/ModularPipelines/Configuration/AdvancedModuleConfigurationBuilder.cs +++ b/src/ModularPipelines/Configuration/AdvancedModuleConfigurationBuilder.cs @@ -1,10 +1,10 @@ +using Kevlar; using ModularPipelines.Context; -using Polly; namespace ModularPipelines.Configuration; /// -/// Provides advanced module configuration that depends on third-party policy abstractions. +/// Provides advanced module configuration using Kevlar resilience shields. /// public sealed class AdvancedModuleConfigurationBuilder { @@ -16,24 +16,24 @@ internal AdvancedModuleConfigurationBuilder(ModuleConfigurationBuilder builder) } /// - /// Sets a custom Polly async policy for module execution. + /// Sets a custom Kevlar shield for module execution. /// - /// The Polly async policy to execute around the module. + /// The Kevlar shield to execute around the module. /// The parent module configuration builder. - public ModuleConfigurationBuilder WithRetryPolicy(IAsyncPolicy policy) + public ModuleConfigurationBuilder WithShield(Shield shield) { - ArgumentNullException.ThrowIfNull(policy); - return _builder.SetAdvancedRetryPolicy(_ => policy); + ArgumentNullException.ThrowIfNull(shield); + return _builder.SetResilienceShield(_ => shield); } /// - /// Sets a factory that creates a custom Polly async policy from the module context. + /// Sets a factory that creates a custom Kevlar shield from the module context. /// - /// The policy factory. + /// The shield factory. /// The parent module configuration builder. - public ModuleConfigurationBuilder WithRetryPolicy(Func factory) + public ModuleConfigurationBuilder WithShield(Func factory) { ArgumentNullException.ThrowIfNull(factory); - return _builder.SetAdvancedRetryPolicy(factory); + return _builder.SetResilienceShield(factory); } } diff --git a/src/ModularPipelines/Configuration/ModuleConfiguration.cs b/src/ModularPipelines/Configuration/ModuleConfiguration.cs index caf74fc73bd..f74183aeeb3 100644 --- a/src/ModularPipelines/Configuration/ModuleConfiguration.cs +++ b/src/ModularPipelines/Configuration/ModuleConfiguration.cs @@ -1,8 +1,8 @@ using System.Collections.Frozen; +using Kevlar; using ModularPipelines.Context; using ModularPipelines.Enums; using ModularPipelines.Models; -using Polly; namespace ModularPipelines.Configuration; @@ -158,7 +158,7 @@ public sealed class ModuleConfiguration internal ModuleRetryConfiguration? RetryConfiguration { get; init; } /// - /// Gets the advanced policy factory for module execution. + /// Gets the resilience shield factory for module execution. /// - internal Func? AdvancedRetryPolicyFactory { get; init; } + internal Func? ResilienceShieldFactory { get; init; } } diff --git a/src/ModularPipelines/Configuration/ModuleConfigurationAttributeAdapter.cs b/src/ModularPipelines/Configuration/ModuleConfigurationAttributeAdapter.cs index 61d835d8bac..00caa267c51 100644 --- a/src/ModularPipelines/Configuration/ModuleConfigurationAttributeAdapter.cs +++ b/src/ModularPipelines/Configuration/ModuleConfigurationAttributeAdapter.cs @@ -39,7 +39,7 @@ public static ModuleConfiguration Apply( PlanningSkipCondition = configured.PlanningSkipCondition, Timeout = configured.Timeout, RetryConfiguration = configured.RetryConfiguration, - AdvancedRetryPolicyFactory = configured.AdvancedRetryPolicyFactory, + ResilienceShieldFactory = configured.ResilienceShieldFactory, IgnoreFailuresCondition = configured.IgnoreFailuresCondition, AlwaysRun = configured.AlwaysRun, ParallelConstraintKeys = configured.ParallelConstraintKeys diff --git a/src/ModularPipelines/Configuration/ModuleConfigurationBuilder.cs b/src/ModularPipelines/Configuration/ModuleConfigurationBuilder.cs index 7047f8f315f..7d3b8708bbd 100644 --- a/src/ModularPipelines/Configuration/ModuleConfigurationBuilder.cs +++ b/src/ModularPipelines/Configuration/ModuleConfigurationBuilder.cs @@ -1,10 +1,10 @@ +using Kevlar; using ModularPipelines.Context; using ModularPipelines.Engine; using ModularPipelines.Enums; using ModularPipelines.Exceptions; using ModularPipelines.Models; using ModularPipelines.Modules; -using Polly; namespace ModularPipelines.Configuration; @@ -45,7 +45,7 @@ public sealed class ModuleConfigurationBuilder private string? _cacheAssemblyVersionKey; private TimeSpan? _timeout; private ModuleRetryConfiguration? _retryConfiguration; - private Func? _advancedRetryPolicyFactory; + private Func? _resilienceShieldFactory; private Func>? _ignoreFailuresCondition; private bool _alwaysRun; private string[]? _parallelConstraintKeys; @@ -364,7 +364,7 @@ public ModuleConfigurationBuilder WithRetry( } _retryConfiguration = new ModuleRetryConfiguration(count, retryBaseDelay, shouldRetry); - _advancedRetryPolicyFactory = null; + _resilienceShieldFactory = null; return this; } @@ -432,7 +432,7 @@ public ModuleConfiguration Build() PlanningSkipCondition = ComposePlanningSkipConditions(), Timeout = _timeout, RetryConfiguration = _retryConfiguration, - AdvancedRetryPolicyFactory = _advancedRetryPolicyFactory, + ResilienceShieldFactory = _resilienceShieldFactory, IgnoreFailuresCondition = _ignoreFailuresCondition, AlwaysRun = _alwaysRun, ParallelConstraintKeys = _parallelConstraintKeys, @@ -447,9 +447,9 @@ public ModuleConfiguration Build() }; } - internal ModuleConfigurationBuilder SetAdvancedRetryPolicy(Func factory) + internal ModuleConfigurationBuilder SetResilienceShield(Func factory) { - _advancedRetryPolicyFactory = factory; + _resilienceShieldFactory = factory; _retryConfiguration = null; return this; } diff --git a/src/ModularPipelines/Configuration/ModuleRetryPolicyFactory.cs b/src/ModularPipelines/Configuration/ModuleRetryShieldFactory.cs similarity index 63% rename from src/ModularPipelines/Configuration/ModuleRetryPolicyFactory.cs rename to src/ModularPipelines/Configuration/ModuleRetryShieldFactory.cs index e1a162abbfe..659601b3ed7 100644 --- a/src/ModularPipelines/Configuration/ModuleRetryPolicyFactory.cs +++ b/src/ModularPipelines/Configuration/ModuleRetryShieldFactory.cs @@ -1,21 +1,19 @@ -using Polly; +using Kevlar; namespace ModularPipelines.Configuration; -internal static class ModuleRetryPolicyFactory +internal static class ModuleRetryShieldFactory { - internal static IAsyncPolicy Create(ModuleRetryConfiguration configuration) + internal static Shield Create(ModuleRetryConfiguration configuration) { - var policyBuilder = configuration.ShouldRetry is null - ? Policy.Handle() - : Policy.Handle(configuration.ShouldRetry); - - return policyBuilder.WaitAndRetryAsync( - configuration.Count, - retryAttempt => CalculateDelay( - retryAttempt, - configuration.BaseDelay, - Random.Shared.NextDouble())); + return Shield + .When(configuration.ShouldRetry ?? (static _ => true)) + .Retry( + configuration.Count, + Backoff.Custom(retryAttempt => CalculateDelay( + retryAttempt, + configuration.BaseDelay, + Random.Shared.NextDouble()))); } internal static TimeSpan CalculateDelay( diff --git a/src/ModularPipelines/Engine/ModuleExecutionPipeline.cs b/src/ModularPipelines/Engine/ModuleExecutionPipeline.cs index ed89eb59f28..8997f5e779a 100644 --- a/src/ModularPipelines/Engine/ModuleExecutionPipeline.cs +++ b/src/ModularPipelines/Engine/ModuleExecutionPipeline.cs @@ -1,3 +1,4 @@ +using Kevlar; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ModularPipelines.Caching; @@ -12,7 +13,6 @@ using ModularPipelines.Modules; using ModularPipelines.Options; using ModularPipelines.Tracing; -using Polly; namespace ModularPipelines.Engine; @@ -486,8 +486,8 @@ private async Task ExecuteWithPolicies( var cancellationToken = executionContext.ModuleCancellationTokenSource.Token; - // Get retry policy if applicable - var retryPolicy = GetRetryPolicy(config, moduleContext); + // Get resilience shield if applicable + var resilienceShield = GetResilienceShield(config, moduleContext); var moduleAttemptCount = 0; var moduleAttemptRespondedToCancellation = 0; @@ -507,9 +507,11 @@ async Task ExecuteModuleAttempt(CancellationToken ct) } } - // Create the execution function that optionally includes retry - Func> executeFunc = retryPolicy != null - ? ct => retryPolicy.ExecuteAsync(ExecuteModuleAttempt, ct) + // Create the execution function that optionally includes resilience strategies + Func> executeFunc = resilienceShield != null + ? async ct => await resilienceShield.ExecuteAsync( + async shieldToken => await ExecuteModuleAttempt(shieldToken).ConfigureAwait(false), + ct).ConfigureAwait(false) : ExecuteModuleAttempt; // Use TimeoutHelper with detailed results to get information about token cooperation @@ -532,7 +534,7 @@ async Task ExecuteModuleAttempt(CancellationToken ct) if (timeoutResult.TimedOut) { var wasCancellationTokenRespected = timeoutResult.WasCancellationTokenRespected - && (retryPolicy is null + && (resilienceShield is null || Volatile.Read(ref moduleAttemptRespondedToCancellation) == 1); // Create a detailed timeout exception with information about token cooperation @@ -551,25 +553,25 @@ private TimeSpan GetTimeout(ModuleConfiguration config) return config.Timeout ?? _pipelineOptions.Value.DefaultModuleTimeout; } - private static IAsyncPolicy? GetRetryPolicy( + private static Shield? GetResilienceShield( ModuleConfiguration config, IModuleContext moduleContext) { - if (config.AdvancedRetryPolicyFactory != null) + if (config.ResilienceShieldFactory != null) { - return config.AdvancedRetryPolicyFactory(moduleContext); + return config.ResilienceShieldFactory(moduleContext); } if (config.RetryConfiguration != null) { - return ModuleRetryPolicyFactory.Create(config.RetryConfiguration); + return ModuleRetryShieldFactory.Create(config.RetryConfiguration); } // Check if default retry count is configured var defaultRetryCount = moduleContext.Services.Options.DefaultRetryCount; if (defaultRetryCount > 0) { - return ModuleRetryPolicyFactory.Create(new ModuleRetryConfiguration( + return ModuleRetryShieldFactory.Create(new ModuleRetryConfiguration( defaultRetryCount, ModuleRetryConfiguration.DefaultBaseDelay, ShouldRetry: null)); diff --git a/src/ModularPipelines/Helpers/DefaultRetryPolicyProvider.cs b/src/ModularPipelines/Helpers/DefaultRetryPolicyProvider.cs deleted file mode 100644 index 679decd921f..00000000000 --- a/src/ModularPipelines/Helpers/DefaultRetryPolicyProvider.cs +++ /dev/null @@ -1,18 +0,0 @@ -using ModularPipelines.Context; -using Polly; -using Polly.Retry; - -namespace ModularPipelines.Helpers; - -internal static class DefaultRetryPolicyProvider -{ - public static AsyncRetryPolicy GetDefaultRetryPolicy(IPipelineContext context) - { - var retryCount = context.Services.Options.DefaultRetryCount; - - return Policy.Handle() - .WaitAndRetryAsync(retryCount, - i => TimeSpan.FromMilliseconds(i * i * 100) - ); - } -} \ No newline at end of file diff --git a/src/ModularPipelines/Http/ResilienceHttpHandler.cs b/src/ModularPipelines/Http/ResilienceHttpHandler.cs index 83f32090c74..2acd651fec6 100644 --- a/src/ModularPipelines/Http/ResilienceHttpHandler.cs +++ b/src/ModularPipelines/Http/ResilienceHttpHandler.cs @@ -1,14 +1,13 @@ +using Kevlar; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ModularPipelines.Logging; using ModularPipelines.Options; -using Polly; -using Polly.Retry; namespace ModularPipelines.Http; /// -/// A delegating handler that adds retry resilience to HTTP requests using Polly. +/// A delegating handler that adds retry resilience to HTTP requests using Kevlar. /// Handles transient failures including network errors and server errors (5xx). /// internal class ResilienceHttpHandler : DelegatingHandler @@ -41,25 +40,27 @@ protected override async Task SendAsync(HttpRequestMessage contentBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); } - var retryPolicy = BuildRetryPolicy(options); + var retryShield = BuildRetryShield(options); - return await retryPolicy.ExecuteAsync( + return await retryShield.ExecuteAsync( async ct => await base.SendAsync(CloneRequest(request, contentBytes), ct).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); } - private AsyncRetryPolicy BuildRetryPolicy(HttpResilienceOptions options) + private Shield BuildRetryShield(HttpResilienceOptions options) { - return Policy - .Handle(ex => options.RetryOnHttpRequestException) - .Or(ex => + return Shield.For() + .When(_ => options.RetryOnHttpRequestException) + .Or(exception => options.RetryOnTimeout && - ex.InnerException is TimeoutException) + exception.InnerException is TimeoutException) .OrResult(response => ShouldRetryStatusCode(response, options)) - .WaitAndRetryAsync( - options.MaxRetryAttempts, - retryAttempt => CalculateDelay(retryAttempt, options), - OnRetry); + .Retry(retryOptions => + { + retryOptions.MaxRetries = options.MaxRetryAttempts; + retryOptions.Backoff = Backoff.Custom(retryAttempt => CalculateDelay(retryAttempt, options)); + retryOptions.OnRetry = OnRetry; + }); } private static bool ShouldRetryStatusCode(HttpResponseMessage response, HttpResilienceOptions options) @@ -93,17 +94,18 @@ private TimeSpan CalculateDelay(int retryAttempt, HttpResilienceOptions options) return baseDelay; } - private Task OnRetry(DelegateResult outcome, TimeSpan delay, int retryAttempt, Polly.Context context) + private void OnRetry(RetryEvent retryEvent) { var logger = _loggerProvider.GetLogger(); + var outcome = retryEvent.Outcome; if (outcome.Exception != null) { logger.LogWarning("HTTP request failed with {ExceptionType}: {Message}. Retry attempt {RetryAttempt} after {Delay}ms", outcome.Exception.GetType().Name, outcome.Exception.Message, - retryAttempt, - (int) delay.TotalMilliseconds); + retryEvent.Attempt, + (int) retryEvent.Delay.TotalMilliseconds); } else if (outcome.Result != null) { @@ -111,16 +113,14 @@ private Task OnRetry(DelegateResult outcome, TimeSpan delay { logger.LogWarning("HTTP request returned {StatusCode}. Retry attempt {RetryAttempt} after {Delay}ms", (int) outcome.Result.StatusCode, - retryAttempt, - (int) delay.TotalMilliseconds); + retryEvent.Attempt, + (int) retryEvent.Delay.TotalMilliseconds); } finally { outcome.Result.Dispose(); } } - - return Task.CompletedTask; } /// diff --git a/src/ModularPipelines/ModularPipelines.csproj b/src/ModularPipelines/ModularPipelines.csproj index 5bd0eeae4b5..d0ef6bb570d 100644 --- a/src/ModularPipelines/ModularPipelines.csproj +++ b/src/ModularPipelines/ModularPipelines.csproj @@ -136,7 +136,7 @@ compile;contentfiles;build;analyzers - + diff --git a/src/ModularPipelines/Options/PipelineOptions.cs b/src/ModularPipelines/Options/PipelineOptions.cs index 063df831bc8..ac730f6a621 100644 --- a/src/ModularPipelines/Options/PipelineOptions.cs +++ b/src/ModularPipelines/Options/PipelineOptions.cs @@ -40,7 +40,7 @@ namespace ModularPipelines.Options; /// /// Module Behaviors: /// Module-level configuration uses . A module with -/// configured will use its custom retry policy instead of +/// configured will use its custom retry configuration instead of /// . Modules without configuration fall back to global settings. /// /// diff --git a/test/ModularPipelines.UnitTests/Configuration/ModuleConfigurationTests.cs b/test/ModularPipelines.UnitTests/Configuration/ModuleConfigurationTests.cs index ba9ec3456fe..046d1a6fc01 100644 --- a/test/ModularPipelines.UnitTests/Configuration/ModuleConfigurationTests.cs +++ b/test/ModularPipelines.UnitTests/Configuration/ModuleConfigurationTests.cs @@ -1,10 +1,10 @@ +using Kevlar; using ModularPipelines.Configuration; using ModularPipelines.Context; using ModularPipelines.Enums; using ModularPipelines.Models; using ModularPipelines.Modules; using Moq; -using Polly; namespace ModularPipelines.UnitTests.Configuration; @@ -42,7 +42,7 @@ public async Task Default_RetryConfiguration_IsNull() using (Assert.Multiple()) { await Assert.That(config.RetryConfiguration).IsNull(); - await Assert.That(config.AdvancedRetryPolicyFactory).IsNull(); + await Assert.That(config.ResilienceShieldFactory).IsNull(); } } @@ -344,7 +344,7 @@ public async Task WithRetry_UsesDefaultBaseDelay() await Assert.That(config.RetryConfiguration!.Count).IsEqualTo(3); await Assert.That(config.RetryConfiguration.BaseDelay).IsEqualTo(TimeSpan.FromMilliseconds(100)); await Assert.That(config.RetryConfiguration.ShouldRetry).IsNull(); - await Assert.That(config.AdvancedRetryPolicyFactory).IsNull(); + await Assert.That(config.ResilienceShieldFactory).IsNull(); } } @@ -383,43 +383,43 @@ public async Task WithRetry_RejectsNegativeValues() } [Test] - public async Task Advanced_WithRetryPolicy_Direct_SetsAdvancedRetryPolicyFactory() + public async Task Advanced_WithShield_Direct_SetsResilienceShieldFactory() { - var policy = Policy.NoOpAsync(); + var shield = Shield.Retry(0); var config = ModuleConfiguration.Create() .Advanced - .WithRetryPolicy(policy) + .WithShield(shield) .Build(); - await Assert.That(config.AdvancedRetryPolicyFactory).IsNotNull(); + await Assert.That(config.ResilienceShieldFactory).IsNotNull(); var context = Mock.Of(); - var result = config.AdvancedRetryPolicyFactory!(context); + var result = config.ResilienceShieldFactory!(context); - await Assert.That(result).IsEqualTo(policy); + await Assert.That(result).IsEqualTo(shield); } [Test] - public async Task Advanced_WithRetryPolicy_Factory_SetsAdvancedRetryPolicyFactory() + public async Task Advanced_WithShield_Factory_SetsResilienceShieldFactory() { - var policy = Policy.NoOpAsync(); + var shield = Shield.Retry(0); var config = ModuleConfiguration.Create() .Advanced - .WithRetryPolicy(_ => policy) + .WithShield(_ => shield) .Build(); - await Assert.That(config.AdvancedRetryPolicyFactory).IsNotNull(); + await Assert.That(config.ResilienceShieldFactory).IsNotNull(); var context = Mock.Of(); - var result = config.AdvancedRetryPolicyFactory!(context); + var result = config.ResilienceShieldFactory!(context); - await Assert.That(result).IsEqualTo(policy); + await Assert.That(result).IsEqualTo(shield); } [Test] - public async Task StandardConfigurationSurface_DoesNotExposePollyTypes() + public async Task StandardConfigurationSurface_DoesNotExposeKevlarTypes() { var publicSurfaceTypes = typeof(ModuleConfigurationBuilder) .GetMethods() @@ -430,7 +430,7 @@ public async Task StandardConfigurationSurface_DoesNotExposePollyTypes() .GetProperties() .Select(property => property.PropertyType)); - await Assert.That(publicSurfaceTypes.Any(ContainsPollyType)).IsFalse(); + await Assert.That(publicSurfaceTypes.Any(ContainsKevlarType)).IsFalse(); } [Test] @@ -438,7 +438,7 @@ public async Task StandardConfigurationSurface_DoesNotExposePollyTypes() [Arguments(1, 200)] public async Task RetryDelayCalculator_AddsBoundedJitter(double jitterFactor, int expectedMilliseconds) { - var delay = ModuleRetryPolicyFactory.CalculateDelay( + var delay = ModuleRetryShieldFactory.CalculateDelay( retryAttempt: 2, baseDelay: TimeSpan.FromMilliseconds(100), jitterFactor); @@ -580,7 +580,7 @@ public async Task Builder_FluentChaining_AllMethodsChain() #endregion - private static bool ContainsPollyType(Type type) => - type.Namespace?.StartsWith("Polly", StringComparison.Ordinal) == true - || type.GetGenericArguments().Any(ContainsPollyType); + private static bool ContainsKevlarType(Type type) => + type.Namespace?.StartsWith("Kevlar", StringComparison.Ordinal) == true + || type.GetGenericArguments().Any(ContainsKevlarType); } diff --git a/test/ModularPipelines.UnitTests/Execution/ComposableModuleTests.cs b/test/ModularPipelines.UnitTests/Execution/ComposableModuleTests.cs index 2bdc9330a11..4aa370cc6f2 100644 --- a/test/ModularPipelines.UnitTests/Execution/ComposableModuleTests.cs +++ b/test/ModularPipelines.UnitTests/Execution/ComposableModuleTests.cs @@ -6,8 +6,6 @@ using ModularPipelines.Models; using ModularPipelines.Modules; using ModularPipelines.TestHelpers; -using Polly; -using Polly.Retry; using TUnit.Assertions.Extensions; namespace ModularPipelines.UnitTests.Execution; diff --git a/test/ModularPipelines.UnitTests/Execution/RetryTests.cs b/test/ModularPipelines.UnitTests/Execution/RetryTests.cs index 00e7a52414f..327caffa354 100644 --- a/test/ModularPipelines.UnitTests/Execution/RetryTests.cs +++ b/test/ModularPipelines.UnitTests/Execution/RetryTests.cs @@ -1,3 +1,4 @@ +using Kevlar; using Microsoft.Extensions.DependencyInjection; using ModularPipelines.Configuration; using ModularPipelines.Context; @@ -7,8 +8,6 @@ using ModularPipelines.Models; using ModularPipelines.Modules; using ModularPipelines.TestHelpers; -using Polly; -using Polly.Retry; namespace ModularPipelines.UnitTests.Execution; @@ -118,9 +117,9 @@ private class FailedModuleWithCustomRetryPolicy : Module protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() .Advanced - .WithRetryPolicy(Policy - .Handle() - .WaitAndRetryAsync(DefaultRetryCount, _ => TimeSpan.Zero)) + .WithShield(Shield + .When() + .Retry(DefaultRetryCount, Backoff.None)) .Build(); protected internal override async Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) @@ -144,9 +143,9 @@ private class FailedModuleWithTimeout : Module protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() .WithTimeout(TimeSpan.FromMilliseconds(ModuleTimeoutMs)) .Advanced - .WithRetryPolicy(Policy - .Handle() - .WaitAndRetryAsync(DefaultRetryCount, _ => TimeSpan.FromMilliseconds(RetryDelayMs))) + .WithShield(Shield + .When() + .Retry(DefaultRetryCount, Backoff.Constant(TimeSpan.FromMilliseconds(RetryDelayMs)))) .Build(); protected internal override Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) diff --git a/test/ModularPipelines.UnitTests/Execution/SubModuleTests.cs b/test/ModularPipelines.UnitTests/Execution/SubModuleTests.cs index 5bfe62074c3..dd7b2308dfe 100644 --- a/test/ModularPipelines.UnitTests/Execution/SubModuleTests.cs +++ b/test/ModularPipelines.UnitTests/Execution/SubModuleTests.cs @@ -1,3 +1,4 @@ +using Kevlar; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using ModularPipelines.Configuration; @@ -8,8 +9,6 @@ using EnumerableAsyncProcessor.Extensions; using ModularPipelines.Exceptions; using ModularPipelines.TestHelpers; -using Polly; -using Polly.Retry; namespace ModularPipelines.UnitTests.Execution; @@ -171,7 +170,7 @@ private class SucceedingSubModulesDoNotRetryModule : Module protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() .Advanced - .WithRetryPolicy(Policy.Handle().RetryAsync(3)) + .WithShield(Shield.When().Retry(3, Backoff.None)) .Build(); protected internal override async Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) @@ -209,7 +208,7 @@ private class SucceedingSubModulesDoNotRetryModule_WithReturnType : Module ModuleConfiguration.Create() .Advanced - .WithRetryPolicy(Policy.Handle().RetryAsync(3)) + .WithShield(Shield.When().Retry(3, Backoff.None)) .Build(); protected internal override async Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) @@ -401,8 +400,8 @@ public async Task Module_With_Retry_Policy_Retries_Entire_Execution() var module = host.Services.GetServices().OfType().First(); - // Polly retries the entire module execution, so all counters increment on each retry - // With RetryAsync(3), we get 1 original + 3 retries = 4 total executions + // Kevlar retries the entire module execution, so all counters increment on each retry. + // With Retry(3), we get 1 original + 3 retries = 4 total executions. using (Assert.Multiple()) { await Assert.That(module._oneCount).IsEqualTo(4); @@ -423,8 +422,8 @@ public async Task Module_With_Retry_Policy_Retries_Entire_Execution_With_Return_ var module = host.Services.GetServices().OfType().First(); - // Polly retries the entire module execution, so all counters increment on each retry - // With RetryAsync(3), we get 1 original + 3 retries = 4 total executions + // Kevlar retries the entire module execution, so all counters increment on each retry. + // With Retry(3), we get 1 original + 3 retries = 4 total executions. using (Assert.Multiple()) { await Assert.That(module._oneCount).IsEqualTo(4); diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/ModularPipelines.OptionsGenerator.csproj b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/ModularPipelines.OptionsGenerator.csproj index 4bfdc24b084..f343dd3935e 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/ModularPipelines.OptionsGenerator.csproj +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/ModularPipelines.OptionsGenerator.csproj @@ -22,7 +22,7 @@ - + diff --git a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/TypeDetection/ResilientCliCommandExecutor.cs b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/TypeDetection/ResilientCliCommandExecutor.cs index 46a06918f5c..f270e328417 100644 --- a/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/TypeDetection/ResilientCliCommandExecutor.cs +++ b/tools/ModularPipelines.OptionsGenerator/src/ModularPipelines.OptionsGenerator/TypeDetection/ResilientCliCommandExecutor.cs @@ -1,13 +1,11 @@ +using Kevlar; using Microsoft.Extensions.Logging; -using Polly; -using Polly.CircuitBreaker; -using Polly.Retry; namespace ModularPipelines.OptionsGenerator.TypeDetection; /// /// Decorator that adds resilience patterns (retry and circuit breaker) to CLI command execution. -/// Uses Polly for: +/// Uses Kevlar for: /// - Exponential backoff retry (3 attempts) for transient failures /// - Circuit breaker (5 failures opens circuit for 30 seconds) to prevent cascading failures /// @@ -15,7 +13,7 @@ public sealed class ResilientCliCommandExecutor : ICliCommandExecutor { private readonly ICliCommandExecutor _inner; private readonly ILogger _logger; - private readonly ResiliencePipeline _pipeline; + private readonly Shield _shield; /// /// Default retry count for transient failures. @@ -58,51 +56,45 @@ public ResilientCliCommandExecutor( _inner = inner; _logger = logger; - _pipeline = new ResiliencePipelineBuilder() - .AddRetry(new RetryStrategyOptions + _shield = Shield.For() + .WhenResult(IsTransientFailure) + .Retry(options => { - MaxRetryAttempts = maxRetries, - BackoffType = DelayBackoffType.Exponential, - Delay = baseDelay, - ShouldHandle = new PredicateBuilder() - .HandleResult(r => IsTransientFailure(r)), - OnRetry = args => + options.MaxRetries = maxRetries; + options.Backoff = Backoff.Exponential(baseDelay, jitter: false); + options.OnRetry = retryEvent => { _logger.LogWarning( "CLI command failed (attempt {Attempt}/{MaxAttempts}), retrying in {Delay}ms...", - args.AttemptNumber, + retryEvent.Attempt, maxRetries, - args.RetryDelay.TotalMilliseconds); - return default; - } + retryEvent.Delay.TotalMilliseconds); + }; }) - .AddCircuitBreaker(new CircuitBreakerStrategyOptions + .CircuitBreaker(options => { - FailureRatio = 0.5, - MinimumThroughput = circuitBreakerThreshold, - SamplingDuration = TimeSpan.FromSeconds(30), - BreakDuration = circuitBreakerDuration, - ShouldHandle = new PredicateBuilder() - .HandleResult(r => IsTransientFailure(r)), - OnOpened = args => + options.FailureRatio = 0.5; + options.MinimumThroughput = circuitBreakerThreshold; + options.SamplingWindow = TimeSpan.FromSeconds(30); + options.BreakDuration = circuitBreakerDuration; + options.OnStateChanged = stateChangedEvent => { - _logger.LogError( - "Circuit breaker OPENED - CLI commands are failing. Will retry after {Duration}s", - args.BreakDuration.TotalSeconds); - return default; - }, - OnClosed = _ => - { - _logger.LogInformation("Circuit breaker CLOSED - CLI commands are healthy again"); - return default; - }, - OnHalfOpened = _ => - { - _logger.LogInformation("Circuit breaker HALF-OPEN - Testing if CLI commands are healthy..."); - return default; - } - }) - .Build(); + switch (stateChangedEvent.To) + { + case CircuitState.Open: + _logger.LogError( + "Circuit breaker OPENED - CLI commands are failing. Will retry after {Duration}s", + circuitBreakerDuration.TotalSeconds); + break; + case CircuitState.Closed: + _logger.LogInformation("Circuit breaker CLOSED - CLI commands are healthy again"); + break; + case CircuitState.HalfOpen: + _logger.LogInformation("Circuit breaker HALF-OPEN - Testing if CLI commands are healthy..."); + break; + } + }; + }); } public async Task ExecuteAsync( @@ -113,11 +105,11 @@ public async Task ExecuteAsync( { try { - return await _pipeline.ExecuteAsync( + return await _shield.ExecuteAsync( async token => await _inner.ExecuteAsync(command, arguments, token, workingDirectory), cancellationToken); } - catch (BrokenCircuitException ex) + catch (CircuitOpenException ex) { _logger.LogError("Circuit breaker is open - CLI execution rejected: {Command} {Arguments}", command, arguments); return new CliCommandResult