Skip to content

Commit 575bca2

Browse files
rian-berian-be
committed
benchmarks: add engine throughput suite for #56 
1 parent 539395e commit 575bca2

7 files changed

Lines changed: 438 additions & 162 deletions

‎Benchmarks/Engine/MutationEngineBatchBenchmarks.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions;
3+
using ModularityKit.Mutator.Abstractions.Context;
4+
using ModularityKit.Mutator.Abstractions.Engine;
5+
6+
namespace ModularityKit.Mutator.Benchmarks.Engine;
7+
8+
/// <summary>
9+
/// Benchmarks batch commit execution overhead for the performance-oriented engine path.
10+
/// </summary>
11+
[MemoryDiagnoser]
12+
[InProcess]
13+
public class MutationEngineBatchBenchmarks
14+
{
15+
private IMutationEngine _performanceEngine = null!;
16+
private MutationEngineBenchmarkSupport.CounterState _state = null!;
17+
private IReadOnlyList<IMutation<MutationEngineBenchmarkSupport.CounterState>> _batchMutations = null!;
18+
19+
/// <summary>
20+
/// Controls how many commit mutations are executed in a single batch benchmark iteration.
21+
/// </summary>
22+
[Params(10, 100)]
23+
public int BatchSize { get; set; }
24+
25+
/// <summary>
26+
/// Prepares the engine, base state, and batch mutation list for the selected batch size.
27+
/// </summary>
28+
[GlobalSetup]
29+
public void Setup()
30+
{
31+
_performanceEngine = MutationEngineBenchmarkSupport.BuildEngine(MutationEngineOptions.Performance);
32+
_state = new MutationEngineBenchmarkSupport.CounterState(42);
33+
_batchMutations = [.. Enumerable.Range(0, BatchSize)
34+
.Select(i => MutationEngineBenchmarkSupport.CreateCounterMutation(MutationMode.Commit, $"batch-{i}"))];
35+
}
36+
37+
/// <summary>
38+
/// Measures sequential batch commit execution without policy pressure.
39+
/// </summary>
40+
[Benchmark]
41+
public async Task Batch_Commit_Performance_NoPolicy()
42+
{
43+
var result = await _performanceEngine.ExecuteBatchAsync(_batchMutations, _state);
44+
GC.KeepAlive(result);
45+
}
46+
}

‎Benchmarks/Engine/MutationEngineBenchmarkSupport.cs‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
using ModularityKit.Mutator.Abstractions;
3+
using ModularityKit.Mutator.Abstractions.Changes;
4+
using ModularityKit.Mutator.Abstractions.Context;
5+
using ModularityKit.Mutator.Abstractions.Engine;
6+
using ModularityKit.Mutator.Abstractions.Intent;
7+
using ModularityKit.Mutator.Abstractions.Policies;
8+
using ModularityKit.Mutator.Abstractions.Results;
9+
using ModularityKit.Mutator.Runtime;
10+
11+
namespace ModularityKit.Mutator.Benchmarks.Engine;
12+
13+
/// <summary>
14+
/// Shared support types for core engine benchmark scenarios.
15+
/// </summary>
16+
internal static class MutationEngineBenchmarkSupport
17+
{
18+
public const string CounterStateId = "benchmark-counter";
19+
20+
public static IMutationEngine BuildEngine(
21+
MutationEngineOptions options,
22+
Action<IMutationEngine>? configure = null)
23+
{
24+
var services = new ServiceCollection();
25+
services.AddMutators(options);
26+
27+
var engine = services
28+
.BuildServiceProvider()
29+
.GetRequiredService<IMutationEngine>();
30+
31+
configure?.Invoke(engine);
32+
return engine;
33+
}
34+
35+
public static IncrementCounterMutation CreateCounterMutation(MutationMode mode, string operationSuffix)
36+
{
37+
var context = MutationContext.System("benchmark")
38+
with
39+
{
40+
StateId = CounterStateId,
41+
Mode = mode,
42+
CorrelationId = $"{CounterStateId}:{operationSuffix}"
43+
};
44+
45+
return new IncrementCounterMutation(context);
46+
}
47+
48+
/// <summary>
49+
/// Minimal counter state used by engine benchmark scenarios.
50+
/// </summary>
51+
/// <param name="Value">The current counter value.</param>
52+
public sealed record CounterState(int Value);
53+
54+
/// <summary>
55+
/// Minimal counter mutation shared by core engine benchmark scenarios.
56+
/// </summary>
57+
public sealed class IncrementCounterMutation(MutationContext context) : IMutation<CounterState>
58+
{
59+
public MutationIntent Intent { get; } = new()
60+
{
61+
OperationName = "IncrementCounter",
62+
Category = "Benchmark",
63+
Description = "Increment the benchmark counter by one",
64+
RiskLevel = MutationRiskLevel.Low,
65+
IsReversible = true
66+
};
67+
68+
public MutationContext Context { get; } = context;
69+
70+
public MutationResult<CounterState> Apply(CounterState state)
71+
{
72+
var next = state with { Value = state.Value + 1 };
73+
74+
return MutationResult<CounterState>.Success(
75+
next,
76+
ChangeSet.Single(StateChange.Modified(nameof(CounterState.Value), state.Value, next.Value)));
77+
}
78+
79+
public ValidationResult Validate(CounterState state)
80+
{
81+
var result = ValidationResult.Success();
82+
83+
if (state.Value < 0)
84+
result.AddError(nameof(CounterState.Value), "Counter value must be non-negative.");
85+
86+
return result;
87+
}
88+
89+
public MutationResult<CounterState> Simulate(CounterState state)
90+
{
91+
var next = state with { Value = state.Value + 1 };
92+
93+
return MutationResult<CounterState>.Success(
94+
next,
95+
ChangeSet.Single(StateChange.Modified(nameof(CounterState.Value), state.Value, next.Value)));
96+
}
97+
}
98+
99+
/// <summary>
100+
/// Trivial allow policy used to measure policy-aware engine paths.
101+
/// </summary>
102+
public sealed class AllowAllCounterPolicy : IMutationPolicy<CounterState>
103+
{
104+
public string Name => nameof(AllowAllCounterPolicy);
105+
106+
public int Priority => 0;
107+
108+
public string? Description => "Always allows the benchmark counter mutation.";
109+
110+
public PolicyDecision Evaluate(IMutation<CounterState> mutation, CounterState state)
111+
=> PolicyDecision.Allow(Name, "Benchmark policy allows all mutations.");
112+
}
113+
}

‎Benchmarks/Engine/MutationEngineCommitBenchmarks.cs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions;
3+
using ModularityKit.Mutator.Abstractions.Engine;
4+
5+
namespace ModularityKit.Mutator.Benchmarks.Engine;
6+
7+
/// <summary>
8+
/// Benchmarks single commit execution for the core mutation engine with and without policy evaluation.
9+
/// </summary>
10+
[MemoryDiagnoser]
11+
[InProcess]
12+
public class MutationEngineCommitBenchmarks
13+
{
14+
private IMutationEngine _performanceEngine = null!;
15+
private IMutationEngine _strictEngine = null!;
16+
private MutationEngineBenchmarkSupport.CounterState _state = null!;
17+
private MutationEngineBenchmarkSupport.IncrementCounterMutation _commitMutation = null!;
18+
19+
/// <summary>
20+
/// Prepares the benchmark engines, state snapshot, and commit mutation instance.
21+
/// </summary>
22+
[GlobalSetup]
23+
public void Setup()
24+
{
25+
_performanceEngine = MutationEngineBenchmarkSupport.BuildEngine(MutationEngineOptions.Performance);
26+
_strictEngine = MutationEngineBenchmarkSupport.BuildEngine(
27+
MutationEngineOptions.Strict,
28+
engine => engine.RegisterPolicy(new MutationEngineBenchmarkSupport.AllowAllCounterPolicy()));
29+
30+
_state = new MutationEngineBenchmarkSupport.CounterState(42);
31+
_commitMutation = MutationEngineBenchmarkSupport.CreateCounterMutation(Abstractions.Context.MutationMode.Commit, "commit-one");
32+
}
33+
34+
/// <summary>
35+
/// Measures a commit execution through the performance-oriented runtime path without policies.
36+
/// </summary>
37+
[Benchmark(Baseline = true)]
38+
public async Task Commit_Performance_NoPolicy()
39+
{
40+
var result = await _performanceEngine.ExecuteAsync(_commitMutation, _state);
41+
GC.KeepAlive(result);
42+
}
43+
44+
/// <summary>
45+
/// Measures a commit execution through the strict runtime path with one allow policy.
46+
/// </summary>
47+
[Benchmark]
48+
public async Task Commit_Strict_WithPolicy()
49+
{
50+
var result = await _strictEngine.ExecuteAsync(_commitMutation, _state);
51+
GC.KeepAlive(result);
52+
}
53+
}

‎Benchmarks/Engine/MutationEngineModeBenchmarks.cs‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
using BenchmarkDotNet.Attributes;
2+
using ModularityKit.Mutator.Abstractions;
3+
using ModularityKit.Mutator.Abstractions.Context;
4+
using ModularityKit.Mutator.Abstractions.Engine;
5+
6+
namespace ModularityKit.Mutator.Benchmarks.Engine;
7+
8+
/// <summary>
9+
/// Benchmarks non-commit engine modes to isolate simulate and validate-only execution overhead.
10+
/// </summary>
11+
[MemoryDiagnoser]
12+
[InProcess]
13+
public class MutationEngineModeBenchmarks
14+
{
15+
private IMutationEngine _strictEngine = null!;
16+
private MutationEngineBenchmarkSupport.CounterState _state = null!;
17+
private MutationEngineBenchmarkSupport.IncrementCounterMutation _simulateMutation = null!;
18+
private MutationEngineBenchmarkSupport.IncrementCounterMutation _validateMutation = null!;
19+
20+
/// <summary>
21+
/// Prepares the strict engine and mode-specific mutations used in this suite.
22+
/// </summary>
23+
[GlobalSetup]
24+
public void Setup()
25+
{
26+
_strictEngine = MutationEngineBenchmarkSupport.BuildEngine(
27+
MutationEngineOptions.Strict,
28+
engine => engine.RegisterPolicy(new MutationEngineBenchmarkSupport.AllowAllCounterPolicy()));
29+
30+
_state = new MutationEngineBenchmarkSupport.CounterState(42);
31+
_simulateMutation = MutationEngineBenchmarkSupport.CreateCounterMutation(MutationMode.Simulate, "simulate-one");
32+
_validateMutation = MutationEngineBenchmarkSupport.CreateCounterMutation(MutationMode.Validate, "validate-one");
33+
}
34+
35+
/// <summary>
36+
/// Measures the simulate path with strict engine behavior and one allow policy.
37+
/// </summary>
38+
[Benchmark(Baseline = true)]
39+
public async Task Simulate_Strict_WithPolicy()
40+
{
41+
var result = await _strictEngine.ExecuteAsync(_simulateMutation, _state);
42+
GC.KeepAlive(result);
43+
}
44+
45+
/// <summary>
46+
/// Measures the validate-only path with strict engine behavior and one allow policy.
47+
/// </summary>
48+
[Benchmark]
49+
public async Task ValidateOnly_Strict_WithPolicy()
50+
{
51+
var result = await _strictEngine.ExecuteAsync(_validateMutation, _state);
52+
GC.KeepAlive(result);
53+
}
54+
}

0 commit comments

Comments
 (0)