diff --git a/VERSION b/VERSION index bea438e9ad..4772543317 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.1 +3.3.2 diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/DotNetRuntime/DotNetRuntimeExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/DotNetRuntime/DotNetRuntimeExecutorTests.cs new file mode 100644 index 0000000000..574683eaa7 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/DotNetRuntime/DotNetRuntimeExecutorTests.cs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using AutoFixture; + using VirtualClient; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + + [TestFixture] + [Category("Unit")] + [Platform(Exclude = "Unix,Linux,MacOsX")] + public class DotNetRuntimeExecutorTests + { + private MockFixture fixture; + private DependencyPath mockPath; + private DependencyPath currentDirectoryPath; + private string rawString; + + [SetUp] + public void SetUpTests() + { + this.fixture = new MockFixture(); + this.fixture.Setup(PlatformID.Win32NT); + this.mockPath = this.fixture.Create(); + this.fixture.Parameters = new Dictionary + { + { "PackageName", "DotNetRuntime" } + }; + + this.SetupDefaultMockBehavior(); + } + + [Test] + public async Task DotNetRuntimeExecutorInitializesItsDependenciesAsExpected() + { + using (TestDotNetRuntimeExecutor executor = new TestDotNetRuntimeExecutor(this.fixture)) + { + Assert.IsNull(executor.ExecutablePath); + + await executor.InitializeAsync(EventContext.None, CancellationToken.None) + .ConfigureAwait(false); + + string expectedPath = this.fixture.PlatformSpecifics.Combine( + this.mockPath.Path, "win-x64", "dotnet.bat"); + Assert.AreEqual(expectedPath, executor.ExecutablePath); + } + } + + [Test] + public void DotNetRuntimeExecutorThrowsOnInitializationWhenTheWorkloadPackageIsNotFound() + { + this.fixture.PackageManager.OnGetPackage().ReturnsAsync(null as DependencyPath); + using (TestDotNetRuntimeExecutor executor = new TestDotNetRuntimeExecutor(this.fixture)) + { + DependencyException exception = Assert.ThrowsAsync( + () => executor.InitializeAsync(EventContext.None, CancellationToken.None)); + Assert.AreEqual(ErrorReason.WorkloadDependencyMissing, exception.Reason); + } + } + + [Test] + [Ignore("There is some kind of very unusual and difficult to determine anomaly that causes this method to fail to run due to a call to the IProcessProxy.Kill() method downstream.")] + public async Task DotNetRuntimeExecutorExecutesWorkloadAsExpected() + { + using (TestDotNetRuntimeExecutor executor = new TestDotNetRuntimeExecutor(this.fixture)) + { + string expectedFilePath = this.fixture.PlatformSpecifics.Combine(this.mockPath.Path, "runtimes", "win-x64", "dotnet.bat"); + int executed = 0; + this.fixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) => + { + executed++; + Assert.AreEqual(expectedFilePath, file); + return this.fixture.Process; + }; + + await executor.ExecuteAsync(EventContext.None, CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(1, executed); + } + } + + [Test] + [Ignore("There is some kind of very unusual and difficult to determine anomaly that causes this method to fail to run due to a call to the IProcessProxy.Kill() method downstream.")] + public void DotNetRuntimeExecutorThrowsWorkloadExceptionWhenTheResultsFileIsNotGenerated() + { + using (TestDotNetRuntimeExecutor executor = new TestDotNetRuntimeExecutor(this.fixture)) + { + this.fixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) => + { + this.fixture.FileSystem.Setup(fe => fe.File.Exists(executor.ResultsFilePath)).Returns(false); + return this.fixture.Process; + }; + + WorkloadException exception = Assert.ThrowsAsync( + () => executor.ExecuteAsync(EventContext.None, CancellationToken.None)); + Assert.AreEqual(ErrorReason.WorkloadFailed, exception.Reason); + } + } + + private void SetupDefaultMockBehavior() + { + string currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + this.currentDirectoryPath = new DependencyPath("DotNetRuntime", currentDirectory); + string resultsPath = this.fixture.PlatformSpecifics.Combine(this.currentDirectoryPath.Path, "Examples", "DotNetRuntimeResultsExample.txt"); + this.rawString = File.ReadAllText(resultsPath); + this.fixture.FileSystem.Setup(fe => fe.File.Exists(It.IsAny())).Returns(true); + this.fixture.FileSystem.Setup(fe => fe.File.Exists(null)).Returns(false); + this.fixture.FileSystem.Setup(fc => fc.File.Copy(It.IsAny(), It.IsAny())); + this.fixture.Directory.Setup(d => d.Exists(It.IsAny())) + .Returns(true); + + this.fixture.FileSystem.Setup(rt => rt.File.ReadAllTextAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(this.rawString); + + this.fixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPath); + this.fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => this.fixture.Process; + } + + private class TestDotNetRuntimeExecutor : DotNetRuntimeExecutor + { + public TestDotNetRuntimeExecutor(MockFixture fixture) + : base(fixture.Dependencies, fixture.Parameters) + { + } + + public TestDotNetRuntimeExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + public new Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.InitializeAsync(telemetryContext, cancellationToken); + } + + public new Task ExecuteAsync(EventContext context, CancellationToken cancellationToken) + { + this.InitializeAsync(context, cancellationToken).GetAwaiter().GetResult(); + return base.ExecuteAsync(context, cancellationToken); + } + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/DotNetRuntime/DotNetRuntimeMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/DotNetRuntime/DotNetRuntimeMetricsParserTests.cs new file mode 100644 index 0000000000..3292e16837 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/DotNetRuntime/DotNetRuntimeMetricsParserTests.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using VirtualClient.Contracts; +using NUnit.Framework; +using VirtualClient; + +namespace VirtualClient.Actions +{ + + [TestFixture] + [Category("Unit")] + internal class DotNetRuntimeMetricsParserUnitTests + { + private string rawText; + private DotNetRuntimeMetricsParser testParser; + + [SetUp] + public void Setup() + { + string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string outputPath = Path.Combine(workingDirectory, @"Examples", "DotNetRuntimeResultsExample.txt"); + this.rawText = File.ReadAllText(outputPath); + this.testParser = new DotNetRuntimeMetricsParser(this.rawText); + } + + [Test] + public void DotNetRuntimeParserVerifyThroughputResult() + { + this.testParser.Parse(); + this.testParser.ThroughputResult.PrintDataTableFormatted(); + Assert.AreEqual(4, this.testParser.ThroughputResult.Columns.Count); + } + + [Test] + public void DotNetRuntimeParserVerifyMetrics() + { + IList metrics = this.testParser.Parse(); + MetricAssert.Exists(metrics, "throughput", 11284.51, "bops"); + } + + [Test] + public void DotNetRuntimeParserThrowIfInvalidOutputFormat() + { + string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string IncorrectDotNetoutputPath =Path.Combine(workingDirectory, @"Examples", "IncorrectDotNetRuntimeResultsExample.txt"); + this.rawText = File.ReadAllText(IncorrectDotNetoutputPath); + this.testParser = new DotNetRuntimeMetricsParser(this.rawText); + SchemaException exception = Assert.Throws(() => this.testParser.Parse()); + StringAssert.Contains("The DotNetRuntime output file has incorrect format for parsing", exception.Message); + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/DotNetRuntimeResultsExample.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/DotNetRuntimeResultsExample.txt new file mode 100644 index 0000000000..886b074cd9 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/DotNetRuntimeResultsExample.txt @@ -0,0 +1,12 @@ +=============================================================================== +TOTALS FOR: COMPANY with 8 warehouses +........ .NET fixed throughput benchmark 1.0 Results (time in seconds) ........ + Count Total Min Max Avg Heap Space + New Order: 9142643 4568.99 0.000 ****** 0.000 total 0.0MB + Payment: 6101520 2572.06 0.000 ****** 0.000 used 0.0MB + OrderStatus: 691503 368.25 0.000 39.781 0.001 + Delivery: 671169 3068.53 0.000 ****** 0.005 + Stock Level: 671171 693.12 0.000 79.547 0.001 + Cust Report: 3060386 1945.34 0.000 ****** 0.001 + + throughput = 11284.51 bops diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/IncorrectDotNetRuntimeResultsExample.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/IncorrectDotNetRuntimeResultsExample.txt new file mode 100644 index 0000000000..05fa4620f9 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/IncorrectDotNetRuntimeResultsExample.txt @@ -0,0 +1 @@ +This file is IncorrectDotNetRuntime example. \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/DotNetRuntime/DotNetRuntimeExecutor.cs b/src/VirtualClient/VirtualClient.Actions/DotNetRuntime/DotNetRuntimeExecutor.cs new file mode 100644 index 0000000000..5e41dd75d5 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/DotNetRuntime/DotNetRuntimeExecutor.cs @@ -0,0 +1,378 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Platform; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// DotNetRuntime workload executor. + /// + [WindowsCompatible] + public class DotNetRuntimeExecutor : VirtualClientComponent + { + private IFileSystem fileSystem; + private ProcessManager processManager; + private ISystemManagement systemManagement; + + /// + /// The path to the dotNet.props file in DotNetRuntime workload. + /// + private string propsFilePath; + + /// + /// Constructor. + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public DotNetRuntimeExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.fileSystem = dependencies.GetService(); + this.processManager = dependencies.GetService(); + this.systemManagement = dependencies.GetService(); + this.SupportingExecutables = new List(); + } + + /// + /// Path to DotNetRuntime Package. + /// + public string PackagePath { get; set; } + + /// + /// The path to the DotNetRuntime executable. + /// + public string ExecutablePath { get; set; } + + /// + /// The number of JVM(Java Virtual Machine) instances of the DotNetRuntime workload to be intialized. + /// + public string NumberOfJvmInstances + { + get + { + this.Parameters.TryGetValue(nameof(DotNetRuntimeExecutor.NumberOfJvmInstances), out IConvertible numberOfJvmInstances); + return numberOfJvmInstances?.ToString(); + } + } + + /// + /// The number of Warehouses to be intialized. + /// + public string NumberOfWarehouses + { + get + { + this.Parameters.TryGetValue(nameof(DotNetRuntimeExecutor.NumberOfWarehouses), out IConvertible numberOfWarehouses); + return numberOfWarehouses?.ToString(); + } + } + + /// + /// The time in seconds that the workload will ramp up/warm up. + /// + public string RampUpSeconds + { + get + { + this.Parameters.TryGetValue(nameof(DotNetRuntimeExecutor.RampUpSeconds), out IConvertible rampUpSeconds); + return rampUpSeconds?.ToString(); + } + } + + /// + /// The time in seconds that the workload will run. + /// + public string MeasurementSeconds + { + get + { + this.Parameters.TryGetValue(nameof(DotNetRuntimeExecutor.MeasurementSeconds), out IConvertible measurementSeconds); + return measurementSeconds?.ToString(); + } + } + + /// + /// The the target throughput when we run the workload. + /// + public string FixedThroughput + { + get + { + this.Parameters.TryGetValue(nameof(DotNetRuntimeExecutor.FixedThroughput), out IConvertible fixedThroughput); + return fixedThroughput?.ToString(); + } + } + + /// + /// Path to the results file after executing dotNetRuntime workload. + /// + public string ResultsFilePath { get; set; } + + /// + /// A set of paths for supporting executables of the main process + /// cleaned up/terminated at the end of each round of processing. + /// + protected IList SupportingExecutables { get; } + + /// + /// Executes DotNet Runtime. + /// + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.ResetResultsFile(telemetryContext); + + try + { + DateTime startTime = DateTime.UtcNow; + await this.ExecuteWorkloadAsync(this.ExecutablePath, telemetryContext, cancellationToken) + .ConfigureAwait(false); + + DateTime endTime = DateTime.UtcNow; + } + finally + { + // Ensure any dotnet.exe instances running are stopped. + this.processManager.SafeKill(this.SupportingExecutables, this.Logger); + } + } + + /// + /// Initializes the environment and dependencies for running the DotNetRuntime workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + IPackageManager packageManager = this.Dependencies.GetService(); + DependencyPath workloadPackage = await packageManager.GetPackageAsync(this.PackageName, cancellationToken) + .ConfigureAwait(false); + + if (workloadPackage == null) + { + throw new DependencyException( + $"The expected package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + workloadPackage = this.PlatformSpecifics.ToPlatformSpecificPath(workloadPackage, this.Platform, this.CpuArchitecture); + + this.PackagePath = workloadPackage.Path; + this.ExecutablePath = this.PlatformSpecifics.Combine(workloadPackage.Path, "dotnet.bat"); + this.SupportingExecutables.Add(this.PlatformSpecifics.Combine(workloadPackage.Path, "dotnet.exe")); + this.propsFilePath = this.PlatformSpecifics.Combine(workloadPackage.Path, @"dotNet.props"); + + await this.UpdatePropsFile(this.propsFilePath); + + if (!this.fileSystem.File.Exists(this.ExecutablePath)) + { + throw new DependencyException( + $"DotNetRuntime executable not found at path '{this.ExecutablePath}'", + ErrorReason.WorkloadDependencyMissing); + } + + this.ResultsFilePath = this.PlatformSpecifics.Combine(this.PackagePath, @"results\results.txt"); + } + + /// + /// Updates the parameters value in dotnet.props file with the values provided through command line. + /// + private async Task UpdatePropsFile(string propsFilePath) + { + if (this.NumberOfJvmInstances != null) + { + await this.ReplaceInFileAsync( + propsFilePath, @"input.jvm_instances=[0-9]+", $"input.jvm_instances={this.NumberOfJvmInstances}"); + } + + if (this.NumberOfWarehouses != null) + { + await this.ReplaceInFileAsync( + propsFilePath, + @"input.sequence_of_number_of_warehouses=[0-9]+", + $"input.sequence_of_number_of_warehouses={this.NumberOfWarehouses}"); + } + + if (this.RampUpSeconds != null) + { + await this.ReplaceInFileAsync( + propsFilePath, + @"input.ramp_up_seconds=[0-9]+", + $"input.ramp_up_seconds={this.RampUpSeconds}"); + } + + if (this.MeasurementSeconds != null) + { + await this.ReplaceInFileAsync( + propsFilePath, + @"input.measurement_seconds=[0-9]+", + $"input.measurement_seconds={this.MeasurementSeconds}"); + } + + if (this.FixedThroughput != null) + { + await this.ReplaceInFileAsync( + propsFilePath, + @"input.fixed_throughput=[0-9]+", + $"input.fixed_throughput={this.FixedThroughput}"); + } + } + + /// + /// Replaces text in a file. + /// + /// Path of the text file. + /// Text to search for. + /// Text to replace the search text. + private async Task ReplaceInFileAsync(string filePath, string searchText, string replaceText) + { + using (Stream stream = this.fileSystem.FileStream.New(filePath, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + { + byte[] fileContent = new byte[stream.Length]; + await stream.ReadAsync(fileContent, 0, fileContent.Length).ConfigureAwait(false); + + string content = Regex.Replace(Encoding.Default.GetString(fileContent), searchText, replaceText); + + stream.SetLength(0); + await stream.WriteAsync(Encoding.UTF8.GetBytes(content), 0, content.Length); + } + } + + private void ResetResultsFile(EventContext telemetryContext) + { + try + { + if (this.fileSystem.File.Exists(this.ResultsFilePath)) + { + this.fileSystem.File.Delete(this.ResultsFilePath); + } + + string resultsDirectory = Path.GetDirectoryName(this.ResultsFilePath); + if (!this.fileSystem.Directory.Exists(resultsDirectory)) + { + this.fileSystem.Directory.CreateDirectory(resultsDirectory); + } + + this.fileSystem.File.WriteAllText(this.ResultsFilePath, string.Empty); + } + catch (IOException exc) + { + this.Logger.LogErrorMessage(exc, telemetryContext); + } + } + + private Task ExecuteWorkloadAsync(string executablePath, EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone() + .AddContext("executable", executablePath); + + return this.Logger.LogMessageAsync($"{nameof(DotNetRuntimeExecutor)}.ExecuteWorkload", relatedContext, async () => + { + DateTime startTime = DateTime.UtcNow; + ISystemManagement systemManagement = this.Dependencies.GetService(); + + using (IProcessProxy process = systemManagement.ProcessManager.CreateElevatedProcess(this.Platform, executablePath)) + { + this.CleanupTasks.Add(() => process.SafeKill()); + await process.StartAndWaitAsync(cancellationToken); + + if (!cancellationToken.IsCancellationRequested) + { + try + { + process.ThrowIfWorkloadFailed(); + + // Allow the dotnet.exe application to finish its work calculating results and writing them to + // the results file. + KeyValuePair results = await this.WaitForResultsAsync(this.ResultsFilePath, TimeSpan.FromMinutes(30), cancellationToken); + + await this.LogProcessDetailsAsync(process, telemetryContext, "DotNetRuntime", logToFile: true, results: results); + this.CaptureMetrics(results.Value, executablePath, startTime, DateTime.UtcNow, telemetryContext, cancellationToken); + } + catch + { + await this.LogProcessDetailsAsync(process, telemetryContext, "DotNetRuntime", logToFile: true); + } + } + } + }); + } + + private void CaptureMetrics(string results, string commandArguments, DateTime startTime, DateTime endTime, EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + if (!string.IsNullOrWhiteSpace(results)) + { + try + { + this.MetadataContract.AddForScenario( + "DotNetRuntime", + commandArguments, + toolVersion: null); + + this.MetadataContract.Apply(telemetryContext); + + DotNetRuntimeMetricsParser dotNetParser = new DotNetRuntimeMetricsParser(results); + IList metrics = dotNetParser.Parse(); + + this.Logger.LogMetrics( + "DotNetRuntime", + "DotNetRuntime", + startTime, + endTime, + metrics, + metricCategorization: string.Empty, + scenarioArguments: commandArguments, + this.Tags, + telemetryContext); + } + catch (SchemaException exc) + { + throw new WorkloadException($"Failed to parse workload results file.", exc, ErrorReason.WorkloadFailed); + } + } + else + { + throw new WorkloadException( + $"Missing results. Workload results were not emitted by the workload.", + ErrorReason.WorkloadFailed); + } + } + } + + private async Task> WaitForResultsAsync(string resultsFilePath, TimeSpan timeout, CancellationToken cancellationToken) + { + KeyValuePair results = default; + DateTime waitTimeout = DateTime.UtcNow.Add(timeout); + + while (!cancellationToken.IsCancellationRequested && DateTime.UtcNow < waitTimeout) + { + string content = await this.fileSystem.File.ReadAllTextAsync(resultsFilePath, cancellationToken); + + if (!string.IsNullOrWhiteSpace(content)) + { + results = new KeyValuePair(resultsFilePath, content); + break; + } + + await Task.Delay(500); + } + + return results; + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/DotNetRuntime/DotNetRuntimeMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/DotNetRuntime/DotNetRuntimeMetricsParser.cs new file mode 100644 index 0000000000..18277dba94 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/DotNetRuntime/DotNetRuntimeMetricsParser.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Data; + using System.Text.RegularExpressions; + using VirtualClient; + using VirtualClient.Contracts; + using DataTableExtensions = VirtualClient.Contracts.DataTableExtensions; + + /// + /// Parser for DotNetRuntime output document. + /// + public class DotNetRuntimeMetricsParser : MetricsParser + { + /// + /// Identifies string starting with digits and ending with digits. + /// + private static readonly Regex ValueUnitSplitRegex = new Regex(@"(?<=\d)( )(?=\w)", RegexOptions.ExplicitCapture); + + /// + /// Sectionize by one or more empty lines. + /// + private static readonly Regex DotNetRuntimeSectionDelimiter = new Regex(@"(\n)(\s)*(\n)", RegexOptions.ExplicitCapture); + + /// + /// Separate the column values by 2 or more spaces. + /// + private static readonly Regex DotNetRuntimeDataTableDelimiter = new Regex(@"(\s){2,}", RegexOptions.ExplicitCapture); + + /// + /// Regex for removing "==========" lines. + /// + private static readonly Regex EqualLineRegex = new Regex(@"(=){2,}(\s)*", RegexOptions.ExplicitCapture); + + /// + /// constructor for . + /// + /// Raw text to parse. + public DotNetRuntimeMetricsParser(string rawText) + : base(rawText) + { + } + + /// + /// Throughput result for DotNet Runtime. + /// + public DataTable ThroughputResult { get; set; } + + /// + public override IList Parse() + { + this.Preprocess(); + this.Sections = TextParsingExtensions.Sectionize(this.PreprocessedText, DotNetRuntimeSectionDelimiter); + this.ThrowIfInvalidOutputFormat(); + this.CalculateThroughputResult(); + + List metrics = new List(); + metrics.AddRange(this.ThroughputResult.GetMetrics(nameIndex: 0, valueIndex: 2, unitIndex: 3, metricRelativity: MetricRelativity.HigherIsBetter)); + return metrics; + } + + /// + protected override void Preprocess() + { + this.PreprocessedText = TextParsingExtensions.RemoveRows(this.RawText, DotNetRuntimeMetricsParser.EqualLineRegex); + this.PreprocessedText = this.PreprocessedText.Replace("TOTALS FOR:", $"DotNetSummary{Environment.NewLine}"); + this.PreprocessedText = Regex.Replace(this.PreprocessedText, @"throughput =", $"{Environment.NewLine}Throughput{Environment.NewLine}throughput"); + } + + private void CalculateThroughputResult() + { + string sectionName = "Throughput"; + IList columnNames = new List { "Name", "Measurement" }; + this.ThroughputResult = DataTableExtensions.ConvertToDataTable( + this.Sections[sectionName], DotNetRuntimeMetricsParser.DotNetRuntimeDataTableDelimiter, sectionName, columnNames); + + IList splitColumnNames = new List { "Value", "Unit" }; + this.ThroughputResult.SplitDataColumn(columnIndex: 1, DotNetRuntimeMetricsParser.ValueUnitSplitRegex, splitColumnNames); + } + + /// + private void ThrowIfInvalidOutputFormat() + { + if (this.Sections.Count <= 0 || !this.Sections.ContainsKey("Throughput")) + { + throw new SchemaException("The DotNetRuntime output file has incorrect format for parsing"); + } + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-CPU-DOTNETRUNTIME.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-CPU-DOTNETRUNTIME.json new file mode 100644 index 0000000000..e127fc1317 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-CPU-DOTNETRUNTIME.json @@ -0,0 +1,36 @@ +{ + "Description": "DotNetRuntime CPU Performance workload", + "Parameters": { + "NumberOfJvmInstances": "1", + "NumberOfWarehouses": "8", + "RampUpSeconds": "60", + "MeasurementSeconds": "3600", + "FixedThroughput": "16000" + }, + "Actions": [ + { + "Type": "DotNetRuntimeExecutor", + "Parameters": { + "Scenario": "CpuOperationThroughput", + "NumberOfJvmInstances": "$.Parameters.NumberOfJvmInstances", + "PackageName": "dotnetruntime", + "NumberOfWarehouses": "$.Parameters.NumberOfWarehouses", + "RampUpSeconds": "$.Parameters.RampUpSeconds", + "MeasurementSeconds": "$.Parameters.MeasurementSeconds", + "FixedThroughput": "$.Parameters.FixedThroughput" + } + } + ], + "Dependencies": [ + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallDotNetRuntimePackage", + "BlobContainer": "packages", + "BlobName": "dotnetruntime.4.6.27817.3-1.zip", + "PackageName": "dotnetruntime", + "Extract": true + } + } + ] +} \ No newline at end of file diff --git a/website/docs/workloads/dotnetruntime/dotnetruntime-profiles.md b/website/docs/workloads/dotnetruntime/dotnetruntime-profiles.md new file mode 100644 index 0000000000..463a08bba0 --- /dev/null +++ b/website/docs/workloads/dotnetruntime/dotnetruntime-profiles.md @@ -0,0 +1,56 @@ +# DotNetRuntime Workload Profiles +The following profiles run customer-representative or benchmarking scenarios using the .NET Runtime workload. + +* [Getting Started](https://microsoft.github.io/VirtualClient/) +* [Workload Details](./dotnetruntime.md) + +## PERF-CPU-DOTNETRUNTIME.json +Runs a CPU-intensive workload using the DotNetRuntime toolset to test the performance of the CPU in processing transactions to the database(warehouse). +This profile is designed by the Intel team as part of Cloud R1 Workload to identify general/broad regressions when compared against a baseline. + +* [Workload Profile](https://msazure.visualstudio.com/One/_git/CRC-AIR-Workloads?path=/src/VirtualClient/CRC.VirtualClient.Packaging/profiles/PERF-CPU-DOTNETRUNTIME.json) + +* **Supported Platform/Architectures** + * win-x64 + * win-arm64 + +* **Supported Operating Systems** + * Windows + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + The following parameters can be optionally supplied on the command line to modify the behaviors of the workload. + + | Parameter | Purpose | Default value | + |---------------------------|---------------------------------------------------------------------------------|---------------| + | NumberOfJvmInstances | Optional. This specifies the number of JVM(Java Virtual Machine) application instances to run. | 1 + | NumberOfWarehouses | Optional. This specifies the number of warehouses that we can have. | 8 + | RampUpSeconds | Optional. This specifies the time in seconds that the workload will ramp up/warm up. | 60 + | MeasurementSeconds | Optional. This specifies the time in seconds that the workload will run. | 3600 + | FixedThroughput | Optional. This specifies the target throughput. The workload will try to target the specified throughput. | 16000 | + +* **Profile Runtimes** + The following timings represent the length of time required to run a single round of profile actions. These timings can be used to determine + minimum required runtimes for the Virtual Client in order to get results. These are estimates based on the number of system cores. + This particular workload runtime is affected directly by the value of the 'MeasurementSeconds' parameter of the profile. The larger the value, + the longer the runtime. + + * Expected Runtime = 1 hour + +* **Usage Examples** + The following section provides a few basic examples of how to use the workload profile. + + ``` bash + # Execute the workload profile + VirtualClient.exe --profile=PERF-CPU-DOTNETRUNTIME.json --system=Juno --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" + + # Increase the scale factors. + VirtualClient.exe --profile=PERF-CPU-DOTNETRUNTIME.json --system=Juno --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" --parameters=NumberOfJvmInstances=2 + VirtualClient.exe --profile=PERF-CPU-DOTNETRUNTIME.json --system=Juno --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" --parameters=NumberOfJvmInstances=2,,,NumberOfWarehouses=16 + ``` \ No newline at end of file diff --git a/website/docs/workloads/dotnetruntime/dotnetruntime-supplemental.md b/website/docs/workloads/dotnetruntime/dotnetruntime-supplemental.md new file mode 100644 index 0000000000..bc3dad5e70 --- /dev/null +++ b/website/docs/workloads/dotnetruntime/dotnetruntime-supplemental.md @@ -0,0 +1,72 @@ +# DotNetRuntime Workload Supplemental +The following information is additional/supplemental to the documentation available for the DotNetRuntime workload. This information is intended for +use by teams internal to Microsoft and their affiliates. + +## System Recommendations +The following sections provide recommendations to consider when running Virtual Client profiles (workloads, monitors and tests) on +a system. + +### PERF-CPU-DOTNETRUNTIME.json +The following configurations are general recommendations for use when running this profile on cloud hardware systems and virtual machines. + +* **Recommended Configurations (Azure Cloud)** + Note that the term "cores" as used below in describing VM specifications should be inferred as synonymous with the term virtual CPU (vCPU). The configurations + below cover those used by the CRC team for running this workload as part of the Virtual Client platform. These come from recommendations and empirical + evidence from running on Azure cloud systems and are designed to mimic "customer-representative" scenarios or to utilize/stress the physical nodes/systems. + These configurations have generally proven to be well-suited for net impact analysis on systems where a change is being applied to the physical hardware + (e.g. a firmware update). + + * Operating System (unless otherwise specified below) + * Windows Scenarios + * Publisher: MicrosoftWindowsServer + * Offer: WindowsServer + * Sku: 2019-Datacenter + * Version: latest +

+ * AMD Gen6 (Naples) Hardware + * Virtual Machines (per node) + * Firmware/Hardware Validations = 10 x 2-core -> Standard_L2_v2 + * Firmware/Hardware Validations (Intel R1) = TBD + * Test/QoS = 1 x 2-core -> Standard_L2_v2 +

+ * AMD Gen7 (Rome) Hardware + * Virtual Machines (per node) + * Firmware/Hardware Validations = 10 x 2-core -> Standard_D2a_v4, Standard_E2a_v4 + * Firmware/Hardware Validations (Intel R1) = TBD + * Test/QoS = 1 x 2-core -> Standard_D2a_v4, Standard_E2a_v4 +

+ * AMD Gen8 (Milan) Hardware + * Virtual Machines (per node) + * Firmware/Hardware Validations = 10 x 2-core -> Standard_D2a_v4/v5, Standard_E2a_v4/v5 + * Firmware/Hardware Validations (Intel R1) = TBD + * Test/QoS = 1 x 2-core -> Standard_D2a_v4/v5, Standard_E2a_v4/v5 +

+ * Intel Gen5 (Broadwell) Hardware + * Virtual Machines (per node) + * Firmware/Hardware Validations = 10 x 2-core -> Standard_D2_v3, Standard_E2_v3, Standard_F2_v2 + * Firmware/Hardware Validations (Intel R1) = TBD + * Test/QoS = 1 x 2-core -> Standard_D2_v3, Standard_E2_v3, Standard_F2 +

+ * Intel Gen6 (Coffee Lake) Hardware + * Virtual Machines (per node) + * Firmware/Hardware Validations = 10 x 2-core -> Standard_D2_v3, Standard_E2_v3, Standard_F2_v2 + * Firmware/Hardware Validations (Intel R1) = TBD + * Test/QoS = 1 x 2-core -> Standard_D2_v3, Standard_E2_v3, Standard_F2_v2 +

+ * Intel Gen6 (Skylake) Hardware + * Virtual Machines (per node) + * Firmware/Hardware Validations = 10 x 2-core -> Standard_D2_v3, Standard_E2_v3, Standard_F2_v2 + * Firmware/Hardware Validations (Intel R1) = TBD + * Test/QoS = 1 x 2-core -> Standard_D2_v3, Standard_E2_v3, Standard_F2_v2 +

+ * Intel Gen7 (Cascade Lake) Hardware + * Virtual Machines (per node) + * Firmware/Hardware Validations = 10 x 2-core -> Standard_D2_v3/v4, Standard_E2_v3/v4, Standard_F2_v2 + * Firmware/Hardware Validations (Intel R1) = TBD + * Test/QoS = 1 x 2-core -> Standard_D2_v3/v4, Standard_E2_v3/v4, Standard_F2_v2 +

+ * Intel Gen8 (Icelake) Hardware + * Virtual Machines (per node) + * Firmware/Hardware Validations = 10 x 2-core -> Standard_D2_v4/v5, Standard_E2_v4/v5, Standard_F2_v2 + * Firmware/Hardware Validations (Intel R1) = TBD + * Test/QoS = 1 x 2-core -> Standard_D2_v4/v5, Standard_E2_v4/v5, Standard_F2_v2 \ No newline at end of file diff --git a/website/docs/workloads/dotnetruntime/dotnetruntime-workload-overview.docx b/website/docs/workloads/dotnetruntime/dotnetruntime-workload-overview.docx new file mode 100644 index 0000000000..86c563c0de Binary files /dev/null and b/website/docs/workloads/dotnetruntime/dotnetruntime-workload-overview.docx differ diff --git a/website/docs/workloads/dotnetruntime/dotnetruntime.md b/website/docs/workloads/dotnetruntime/dotnetruntime.md new file mode 100644 index 0000000000..8dc6feaf4b --- /dev/null +++ b/website/docs/workloads/dotnetruntime/dotnetruntime.md @@ -0,0 +1,27 @@ +# DotNetRuntime +The DotNetRuntime workload is a .NET program which mimics a 3-tier system with emphasis on the middle tier.The first tier is a set +of random input selections. This workload is a compute intensive workload and drives up the CPU utilization of the system that it is +running on. It is representative of a middle tier system and is simplified for easy benchmarking. + +* [DotNetRuntime Documentation](./dotnetruntime-workload-overview.docx) + +## What is Being Measured? +DotNetRuntime workload is designed to be a very simple benchmarking tool. It produces a measurement of **throughput** for each run of the workload +on the system. + +* Throughput in bops (billions of operations per second) + +## Workload Metrics +The following metrics are produced by the DotNetRuntime workload itself. + +| Metric Name | Example Value (min) | Example Value (max) | Example Value (avg) | Unit | +|-------------|---------------------|------| +| throughput | 15937.97 | 16141.61 | 16047.287142857143 | bops | + +## System Metrics +Different metrics are captured from the system depending upon which monitor profiles are used. If a monitor profile is not +defined, the default MONITORS-DEFAULT.json profile is used. See the following documentation to determine monitor profiles +that are available. + +* [Monitor Profiles](https://github.com/microsoft/VirtualClient/blob/main/website/docs/monitors/monitor-profiles.md) +* [Monitor Profiles (internal only)](../../monitors/monitor-profiles.md) \ No newline at end of file