diff --git a/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.PAVcalibration-rp.txt b/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.PAVcalibration-rp.txt new file mode 100644 index 0000000000..e852d92116 --- /dev/null +++ b/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.PAVcalibration-rp.txt @@ -0,0 +1,4 @@ +AveragedPerceptron +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.996146 0.97511 0.95122 0.979079 0.988558 0.972973 0.084507 0.909522 0.964948 0.992065 AveragedPerceptron %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=AveragedPerceptron cali=PAV dout=%Output% data=%Data% out=%Output% seed=1 + diff --git a/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.calibrateRandom-rp.txt b/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.calibrateRandom-rp.txt new file mode 100644 index 0000000000..0860eb7030 --- /dev/null +++ b/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.calibrateRandom-rp.txt @@ -0,0 +1,4 @@ +AveragedPerceptron +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.996146 0.97511 0.95122 0.979079 0.988558 0.972973 0.120617 0.87086 0.964948 0.992065 AveragedPerceptron %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=AveragedPerceptron numcali=200 dout=%Output% data=%Data% out=%Output% seed=1 + diff --git a/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.nocalibration-rp.txt b/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.nocalibration-rp.txt new file mode 100644 index 0000000000..ac4bbeaf12 --- /dev/null +++ b/test/BaselineOutput/Common/AveragedPerceptron/osx-arm64/AveragedPerceptron-TrainTest-breast-cancer.nocalibration-rp.txt @@ -0,0 +1,4 @@ +AveragedPerceptron +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.996146 0.97511 0.95122 0.979079 0.988558 0.972973 NaN 0 0.964948 0.992065 AveragedPerceptron %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=AveragedPerceptron cali={} dout=%Output% data=%Data% out=%Output% seed=1 + diff --git a/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-def-CV-breast-cancer-rp.txt b/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-def-CV-breast-cancer-rp.txt new file mode 100644 index 0000000000..480e3e3ca1 --- /dev/null +++ b/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-def-CV-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +LdSvm +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /iter Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.973077 0.963079 0.94487 0.947619 0.97619 0.968344 0.204103 0.777959 0.945069 0.974864 1000 LdSvm %Data% %Output% 99 0 0 maml.exe CV tr=LdSvm{iter=1000} threads=- dout=%Output% data=%Data% seed=1 /iter:1000 + diff --git a/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-nob-CV-breast-cancer-rp.txt b/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-nob-CV-breast-cancer-rp.txt new file mode 100644 index 0000000000..00dfdd4bd7 --- /dev/null +++ b/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-nob-CV-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +LdSvm +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bias /iter Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.982598 0.914001 0.827025 0.960625 0.976218 0.89237 0.24697 0.734766 0.885366 0.972638 - 1000 LdSvm %Data% %Output% 99 0 0 maml.exe CV tr=LdSvm{iter=1000 bias=-} threads=- dout=%Output% data=%Data% seed=1 /bias:-;/iter:1000 + diff --git a/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-nob-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-nob-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..c9e5f4b580 --- /dev/null +++ b/test/BaselineOutput/Common/LdSvm/osx-arm64/LDSVM-nob-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +LdSvm +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bias /iter Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.969024 0.951684 0.955752 0.903766 0.949672 0.977477 0.25164 0.730579 0.929032 0.965033 - 1000 LdSvm %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LdSvm{iter=1000 bias=-} dout=%Output% data=%Data% out=%Output% seed=1 /bias:-;/iter:1000 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/osx-arm64/WE-AvgPer-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/osx-arm64/WE-AvgPer-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..5a83bbedba --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/osx-arm64/WE-AvgPer-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.99624 0.973646 0.958506 0.966527 0.9819 0.977477 0.112168 0.879907 0.9625 0.9924 AvgPer 3 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{bp=AvgPer nm=3 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:AvgPer;/nm:3 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/osx-arm64/WE-Hetero-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/osx-arm64/WE-Hetero-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..a14165fac2 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/osx-arm64/WE-Hetero-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.996249 0.97511 0.9625 0.966527 0.981941 0.97973 0.112863 0.879162 0.964509 0.992435 svm,ap 20 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{bp=svm bp=ap nm=20 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:svm,ap;/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/osx-arm64/WE-Average-TrainTest-iris-rp.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/osx-arm64/WE-Average-TrainTest-iris-rp.txt new file mode 100644 index 0000000000..e4fb552ba6 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/osx-arm64/WE-Average-TrainTest-iris-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsembleMulticlass +Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.946667 0.946667 0.433342 0.605555 MultiAverage mlr{t-} 5 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=5 oc=MultiAverage tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /oc:MultiAverage;/bp:mlr{t-};/nm:5 + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/osx-arm64/WE-Bootstrap-TrainTest-iris-rp.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/osx-arm64/WE-Bootstrap-TrainTest-iris-rp.txt new file mode 100644 index 0000000000..b0472d02ed --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/osx-arm64/WE-Bootstrap-TrainTest-iris-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsembleMulticlass +Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.94 0.94 0.435527 0.603567 mlr{t-} 20 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=20 st=BootstrapSelector{} tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /bp:mlr{t-};/nm:20 + diff --git a/test/Microsoft.ML.Predictor.Tests/ResultProcessor/TestResultProcessor.cs b/test/Microsoft.ML.Predictor.Tests/ResultProcessor/TestResultProcessor.cs index 1a0e3cb737..c3c6da7d62 100644 --- a/test/Microsoft.ML.Predictor.Tests/ResultProcessor/TestResultProcessor.cs +++ b/test/Microsoft.ML.Predictor.Tests/ResultProcessor/TestResultProcessor.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; +using System.Linq; using System.Reflection; using Xunit; using Xunit.Abstractions; @@ -22,6 +23,37 @@ public TestResultProcessor(ITestOutputHelper helper) : base(helper) { } + [Theory] + [InlineData("absolute")] + [InlineData("with spaces")] + [InlineData("with {braces}")] + [InlineData("with {unmatched brace")] + public void ResultProcessorWithAbsolutePaths(string directory) + { + string outputDirectory = Path.Combine(SubDirectory, directory); + string[] commands = { "TrainTest", "CV" }; + string[] inputPaths = new string[commands.Length]; + for (int i = 0; i < commands.Length; i++) + { + string name = $"LogisticRegression-norm-{commands[i]}-breast-cancer-out.txt"; + inputPaths[i] = DeleteOutputPath(outputDirectory, name); + File.Copy(GetBaselinePath("LogisticRegression", name), inputPaths[i]); + } + + string outputPath = DeleteOutputPath(outputDirectory, "results-rp.txt"); + RunResultProcessorTest(inputPaths, outputPath, null); + + Assert.True(File.Exists(outputPath)); + string[] lines = File.ReadAllLines(outputPath).Where(line => !string.IsNullOrWhiteSpace(line)).ToArray(); + Assert.Equal(4, lines.Length); + Assert.Equal("LogisticRegression", lines[0]); + Assert.Contains("Accuracy", lines[1]); + for (int i = 0; i < inputPaths.Length; i++) + Assert.Contains(inputPaths[i], lines[i + 2]); + + Done(); + } + // Worker method for running the tests private void RunTestCore(string name, string fileName, string[] testDataNames, string[] extraArgs = null) { diff --git a/test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs b/test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs index 0cff1a5fb3..e6b9c35970 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs @@ -5,7 +5,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.InteropServices; +using System.Text; +using Microsoft.ML.CommandLine; using Microsoft.ML.Runtime; using Microsoft.ML.TestFrameworkCommon; @@ -199,19 +200,16 @@ protected void Run(RunContext ctx, int digitsOfPrecision = DigitsOfPrecision, Nu return; // ResultProcessor output - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // -rp.txt files are not getting generated for Non-Windows Os - { - string rpName = basePrefix + "-rp.txt"; - string rpOutPath = DeleteOutputPath(dir, rpName); + string rpName = basePrefix + "-rp.txt"; + string rpOutPath = DeleteOutputPath(dir, rpName); - string[] rpArgs = null; - if (ctx.Command == Cmd.CV && ctx.ExtraArgs != null && ctx.ExtraArgs.Any(arg => arg.Contains("opf+"))) - rpArgs = new string[] { "opf+" }; + string[] rpArgs = null; + if (ctx.Command == Cmd.CV && ctx.ExtraArgs != null && ctx.ExtraArgs.Any(arg => arg.Contains("opf+"))) + rpArgs = new string[] { "opf+" }; - // Run result processor on the console output. - RunResultProcessorTest(new string[] { consOutPath.Path }, rpOutPath, rpArgs); - CheckEqualityNormalized(dir, rpName, digitsOfPrecision: digitsOfPrecision, parseOption: parseOption); - } + // Run result processor on the console output. + RunResultProcessorTest(new string[] { consOutPath.Path }, rpOutPath, rpArgs); + CheckEqualityNormalized(dir, rpName, digitsOfPrecision: digitsOfPrecision, parseOption: parseOption); // Check the prediction output against its baseline. Contracts.Assert(predOutPath != null); @@ -288,17 +286,22 @@ protected void RunResultProcessorTest(string[] dataFiles, string outPath, string File.Delete(outPath); List args = new List(); + StringBuilder arg = new StringBuilder(); for (int i = 0; i < dataFiles.Length; i++) { - args.Add("\"" + dataFiles[i] + "\""); + arg.Clear().Append("in="); + CmdQuoter.QuoteValue(dataFiles[i], arg); + args.Add(arg.ToString()); } - args.Add("/o"); - args.Add(outPath); + arg.Clear().Append("o="); + CmdQuoter.QuoteValue(outPath, arg); + args.Add(arg.ToString()); args.Add("/calledFromUnitTestSuite+"); if (extraArgs != null) args.AddRange(extraArgs); - ResultProcessor.Main(Env, args.ToArray()); + int result = ResultProcessor.Main(Env, args.ToArray()); + Check(result == 0, "Result processor failed with exit code {0}.", result); } private static string GetNamePrefix(string testType, PredictorAndArgs predictor, TestDataset dataset, string extraTag = "")