Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
AveragedPerceptron
AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.996146 0.97511 0.95122 0.979079 0.988558 0.972973 0.084507 0.909522 0.964948 0.992065 AveragedPerceptron %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=AveragedPerceptron cali=PAV dout=%Output% data=%Data% out=%Output% seed=1

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
AveragedPerceptron
AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.996146 0.97511 0.95122 0.979079 0.988558 0.972973 0.120617 0.87086 0.964948 0.992065 AveragedPerceptron %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=AveragedPerceptron numcali=200 dout=%Output% data=%Data% out=%Output% seed=1

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
AveragedPerceptron
AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.996146 0.97511 0.95122 0.979079 0.988558 0.972973 NaN 0 0.964948 0.992065 AveragedPerceptron %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=AveragedPerceptron cali={} dout=%Output% data=%Data% out=%Output% seed=1

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
LdSvm
AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /iter Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.973077 0.963079 0.94487 0.947619 0.97619 0.968344 0.204103 0.777959 0.945069 0.974864 1000 LdSvm %Data% %Output% 99 0 0 maml.exe CV tr=LdSvm{iter=1000} threads=- dout=%Output% data=%Data% seed=1 /iter:1000

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
LdSvm
AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bias /iter Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.982598 0.914001 0.827025 0.960625 0.976218 0.89237 0.24697 0.734766 0.885366 0.972638 - 1000 LdSvm %Data% %Output% 99 0 0 maml.exe CV tr=LdSvm{iter=1000 bias=-} threads=- dout=%Output% data=%Data% seed=1 /bias:-;/iter:1000

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
LdSvm
AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bias /iter Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.969024 0.951684 0.955752 0.903766 0.949672 0.977477 0.25164 0.730579 0.929032 0.965033 - 1000 LdSvm %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LdSvm{iter=1000 bias=-} dout=%Output% data=%Data% out=%Output% seed=1 /bias:-;/iter:1000

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
WeightedEnsemble
AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.99624 0.973646 0.958506 0.966527 0.9819 0.977477 0.112168 0.879907 0.9625 0.9924 AvgPer 3 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{bp=AvgPer nm=3 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:AvgPer;/nm:3

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
WeightedEnsemble
AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.996249 0.97511 0.9625 0.966527 0.981941 0.97973 0.112863 0.879162 0.964509 0.992435 svm,ap 20 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{bp=svm bp=ap nm=20 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:svm,ap;/nm:20

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
WeightedEnsembleMulticlass
Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.946667 0.946667 0.433342 0.605555 MultiAverage mlr{t-} 5 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=5 oc=MultiAverage tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /oc:MultiAverage;/bp:mlr{t-};/nm:5

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
WeightedEnsembleMulticlass
Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings
0.94 0.94 0.435527 0.603567 mlr{t-} 20 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=20 st=BootstrapSelector{} tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /bp:mlr{t-};/nm:20

Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using Xunit;
using Xunit.Abstractions;
Expand All @@ -22,6 +23,37 @@ public TestResultProcessor(ITestOutputHelper helper) : base(helper)
{
}

[Theory]
[InlineData("absolute")]
[InlineData("with spaces")]
[InlineData("with {braces}")]
[InlineData("with {unmatched brace")]
public void ResultProcessorWithAbsolutePaths(string directory)
{
string outputDirectory = Path.Combine(SubDirectory, directory);
string[] commands = { "TrainTest", "CV" };
string[] inputPaths = new string[commands.Length];
for (int i = 0; i < commands.Length; i++)
{
string name = $"LogisticRegression-norm-{commands[i]}-breast-cancer-out.txt";
inputPaths[i] = DeleteOutputPath(outputDirectory, name);
File.Copy(GetBaselinePath("LogisticRegression", name), inputPaths[i]);
}

string outputPath = DeleteOutputPath(outputDirectory, "results-rp.txt");
RunResultProcessorTest(inputPaths, outputPath, null);

Assert.True(File.Exists(outputPath));
string[] lines = File.ReadAllLines(outputPath).Where(line => !string.IsNullOrWhiteSpace(line)).ToArray();
Assert.Equal(4, lines.Length);
Assert.Equal("LogisticRegression", lines[0]);
Comment on lines +47 to +49
Assert.Contains("Accuracy", lines[1]);
for (int i = 0; i < inputPaths.Length; i++)
Assert.Contains(inputPaths[i], lines[i + 2]);

Done();
}

// Worker method for running the tests
private void RunTestCore(string name, string fileName, string[] testDataNames, string[] extraArgs = null)
{
Expand Down
35 changes: 19 additions & 16 deletions test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using Microsoft.ML.CommandLine;
using Microsoft.ML.Runtime;
using Microsoft.ML.TestFrameworkCommon;

Expand Down Expand Up @@ -199,19 +200,16 @@ protected void Run(RunContext ctx, int digitsOfPrecision = DigitsOfPrecision, Nu
return;

// ResultProcessor output
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // -rp.txt files are not getting generated for Non-Windows Os
{
string rpName = basePrefix + "-rp.txt";
string rpOutPath = DeleteOutputPath(dir, rpName);
string rpName = basePrefix + "-rp.txt";
string rpOutPath = DeleteOutputPath(dir, rpName);

string[] rpArgs = null;
if (ctx.Command == Cmd.CV && ctx.ExtraArgs != null && ctx.ExtraArgs.Any(arg => arg.Contains("opf+")))
rpArgs = new string[] { "opf+" };
string[] rpArgs = null;
if (ctx.Command == Cmd.CV && ctx.ExtraArgs != null && ctx.ExtraArgs.Any(arg => arg.Contains("opf+")))
rpArgs = new string[] { "opf+" };

// Run result processor on the console output.
RunResultProcessorTest(new string[] { consOutPath.Path }, rpOutPath, rpArgs);
CheckEqualityNormalized(dir, rpName, digitsOfPrecision: digitsOfPrecision, parseOption: parseOption);
}
// Run result processor on the console output.
RunResultProcessorTest(new string[] { consOutPath.Path }, rpOutPath, rpArgs);
CheckEqualityNormalized(dir, rpName, digitsOfPrecision: digitsOfPrecision, parseOption: parseOption);

// Check the prediction output against its baseline.
Contracts.Assert(predOutPath != null);
Expand Down Expand Up @@ -288,17 +286,22 @@ protected void RunResultProcessorTest(string[] dataFiles, string outPath, string
File.Delete(outPath);

List<string> args = new List<string>();
StringBuilder arg = new StringBuilder();
for (int i = 0; i < dataFiles.Length; i++)
{
args.Add("\"" + dataFiles[i] + "\"");
arg.Clear().Append("in=");
CmdQuoter.QuoteValue(dataFiles[i], arg);
args.Add(arg.ToString());
}
args.Add("/o");
args.Add(outPath);
arg.Clear().Append("o=");
CmdQuoter.QuoteValue(outPath, arg);
args.Add(arg.ToString());
args.Add("/calledFromUnitTestSuite+");

if (extraArgs != null)
args.AddRange(extraArgs);
ResultProcessor.Main(Env, args.ToArray());
int result = ResultProcessor.Main(Env, args.ToArray());
Check(result == 0, "Result processor failed with exit code {0}.", result);
}

private static string GetNamePrefix(string testType, PredictorAndArgs predictor, TestDataset dataset, string extraTag = "")
Expand Down
Loading