diff --git a/README.md b/README.md index 0c50205..6194ce8 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,73 @@ OPTIONS: > dotnet verify reject ``` +## How snapshots are paired + +Accepting a snapshot means moving a `.received.` file over the `.verified.` file it belongs to. Those two names are not always the same, so the tool has to work out which verified file each received file maps to. For example, a multi targeted project puts the runtime and version on the received name only: + +``` +MyTests.MyTest.DotNet11_0.received.txt -> MyTests.MyTest.verified.txt +``` + +Each received file is resolved in this order: + +```mermaid +flowchart TD + Start["A .received. file"] --> Recorded{"Did Verify record
the pairing?"} + Recorded -->|yes| UseRecord["Use the recorded
.verified. path"] + + subgraph Fallback["Fallback, when there is no record"] + SameName{"A .verified. file
with same name?"} -->|yes| UseSame["Use it"] + SameName -->|no| Reduces{"Does name reduce
to a .verified. file
beside it?"} + Reduces -->|yes| UseReduced["Use it, shown as rerouted"] + Reduces -->|no| Derived["Use received-derived name, which can be wrong for a new snapshot"] + end + + Recorded -->|no| SameName +``` + +### Recorded pairings + +From [Verify 31.27.0](https://github.com/VerifyTests/Verify/issues/1809), whenever a received file is left on disk, Verify records the verified file it belongs to. This tool reads those records, so the pairing is exact rather than guessed. + +A record is a two line text file, the received path then the verified path, both absolute: + +``` +C:\code\MyProject\Tests\MyTests.MyTest.DotNet11_0.received.txt +C:\code\MyProject\Tests\MyTests.MyTest.verified.txt +``` + +Records go in the intermediate (`obj`) directory of the test project rather than beside the snapshot, so they neither clutter the directory holding the code and snapshots nor get picked up by the `*.received.*` glob used to find snapshots: + +``` +{IntermediateDirectory}/VerifyReceived/{hash}.txt +``` + + * `{IntermediateDirectory}` is the project's `IntermediateOutputPath`, captured at build time by Verify's MSBuild props and emitted into the test assembly as a `Verify.IntermediateDirectory` metadata attribute. It is per configuration and per target framework, so eg `obj/Debug/net10.0/`. A project that does not consume those props has no directory to write to, and so records nothing. + * `{hash}` is an FNV-1a hash of the received path, as 16 hex characters. Deriving the name from the path means re running a test overwrites its record rather than accumulating one per run. + +Since a record is only written when a received file is left on disk, none are written by a passing test, by [AutoVerify](https://github.com/VerifyTests/Verify/blob/main/docs/autoverify.md), which accepts in process, or on a [build server](https://github.com/VerifyTests/Verify/blob/main/docs/build-server.md), where nothing consumes them and the recorded paths do not apply off the agent. + +Records outlive the received files they describe, for example once a snapshot has been accepted or its test deleted. Stale records are ignored rather than acted on, since a record is only used when the received file it names still exists. They are cleared whenever `obj` is. + +To find the records, the tool scans down from the working directory for `VerifyReceived` directories, skipping `.git` and `node_modules`. So the working directory has to contain the `obj` directory, which is the case when running from a project or repository root. Pointing `-w` at a snapshot subdirectory alone means the records are not seen. + +### Fallback + +Where no record exists, the tool falls back to matching each received file against the verified files that sit next to it. This applies to: + + * snapshots produced by a Verify older than 31.27.0 + * an `obj` directory that is not under the working directory, as above, or that has been removed since the test run + +The fallback handles the common cases, including multi targeting, `UniqueFor*`, and a trailing ignored parameter. It cannot cover everything though: + + * A brand new snapshot has no verified file to match against, so a runtime suffix cannot be removed. Accepting it produces a verified file that Verify will not read back. + * A leading or middle ignored parameter cannot be reconstructed, since the verified name is not a truncation of the received name. + +When a received file is paired with a differently named verified file, `review` shows it as `(rerouted)`. + +See [Verify's file naming docs](https://github.com/VerifyTests/Verify/blob/main/docs/naming.md) for how the names are built. + ## Building ``` diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index ee72b11..994cd9b 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -14,6 +14,7 @@ + diff --git a/src/Verify.Terminal.IntegrationTests/GlobalUsings.cs b/src/Verify.Terminal.IntegrationTests/GlobalUsings.cs new file mode 100644 index 0000000..71b74b9 --- /dev/null +++ b/src/Verify.Terminal.IntegrationTests/GlobalUsings.cs @@ -0,0 +1,11 @@ +global using System.Reflection; +global using Shouldly; +global using Spectre.IO; +global using Verify.Terminal; +global using VerifyTests; +global using VerifyXunit; +global using Xunit; + +// Verify keeps global static naming/uniqueness state and these tests hit the real filesystem, so run +// them serially to keep scenarios isolated. +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/src/Verify.Terminal.IntegrationTests/Harness.cs b/src/Verify.Terminal.IntegrationTests/Harness.cs new file mode 100644 index 0000000..02c4354 --- /dev/null +++ b/src/Verify.Terminal.IntegrationTests/Harness.cs @@ -0,0 +1,99 @@ +namespace Verify.Terminal.IntegrationTests; + +// An isolated temp directory that real Verify writes into and the real SnapshotFinder scans. +public sealed class Harness : IDisposable +{ + private readonly string _directory; + + public Harness(string name) + { + // Verify writes no received maps on a build server, so force it off to keep these scenarios + // deterministic locally and on CI. The assembly disables test parallelization, so this is safe. + DiffEngine.BuildServerDetector.Detected = false; + + _directory = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + "verify-terminal-it", + $"{name}-{Guid.NewGuid():N}"); + System.IO.Directory.CreateDirectory(_directory); + Directory = new(_directory); + } + + public DirectoryPath Directory { get; } + + // Verify writes maps to this test project's obj directory, but a real run scans a root that + // contains obj. So copy this scenario's maps under the harness directory to match that layout. + // Returns how many were copied, so a test can assert the map path is actually set up rather than + // silently falling back. + public int PublishMaps() + { + var copied = 0; + var source = System.IO.Path.Combine( + AttributeReader.GetIntermediateDirectory(typeof(Harness).Assembly), + "VerifyReceived"); + if (!System.IO.Directory.Exists(source)) + { + return copied; + } + + var target = System.IO.Path.Combine(_directory, "obj", "VerifyReceived"); + System.IO.Directory.CreateDirectory(target); + + foreach (var file in System.IO.Directory.GetFiles(source)) + { + var lines = System.IO.File.ReadAllLines(file); + if (lines.Length > 0 && + lines[0].StartsWith(_directory, StringComparison.OrdinalIgnoreCase)) + { + System.IO.File.Copy(file, System.IO.Path.Combine(target, System.IO.Path.GetFileName(file)), true); + copied++; + } + } + + return copied; + } + + public VerifySettings CreateSettings() + { + var settings = new VerifySettings(); + settings.UseDirectory(_directory); + // No diff tool, and allow the same prefix to be verified twice (generate then re-verify). + settings.DisableDiff(); + settings.DisableRequireUniquePrefix(); + return settings; + } + + public void SeedVerified(string fileName, string content) => + System.IO.File.WriteAllText(System.IO.Path.Combine(_directory, fileName), content); + + public IReadOnlyList ReceivedFileNames() => + System.IO.Directory + .GetFiles(_directory, "*.received.*") + .Select(System.IO.Path.GetFileName) + .ToList(); + + // Runs the real SnapshotFinder (real globber, real filesystem) over the temp directory. + public Snapshot FindSingle() + { + var environment = new Spectre.IO.Environment(); + var fileSystem = new FileSystem(); + var globber = new Globber(fileSystem, environment); + var finder = new SnapshotFinder(globber, environment); + return finder.Find(Directory).Single(); + } + + public bool Accept(Snapshot snapshot) => + new SnapshotManager(new FileSystem()).Accept(snapshot); + + public void Dispose() + { + try + { + System.IO.Directory.Delete(_directory, recursive: true); + } + catch + { + // Best effort cleanup of the temp directory. + } + } +} diff --git a/src/Verify.Terminal.IntegrationTests/IntegrationTestBase.cs b/src/Verify.Terminal.IntegrationTests/IntegrationTestBase.cs new file mode 100644 index 0000000..ea73e0d --- /dev/null +++ b/src/Verify.Terminal.IntegrationTests/IntegrationTestBase.cs @@ -0,0 +1,185 @@ +namespace Verify.Terminal.IntegrationTests; + +// Base for the naming integration tests. Drives real Verify to produce received files, then asserts +// how the real SnapshotFinder pairs them with verified files. +// +// The purpose is to pin the assumptions Verify.Terminal makes about Verify's received/verified naming, +// so a future Verify version that changes naming breaks these tests rather than silently misbehaving. +public abstract class IntegrationTestBase +{ + // The value verified in every scenario. After a successful accept the verified file holds this, + // so a re-verify with the same value passes. + protected const string Value = "the-received-value"; + + // Runs Verify expecting failure (new or changed snapshot) and returns the verified file name Verify + // itself reports it wants. That name is the ground truth for the verified naming. + protected static async Task ProduceReceived(VerifySettings settings) + { + var exception = await Record.ExceptionAsync(async () => await Verifier.Verify(Value, settings)); + exception.ShouldNotBeNull("Verify was expected to fail and produce a received file."); + return ParseVerifiedFileName(exception.Message); + } + + // Runs Verify and reports whether it passed (no exception). + protected static async Task Verifies(VerifySettings settings) + { + var exception = await Record.ExceptionAsync(async () => await Verifier.Verify(Value, settings)); + return exception is null; + } + + protected static string AssemblyConfiguration() => + typeof(IntegrationTestBase).Assembly + .GetCustomAttribute()! + .Configuration; + + // A scenario where the correct verified file exists, so the finder should pair the received file + // with it (rerouting when the names differ) and an accept round-trips cleanly. + // Run against both, since the map is how Verify behaves now, while the fallback still applies to + // snapshots from an older Verify, or when obj is not scanned. Both have to reach the same file. + // Looped rather than a [Theory], since a test method parameter would be appended to the snapshot + // name by Verify and change the very names under assertion. + protected async Task AssertExistingVerifiedIsDetected( + string method, + Action configure, + string expectedVerified) + { + await Run(withMap: true); + await Run(withMap: false); + + return; + + async Task Run(bool withMap) + { + using var harness = new Harness(method); + + VerifySettings Settings() + { + var settings = harness.CreateSettings(); + settings.UseTypeName("N"); + settings.UseMethodName(method); + configure(settings); + return settings; + } + + var because = withMap ? "with map" : "without map"; + + var correctVerified = await ProduceReceived(Settings()); + + // Verify's own verified name matches what Verify.Terminal assumes. + correctVerified.ShouldBe(expectedVerified, because); + + // In a multi-targeted project the received file always ends with the runtime and version. + var received = harness.ReceivedFileNames().ShouldHaveSingleItem(); + received.ShouldEndWith($".{Namer.RuntimeAndVersion}.received.txt", customMessage: because); + + // Make the correct verified file exist, then let the finder pair against it. + harness.SeedVerified(correctVerified, "old-verified"); + if (withMap) + { + // Both paths reach the same file here, so assert the map really was published, + // otherwise this would silently be testing the fallback twice. + harness.PublishMaps().ShouldBeGreaterThan(0); + } + + var snapshot = harness.FindSingle(); + + System.IO.Path.GetFileName(snapshot.Verified.FullPath).ShouldBe(correctVerified, because); + var literal = received.Replace(".received.", ".verified."); + snapshot.IsRerouted.ShouldBe(correctVerified != literal, because); + + harness.Accept(snapshot).ShouldBeTrue(because); + + // The received value now lives at the correct verified name, so Verify passes. + (await Verifies(Settings())).ShouldBeTrue(because); + } + } + + // A brand new snapshot with no verified file on disk. With nothing to pair against, the finder + // falls back to the received-derived name. Whether that is correct depends on whether the correct + // verified name equals the received-derived name. + protected async Task AssertNewSnapshot( + string method, + Action configure, + string expectedVerified, + bool expectRoundTrips) + { + using var harness = new Harness(method); + + VerifySettings Settings() + { + var settings = harness.CreateSettings(); + settings.UseTypeName("N"); + settings.UseMethodName(method); + configure(settings); + return settings; + } + + var correctVerified = await ProduceReceived(Settings()); + correctVerified.ShouldBe(expectedVerified); + + var received = harness.ReceivedFileNames().ShouldHaveSingleItem(); + received.ShouldEndWith($".{Namer.RuntimeAndVersion}.received.txt"); + + // No verified file exists, so the finder can only fall back to the received-derived name. + var snapshot = harness.FindSingle(); + var literal = received.Replace(".received.", ".verified."); + System.IO.Path.GetFileName(snapshot.Verified.FullPath).ShouldBe(literal); + snapshot.IsRerouted.ShouldBeFalse(); + + harness.Accept(snapshot).ShouldBeTrue(); + + (await Verifies(Settings())).ShouldBe(expectRoundTrips); + } + + // A brand new snapshot, but with Verify's received map available. The map names the verified file, + // so the finder places it correctly instead of falling back to the received-derived name. + protected async Task AssertNewSnapshotWithMap( + string method, + Action configure, + string expectedVerified) + { + using var harness = new Harness(method); + + VerifySettings Settings() + { + var settings = harness.CreateSettings(); + settings.UseTypeName("N"); + settings.UseMethodName(method); + configure(settings); + return settings; + } + + var correctVerified = await ProduceReceived(Settings()); + correctVerified.ShouldBe(expectedVerified); + + var received = harness.ReceivedFileNames().ShouldHaveSingleItem(); + received.ShouldEndWith($".{Namer.RuntimeAndVersion}.received.txt"); + + harness.PublishMaps().ShouldBeGreaterThan(0); + var snapshot = harness.FindSingle(); + + System.IO.Path.GetFileName(snapshot.Verified.FullPath).ShouldBe(correctVerified); + snapshot.IsRerouted.ShouldBe(correctVerified != received.Replace(".received.", ".verified.")); + + harness.Accept(snapshot).ShouldBeTrue(); + + // The accept landed where Verify expects, so the next run passes. + (await Verifies(Settings())).ShouldBeTrue(); + } + + private static string ParseVerifiedFileName(string message) + { + const string marker = "Verified:"; + foreach (var line in message.Split('\n')) + { + var index = line.IndexOf(marker, StringComparison.Ordinal); + if (index >= 0) + { + var path = line[(index + marker.Length)..].Trim(); + return System.IO.Path.GetFileName(path); + } + } + + throw new InvalidOperationException($"No 'Verified:' line found in Verify exception message:\n{message}"); + } +} diff --git a/src/Verify.Terminal.IntegrationTests/ParameterNamingTests.cs b/src/Verify.Terminal.IntegrationTests/ParameterNamingTests.cs new file mode 100644 index 0000000..b272957 --- /dev/null +++ b/src/Verify.Terminal.IntegrationTests/ParameterNamingTests.cs @@ -0,0 +1,151 @@ +namespace Verify.Terminal.IntegrationTests; + +// Covers the parameter naming axis. These need real method parameters (so Verify appends `_name=value`), +// which requires a [Theory]. The received file keeps all parameters; the verified file drops the ignored +// ones. The finder can undo a dropped *trailing* parameter, but not a dropped leading/middle one. +public class ParameterNamingTests : IntegrationTestBase +{ + [Theory] + [InlineData("1", "2")] + public Task Parameters_ExistingVerified_IsDetected(string a, string b) => + // Nothing ignored, so the verified name keeps both parameters. + AssertParametersAreDetected( + nameof(Parameters_ExistingVerified_IsDetected), + a, + b, + ignored: null, + expectedVerified: "N.Params_a=1_b=2.verified.txt"); + + [Theory] + [InlineData("1", "2")] + public Task IgnoreTrailingParameter_ExistingVerified_IsDetected(string a, string b) => + // The trailing parameter `b` is dropped from the verified name, which the finder can undo. + AssertParametersAreDetected( + nameof(IgnoreTrailingParameter_ExistingVerified_IsDetected), + a, + b, + ignored: "b", + expectedVerified: "N.Params_a=1.verified.txt"); + + // Run against both the map, which is how Verify behaves now, and the fallback, which still applies + // to an older Verify, or when obj is not scanned. Looped rather than another [InlineData], since an extra + // test method parameter would be appended to the snapshot name by Verify. + async Task AssertParametersAreDetected(string name, string a, string b, string ignored, string expectedVerified) + { + await Run(withMap: true); + await Run(withMap: false); + + return; + + async Task Run(bool withMap) + { + using var harness = new Harness(name); + + VerifySettings Settings() + { + var settings = harness.CreateSettings(); + settings.UseTypeName("N"); + settings.UseMethodName("Params"); + settings.UseParameters(a, b); + if (ignored != null) + { + settings.IgnoreParameters(ignored); + } + + return settings; + } + + var because = withMap ? "with map" : "without map"; + + var correctVerified = await ProduceReceived(Settings()); + correctVerified.ShouldBe(expectedVerified, because); + + var received = harness.ReceivedFileNames().ShouldHaveSingleItem(); + received.ShouldBe($"N.Params_a=1_b=2.{Namer.RuntimeAndVersion}.received.txt", because); + + harness.SeedVerified(correctVerified, "old-verified"); + if (withMap) + { + // Both paths reach the same file here, so assert the map really was published. + harness.PublishMaps().ShouldBeGreaterThan(0); + } + + var snapshot = harness.FindSingle(); + System.IO.Path.GetFileName(snapshot.Verified.FullPath).ShouldBe(correctVerified, because); + snapshot.IsRerouted.ShouldBeTrue(because); + + harness.Accept(snapshot).ShouldBeTrue(because); + (await Verifies(Settings())).ShouldBeTrue(because); + } + } + + [Theory] + [InlineData("1", "2")] + public async Task IgnoreLeadingParameter_WithMap_IsPlaced(string a, string b) + { + using var harness = new Harness(nameof(IgnoreLeadingParameter_WithMap_IsPlaced)); + + VerifySettings Settings() + { + var settings = harness.CreateSettings(); + settings.UseTypeName("N"); + settings.UseMethodName("Params"); + settings.UseParameters(a, b); + settings.IgnoreParameters("a"); + return settings; + } + + var correctVerified = await ProduceReceived(Settings()); + correctVerified.ShouldBe("N.Params_b=2.verified.txt"); + + harness.SeedVerified(correctVerified, "old-verified"); + harness.PublishMaps().ShouldBeGreaterThan(0); + var snapshot = harness.FindSingle(); + + // The map names the verified file, so the leading ignored parameter no longer matters. + System.IO.Path.GetFileName(snapshot.Verified.FullPath).ShouldBe(correctVerified); + snapshot.IsRerouted.ShouldBeTrue(); + + harness.Accept(snapshot).ShouldBeTrue(); + (await Verifies(Settings())).ShouldBeTrue(); + } + + [Theory] + [InlineData("1", "2")] + public async Task IgnoreLeadingParameter_WithoutMap_CannotBePaired(string a, string b) + { + using var harness = new Harness(nameof(IgnoreLeadingParameter_WithoutMap_CannotBePaired)); + + VerifySettings Settings() + { + var settings = harness.CreateSettings(); + settings.UseTypeName("N"); + settings.UseMethodName("Params"); + settings.UseParameters(a, b); + settings.IgnoreParameters("a"); + return settings; + } + + var correctVerified = await ProduceReceived(Settings()); + // The leading parameter `a` is dropped, so the verified name is not a prefix of the received name. + correctVerified.ShouldBe("N.Params_b=2.verified.txt"); + + var received = harness.ReceivedFileNames().ShouldHaveSingleItem(); + received.ShouldBe($"N.Params_a=1_b=2.{Namer.RuntimeAndVersion}.received.txt"); + + // The correct verified file exists, but the finder cannot reduce a non-trailing parameter and + // falls back to the received-derived name. + harness.SeedVerified(correctVerified, "old-verified"); + var snapshot = harness.FindSingle(); + var literal = received.Replace(".received.", ".verified."); + System.IO.Path.GetFileName(snapshot.Verified.FullPath).ShouldBe(literal); + snapshot.IsRerouted.ShouldBeFalse(); + + harness.Accept(snapshot).ShouldBeTrue(); + + // A non-trailing ignored parameter cannot be reconstructed from the received name, so the + // accept lands at the wrong verified file and Verify still fails. This is only reachable + // without a map, ie. an older Verify, or when obj is not scanned. See the WithMap case above. + (await Verifies(Settings())).ShouldBeFalse(); + } +} diff --git a/src/Verify.Terminal.IntegrationTests/UniquenessNamingTests.cs b/src/Verify.Terminal.IntegrationTests/UniquenessNamingTests.cs new file mode 100644 index 0000000..245aaeb --- /dev/null +++ b/src/Verify.Terminal.IntegrationTests/UniquenessNamingTests.cs @@ -0,0 +1,111 @@ +namespace Verify.Terminal.IntegrationTests; + +// Covers the UniqueFor* naming axis. In a multi-targeted project the received file always gets the +// runtime and version; the verified file gets whatever UniqueFor* the test asked for. +public class UniquenessNamingTests : IntegrationTestBase +{ + // Each of these runs twice: once with Verify's received map, which is the path used now, and once + // without it, which is the fallback for an older Verify, or when obj is not scanned. + [Fact] + public Task Plain_ExistingVerified_IsDetected() => + // received `N.Plain.{RaV}` -> verified `N.Plain` + AssertExistingVerifiedIsDetected( + "Plain", + _ => { }, + "N.Plain.verified.txt"); + + [Fact] + public Task UniqueForRuntime_ExistingVerified_IsDetected() => + // received `N.UniqueForRuntime.{RaV}` -> verified `N.UniqueForRuntime.{Runtime}` + AssertExistingVerifiedIsDetected( + "UniqueForRuntime", + _ => _.UniqueForRuntime(), + $"N.UniqueForRuntime.{Namer.Runtime}.verified.txt"); + + [Fact] + public Task UniqueForRuntimeAndVersion_ExistingVerified_IsDetected() => + // received and verified are identical: `N.UniqueForRuntimeAndVersion.{RaV}` + AssertExistingVerifiedIsDetected( + "UniqueForRuntimeAndVersion", + _ => _.UniqueForRuntimeAndVersion(), + $"N.UniqueForRuntimeAndVersion.{Namer.RuntimeAndVersion}.verified.txt"); + + [Fact] + public Task UniqueForArchitecture_ExistingVerified_IsDetected() => + // received `N.UniqueForArchitecture.{Arch}.{RaV}` -> verified `N.UniqueForArchitecture.{Arch}` + AssertExistingVerifiedIsDetected( + "UniqueForArchitecture", + _ => _.UniqueForArchitecture(), + $"N.UniqueForArchitecture.{Namer.Architecture}.verified.txt"); + + [Fact] + public Task UniqueForOSPlatform_ExistingVerified_IsDetected() => + AssertExistingVerifiedIsDetected( + "UniqueForOSPlatform", + _ => _.UniqueForOSPlatform(), + $"N.UniqueForOSPlatform.{Namer.OperatingSystemPlatform}.verified.txt"); + + [Fact] + public Task UniqueForAssemblyConfiguration_ExistingVerified_IsDetected() => + AssertExistingVerifiedIsDetected( + "UniqueForAssemblyConfiguration", + _ => _.UniqueForAssemblyConfiguration(), + $"N.UniqueForAssemblyConfiguration.{AssemblyConfiguration()}.verified.txt"); + + [Fact] + public Task IgnoreParametersForVerified_ExistingVerified_IsDetected() => + // received keeps the parameter text, verified drops it: `N.IgnoreAll_p.{RaV}` -> `N.IgnoreAll` + AssertExistingVerifiedIsDetected( + "IgnoreAll", + _ => + { + _.UseTextForParameters("p"); + _.IgnoreParametersForVerified(); + }, + "N.IgnoreAll.verified.txt"); + + [Fact] + public Task Plain_NewSnapshot_WithoutMap_CannotBePlaced() => + // Without a map there is no verified file to pair against, so the finder keeps the runtime + // suffix while the correct verified name has none. This is the fallback used when no map is + // available, ie. an older Verify, or an obj that is not scanned. + AssertNewSnapshot( + "Plain", + _ => { }, + "N.Plain.verified.txt", + expectRoundTrips: false); + + [Fact] + public Task Plain_NewSnapshot_WithMap_IsPlaced() => + AssertNewSnapshotWithMap( + "Plain", + _ => { }, + "N.Plain.verified.txt"); + + [Fact] + public Task UniqueForRuntime_NewSnapshot_WithoutMap_CannotBePlaced() => + // Without a map the finder cannot know to collapse the received `{RaV}` to the verified + // `{Runtime}`. + AssertNewSnapshot( + "UniqueForRuntime", + _ => _.UniqueForRuntime(), + $"N.UniqueForRuntime.{Namer.Runtime}.verified.txt", + expectRoundTrips: false); + + [Fact] + public Task UniqueForRuntime_NewSnapshot_WithMap_IsPlaced() => + AssertNewSnapshotWithMap( + "UniqueForRuntime", + _ => _.UniqueForRuntime(), + $"N.UniqueForRuntime.{Namer.Runtime}.verified.txt"); + + [Fact] + public Task UniqueForRuntimeAndVersion_NewSnapshot_Succeeds() => + // The only new-snapshot case that works: the received-derived name already equals the correct + // verified name, because UniqueForRuntimeAndVersion keeps the runtime and version on both. + AssertNewSnapshot( + "UniqueForRuntimeAndVersion", + _ => _.UniqueForRuntimeAndVersion(), + $"N.UniqueForRuntimeAndVersion.{Namer.RuntimeAndVersion}.verified.txt", + expectRoundTrips: true); +} diff --git a/src/Verify.Terminal.IntegrationTests/Verify.Terminal.IntegrationTests.csproj b/src/Verify.Terminal.IntegrationTests/Verify.Terminal.IntegrationTests.csproj new file mode 100644 index 0000000..e144579 --- /dev/null +++ b/src/Verify.Terminal.IntegrationTests/Verify.Terminal.IntegrationTests.csproj @@ -0,0 +1,19 @@ + + + + net8.0;net10.0 + disable + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/src/Verify.Terminal.Tests/SnapshotFinderTests.cs b/src/Verify.Terminal.Tests/SnapshotFinderTests.cs index 7c852c3..7ad07de 100644 --- a/src/Verify.Terminal.Tests/SnapshotFinderTests.cs +++ b/src/Verify.Terminal.Tests/SnapshotFinderTests.cs @@ -5,18 +5,10 @@ public sealed class SnapshotFinderTests [Fact] public void Should_Return_Expected_Snapshot() { - // Given - var environment = new FakeEnvironment(Spectre.IO.PlatformFamily.Linux); - var filesystem = new FakeFileSystem(environment); - var globber = new Globber(filesystem, environment); - filesystem.CreateFile("/Working/lol.received.txt"); - filesystem.CreateFile("/Working/lol.verified.txt"); - var finder = new SnapshotFinder(filesystem, globber, environment); - - // When - var result = finder.Find().SingleOrDefault(); + var result = Find( + "/Working/lol.received.txt", + "/Working/lol.verified.txt"); - // Then result.ShouldNotBeNull(); result.IsRerouted.ShouldBeFalse(); result.Received.FullPath.ShouldBe("/Working/lol.received.txt"); @@ -26,18 +18,10 @@ public void Should_Return_Expected_Snapshot() [Fact] public void Should_Return_Expected_Snapshot_For_Non_Framework_Specific_File() { - // Given - var environment = new FakeEnvironment(Spectre.IO.PlatformFamily.Linux); - var filesystem = new FakeFileSystem(environment); - var globber = new Globber(filesystem, environment); - filesystem.CreateFile("/Working/lol.DotNet6_0.received.txt"); - filesystem.CreateFile("/Working/lol.verified.txt"); - var finder = new SnapshotFinder(filesystem, globber, environment); - - // When - var result = finder.Find().SingleOrDefault(); + var result = Find( + "/Working/lol.DotNet6_0.received.txt", + "/Working/lol.verified.txt"); - // Then result.ShouldNotBeNull(); result.IsRerouted.ShouldBeTrue(); result.Received.FullPath.ShouldBe("/Working/lol.DotNet6_0.received.txt"); @@ -47,21 +31,218 @@ public void Should_Return_Expected_Snapshot_For_Non_Framework_Specific_File() [Fact] public void Should_Return_Expected_Snapshot_For_Framework_Specific_File() { - // Given - var environment = new FakeEnvironment(Spectre.IO.PlatformFamily.Linux); - var filesystem = new FakeFileSystem(environment); - var globber = new Globber(filesystem, environment); - filesystem.CreateFile("/Working/lol.DotNet6_0.received.txt"); - filesystem.CreateFile("/Working/lol.DotNet6_0.verified.txt"); - var finder = new SnapshotFinder(filesystem, globber, environment); + var result = Find( + "/Working/lol.DotNet6_0.received.txt", + "/Working/lol.DotNet6_0.verified.txt"); - // When - var result = finder.Find().SingleOrDefault(); + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeFalse(); + result.Received.FullPath.ShouldBe("/Working/lol.DotNet6_0.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/lol.DotNet6_0.verified.txt"); + } + + [Fact] + public void Should_Return_Expected_Snapshot_For_Runtime_Specific_File() + { + var result = Find( + "/Working/lol.DotNet6_0.received.txt", + "/Working/lol.DotNet.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/lol.DotNet6_0.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/lol.DotNet.verified.txt"); + } + + [Fact] + public void Should_Return_Expected_Snapshot_For_Net_Framework_Runtime_Specific_File() + { + var result = Find( + "/Working/lol.Net4_8.received.txt", + "/Working/lol.Net.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/lol.Net4_8.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/lol.Net.verified.txt"); + } + + [Fact] + public void Should_Return_Expected_Snapshot_For_Runtime_Specific_File_Targeting_A_Single_Framework() + { + var result = Find( + "/Working/lol.received.txt", + "/Working/lol.DotNet.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/lol.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/lol.DotNet.verified.txt"); + } + + [Fact] + public void Should_Prefer_Framework_Specific_File_Over_Runtime_Specific_File() + { + var result = Find( + "/Working/lol.DotNet6_0.received.txt", + "/Working/lol.DotNet6_0.verified.txt", + "/Working/lol.DotNet.verified.txt"); - // Then result.ShouldNotBeNull(); result.IsRerouted.ShouldBeFalse(); result.Received.FullPath.ShouldBe("/Working/lol.DotNet6_0.received.txt"); result.Verified.FullPath.ShouldBe("/Working/lol.DotNet6_0.verified.txt"); } + + [Fact] + public void Should_Prefer_Runtime_Specific_File_Over_Non_Framework_Specific_File() + { + var result = Find( + "/Working/lol.DotNet6_0.received.txt", + "/Working/lol.DotNet.verified.txt", + "/Working/lol.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/lol.DotNet6_0.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/lol.DotNet.verified.txt"); + } + + [Fact] + public void Should_Not_Reroute_Snapshot_That_Only_Looks_Like_A_Runtime() + { + var result = Find( + "/Working/lol.Networking.received.txt", + "/Working/lol.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeFalse(); + result.Received.FullPath.ShouldBe("/Working/lol.Networking.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/lol.Networking.verified.txt"); + } + + [Fact] + public void Should_Return_Verified_When_All_Parameters_Ignored() + { + // IgnoreParametersForVerified drops all parameters from the verified name. + var result = Find( + "/Working/Foo_a=1_b=2.received.txt", + "/Working/Foo.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/Foo_a=1_b=2.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/Foo.verified.txt"); + } + + [Fact] + public void Should_Return_Verified_When_Trailing_Parameter_Ignored() + { + // IgnoreParameters("b") drops a trailing parameter from the verified name. + var result = Find( + "/Working/Foo_a=1_b=2.received.txt", + "/Working/Foo_a=1.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/Foo_a=1_b=2.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/Foo_a=1.verified.txt"); + } + + [Fact] + public void Should_Prefer_More_Specific_Parameter_Match() + { + var result = Find( + "/Working/Foo_a=1_b=2.received.txt", + "/Working/Foo_a=1.verified.txt", + "/Working/Foo.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/Foo_a=1_b=2.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/Foo_a=1.verified.txt"); + } + + [Fact] + public void Should_Return_Verified_When_Parameters_Ignored_And_Multi_Targeting() + { + var result = Find( + "/Working/Foo_a=1.DotNet11_0.received.txt", + "/Working/Foo.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/Foo_a=1.DotNet11_0.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/Foo.verified.txt"); + } + + [Fact] + public void Should_Return_Runtime_Verified_When_Parameters_Ignored_And_Multi_Targeting() + { + var result = Find( + "/Working/Foo_a=1.DotNet11_0.received.txt", + "/Working/Foo.DotNet.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/Foo_a=1.DotNet11_0.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/Foo.DotNet.verified.txt"); + } + + [Fact] + public void Should_Not_Match_Different_Test_With_Shared_Prefix() + { + // `Foo` is a string prefix of `FooBar` but not a parameter-boundary reduction of it, so it + // must not be rerouted to the unrelated `Foo` snapshot. + var result = Find( + "/Working/FooBar_a=1.received.txt", + "/Working/Foo.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeFalse(); + result.Received.FullPath.ShouldBe("/Working/FooBar_a=1.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/FooBar_a=1.verified.txt"); + } + + [Fact] + public void Should_Reroute_Multi_Target_Indexed_File() + { + var result = Find( + "/Working/Foo.DotNet11_0#00.received.txt", + "/Working/Foo#00.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeTrue(); + result.Received.FullPath.ShouldBe("/Working/Foo.DotNet11_0#00.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/Foo#00.verified.txt"); + } + + [Fact] + public void Should_Not_Cross_Match_Indexed_Files() + { + // The `#index` differs, so these are different targets and must not be paired. + var result = Find( + "/Working/Foo.DotNet11_0#00.received.txt", + "/Working/Foo#01.verified.txt"); + + result.ShouldNotBeNull(); + result.IsRerouted.ShouldBeFalse(); + result.Received.FullPath.ShouldBe("/Working/Foo.DotNet11_0#00.received.txt"); + result.Verified.FullPath.ShouldBe("/Working/Foo.DotNet11_0#00.verified.txt"); + } + + private static Snapshot Find(params string[] files) + { + var environment = new FakeEnvironment(Spectre.IO.PlatformFamily.Linux); + var filesystem = new FakeFileSystem(environment); + var globber = new Globber(filesystem, environment); + + foreach (var file in files) + { + filesystem.CreateFile(file); + } + + var finder = new SnapshotFinder(globber, environment); + return finder.Find().SingleOrDefault(); + } } diff --git a/src/Verify.Terminal.slnx b/src/Verify.Terminal.slnx index 3983191..e60c898 100644 --- a/src/Verify.Terminal.slnx +++ b/src/Verify.Terminal.slnx @@ -8,6 +8,7 @@ + diff --git a/src/Verify.Terminal/GlobalUsings.cs b/src/Verify.Terminal/GlobalUsings.cs index e97c624..01b47b5 100644 --- a/src/Verify.Terminal/GlobalUsings.cs +++ b/src/Verify.Terminal/GlobalUsings.cs @@ -3,4 +3,5 @@ global using System.Runtime.CompilerServices; global using DiffPlex.DiffBuilder; global using DiffPlex.DiffBuilder.Model; -global using Spectre.IO; \ No newline at end of file +global using Spectre.IO; +global using VerifyTests.ExceptionParsing; \ No newline at end of file diff --git a/src/Verify.Terminal/SnapshotFinder.cs b/src/Verify.Terminal/SnapshotFinder.cs index b71a41a..532471c 100644 --- a/src/Verify.Terminal/SnapshotFinder.cs +++ b/src/Verify.Terminal/SnapshotFinder.cs @@ -2,16 +2,21 @@ namespace Verify.Terminal; public sealed class SnapshotFinder { - private readonly IFileSystem _fileSystem; + // The runtime names that Verify appends to a snapshot name. + // `Core` is only emitted by older versions of Verify. + private static readonly string[] _runtimes = ["DotNet", "Net", "Mono", "Core"]; + + // Windows paths are case insensitive. + private static readonly StringComparison _pathComparison = + OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + private readonly IGlobber _globber; private readonly IEnvironment _environment; public SnapshotFinder( - IFileSystem fileSystem, IGlobber globber, IEnvironment environment) { - _fileSystem = fileSystem.NotNull(); _globber = globber.NotNull(); _environment = environment.NotNull(); } @@ -21,71 +26,244 @@ public ISet Find(DirectoryPath? root = null) root ??= _environment.WorkingDirectory; root = root.MakeAbsolute(_environment); - var result = new HashSet(); + // Verify records the verified file each received file belongs to, so prefer that over any + // guess. See https://github.com/VerifyTests/Verify/issues/1809 + var maps = ReceivedMaps.Read(root.FullPath); - root = root.MakeAbsolute(_environment); - var received = _globber.Match("**/*.received.*", new GlobberSettings - { - Root = root, - }).Cast(); + // A map is not always available. Older versions of Verify wrote none, and obj may not be under + // the scanned root, or may have been removed since the test run. So fall back to matching each + // received file against the verified files that exist alongside it. The verified name cannot be + // reliably reconstructed from the received name, so this is a guess. + var verifiedByDirectory = Match(root, "**/*.verified.*", "verified") + .GroupBy(_ => _.Directory, StringComparer.Ordinal) + .ToDictionary(_ => _.Key, _ => _.ToList(), StringComparer.Ordinal); - foreach (var receivedPath in received) + var result = new HashSet(); + foreach (var received in Match(root, "**/*.received.*", "received")) { - var (verifiedPath, isRerouted) = GetVerified(receivedPath); - result.Add(new Snapshot(receivedPath, verifiedPath, isRerouted)); + var (verifiedPath, isRerouted) = GetVerified(received, maps, verifiedByDirectory); + result.Add(new Snapshot(received.Path, verifiedPath, isRerouted)); } return result; } - private (FilePath VerifiedPath, bool IsRerouted) GetVerified(FilePath received) + private IEnumerable Match(DirectoryPath root, string pattern, string marker) => + _globber + .Match(pattern, new GlobberSettings { Root = root }) + .OfType() + .Select(_ => ParsedName.Parse(_, marker)); + + private (FilePath VerifiedPath, bool IsRerouted) GetVerified( + ParsedName received, + ReceivedMaps maps, + Dictionary> verifiedByDirectory) { - var isRerouted = false; - var path = StripExtensions(received, out var originalExtension); + // Verify recorded the pair, so there is nothing to work out. + if (maps.TryGetVerified(received.Path.FullPath, out var mapped)) + { + var verified = new FilePath(mapped); + var isRerouted = !verified.FullPath.Equals(LiteralVerified(received).FullPath, _pathComparison); + return (verified, isRerouted); + } - var extension = path.GetExtension(); - if (extension != null) + var candidates = verifiedByDirectory.TryGetValue(received.Directory, out var inDirectory) + ? inDirectory.Where(_ => _.Extension == received.Extension).ToList() + : []; + + // An exact match is never rerouted. + var exact = candidates.FirstOrDefault(_ => _.Stem == received.Stem); + if (exact != null) { - if (extension.StartsWith(".DotNet") || - extension.StartsWith(".Mono") || - extension.StartsWith(".Net") || - extension.StartsWith(".Core")) - { - var temp = path.RemoveExtension() - .AppendExtension(".verified") - .AppendExtensionIfNotNull(originalExtension); + return (exact.Path, false); + } - if (_fileSystem.File.Exists(temp)) + // Otherwise find the most specific verified file that the received file reduces to, ie. the + // verified file whose name is the received name with a less specific runtime and/or with some + // parameters dropped (`IgnoreParameters`, `IgnoreParametersForVerified`). + var best = candidates + .Select(_ => (Verified: _, Score: ReductionScore(received, _))) + .Where(_ => _.Score >= 0) + .OrderByDescending(_ => _.Score) + .ThenBy(_ => _.Verified.Stem, StringComparer.Ordinal) + .FirstOrDefault(); + if (best.Verified != null) + { + return (best.Verified.Path, true); + } + + // Older versions of Verify left the runtime out of the received file when the test project + // targets a single framework, eg. `Foo.received.txt` -> `Foo.DotNet.verified.txt` + if (received.RuntimeFull.Length == 0) + { + foreach (var runtime in _runtimes) + { + var appended = candidates.FirstOrDefault(_ => _.Stem == $"{received.Stem}.{runtime}"); + if (appended != null) { - isRerouted = true; - path = path.RemoveExtension(); + return (appended.Path, true); } } } - path = path - .AppendExtension(".verified") - .AppendExtensionIfNotNull(originalExtension); + // No verified file exists yet: fall back to the name the received file maps to directly. + return (LiteralVerified(received), false); + } + + // How specifically `received` reduces to `verified`. Higher is more specific; -1 if incompatible. + private static int ReductionScore(ParsedName received, ParsedName verified) + { + // The `#name`/`#index` suffix for multi-target files is identical on both sides. + if (received.Index != verified.Index) + { + return -1; + } + + // The verified name keeps a subset of the received parameters, so its head has to be the + // received head with zero or more trailing parameters removed. + if (!IsBoundedPrefix(received.Head, verified.Head)) + { + return -1; + } + + var runtimeScore = RuntimeScore(received, verified); + if (runtimeScore < 0) + { + return -1; + } + + // Prefer retaining more of the received name (more parameters), then a more specific runtime. + return (verified.Head.Length * 10) + runtimeScore; + } + + // Whether `prefix` is `full`, or `full` truncated at a parameter (`_`) boundary. The boundary + // check stops `Foo` from matching `FooBar`. + private static bool IsBoundedPrefix(string full, string prefix) + { + if (prefix.Length > full.Length || + !full.StartsWith(prefix, StringComparison.Ordinal)) + { + return false; + } + + return prefix.Length == full.Length || + full[prefix.Length] == '_'; + } + + // Whether the verified runtime is a reduction of the received runtime, and how close. + private static int RuntimeScore(ParsedName received, ParsedName verified) + { + // Same runtime, eg. received and verified both `.DotNet11_0`, or both have none. + if (received.RuntimeFull == verified.RuntimeFull) + { + return 2; + } + + // Verified dropped the runtime, eg. received `.DotNet11_0` -> verified none. + if (verified.RuntimeFull.Length == 0) + { + return 0; + } + + // `UniqueForRuntime`, eg. received `.DotNet11_0` -> verified `.DotNet`. + if (verified.RuntimeFull == received.RuntimeBare) + { + return 1; + } - return (path, isRerouted); + return -1; } - private static FilePath StripExtensions(FilePath path, out string? originalExtension) + private static FilePath LiteralVerified(ParsedName received) => + new($"{received.Directory}/{received.Stem}.verified.{received.Extension}"); + + // Verify formats the version as eg. `10_0` in `.DotNet10_0`, or `4_8` in `.Net4_8` + private static bool IsVersion(string version) { - originalExtension = path.GetExtension(); + var parts = version.Split('_'); + return parts.Length == 2 && + parts.All(part => part.Length > 0 && part.All(char.IsAsciiDigit)); + } + + // A received or verified file name split into the parts that can differ between the two. + private sealed class ParsedName + { + public FilePath Path { get; } + public string Directory { get; } + public string Stem { get; } + public string Extension { get; } + + // The trailing `#name`/`#index` for multi-target files, or empty. + public string Index { get; } + + // The name without the runtime token and index, ie. `{TypeAndMethod}{Parameters}`. + public string Head { get; } + + // The runtime token including any version, eg. `.DotNet11_0`, `.DotNet`, or empty. + public string RuntimeFull { get; } - while (path.HasExtension) + // The runtime token without the version, eg. `.DotNet`, or empty. + public string RuntimeBare { get; } + + private ParsedName( + FilePath path, string stem, string extension, + string index, string head, string runtimeFull, string runtimeBare) + { + Path = path; + Directory = path.GetDirectory().FullPath; + Stem = stem; + Extension = extension; + Index = index; + Head = head; + RuntimeFull = runtimeFull; + RuntimeBare = runtimeBare; + } + + public static ParsedName Parse(FilePath path, string marker) + { + var filename = path.GetFilename().FullPath; + + var token = $".{marker}."; + var markerIndex = filename.IndexOf(token, StringComparison.Ordinal); + var stem = filename[..markerIndex]; + var extension = filename[(markerIndex + token.Length)..]; + + var (index, head, runtimeFull, runtimeBare) = Decompose(stem); + return new(path, stem, extension, index, head, runtimeFull, runtimeBare); + } + + private static (string Index, string Head, string RuntimeFull, string RuntimeBare) Decompose(string stem) { - var current = path.GetExtension(); - if (current == ".received") + var index = string.Empty; + var core = stem; + + var hash = stem.IndexOf('#'); + if (hash >= 0) { - path = path.RemoveExtension(); - break; + index = stem[hash..]; + core = stem[..hash]; } - path = path.RemoveExtension(); - } + var lastDot = core.LastIndexOf('.'); + if (lastDot >= 0) + { + var segment = core[(lastDot + 1)..]; + foreach (var runtime in _runtimes) + { + if (!segment.StartsWith(runtime, StringComparison.Ordinal)) + { + continue; + } - return path; + var version = segment[runtime.Length..]; + if (version.Length == 0 || IsVersion(version)) + { + return (index, core[..lastDot], $".{segment}", $".{runtime}"); + } + } + } + + return (index, core, string.Empty, string.Empty); + } } -} \ No newline at end of file +} diff --git a/src/Verify.Terminal/Verify.Terminal.csproj b/src/Verify.Terminal/Verify.Terminal.csproj index 4c71df2..cd80d89 100644 --- a/src/Verify.Terminal/Verify.Terminal.csproj +++ b/src/Verify.Terminal/Verify.Terminal.csproj @@ -19,6 +19,7 @@ +