From 2c660c87e2da9dd539ad3596bb986c625936920f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Thu, 29 Oct 2026 16:20:23 +0100 Subject: [PATCH] fix: oracles seeds credits a seed only against a passing baseline `oracles seeds` read any failed test as a catch, so a SQL Server fixture that never started (2022 under QEMU on an arm64 Mac) credited both SQL Server seeds while every test failed in `SqlServer.InitializeAsync`, and the tool exited 0. The tool now runs every oracle the selected seeds name on a copy with no seed first, built and filtered as the seeded runs are. A seed is CAUGHT only when its oracle passes there and fails with the seed in a test of its own. A run that never reached the tests is ERROR, exit 4, with or without the seed. The TRX decides that, as measured with xunit.v3 4.0.2: - a fixture that fails to set up fails each test with one of xunit's own messages; - a fixture that fails in Dispose is only an `error` counter; - a crashed host writes no TRX and exits other than 0 or 2; - a run whose every test is skipped ran nothing. A failing baseline is ERROR too. Decision 118 records the rule, and the README, CONTRIBUTING and AGENTS say what ERROR means at release. Input class: the ways a run fails outside a test's own code. SeedVerdictTests covers each kind measured above and every setup message xunit.v3.core 4.0.2 holds. With the setup check and the baseline seeded out, three of those tests fail, the dead fixture of #151 among them. Closes #151. Defect-Class: TOOL Found-By: OR Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 8 +- CONTRIBUTING.md | 6 +- ...ed-is-caught-against-a-passing-baseline.md | 54 +++++++ design/adr/README.md | 1 + .../SeedVerdictTests.cs | 135 +++++++++++++++++- test/oracles/README.md | 24 +++- test/oracles/SeedVerdict.cs | 102 ++++++++++++- tools/Oracles/Program.cs | 134 ++++++++++------- 8 files changed, 395 insertions(+), 69 deletions(-) create mode 100644 design/adr/0117-a-seed-is-caught-against-a-passing-baseline.md diff --git a/AGENTS.md b/AGENTS.md index 35a2490..4ce25db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -266,7 +266,10 @@ probe. ## Releasing -Before `git tag`, show every seed caught, locally, a build of one project per seed: +Before `git tag`, show every seed caught, locally: a baseline with no seed, then a build of one project per seed. +A seed counts only where its oracle passes without it and fails with it in a test of its own; a fixture that never +started, a crashed host or a failing baseline is `ERROR`, exit 4 (decision 117). On an arm64 Mac set +`TYPEKIT_MSSQL_IMAGE=mcr.microsoft.com/azure-sql-edge:latest`, or the SQL Server oracles are `ERROR`: ```bash dotnet tools/Oracles/bin/Release/net10.0/oracles.dll seeds @@ -283,4 +286,5 @@ It reads api.nuget.org, needs no model, and is never a CI check: time passing fa The release workflow runs the suite, which holds the catalogue, before it packs; there is no gate beside it (decision 112 withdrew decision 97's). A seed that is not caught is answered by a stronger oracle or a better -seed, never by deleting the seed. +seed, never by deleting the seed, and an `ERROR` by fixing the environment or the baseline, never by reading it as +caught. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b1536f..e9fbb81 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -173,8 +173,10 @@ The packages share one version and are released together, only through `.github/workflows/release.yml` (see [SECURITY.md](SECURITY.md)). 1. Check what the documents rest on, locally: - - Show every oracle seed caught: `dotnet tools/Oracles/bin/Release/net10.0/oracles.dll seeds` (a build of one - project per seed; [test/oracles/README.md](test/oracles/README.md)). + - Show every oracle seed caught: `dotnet tools/Oracles/bin/Release/net10.0/oracles.dll seeds` (a baseline with + no seed, then a build of one project per seed; [test/oracles/README.md](test/oracles/README.md)). It exits 0 + only when every oracle passes without its seed and fails with it. An `ERROR` (exit 4) is an environment or a + baseline to fix, such as a SQL Server container that never started, never a seed to accept. - Re-read the competitor facts that are due: `dotnet tools/DocsQuery/bin/Release/net10.0/docs-query.dll due --nuget` lists each `design/knowledge/` entry pinned `external` whose `verified:` date is more than six months old, or whose pinned package NuGet has a later release of. Read each again at the latest release, from its source and diff --git a/design/adr/0117-a-seed-is-caught-against-a-passing-baseline.md b/design/adr/0117-a-seed-is-caught-against-a-passing-baseline.md new file mode 100644 index 0000000..637f323 --- /dev/null +++ b/design/adr/0117-a-seed-is-caught-against-a-passing-baseline.md @@ -0,0 +1,54 @@ +# 0117. A seed is caught against a passing baseline, by a test of its own + +Date: 2026-10-29 +Status: accepted + +Context: [#151](https://github.com/CaffeinatedCoder/CodoMetis.TypeKit/issues/151), found by the 2.3.0 pre-release +review. [Decision 112](0112-the-prober-gives-way-to-valueranges-oracle-kit.md) made `oracles seeds` the release's +proof that every oracle can fail; [decision 108](0108-a-probe-check-earns-credit-against-a-baseline-and-ws2-closes.md) +had held the prober it replaced to a baseline, and the port did not carry that over. + +`oracles seeds` credited a seed whenever one test of its oracle failed with a row that was not a known candidate, +whatever the failure was. On an arm64 Mac, SQL Server 2022 under QEMU stops before it is ready, so the SQL Server +collection fixture throws `ContainerNotRunningException` in `InitializeAsync` and every test of both SQL Server +oracles fails, with and without a seed. The tool reported both seeds `CAUGHT` (10,071 of 10,071 and 170 of 170 tests +failing) and exited 0, ending `every seed is caught`, while no query or insert under review had run. + +**A seed is caught only when its oracle passes without it and fails with it, in a test of its own.** `oracles seeds` +first runs every oracle the selected seeds name on a copy of the tree with no seed, built and filtered as the seeded +runs are, in the same process and environment. An oracle that does not pass there (a run that never reached its +tests, or a test that fails on a row that is not a known candidate) makes each of its pairs `ERROR`, and is not run +with the seed. A baseline whose only failures are known candidates passes, since an unfiled candidate fails its sweep +until it is filed. + +**A run that did not reach the tests' own code is `ERROR`, never `CAUGHT`, with or without the seed.** What the TRX +holds decides it, measured with xunit.v3 4.0.2 on Microsoft Testing Platform on 2026-10-29, one class per kind of +failure: + +- A class, collection or assembly fixture that throws in its constructor or `InitializeAsync` fails every test that + uses it, each with xunit's own message (`Collection fixture type 'T' threw in InitializeAsync`, the fixture's + exception on the next line, the stack only an inner one). xunit.v3.core 4.0.2 holds four more setup messages of + the same kind: unresolved constructor arguments, more than one public constructor, constructor parameters without + fixture data, and a type it cannot create. A failed test whose message begins with one of them is setup. +- A fixture that throws in `Dispose` or `DisposeAsync` fails no test: the TRX counts it as an `error` in its + `Counters` and holds no result for it. A nonzero `error`, `aborted`, `timeout` or `disconnected` counter is an error. +- A test host that crashes (`Environment.FailFast`) writes no TRX and exits 134. Microsoft Testing Platform exits 0 + when every test passes and 2 when one fails; any other exit code, or no TRX, is an error. +- A run whose every test is skipped ran nothing, and is an error. +- A failure in the test class's own constructor or `InitializeAsync` is written like one in the test's body: the + exception's message, and a stack trace under the test's code. It counts as the test's: the oracles build there + what they assert on (`KeyGenerationVariantTests` opens its connection in its constructor), and the baseline has + shown that it passes without the seed. + +The rule is one file, `test/oracles/SeedVerdict.cs`, compiled into the tool and into `SeedVerdictTests`, which hold +it on TRX documents of the measured shape. With the setup check and the baseline taken out, the dead fixture of #151 +reads as caught again and three of those tests fail. + +**`ERROR` exits 4**, beside 1 for a seed missed, so a release script tells a seed that needs a stronger oracle from an +environment that needs fixing. A release on an arm64 Mac runs `oracles seeds` with `TYPEKIT_MSSQL_IMAGE` naming +Azure SQL Edge, as the suite does (CONTRIBUTING's Docker bullet). + +Rejected: telling a setup failure apart by its stack trace (a frame of `InitializeAsync`, a constructor), since a +test's own exception passes through product code whose frames carry the same names, and xunit writes the fixture +cases in fixed words of its own; and requiring the seeded run to fail on the rows the baseline passed, row by row, +since a test's rows are not in the TRX and the baseline already rules out a failure that does not depend on the seed. diff --git a/design/adr/README.md b/design/adr/README.md index 8250177..cb8d9aa 100644 --- a/design/adr/README.md +++ b/design/adr/README.md @@ -123,3 +123,4 @@ per area ([design](../plan.md)) link to the records they rest on. | [0114](0114-a-value-object-s-temporary-number-is-its-wrapped-type-s.md) | A value object over a number gets the temporary values a property of its wrapped type gets, from a factory in its converter's mapping hints, which EF consults after a generator on the property and after HiLo: a default `ulong` key no longer throws at `Add`, and a `char` key configured on add gets a temporary `char` on SQL Server; under `--nativeaot` the factory is lost and a default `ulong` key still throws | 2026-10-29 | accepted | | [0115](0115-value-objects-can-be-inlined-everywhere.md) | `AddTypeKit(typeKit => typeKit.InlineValueObjects = true)` writes every value object in place, by its wrapped type's schema, in every position and version, with no component of its own; off by default, the document unchanged; closed sets and wrapped enums keep their components; an inlined value object claims no name (decision 55) and shares no schema between its JSON positions and its parameters (decision 106) | 2026-10-29 | accepted | | [0116](0116-the-docs-site-is-docfx-built-from-main.md) | The docs site is DocFX, published to GitHub Pages from `main` alone: the guides, the READMEs, the changelog and the API pages of the run-time packages, from the files as they are; a link to a file that is not a page becomes a GitHub address at the built commit; a pull request touching a page builds it with warnings as errors; `site/**` is prose; DocFX 2.81.0 lists the members of C# 14 extension blocks as extension methods | 2026-10-29 | accepted | +| [0117](0117-a-seed-is-caught-against-a-passing-baseline.md) | `oracles seeds` credits a seed only when its oracle passes without it, in a baseline run with the same build and filter, and fails with it in a test of its own; a run that never reached the tests (a fixture xunit could not set up, an error counted outside any test, a crashed host, no TRX, every test skipped) or a failing baseline is `ERROR`, exit 4, never `CAUGHT` | 2026-10-29 | accepted | diff --git a/test/CodoMetis.TypeKit.Oracles.Tests/SeedVerdictTests.cs b/test/CodoMetis.TypeKit.Oracles.Tests/SeedVerdictTests.cs index 7083091..aa42072 100644 --- a/test/CodoMetis.TypeKit.Oracles.Tests/SeedVerdictTests.cs +++ b/test/CodoMetis.TypeKit.Oracles.Tests/SeedVerdictTests.cs @@ -1,10 +1,18 @@ +using System.Xml.Linq; + namespace CodoMetis.TypeKit.Oracles.Tests; /// -/// The rule oracles seeds judges a failed test by (), fed the messages -/// writes: a seed is caught when one row of a failed test is not a known candidate, -/// whatever else shares its message. +/// The rule oracles seeds judges a seed by (): a seed is caught when its oracle passes +/// without it and one of its tests fails with it, with a row that is not a known candidate, whatever else shares its +/// message, and never by a failure xunit wrote for a fixture it could not set up or by a run that did not reach the +/// tests (design/adr/0117). /// +/// +/// The TRX documents here have the shape xunit.v3 4.0.2 writes through --report-xunit-trx, measured on +/// 2026-10-29 with one class per kind of failure: the messages are xunit's own, a fixture's Dispose that throws +/// is a counter and no result, a crashed host writes no file. +/// public sealed class SeedVerdictTests { private static readonly Candidates.Candidate Unfiled = new("AnyOracle", "row 1:", "a known disagreement", Issue: ""); @@ -60,6 +68,127 @@ public void A_known_row_spanning_lines_does_not_catch() public void A_sweep_with_no_failure_and_no_candidate_passes() => Should.NotThrow(() => Candidates.Judge([], [], "a sweep")); + /// What SQL Server 2022 under QEMU wrote for every test of a SQL Server oracle (#151). + private const string DeadSqlServer = + "Collection fixture type 'CodoMetis.TypeKit.EntityFrameworkCore.Tests.SqlServer' threw in InitializeAsync\n" + + "---- DotNet.Testcontainers.Containers.ContainerNotRunningException : Container is not running."; + + // Before Passing: static fields initialize in the order they are written. + private static readonly XNamespace Ns = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + + private static readonly SeedVerdict.Run Passing = Read(Result("Passed"), Result("Passed")); + + /// The defect of #151: a dead fixture failed every test with and without the seed, and the seed read as caught. + [Fact] + public void A_fixture_that_fails_to_initialize_is_an_error_with_or_without_the_seed() + { + var dead = Read(Result("Failed", DeadSqlServer), Result("Failed", DeadSqlServer)); + + dead.Error.ShouldNotBeNull().ShouldContain("2 of 2 tests failed in setup"); + SeedVerdict.Judge(dead, dead).ShouldStartWith("ERROR without the seed"); + SeedVerdict.Judge(Passing, dead).ShouldStartWith("ERROR with the seed"); + } + + /// A seed that breaks a fixture is not credited either: only a test of the oracle's own catches. + [Fact] + public void A_fixture_failure_beside_a_caught_test_is_an_error() + { + var seeded = Read(Result("Failed", "Shouldly.ShouldAssertException : value should be 1 but was 2"), Result("Failed", DeadSqlServer)); + + SeedVerdict.Judge(Passing, seeded).ShouldStartWith("ERROR with the seed"); + } + + /// Every message xunit.v3.core 4.0.2 holds for a test it could not set up. + [Theory] + [InlineData(DeadSqlServer)] + [InlineData("Class fixture type 'Probe.ThrowingCtorFixture' threw in its constructor\n---- System.InvalidOperationException : fixture ctor threw")] + [InlineData("Class fixture type 'Probe.ThrowingInitFixture' threw in InitializeAsync\n---- System.InvalidOperationException : threw")] + [InlineData("Assembly fixture type 'Probe.Fixture' threw in InitializeAsync\n---- System.Exception : threw")] + [InlineData("Collection fixture type 'Probe.Fixture' had one or more unresolved constructor arguments: Probe.Other other")] + [InlineData("Class fixture type 'Probe.Fixture' may only define a single public constructor.")] + [InlineData("The following constructor parameters did not have matching fixture data: SqlServer server")] + [InlineData("Cannot create type 'Probe.Fixture' due to missing constructor arguments: Probe.Other")] + public void A_message_xunit_writes_for_a_fixture_it_could_not_set_up_is_setup(string message) => + SeedVerdict.IsSetupFailure(message).ShouldBeTrue(); + + /// An assertion, an exception from the code a test calls, and a sweep's verdict are the test's own failures. + [Theory] + [InlineData("Shouldly.ShouldAssertException : [Fact] public void Shouldly_fails1\n should be\n2\n but was\n1")] + [InlineData("Assert.Equal() Failure: Values differ\nExpected: 1\nActual: 2")] + [InlineData("System.InvalidOperationException : product threw")] + [InlineData("a sweep: 3 of 170 rows disagree\n(shape: TptRoot, kind: \"ushort\"): the key reads back as 0")] + [InlineData("row 1: the Collection fixture type 'X' threw in InitializeAsync, quoted by a test")] + public void A_failure_in_the_test_itself_is_not_setup(string message) => SeedVerdict.IsSetupFailure(message).ShouldBeFalse(); + + [Fact] + public void An_oracle_that_fails_without_the_seed_is_an_error() + { + var failing = Read(Result("Passed"), Result("Failed", "Assert.Equal() Failure: Values differ")); + + SeedVerdict.Judge(failing, failing).ShouldBe("ERROR 1 of 2 tests fail without the seed, e.g. Probe.Oracle.T1"); + } + + [Fact] + public void A_test_failing_only_with_the_seed_catches() => + SeedVerdict.Judge(Passing, Read(Result("Passed"), Result("Failed", "Assert.Equal() Failure: Values differ"))) + .ShouldBe("CAUGHT 1 of 2 tests fail, e.g. Probe.Oracle.T1"); + + [Fact] + public void An_oracle_that_passes_with_the_seed_misses_it() => + SeedVerdict.Judge(Passing, Passing).ShouldBe("MISSED all 2 tests pass with the defect planted"); + + /// An unfiled candidate fails its sweep without a seed; the baseline passes, and the seed is judged by its other rows. + [Fact] + public void A_baseline_failing_only_on_known_candidates_passes() + { + var known = Read(Result("Passed"), Result("Failed", MessageOf([Unfiled], ["row 1: expected 1, got 2"]))); + var seeded = Read(Result("Passed"), Result("Failed", MessageOf([Unfiled], ["row 1: expected 1, got 2", "row 2: expected 1, got 3"]))); + + known.Error.ShouldBeNull(); + SeedVerdict.Judge(known, known).ShouldBe("MISSED 1 of 2 tests fail, every row a known candidate"); + SeedVerdict.Judge(known, seeded).ShouldStartWith("CAUGHT"); + } + + /// A fixture whose Dispose throws fails no test; the TRX counts it as an error outside any result. + [Fact] + public void An_error_counted_outside_any_test_is_an_error() => + SeedVerdict.Read(2, Trx([Result("Passed")], error: 1), "Probe.Oracle").Error.ShouldNotBeNull().ShouldContain("1 error outside any test"); + + /// A host that crashes writes no TRX; one that exits otherwise than 0 or 2 did not finish its tests. + [Theory] + [InlineData(134, false)] + [InlineData(7, true)] + [InlineData(3, true)] + public void A_host_that_did_not_finish_is_an_error(int exitCode, bool wroteResults) => + SeedVerdict.Read(exitCode, wroteResults ? Trx([Result("Failed", "Assert.Equal() Failure: Values differ")]) : null, "Probe.Oracle") + .Error.ShouldNotBeNull().ShouldContain($"exited {exitCode}"); + + [Fact] + public void A_run_whose_every_test_is_skipped_is_an_error() => + SeedVerdict.Read(0, Trx([Result("NotExecuted"), Result("NotExecuted")]), "Probe.Oracle").Error.ShouldBe("no test of Probe.Oracle ran"); + + private static SeedVerdict.Run Read(params XElement[] results) => + SeedVerdict.Read(results.Any(result => (string?)result.Attribute("outcome") == "Failed") ? 2 : 0, Trx(results), "Probe.Oracle"); + + private static XElement Result(string outcome, string? message = null) => + new(Ns + "UnitTestResult", new XAttribute("testName", "?"), new XAttribute("outcome", outcome), + message is null ? null : new XElement(Ns + "Output", new XElement(Ns + "ErrorInfo", new XElement(Ns + "Message", message), new XElement(Ns + "StackTrace", " at Probe.Oracle.T()")))); + + private static XDocument Trx(IReadOnlyList results, int error = 0) + { + var named = results.Select(result => new XElement(result)).ToList(); + for (var i = 0; i < named.Count; i++) named[i].SetAttributeValue("testName", $"Probe.Oracle.T{i}"); + + int Count(string outcome) => named.Count(result => (string?)result.Attribute("outcome") == outcome); + return new XDocument(new XElement(Ns + "TestRun", + new XElement(Ns + "Results", named), + new XElement(Ns + "ResultSummary", new XAttribute("outcome", Count("Failed") > 0 || error > 0 ? "Failed" : "Completed"), + new XElement(Ns + "Counters", + new XAttribute("total", named.Count), new XAttribute("executed", Count("Passed") + Count("Failed")), + new XAttribute("passed", Count("Passed")), new XAttribute("failed", Count("Failed")), new XAttribute("error", error), + new XAttribute("timeout", 0), new XAttribute("aborted", 0), new XAttribute("notExecuted", Count("NotExecuted")), new XAttribute("disconnected", 0))))); + } + private static string MessageOf(IReadOnlyList known, IReadOnlyList failures, int shown = 25) => Should.Throw(() => Candidates.Judge(known, failures, "a sweep", shown)).Message; } diff --git a/test/oracles/README.md b/test/oracles/README.md index 1f4ba77..deceee8 100644 --- a/test/oracles/README.md +++ b/test/oracles/README.md @@ -28,10 +28,26 @@ dotnet tools/Oracles/bin/Release/net10.0/oracles.dll seeds [name filter] # eac dotnet tools/Oracles/bin/Release/net10.0/oracles.dll status # the worklist: unheld claims, unread waivers, unseeded oracles, candidates ``` -`seeds` copies the working tree (without `bin/`, `obj/` and `.git/`, so uncommitted work is included), applies the -patch, builds each named oracle's project and runs that one class. A seed is caught when one of the class's tests -fails with a row that is not a known candidate (`SeedVerdict`). One line per seed and oracle, `CAUGHT`, `MISSED` or -`ERROR`, and exit code 1 unless every pair is caught. +`seeds` copies the working tree (without `bin/`, `obj/` and `.git/`, so uncommitted work is included), builds each +oracle the selected seeds name and runs that one class: the baseline, with no seed. Then, for each seed, it copies +the tree again, applies the patch, and builds and runs the same classes with the same filter. A seed is caught when +its oracle passes without it and fails with it, in a test of its own: a failed test with a row that is not a known +candidate (`SeedVerdict`, design/adr/0117). It prints one `BASELINE` line per oracle, then one line per seed and +oracle: + +| Verdict | Meaning | Exit code | +|---|---|---| +| `CAUGHT` | The baseline passes, and with the seed a test fails on its own assertion or on what the code it calls throws. | 0 when every pair is caught | +| `MISSED` | The baseline passes, and with the seed every test passes, or fails only on known candidates. | 1 | +| `ERROR` | A run never reached the tests' own code, or the baseline fails. | 4 | + +The TRX decides what reached the tests. A fixture that fails to construct or initialize fails every test with +xunit's own message (`Collection fixture type 'SqlServer' threw in InitializeAsync`). A fixture that throws in +`Dispose` shows up only as an `error` in the TRX's counters. A crashed host writes no TRX and exits with something +other than 0 or 2. A run whose every test was skipped ran nothing. Each of these is `ERROR`, with or without the +seed: a dead fixture fails every test either way, so its failures prove nothing about the seed. On an arm64 Mac, +run it with `TYPEKIT_MSSQL_IMAGE=mcr.microsoft.com/azure-sql-edge:latest`, as the suite runs (CONTRIBUTING's Docker +bullet), or the SQL Server oracles are `ERROR`. To add an oracle: write the test class, register it in `oracles.json`, write a seed by planting the defect it guards against, `git diff` it into `seeds/.patch` under its header lines, undo the edit, and run diff --git a/test/oracles/SeedVerdict.cs b/test/oracles/SeedVerdict.cs index 4fcd1c8..536bce3 100644 --- a/test/oracles/SeedVerdict.cs +++ b/test/oracles/SeedVerdict.cs @@ -1,17 +1,37 @@ +using System.Text.RegularExpressions; +using System.Xml.Linq; + namespace CodoMetis.TypeKit.Oracles; /// -/// Whether a test that failed with a seed planted caught it: the rule oracles seeds applies to each -/// failed test, compiled into tools/Oracles and linked into CodoMetis.TypeKit.Oracles.Tests -/// (SeedVerdictTests), so the two cannot disagree (design/adr/0112). +/// Whether an oracle caught a seed: the rule oracles seeds applies, compiled into tools/Oracles and +/// linked into CodoMetis.TypeKit.Oracles.Tests (SeedVerdictTests), so the two cannot disagree +/// (design/adr/0112, design/adr/0117). /// /// /// +/// A seed is caught when its oracle passes without it and fails with it, in a test of its own. Each oracle class is +/// run twice with the same filter in the same environment: once on a copy of the tree with no seed (the baseline), +/// once with the seed planted, and each run is from the exit code of dotnet test and its +/// TRX. A run that did not reach the tests' own code is an , and an error on either side makes +/// the pair ERROR, never CAUGHT: a dead fixture fails every test, with or without a seed. +/// +/// +/// What the TRX holds, measured with xunit.v3 4.0.2 on Microsoft Testing Platform (2026-10-29): a class, collection +/// or assembly fixture that fails to construct or to initialize fails every test that uses it, each with xunit's own +/// message, Collection fixture type 'T' threw in InitializeAsync, the fixture's exception below it +/// (). A fixture that throws in Dispose fails no test and is counted only as an +/// error in the TRX's counters. A test host that crashes writes no TRX and exits with a code that is not +/// Microsoft Testing Platform's 0 (passed) or 2 (a test failed). A failure in the test's own constructor or +/// InitializeAsync is written like one in its body, the exception's message and the stack trace below the +/// test's code, so it counts as the test's: the oracles build what they assert on there. +/// +/// /// A sweep reports through Candidates.Verdict: a header line, then one line per failing row, where a /// row that matches an unfiled candidate starts with . A failed test catches the -/// seed when at least one of its rows is not a known candidate. Counting a test as known whenever its -/// message held the marker anywhere read a seed that broke a row beside a known one as missed, as -/// CodoMetis.ValueRanges found (its #55). +/// seed when at least one of its rows is not a known candidate (). Counting a test as known +/// whenever its message held the marker anywhere read a seed that broke a row beside a known one as missed, as +/// CodoMetis.ValueRanges found (its #55). A baseline whose only failures are known candidates passes. /// /// /// Candidates.Verdict lists the other rows before the known ones, so a message cut short @@ -27,6 +47,32 @@ public static class SeedVerdict /// The line Candidates.Verdict ends a cut-short message with. public const string MorePrefix = "… and "; + /// The exit codes of a run that reached its tests: Microsoft Testing Platform's success and "a test failed". + private static readonly int[] RanExitCodes = [0, 2]; + + /// The TRX counters of a result outside any test, such as a fixture's Dispose, or of a run cut short. + private static readonly string[] ErrorCounters = ["error", "aborted", "timeout", "disconnected"]; + + private static readonly XNamespace Trx = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + + /// + /// xunit's own messages for a test it could not set up, every one in xunit.v3.core 4.0.2 (its fixture kind is + /// Assembly, Collection or Class). + /// + private static readonly Regex SetupMessage = new( + @"\A(\w+ fixture type '[^']*' (threw in |had one or more unresolved constructor arguments|may only define a single public constructor)" + + @"|The following constructor parameters did not have matching fixture data" + + @"|Cannot create type '[^']*' due to missing constructor arguments)", + RegexOptions.CultureInvariant); + + /// One run of one oracle class, read from the exit code of dotnet test and its TRX. + /// The tests that ran, passed or failed; a skipped test is neither. + /// The tests that failed. + /// The failed tests with a row that is not a known candidate. + /// The first test in , or empty. + /// Why the run never reached the tests' own code, or when it did. + public sealed record Run(int Executed, int Failed, int Caught, string Example, string? Error); + /// Whether a failed test's message holds at least one row that is not a known candidate. public static bool Catches(string message) { @@ -37,5 +83,49 @@ public static bool Catches(string message) return lines.Skip(1).Any(line => !IsKnown(line) && !line.StartsWith(MorePrefix, StringComparison.Ordinal)); } + /// Whether a failed test's message is xunit's own for a fixture or a constructor it could not set up. + public static bool IsSetupFailure(string message) => SetupMessage.IsMatch(message); + + /// Reads one run of : the exit code of dotnet test and its TRX, if it wrote one. + public static Run Read(int exitCode, XDocument? trx, string oracle) + { + if (trx is null) return Failure($"no test results written: the test host exited {exitCode}"); + if (!RanExitCodes.Contains(exitCode)) return Failure($"the test host exited {exitCode}, not 0 or 2"); + + var counters = trx.Descendants(Trx + "Counters").FirstOrDefault(); + var outside = ErrorCounters.Select(name => (Name: name, Count: (int?)counters?.Attribute(name) ?? 0)).Where(counter => counter.Count > 0).ToList(); + if (outside.Count > 0) return Failure($"the run counts {string.Join(", ", outside.Select(counter => $"{counter.Count} {counter.Name}"))} outside any test (a fixture's Dispose, a host cut short)"); + + var results = trx.Descendants(Trx + "UnitTestResult") + .Select(result => (Outcome: (string?)result.Attribute("outcome"), Test: (string?)result.Attribute("testName") ?? "?", Message: result.Descendants(Trx + "Message").FirstOrDefault()?.Value ?? "")) + .Where(result => result.Outcome is "Passed" or "Failed") + .ToList(); + if (results.Count == 0) return Failure($"no test of {oracle} ran"); + + var failed = results.Where(result => result.Outcome == "Failed").ToList(); + var setup = failed.Where(result => IsSetupFailure(result.Message)).ToList(); + if (setup.Count > 0) return Failure($"{setup.Count} of {results.Count} tests failed in setup, not in the test: {setup[0].Message.Split('\n')[0].Trim()}"); + + var caught = failed.Where(result => Catches(result.Message)).ToList(); + return new Run(results.Count, failed.Count, caught.Count, caught.Count > 0 ? caught[0].Test : "", Error: null); + + static Run Failure(string error) => new(0, 0, 0, "", error); + } + + /// + /// The verdict on one seed and one oracle: CAUGHT, MISSED or ERROR, padded to eight + /// characters, then what was seen. + /// + public static string Judge(Run baseline, Run seeded) + { + if (baseline.Error is not null) return $"ERROR without the seed, {baseline.Error}"; + if (baseline.Caught > 0) return $"ERROR {baseline.Caught} of {baseline.Executed} tests fail without the seed, e.g. {baseline.Example}"; + if (seeded.Error is not null) return $"ERROR with the seed, {seeded.Error}"; + if (seeded.Caught > 0) return $"CAUGHT {seeded.Caught} of {seeded.Executed} tests fail, e.g. {seeded.Example}"; + return seeded.Failed == 0 + ? $"MISSED all {seeded.Executed} tests pass with the defect planted" + : $"MISSED {seeded.Failed} of {seeded.Executed} tests fail, every row a known candidate"; + } + private static bool IsKnown(string line) => line.StartsWith(KnownMarker, StringComparison.Ordinal); } diff --git a/tools/Oracles/Program.cs b/tools/Oracles/Program.cs index f332128..95ceccf 100644 --- a/tools/Oracles/Program.cs +++ b/tools/Oracles/Program.cs @@ -3,14 +3,18 @@ // dotnet tools/Oracles/bin/Release/net10.0/oracles.dll seeds [name filter] show that every seed is caught // dotnet tools/Oracles/bin/Release/net10.0/oracles.dll status the catalogue's worklist // -// `seeds`: for each test/oracles/seeds/.patch whose file name holds the filter, a copy of the working tree -// (bin/, obj/, .git/ and the agents' worktrees left out, so uncommitted work is included) under the temp directory, -// the patch applied, each oracle project its `Oracles:` line names built once in Release, and each named class run -// alone. An oracle catches a seed when one of its tests fails with a row that is not a known candidate -// (SeedVerdict). One line per (seed, oracle), CAUGHT, MISSED or ERROR, and exit code 1 unless every pair is caught. +// `seeds`: first a baseline, a copy of the working tree with no seed (bin/, obj/, .git/ and the agents' worktrees left +// out, so uncommitted work is included) under the temp directory, each oracle the selected seeds name built in Release +// and run alone; then, for each test/oracles/seeds/.patch whose file name holds the filter, another copy, the +// patch applied, each oracle project its `Oracles:` line names built once and each named class run alone, with the +// same filter. An oracle catches a seed when it passes without it and fails with it in a test of its own: a test +// that fails with a row that is not a known candidate, never one xunit could not set up, a test host that crashed or +// a run that wrote no results, which are ERROR (SeedVerdict, design/adr/0117). One line per baseline and one per +// (seed, oracle), CAUGHT, MISSED or ERROR. // // Exit 0 is a pass, 1 a seed not caught (or a status that could not read the catalogue), 2 a usage error, 3 a tool -// built from other sources than the checkout holds, which names the command that rebuilds it. +// built from other sources than the checkout holds, which names the command that rebuilds it, 4 an oracle that could +// not be judged: its baseline does not pass, or a run with the seed did not reach the tests' own code. using System.Diagnostics; using System.Text.Json; using System.Text.RegularExpressions; @@ -51,6 +55,7 @@ int Seeds(string filter) var seeds = Directory.GetFiles(Path.Combine(oraclesDirectory, "seeds"), "*.patch") .Where(seed => Path.GetFileName(seed).Contains(filter, StringComparison.OrdinalIgnoreCase)) .Order(StringComparer.Ordinal) + .Select(seed => (Path: seed, Name: Path.GetFileName(seed), Named: Header(File.ReadAllText(seed), "Oracles").Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))) .ToList(); if (seeds.Count == 0) { @@ -58,66 +63,88 @@ int Seeds(string filter) return 1; } - var allCaught = true; - foreach (var seed in seeds) + // The baseline: every registered oracle a seed names, run once on a copy with no seed, built and filtered as the + // seeded runs are. A seed is judged against it (SeedVerdict.Judge), so an oracle that cannot pass here, a dead + // fixture or a failing test, makes its pairs ERROR instead of crediting every seed with its failures. + var started = Stopwatch.StartNew(); + var baselines = RunOracles(seed: null, [.. seeds.SelectMany(seed => seed.Named).Distinct(StringComparer.Ordinal).Where(oracles.ContainsKey).Select(oracle => oracles[oracle])]); + foreach (var (oracle, run) in baselines.OrderBy(pair => pair.Key, StringComparer.Ordinal)) { - var name = Path.GetFileName(seed); - var named = Header(File.ReadAllText(seed), "Oracles").Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); - var scratch = Path.Combine(Path.GetTempPath(), $"typekit-seed-{Guid.NewGuid():N}"); - var started = Stopwatch.StartNew(); + Console.WriteLine(run.Error is null && run.Caught == 0 + ? $"BASELINE {oracle}: passes without a seed, {run.Executed} tests ({started.Elapsed.TotalSeconds:F0} s)" + : $"BASELINE {oracle}: DOES NOT PASS without a seed, {run.Error ?? $"{run.Caught} of {run.Executed} tests fail, e.g. {run.Example}"}"); + } + + var notRun = new SeedVerdict.Run(0, 0, 0, "", "not run, since it does not pass without the seed"); + var missed = false; + var errors = false; + foreach (var (seed, name, named) in seeds) + { + started.Restart(); try { - CopyTree(root!, scratch); - Require(RunProcess(scratch, "git", null, ["apply", "--whitespace=nowarn", seed]), "apply the patch"); - - foreach (var group in named.Select(oracle => oracles.TryGetValue(oracle, out var registered) ? registered : throw new SeedError($"'{oracle}' is not in oracles.json")) - .GroupBy(oracle => oracle.Project)) + var registered = named.Select(oracle => oracles.TryGetValue(oracle, out var found) ? found : throw new SeedError($"'{oracle}' is not in oracles.json")).ToList(); + var seeded = RunOracles(seed, [.. registered.Where(oracle => baselines[oracle.Class] is { Error: null, Caught: 0 })]); + foreach (var oracle in registered) { - var project = Path.Combine("test", group.Key); - Require(RunDotnet(scratch, "build", project, "-c", "Release", "-nologo", "-v", "q"), $"build {group.Key}"); - foreach (var oracle in group) - { - var results = Path.Combine(scratch, "SeedResults", oracle.Class); - RunDotnet(scratch, "test", "--project", project, "-c", "Release", "--no-build", - "--filter-class", $"*.{oracle.Class}", "--report-xunit-trx", "--report-xunit-trx-filename", $"{oracle.Class}.trx", - "--results-directory", results); - var verdict = Verdict(Path.Combine(results, $"{oracle.Class}.trx"), oracle.Class); - allCaught &= verdict.StartsWith("CAUGHT", StringComparison.Ordinal); - Console.WriteLine($"{verdict[..8]}{name} -> {oracle.Class}: {verdict[8..]} ({started.Elapsed.TotalSeconds:F0} s)"); - } + var verdict = SeedVerdict.Judge(baselines[oracle.Class], seeded.GetValueOrDefault(oracle.Class) ?? notRun); + missed |= verdict.StartsWith("MISSED", StringComparison.Ordinal); + errors |= verdict.StartsWith("ERROR", StringComparison.Ordinal); + Console.WriteLine($"{verdict[..8]}{name} -> {oracle.Class}: {verdict[8..]} ({started.Elapsed.TotalSeconds:F0} s)"); } } catch (SeedError error) { - allCaught = false; + errors = true; Console.WriteLine($"ERROR {name}: {error.Message}"); } - finally - { - try { Directory.Delete(scratch, recursive: true); } catch (IOException) { } catch (UnauthorizedAccessException) { } - } } - Console.WriteLine(allCaught ? "every seed is caught" : "A SEED IS NOT CAUGHT: the oracle it names cannot fail for that defect"); - return allCaught ? 0 : 1; + if (missed) Console.WriteLine("A SEED IS NOT CAUGHT: the oracle it names cannot fail for that defect"); + if (errors) Console.WriteLine("AN ORACLE COULD NOT BE JUDGED: its baseline or its run did not reach the tests' own code; fix that and run again"); + if (!missed && !errors) Console.WriteLine("every seed is caught"); + return errors ? 4 : missed ? 1 : 0; } -string Verdict(string trx, string oracle) +// A copy of the working tree with the seed applied, or none, each oracle's project built once in Release and each +// class run alone. A project that does not build is an error run for each of its oracles. +Dictionary RunOracles(string? seed, IReadOnlyList selected) { - if (!File.Exists(trx)) return "ERROR no test results written (did the class run?)"; - - XNamespace ns = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; - var results = XDocument.Load(trx).Descendants(ns + "UnitTestResult").ToList(); - if (results.Count == 0) return $"ERROR no test of {oracle} ran"; - - var failed = results.Where(result => (string?)result.Attribute("outcome") == "Failed") - .Select(result => (Test: (string?)result.Attribute("testName") ?? "?", Message: result.Descendants(ns + "Message").FirstOrDefault()?.Value ?? "")) - .ToList(); - var caught = failed.Where(failure => SeedVerdict.Catches(failure.Message)).ToList(); - if (caught.Count > 0) return $"CAUGHT {caught.Count} of {results.Count} tests fail, e.g. {caught[0].Test}"; - return failed.Count == 0 - ? $"MISSED all {results.Count} tests pass with the defect planted" - : $"MISSED {failed.Count} of {results.Count} tests fail, every row a known candidate"; + var runs = new Dictionary(StringComparer.Ordinal); + if (selected.Count == 0) return runs; + + var scratch = Path.Combine(Path.GetTempPath(), $"typekit-seed-{Guid.NewGuid():N}"); + try + { + CopyTree(root!, scratch); + if (seed is not null) Require(RunProcess(scratch, "git", null, ["apply", "--whitespace=nowarn", seed]), "apply the patch"); + + foreach (var group in selected.GroupBy(oracle => oracle.Project)) + { + var project = Path.Combine("test", group.Key); + if (Failure(RunDotnet(scratch, "build", project, "-c", "Release", "-nologo", "-v", "q"), $"build {group.Key}") is { } failure) + { + foreach (var oracle in group) runs[oracle.Class] = new SeedVerdict.Run(0, 0, 0, "", failure); + continue; + } + + foreach (var oracle in group) + { + var results = Path.Combine(scratch, "SeedResults", oracle.Class); + var (exit, _) = RunDotnet(scratch, "test", "--project", project, "-c", "Release", "--no-build", + "--filter-class", $"*.{oracle.Class}", "--report-xunit-trx", "--report-xunit-trx-filename", $"{oracle.Class}.trx", + "--results-directory", results); + var trx = Path.Combine(results, $"{oracle.Class}.trx"); + runs[oracle.Class] = SeedVerdict.Read(exit, File.Exists(trx) ? XDocument.Load(trx) : null, oracle.Class); + } + } + + return runs; + } + finally + { + try { Directory.Delete(scratch, recursive: true); } catch (IOException) { } catch (UnauthorizedAccessException) { } + } } int Status() @@ -203,9 +230,12 @@ static void CopyClaude(string from, string to) static void Require((int Exit, string Output) result, string step) { - if (result.Exit != 0) throw new SeedError($"{step} failed (exit {result.Exit}): {string.Join(" | ", result.Output.Split('\n').Where(line => line.Contains("error", StringComparison.OrdinalIgnoreCase)).Take(3).Select(line => line.Trim()))}"); + if (Failure(result, step) is { } failure) throw new SeedError(failure); } +static string? Failure((int Exit, string Output) result, string step) => + result.Exit == 0 ? null : $"{step} failed (exit {result.Exit}): {string.Join(" | ", result.Output.Split('\n').Where(line => line.Contains("error", StringComparison.OrdinalIgnoreCase)).Take(3).Select(line => line.Trim()))}"; + internal sealed record Claim(string Id, string Source, IReadOnlyList? HeldBy); internal sealed record Oracle(string Class, string Project, string Kind, string Against);