From f094609f8dd440bb15e7f9877670972825502157 Mon Sep 17 00:00:00 2001 From: Sebastian Henn <138793908+SebHenn@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:13:37 +0100 Subject: [PATCH] .NET rules for analyzer, format, test-host, audit and platform failures (#62) dotnet-analyzer-error, dotnet-format-verify, vstest-testhost-crashed, vstest-blame-hang, nuget-audit-vulnerability, netsdk1100-windows-targeting and msbuild-file-locked, each with a fixture. A TRX whose run aborted now yields a "Test run aborted" failure from its error RunInfo, which report mode used to read as passing. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 18 +++ CONTRIBUTING.md | 2 +- src/CiFail.Core/Ingest/EcosystemDetector.cs | 2 +- .../Ingest/Reports/TestReportParser.cs | 15 +++ src/CiFail.Core/rulepacks/dotnet.yaml | 103 ++++++++++++++++++ .../Ingest/TestReportParserTests.cs | 21 ++++ .../Rules/RulePackBreadthTests.cs | 33 ++++++ .../fixtures/dotnet-analyzer-error.log | 5 + .../fixtures/dotnet-format-verify.log | 5 + .../fixtures/msbuild-file-locked.log | 5 + .../fixtures/netsdk1100-windows-targeting.log | 4 + .../fixtures/nuget-audit-vulnerability.log | 4 + .../fixtures/vstest-blame-hang.log | 9 ++ .../fixtures/vstest-testhost-crashed.log | 8 ++ 14 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 tests/CiFail.Core.Tests/fixtures/dotnet-analyzer-error.log create mode 100644 tests/CiFail.Core.Tests/fixtures/dotnet-format-verify.log create mode 100644 tests/CiFail.Core.Tests/fixtures/msbuild-file-locked.log create mode 100644 tests/CiFail.Core.Tests/fixtures/netsdk1100-windows-targeting.log create mode 100644 tests/CiFail.Core.Tests/fixtures/nuget-audit-vulnerability.log create mode 100644 tests/CiFail.Core.Tests/fixtures/vstest-blame-hang.log create mode 100644 tests/CiFail.Core.Tests/fixtures/vstest-testhost-crashed.log diff --git a/CHANGELOG.md b/CHANGELOG.md index 079c484..680a5c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,24 @@ after 1.0. ### Added +- **.NET rules for analyzer, format, test-host, audit and platform failures** + ([#62](https://github.com/SebHenn/ci-failure-intelligence/issues/62)): + `dotnet-analyzer-error` (CA, IDE, SA, RS, xUnit, MSTEST and NUnit diagnostics raised to + errors), `dotnet-format-verify`, `vstest-testhost-crashed`, `vstest-blame-hang`, + `nuget-audit-vulnerability` (NU1901-NU1904 as errors), `netsdk1100-windows-targeting` and + `msbuild-file-locked` (MSB3027/MSB3021). `csc-compile-error` matched CS codes only, so the + commonest of these fell through to "No known rule matched". `dotnet format` now marks a log as + .NET. + +### Fixed + +- **A TRX report of a crashed test host is a failure** + ([#62](https://github.com/SebHenn/ci-failure-intelligence/issues/62)). A run that aborted + leaves no failed result, only an error in `RunInfos`, so report mode read it as passing. That + error is now a failure named "Test run aborted", which the new test-host rules explain. + +### Added + - **`selection-misses` sees failures in test projects the selection skipped whole** ([#57](https://github.com/SebHenn/ci-failure-intelligence/issues/57)). dotnet-tia never names the tests of an F# or VB test project, so when its selection left one out, a failure there diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 48802fb..b2b22c2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -124,7 +124,7 @@ Notes: confidently-identified deprecation notice (`severity: note`, `confidence: 0.9`) or a tentative but fatal OOM. - **`docs:` is expected**, not decorative: it's where a reader goes when the `fix` text isn't - enough. All 136 shipped rules have one and `cifail rules validate` warns when a rule doesn't. + enough. All 143 shipped rules have one and `cifail rules validate` warns when a rule doesn't. - New ecosystem? Add an enum value to `Models/Ecosystem.cs`, marker regexes to `Ingest/EcosystemDetector.cs`, and a new `rulepacks/.yaml`. That's the only case that touches C#. Note `ecosystems:` is *not* the way to express "Android builds are JVM diff --git a/src/CiFail.Core/Ingest/EcosystemDetector.cs b/src/CiFail.Core/Ingest/EcosystemDetector.cs index 6842839..ff2c971 100644 --- a/src/CiFail.Core/Ingest/EcosystemDetector.cs +++ b/src/CiFail.Core/Ingest/EcosystemDetector.cs @@ -205,7 +205,7 @@ private static Regex Compile(string pattern) => { Strong(@"(?:error|warning) (?:NU|MSB|CS|NETSDK)\d{3,5}"), Strong(@"\.csproj"), - Strong(@"\bdotnet (?:build|test|restore|publish|run|pack|tool)\b"), + Strong(@"\bdotnet (?:build|test|restore|publish|run|pack|tool|format)\b"), Weak(@"\bnuget\b"), Weak(@"\bmsbuild\b"), Weak(@"\.sln\b"), diff --git a/src/CiFail.Core/Ingest/Reports/TestReportParser.cs b/src/CiFail.Core/Ingest/Reports/TestReportParser.cs index 144c17f..22df18d 100644 --- a/src/CiFail.Core/Ingest/Reports/TestReportParser.cs +++ b/src/CiFail.Core/Ingest/Reports/TestReportParser.cs @@ -198,6 +198,21 @@ private static List ParseTrx(string content) Assembly = assembly, }); } + + // A crashed or hung test host leaves no failed result: the tests after it never report. + // What it leaves is an error in the run's information, and that is a failure too (#62). + foreach (var info in document.Descendants().Where(e => e.Name.LocalName == "RunInfo")) + { + if (!string.Equals(info.Attribute("outcome")?.Value, "Error", StringComparison.OrdinalIgnoreCase)) + continue; + + var text = info.Elements().FirstOrDefault(e => e.Name.LocalName == "Text")?.Value.Trim(); + if (string.IsNullOrEmpty(text)) + continue; + + failures.Add(new TestFailure(Name: "Test run aborted", Classname: null, Message: text, Details: null)); + } + return failures; } diff --git a/src/CiFail.Core/rulepacks/dotnet.yaml b/src/CiFail.Core/rulepacks/dotnet.yaml index 8380cfa..b799ada 100644 --- a/src/CiFail.Core/rulepacks/dotnet.yaml +++ b/src/CiFail.Core/rulepacks/dotnet.yaml @@ -149,3 +149,106 @@ The failing test's name is on the `Failed ...` line just above. Re-run only that one with `dotnet test --filter FullyQualifiedName~`. docs: https://docs.nunit.org/articles/nunit/writing-tests/assertions/assertions.html + +# ---- Analyzers, test hosts, audit and platform (#62) ----------------------------------------- +# csc-compile-error matches CS codes only, so an analyzer or style diagnostic raised to an error - +# among the commonest .NET CI failures - fell through to generic rules or to nothing. + +- id: dotnet-analyzer-error + ecosystem: dotnet + category: compile + title: Analyzer rule raised to an error + match: "error (?(?:CA|IDE|SA|RS|RCS|VSTHRD|xUnit|MSTEST|NUnit)\\d+): (?.+)" + confidence: 0.75 + fix: | + An analyzer reported {code} as an error: {message} + It fails the build because its severity is error, through .editorconfig + (dotnet_diagnostic.{code}.severity), TreatWarningsAsErrors, or AnalysisLevel / + EnforceCodeStyleInBuild. Fix the code it points at; many style rules are fixed by + `dotnet format`. To change the rule rather than the code, set its severity in + .editorconfig, and keep the change reviewed like any other. + docs: https://learn.microsoft.com/dotnet/fundamentals/code-analysis/configuration-options + +- id: dotnet-format-verify + ecosystem: dotnet + category: build + title: dotnet format found unformatted code + match: "error (?WHITESPACE|CHARSET|FINALNEWLINE|ENDOFLINE|IMPORTS): (?.+)" + confidence: 0.85 + fix: | + `dotnet format --verify-no-changes` found code that is not formatted ({code}). + Run `dotnet format` locally and commit the result. If it disagrees with your editor, + the .editorconfig the build sees is the one to fix; line endings often differ + between Windows and Linux checkouts (core.autocrlf, or a .gitattributes eol rule). + docs: https://learn.microsoft.com/dotnet/core/tools/dotnet-format + +- id: vstest-testhost-crashed + ecosystem: dotnet + category: test + title: Test host crashed + match: "The active test run was aborted\\. Reason: (?.+)|Test host process crashed(?.*)|^\\s*Test Run Aborted\\.?\\s*$" + confidence: 0.8 + fix: | + The process running the tests died, so no test after it reported a result. Common + causes: a stack overflow, an unhandled exception on a background thread, an + Environment.Exit or FailFast in code under test, or the runner killed for memory. + Re-run with `dotnet test --blame-crash` (add --blame-crash-dump-type full) to get a + dump and the test that was running, attached as a test result. + docs: https://learn.microsoft.com/dotnet/core/tools/dotnet-test#options + +- id: vstest-blame-hang + ecosystem: dotnet + category: test + title: Test run hung and was stopped + # Outranks vstest-testhost-crashed: a hang dump also ends the run "aborted". + match: "The specified inactivity time of (?.+?) has elapsed" + confidence: 0.85 + fix: | + No test finished within {timeout}, so the run was stopped and hang dumps collected. + The Sequence file and the dump name the test that was running: look for a deadlock, + a sync-over-async wait (.Result, .Wait()), or a wait on a network or service that + never answers. --blame-hang-timeout sets the limit; give a slow test its own timeout + rather than raising it for every test. + docs: https://learn.microsoft.com/dotnet/core/tools/dotnet-test#options + +- id: nuget-audit-vulnerability + ecosystem: dotnet + category: dependency + title: Package with a known vulnerability + # An error only: NuGetAudit warns by default, and a warning beside the real failure is not it. + match: "error NU190[1-4]: Package '(?[^']+)' (?\\S+) has a known (?\\w+) severity vulnerability" + confidence: 0.85 + fix: | + '{package}' {version} has a known {severity} severity vulnerability, and warnings are + errors in this build. Upgrade it (`dotnet list package --vulnerable --include-transitive` + shows which version is fixed). A transitive one is upgraded by referencing the fixed + version directly. Suppress one advisory only after reviewing it, with + . + docs: https://learn.microsoft.com/nuget/concepts/auditing-packages + +- id: netsdk1100-windows-targeting + ecosystem: dotnet + category: environment + title: Windows-only project built on Linux or macOS + match: "error NETSDK1100: " + confidence: 0.9 + fix: | + A project targets Windows (WPF, WinForms, or a -windows framework) and is building on + a non-Windows runner. Set true (in + Directory.Build.props, or -p:EnableWindowsTargeting=true) to build it there, or build + that project on windows-latest. Running its tests still needs Windows. + docs: https://learn.microsoft.com/dotnet/core/tools/sdk-errors/netsdk1100 + +- id: msbuild-file-locked + ecosystem: dotnet + category: build + title: Output file locked by another process + match: "error MSB30(?:27|21): (?.+)" + confidence: 0.85 + fix: | + MSBuild could not overwrite an output file because another process holds it: {message} + Usually a test host, a dotnet build server or an app from an earlier step that is + still running, or two builds of one project writing the same folder in parallel. + Stop it first (`dotnet build-server shutdown`), and don't build a project twice + concurrently into the same output path. + docs: https://learn.microsoft.com/visualstudio/msbuild/errors/msb3027 diff --git a/tests/CiFail.Core.Tests/Ingest/TestReportParserTests.cs b/tests/CiFail.Core.Tests/Ingest/TestReportParserTests.cs index 72af760..3222635 100644 --- a/tests/CiFail.Core.Tests/Ingest/TestReportParserTests.cs +++ b/tests/CiFail.Core.Tests/Ingest/TestReportParserTests.cs @@ -125,6 +125,27 @@ public void ParseJUnit_takes_the_assembly_from_the_suite_name(string suite, stri failures.Single().Assembly.Should().Be(assembly); } + [Fact] + public void ParseTrx_reports_a_crashed_test_host_as_a_failure() + { + // The run aborted part way: no result says Failed, and the crash is only in RunInfos (#62). + var failures = TestReportParser.ParseFailures(ReportFormat.Trx, """ + + + + + No test matches the given testcase filter + The active test run was aborted. Reason: Test host process crashed : Stack overflow. + + + + """); + + var failure = failures.Should().ContainSingle().Subject; + failure.Name.Should().Be("Test run aborted"); + failure.Message.Should().Contain("Test host process crashed"); + } + [Fact] public void ParseTrx_takes_the_class_and_method_from_the_test_definition() { diff --git a/tests/CiFail.Core.Tests/Rules/RulePackBreadthTests.cs b/tests/CiFail.Core.Tests/Rules/RulePackBreadthTests.cs index 0ce88ab..7f0a430 100644 --- a/tests/CiFail.Core.Tests/Rules/RulePackBreadthTests.cs +++ b/tests/CiFail.Core.Tests/Rules/RulePackBreadthTests.cs @@ -44,6 +44,14 @@ public class RulePackBreadthTests { "msbuild-msb4018.log", Ecosystem.Dotnet, "msbuild-msb4018" }, { "mstest-assert-failed.log", Ecosystem.Dotnet, "mstest-assert-failed" }, { "nunit-assert-failed.log", Ecosystem.Dotnet, "nunit-assert-failed" }, + // #62: analyzers, dotnet format, a crashed or hung test host, audit, platform, a lock. + { "dotnet-analyzer-error.log", Ecosystem.Dotnet, "dotnet-analyzer-error" }, + { "dotnet-format-verify.log", Ecosystem.Dotnet, "dotnet-format-verify" }, + { "vstest-testhost-crashed.log", Ecosystem.Dotnet, "vstest-testhost-crashed" }, + { "vstest-blame-hang.log", Ecosystem.Dotnet, "vstest-blame-hang" }, + { "nuget-audit-vulnerability.log", Ecosystem.Dotnet, "nuget-audit-vulnerability" }, + { "netsdk1100-windows-targeting.log", Ecosystem.Dotnet, "netsdk1100-windows-targeting" }, + { "msbuild-file-locked.log", Ecosystem.Dotnet, "msbuild-file-locked" }, // node { "node-eresolve.log", Ecosystem.Node, "npm-eresolve" }, @@ -365,6 +373,31 @@ public void Captures_the_message_from_either_spelling_of_an_actions_error(string analysis.RootCause.Captures["message"].Should().Be(expected); } + [Fact] + public void Names_the_analyzer_rule_and_the_vulnerable_package() + { + var analyzer = Service.Analyze("a", Fixtures.Load("dotnet-analyzer-error.log")).RootCause!; + analyzer.Captures["code"].Should().Be("CA1305"); + analyzer.Fix.Should().Contain("dotnet_diagnostic.CA1305.severity"); + + var audit = Service.Analyze("a", Fixtures.Load("nuget-audit-vulnerability.log")).RootCause!; + audit.Captures["package"].Should().Be("System.Text.Json"); + audit.Captures["version"].Should().Be("8.0.4"); + audit.Captures["severity"].Should().Be("high"); + } + + [Fact] + public void A_vulnerability_warning_is_not_the_failure() + { + // NuGetAudit warns by default; beside a compile error, the compile error is the cause. + var analysis = Service.Analyze("a", """ + /src/App/App.csproj : warning NU1903: Package 'System.Text.Json' 8.0.4 has a known high severity vulnerability + /src/App/A.cs(3,1): error CS0246: The type or namespace name 'Foo' could not be found [/src/App/App.csproj] + """); + + analysis.RootCause!.Rule.Id.Should().Be("csc-compile-error"); + } + [Fact] public void Captures_the_command_that_was_not_found() { diff --git a/tests/CiFail.Core.Tests/fixtures/dotnet-analyzer-error.log b/tests/CiFail.Core.Tests/fixtures/dotnet-analyzer-error.log new file mode 100644 index 0000000..9a50a61 --- /dev/null +++ b/tests/CiFail.Core.Tests/fixtures/dotnet-analyzer-error.log @@ -0,0 +1,5 @@ +Run dotnet build --configuration Release --no-restore + App -> /home/runner/work/app/src/Core/bin/Release/net8.0/Core.dll +/home/runner/work/app/src/App/OrderService.cs(41,27): error CA1305: The behavior of 'int.ToString()' could vary based on the current user's locale settings. Replace this call in 'OrderService.Describe()' with a call to 'int.ToString(IFormatProvider)'. [/home/runner/work/app/src/App/App.csproj] + +Build FAILED. diff --git a/tests/CiFail.Core.Tests/fixtures/dotnet-format-verify.log b/tests/CiFail.Core.Tests/fixtures/dotnet-format-verify.log new file mode 100644 index 0000000..154abd5 --- /dev/null +++ b/tests/CiFail.Core.Tests/fixtures/dotnet-format-verify.log @@ -0,0 +1,5 @@ +Run dotnet format --verify-no-changes --verbosity diagnostic + Formatting code files in workspace '/home/runner/work/app/App.sln'. +/home/runner/work/app/src/App/OrderService.cs(12,1): error WHITESPACE: Fix whitespace formatting. Replace 2 characters with '\n\s\s\s\s'. [/home/runner/work/app/src/App/App.csproj] + Formatted code file '/home/runner/work/app/src/App/OrderService.cs'. +Error: Process completed with exit code 2. diff --git a/tests/CiFail.Core.Tests/fixtures/msbuild-file-locked.log b/tests/CiFail.Core.Tests/fixtures/msbuild-file-locked.log new file mode 100644 index 0000000..6e0be5d --- /dev/null +++ b/tests/CiFail.Core.Tests/fixtures/msbuild-file-locked.log @@ -0,0 +1,5 @@ +Run dotnet build --no-restore +C:\Program Files\dotnet\sdk\8.0.404\Microsoft.Common.CurrentVersion.targets(4807,5): warning MSB3026: Could not copy "obj\Debug\net8.0\App.dll" to "bin\Debug\net8.0\App.dll". Beginning retry 1 in 1000ms. The process cannot access the file 'bin\Debug\net8.0\App.dll' because it is being used by another process. [D:\a\app\src\App\App.csproj] +C:\Program Files\dotnet\sdk\8.0.404\Microsoft.Common.CurrentVersion.targets(4807,5): error MSB3027: Could not copy "obj\Debug\net8.0\App.dll" to "bin\Debug\net8.0\App.dll". Exceeded retry count of 10. Failed. The file is locked by: "testhost (5120)" [D:\a\app\src\App\App.csproj] + +Build FAILED. diff --git a/tests/CiFail.Core.Tests/fixtures/netsdk1100-windows-targeting.log b/tests/CiFail.Core.Tests/fixtures/netsdk1100-windows-targeting.log new file mode 100644 index 0000000..10e377c --- /dev/null +++ b/tests/CiFail.Core.Tests/fixtures/netsdk1100-windows-targeting.log @@ -0,0 +1,4 @@ +Run dotnet build App.sln +/usr/share/dotnet/sdk/8.0.404/Sdks/Microsoft.NET.Sdk/targets/Microsoft.NET.TargetFrameworkInference.targets(258,5): error NETSDK1100: To build a project targeting Windows on this operating system, set the EnableWindowsTargeting property to true. [/home/runner/work/app/src/Desktop/Desktop.csproj] + +Build FAILED. diff --git a/tests/CiFail.Core.Tests/fixtures/nuget-audit-vulnerability.log b/tests/CiFail.Core.Tests/fixtures/nuget-audit-vulnerability.log new file mode 100644 index 0000000..4b50f71 --- /dev/null +++ b/tests/CiFail.Core.Tests/fixtures/nuget-audit-vulnerability.log @@ -0,0 +1,4 @@ +Run dotnet restore + Determining projects to restore... +/home/runner/work/app/src/App/App.csproj : error NU1903: Package 'System.Text.Json' 8.0.4 has a known high severity vulnerability, https://github.com/advisories/GHSA-8g4q-xg66-9fp4 + Failed to restore /home/runner/work/app/src/App/App.csproj (in 1.2 sec). diff --git a/tests/CiFail.Core.Tests/fixtures/vstest-blame-hang.log b/tests/CiFail.Core.Tests/fixtures/vstest-blame-hang.log new file mode 100644 index 0000000..9cedb5c --- /dev/null +++ b/tests/CiFail.Core.Tests/fixtures/vstest-blame-hang.log @@ -0,0 +1,9 @@ +Run dotnet test --no-build --blame-hang-timeout 10m +Test run for /home/runner/work/app/tests/App.Tests/bin/Debug/net8.0/App.Tests.dll (.NETCoreApp,Version=v8.0) +Starting test execution, please wait... +Blame: Collecting hang dump from 'testhost' (4312) +The specified inactivity time of 10 minutes has elapsed. Collecting hang dumps from testhost and its child processes +Data collector 'Blame' message: The specified inactivity time of 10 minutes has elapsed. Collecting hang dumps from testhost and its child processes. +The active test run was aborted. Reason: Test host process crashed + +Test Run Aborted. diff --git a/tests/CiFail.Core.Tests/fixtures/vstest-testhost-crashed.log b/tests/CiFail.Core.Tests/fixtures/vstest-testhost-crashed.log new file mode 100644 index 0000000..1d7b08c --- /dev/null +++ b/tests/CiFail.Core.Tests/fixtures/vstest-testhost-crashed.log @@ -0,0 +1,8 @@ +Run dotnet test --no-build + Determining projects to restore... +Test run for /home/runner/work/app/tests/App.Tests/bin/Debug/net8.0/App.Tests.dll (.NETCoreApp,Version=v8.0) +Starting test execution, please wait... +The active test run was aborted. Reason: Test host process crashed : Stack overflow. + at App.Tree.Depth(Node) + +Test Run Aborted.