Skip to content

Commit 0863040

Browse files
simonrozsivalCopilotjonathanpeppers
authored
[tests] Improve NUnit runner reporting and add "dry-run" (#11162)
## Summary Improve the Android NUnit harness so trimmable and test-lane investigations can audit discovery and exclusions reliably. ## Changes - fix final Passed/Failed/Skipped summary counts to match NUnit aggregate results - add a discovery-only dry-run mode - add noexclusions=true to bypass built-in exclusions for auditing - surface excluded tests as ignored results with explicit reasons - overwrite logcat output per run so stale logs do not masquerade as current crashes ## Why These generic test-runner improvements were useful while working on #11091. I had problems identifying which tests were discovered, but did not run, as passed + skipped + failed < total. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Jonathan Peppers <jonathan.peppers@microsoft.com>
1 parent 063ffd0 commit 0863040

7 files changed

Lines changed: 242 additions & 31 deletions

File tree

‎build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ public override bool Execute ()
9494
new CommandInfo {
9595
ArgumentsString = $"{AdbTarget} {AdbOptions} logcat -v threadtime -d",
9696
StdoutFilePath = LogcatFilename,
97-
StdoutAppend = true,
97+
StdoutAppend = false,
9898
},
9999

100100
new CommandInfo {

‎build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunUITests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ protected override void AfterCommand (int commandIndex, CommandInfo info)
3636
ArgumentsString = $"{AdbTarget} {AdbOptions} logcat -v threadtime -d",
3737
MergeStdoutAndStderr = false,
3838
StdoutFilePath = LogcatFilename,
39-
StdoutAppend = true,
39+
StdoutAppend = false,
4040
},
4141

4242
new CommandInfo {

‎build-tools/scripts/TestApks.targets‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,8 @@
274274
<PropertyGroup>
275275
<_IncludeCategories Condition=" '$(IncludeCategories)' != '' ">include=$(IncludeCategories)</_IncludeCategories>
276276
<_ExcludeCategories Condition=" '$(ExcludeCategories)' != '' ">exclude=$(ExcludeCategories)</_ExcludeCategories>
277+
<_DryRunTests Condition=" '$(DryRunTests)' == 'true' ">dryrun=true</_DryRunTests>
278+
<_NoTestExclusions Condition=" '$(NoTestExclusions)' == 'true' ">noexclusions=true</_NoTestExclusions>
277279
<PlotDataLabelSuffix Condition=" '$(PlotDataLabelSuffix)' == '' ">$(TestsFlavor)</PlotDataLabelSuffix>
278280
</PropertyGroup>
279281
<RunInstrumentationTests
@@ -287,7 +289,7 @@
287289
Component="%(TestApkInstrumentation.Package)/%(TestApkInstrumentation.Identity)"
288290
NUnit2TestResultsFile="%(TestApkInstrumentation.ResultsPath)"
289291
LogcatFilename="$(_LogcatFilenameBase)-%(TestApkInstrumentation.Package)%(TestApkInstrumentation.LogcatFilenameDistincion).txt"
290-
InstrumentationArguments="$(_IncludeCategories);$(_ExcludeCategories)"
292+
InstrumentationArguments="$(_IncludeCategories);$(_ExcludeCategories);$(_DryRunTests);$(_NoTestExclusions)"
291293
TestFixture="$(TestFixture)"
292294
ToolExe="$(AdbToolExe)"
293295
ToolPath="$(AdbToolPath)"

‎tests/TestRunner.Core/TestInstrumentation.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ protected sealed class KnownArguments
1919
public const string Suite = "suite";
2020
public const string Include = "include";
2121
public const string Exclude = "exclude";
22+
public const string DryRun = "dryrun";
23+
public const string NoExclusions = "noexclusions";
2224
}
2325

2426
const string ResultExecutedTests = "run";
@@ -40,6 +42,8 @@ protected sealed class KnownArguments
4042
protected LogWriter Logger { get; } = new LogWriter ();
4143
protected Dictionary<string, string> StringExtrasInBundle { get; set; } = new Dictionary<string, string> ();
4244
protected string TestSuiteToRun { get; set; }
45+
protected bool DryRunRequested { get; set; }
46+
protected bool NoExclusionsRequested { get; set; }
4347

4448
protected TestInstrumentation ()
4549
{}
@@ -104,6 +108,22 @@ protected virtual void ProcessArguments ()
104108
if (StringExtrasInBundle.ContainsKey (KnownArguments.Suite)) {
105109
TestSuiteToRun = StringExtrasInBundle [KnownArguments.Suite]?.Trim ();
106110
}
111+
112+
if (StringExtrasInBundle.TryGetValue (KnownArguments.DryRun, out string dryRunValue)) {
113+
DryRunRequested = ParseBool (dryRunValue);
114+
}
115+
116+
if (StringExtrasInBundle.TryGetValue (KnownArguments.NoExclusions, out string noExclusionsValue)) {
117+
NoExclusionsRequested = ParseBool (noExclusionsValue);
118+
}
119+
}
120+
121+
static bool ParseBool (string value)
122+
{
123+
string trimmed = value?.Trim ();
124+
return String.Equals (trimmed, "true", StringComparison.OrdinalIgnoreCase) ||
125+
String.Equals (trimmed, "1", StringComparison.OrdinalIgnoreCase) ||
126+
String.Equals (trimmed, "yes", StringComparison.OrdinalIgnoreCase);
107127
}
108128

109129
public override void OnStart ()
@@ -259,8 +279,13 @@ bool RunTests (ref Bundle results)
259279

260280
TRunner runner = CreateRunner (Logger, arguments);
261281
runner.LogTag = LogTag;
282+
runner.DryRun = DryRunRequested;
262283
ConfigureFilters (runner);
263284

285+
if (runner.DryRun) {
286+
Log.Info (LogTag, "Dry-run discovery mode enabled; tests will be enumerated but not executed.");
287+
}
288+
264289
Log.Info (LogTag, "Starting unit tests");
265290
runner.Run (assemblies);
266291
Log.Info (LogTag, "Unit tests completed");

‎tests/TestRunner.Core/TestRunner.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ public abstract class TestRunner
1919
public long ExecutedTests { get; protected set; } = 0;
2020
public long TotalTests { get; protected set; } = 0;
2121
public long FilteredTests { get; protected set; } = 0;
22+
public bool DryRun { get; set; } = false;
2223
public bool RunInParallel { get; set; } = false;
2324
public string TestsRootDirectory { get; set; }
2425
public Context Context { get; }

‎tests/TestRunner.NUnit/NUnitTestInstrumentation.cs‎

Lines changed: 49 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ public abstract class NUnitTestInstrumentation : TestInstrumentation <NUnitTestR
1717
protected IEnumerable<string> IncludedCategories { get; set; }
1818
protected IEnumerable<string> ExcludedCategories { get; set; }
1919
protected IEnumerable<string> ExcludedTestNames { get; set; }
20+
protected IDictionary<string, string> ExcludedCategoryReasons { get; set; }
21+
protected IDictionary<string, string> ExcludedTestReasons { get; set; }
2022
protected string TestsDirectory { get; set; }
2123

2224
protected NUnitTestInstrumentation ()
@@ -67,14 +69,22 @@ protected override void ConfigureFilters (NUnitTestRunner runner)
6769
Log.Info (LogTag, "Configuring test categories to include from extras:");
6870
ChainCategoryFilter (GetFilterValuesFromExtras (KnownArguments.Include), false, ref filter);
6971

70-
Log.Info (LogTag, "Configuring test categories to exclude:");
71-
ChainCategoryFilter (ExcludedCategories, true, ref filter);
72+
if (NoExclusionsRequested) {
73+
Log.Info (LogTag, "Skipping built-in test exclusions due to noexclusions=true.");
74+
} else {
75+
Log.Info (LogTag, "Configuring test categories to exclude:");
76+
RegisterExcludedCategories (runner, ExcludedCategories);
77+
}
7278

7379
Log.Info(LogTag, "Configuring test categories to exclude from extras:");
74-
ChainCategoryFilter (GetFilterValuesFromExtras (KnownArguments.Exclude), true, ref filter);
80+
RegisterExcludedCategories (runner, GetFilterValuesFromExtras (KnownArguments.Exclude));
7581

76-
Log.Info (LogTag, "Configuring tests to exclude (by name):");
77-
ChainTestNameFilter (ExcludedTestNames?.ToArray (), ref filter);
82+
if (NoExclusionsRequested) {
83+
Log.Info (LogTag, "Skipping built-in test-name exclusions due to noexclusions=true.");
84+
} else {
85+
Log.Info (LogTag, "Configuring tests to exclude (by name):");
86+
RegisterExcludedTestNames (runner, ExcludedTestNames?.ToArray ());
87+
}
7888

7989
if (filter.IsEmpty)
8090
return;
@@ -104,7 +114,31 @@ void ChainCategoryFilter (IEnumerable <string> categories, bool negate, ref ITes
104114
Log.Info (LogTag, " none");
105115
}
106116

107-
void ChainTestNameFilter (string[] testNames, ref ITestFilter filter)
117+
void RegisterExcludedCategories (NUnitTestRunner runner, IEnumerable<string> categories)
118+
{
119+
bool gotCategories = false;
120+
if (categories != null) {
121+
foreach (string c in categories) {
122+
Log.Info (LogTag, $" {c}");
123+
runner.AddExcludedCategory (c, GetExcludedCategoryReason (c));
124+
gotCategories = true;
125+
}
126+
}
127+
128+
if (!gotCategories)
129+
Log.Info (LogTag, " none");
130+
}
131+
132+
string GetExcludedCategoryReason (string category)
133+
{
134+
if (ExcludedCategoryReasons != null && !String.IsNullOrEmpty (category) && ExcludedCategoryReasons.TryGetValue (category, out string reason)) {
135+
return reason;
136+
}
137+
138+
return $"Excluded category '{category}'.";
139+
}
140+
141+
void RegisterExcludedTestNames (NUnitTestRunner runner, string[] testNames)
108142
{
109143
if (testNames == null || testNames.Length == 0) {
110144
Log.Info (LogTag, " none");
@@ -115,10 +149,17 @@ void ChainTestNameFilter (string[] testNames, ref ITestFilter filter)
115149
if (String.IsNullOrEmpty (name))
116150
continue;
117151
Log.Info (LogTag, $" {name}");
152+
runner.AddExcludedTestName (name, GetExcludedTestReason (name));
153+
}
154+
}
155+
156+
string GetExcludedTestReason (string testName)
157+
{
158+
if (ExcludedTestReasons != null && !String.IsNullOrEmpty (testName) && ExcludedTestReasons.TryGetValue (testName, out string reason)) {
159+
return reason;
118160
}
119161

120-
var excludeTestNamesFilter = new SimpleNameFilter (testNames);
121-
filter = new AndFilter (filter, new NotFilter (excludeTestNamesFilter));
162+
return $"Excluded test '{testName}'.";
122163
}
123164
}
124165
}

0 commit comments

Comments
 (0)