Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System.Diagnostics;
using System.Text;
using System.Text.RegularExpressions;
using ModelContextProtocol.Tests.Utils;

namespace ModelContextProtocol.ConformanceTests;
Expand Down Expand Up @@ -118,10 +119,32 @@ public async Task RunConformanceTest(string scenario)
);
}

var output = outputBuilder.ToString();
var error = errorBuilder.ToString();
var success = process.ExitCode == 0 || HasOnlyWarnings(output, error);

return (
Success: process.ExitCode == 0,
Output: outputBuilder.ToString(),
Error: errorBuilder.ToString()
Success: success,
Output: output,
Error: error
);
}

/// <summary>
/// Checks if the conformance test output indicates that all checks passed with only
/// warnings (no actual failures). The conformance runner exits with code 1 for warnings,
/// but warnings represent acceptable behavior (e.g., timing tolerances in CI environments).
/// </summary>
private static bool HasOnlyWarnings(string output, string error)
{
// The conformance runner outputs a summary line like:
// "Passed: 2/2, 0 failed, 1 warnings"
// If there are 0 failures but warnings > 0, the test behavior is acceptable.
var combined = output + error;
var match = Regex.Match(combined, @"(?<failed>\d+) failed, (?<warnings>\d+) warnings");
return match.Success
&& match.Groups["failed"].Value == "0"
&& int.TryParse(match.Groups["warnings"].Value, out var warnings)
&& warnings > 0;
}
}