Add retry for StackTraceTests.ToString_ShowILOffset AMSI hang (#125814)

The `ToString_ShowILOffset` test's `Assembly.Load(byte[])` case triggers
an AMSI (Antimalware Scan Interface) scan via Windows Defender RPC. On
some Helix CI machines, this RPC call hangs indefinitely on
`NtAlpcConnectPort`, causing the 60-second RemoteExecutor timeout.

This is an external Windows OS/Defender issue, not a .NET bug (diagnosed
in #122690 by @jkotas).

### Changes

- **Split** the `Assembly.Load(byte[])` case into its own test method
(`ToString_ShowILOffset_ByteArrayLoad`) so an AMSI hang no longer drags
down the other two stable cases (LoadFrom and AssemblyBuilder).
- **Added retry logic** with up to 3 attempts and a 30s timeout. Each
retry spawns a fresh child process via RemoteExecutor, giving the AMSI
RPC a new chance to succeed.

Fixes #125599

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: danmoseley

src/libraries/System.Diagnostics.StackTrace/tests/StackTraceTests.cs

@@ -9,6 +9,7 @@
 using System.Reflection;
 using System.Reflection.Emit;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Text.RegularExpressions;
 using Microsoft.DotNet.RemoteExecutor;
 using System.Threading.Tasks;
@@ -541,24 +542,7 @@ public void ToString_ShowILOffset()
                 }
             }, SourceTestAssemblyPath, AssemblyName, regPattern).Dispose();
 
-            // Assembly.Load(Byte[]) case
-            RemoteExecutor.Invoke((asmPath, asmName, p) =>
-            {
-                AppContext.SetSwitch("Switch.System.Diagnostics.StackTrace.ShowILOffsets", true);
-                var inMemBlob = File.ReadAllBytes(asmPath);
-                var asm2 = Assembly.Load(inMemBlob);
-                try
-                {
-                    asm2.GetType("Program").GetMethod("Foo").Invoke(null, null);
-                }
-                catch (Exception e)
-                {
-                    Assert.Contains(asmName, e.InnerException.StackTrace);
-                    Assert.Matches(p, e.InnerException.StackTrace);
-                }
-            }, SourceTestAssemblyPath, AssemblyName, regPattern).Dispose();
-
-            // AssmblyBuilder.DefineDynamicAssembly() case
+            // AssemblyBuilder.DefineDynamicAssembly() case
             RemoteExecutor.Invoke((p) =>
             {
                 AppContext.SetSwitch("Switch.System.Diagnostics.StackTrace.ShowILOffsets", true);
@@ -583,6 +567,42 @@ public void ToString_ShowILOffset()
             }, regPattern).Dispose();
         }
 
+        // Assembly.Load(byte[]) triggers an AMSI (Antimalware Scan Interface) scan via Windows Defender.
+        // On some Windows x86 CI machines the AMSI RPC call hangs indefinitely,
+        // so we retry with a shorter timeout to work around the transient OS issue.
+        [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+        public void ToString_ShowILOffset_ByteArrayLoad()
+        {
+            string AssemblyName = "ExceptionTestAssembly.dll";
+            string SourceTestAssemblyPath = Path.Combine(Environment.CurrentDirectory, AssemblyName);
+            string regPattern = @":token 0x([a-f0-9]*)\+0x([a-f0-9]*)";
+
+            const int maxAttempts = 3;
+            for (int attempt = 1; ; attempt++)
+            {
+                try
+                {
+                    var options = new RemoteInvokeOptions { TimeOut = 30_000 };
+                    RemoteExecutor.Invoke((asmPath, asmName, p) =>
+                    {
+                        AppContext.SetSwitch("Switch.System.Diagnostics.StackTrace.ShowILOffsets", true);
+                        var inMemBlob = File.ReadAllBytes(asmPath);
+                        var asm = Assembly.Load(inMemBlob);
+                        TargetInvocationException ex = Assert.Throws<TargetInvocationException>(
+                            () => asm.GetType("Program").GetMethod("Foo").Invoke(null, null));
+                        Assert.Contains(asmName, ex.InnerException.StackTrace);
+                        Assert.Matches(p, ex.InnerException.StackTrace);
+                    }, SourceTestAssemblyPath, AssemblyName, regPattern, options).Dispose();
+                    break;
+                }
+                catch (RemoteExecutionException ex) when (OperatingSystem.IsWindows() && RuntimeInformation.ProcessArchitecture == Architecture.X86 && attempt < maxAttempts && ex.Message.Contains("Timed out"))
+                {
+                    // AMSI hang: on some Windows x86 CI machines, Assembly.Load(byte[]) triggers an AMSI scan
+                    // whose RPC call hangs indefinitely. Retry with a fresh process.
+                }
+            }
+        }
+
         // On Android, stack traces do not include file names and line numbers
         // Tracking issue: https://github.com/dotnet/runtime/issues/124087
         private static string FileInfoPattern(string fileLinePattern) =>