aws · GarrettBeatty · Jul 8, 2026 · Jul 8, 2026 · Jul 8, 2026 · Jul 8, 2026
@@ -44,10 +44,16 @@ public async Task VerifyCallbackIsInvoked(LambdaEventSource hostingType)
 
         var serverTask = app.RunAsync();
 
-        // let the server run for a max of 500 ms
-        await Task.WhenAny(
-            serverTask,
-            Task.Delay(TimeSpan.FromMilliseconds(500)));
+        // Poll for the before-snapshot callback to fire rather than racing a single fixed delay.
+        // A fixed 500 ms window is flaky under load (observed intermittently in CI): the
+        // before-snapshot request may not have completed yet, leaving the callback unset. Wait up
+        // to 10 seconds, checking frequently, and stop as soon as the callback has run.
+        var timeout = TimeSpan.FromSeconds(10);
+        var sw = System.Diagnostics.Stopwatch.StartNew();
+        while (!callbackDidTheCallback && sw.Elapsed < timeout && !serverTask.IsCompleted)
+        {
+            await Task.Delay(TimeSpan.FromMilliseconds(50));
+        }
 
         // shut down server
         await app.StopAsync();

@@ -0,0 +1,12 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+using Xunit;
+
+// Many tests in this assembly mutate the process-global AWS_LAMBDA_FUNCTION_NAME (and related)
+// environment variables via EnvironmentVariableHelper to simulate running inside/outside Lambda.
+// That state is shared across the whole process, so running test collections in parallel lets one
+// test's env var leak into another (e.g. AddAWSLambdaHosting_NotInLambda_DoesNotRegisterHostingOptions
+// intermittently sees a value set by a concurrently-running test and fails). Disable in-assembly
+// parallelization so these env-var-dependent tests run one at a time.
+[assembly: CollectionBehavior(DisableTestParallelization = true)]
@@ -0,0 +1,12 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+using Xunit;
+
+// Tests in this assembly share process-global state: TestSnapStartInitialization sets the
+// AWS_LAMBDA_FUNCTION_NAME / AWS_LAMBDA_INITIALIZATION_TYPE environment variables and drives a real
+// LambdaBootstrap polling loop, and SnapStartController.Invoked is a static flag. Running test
+// collections in parallel lets that background bootstrap and the env-var state interfere with other
+// tests (observed in CI as the whole dotnet test process hanging until credentials expired). Disable
+// in-assembly parallelization so these tests run one at a time.
+[assembly: CollectionBehavior(DisableTestParallelization = true)]
@@ -14,7 +14,10 @@ public class BuildStreamingPreludeTests
     {
         // Subclass that skips host startup entirely and
         // just exposes BuildStreamingPrelude directly without needing a running host.
-        private class StandalonePreludeBuilder : APIGatewayHttpApiV2ProxyFunction
+        // Derives from the REST (v1) APIGatewayProxyFunction because these tests assert on the
+        // multi-value MultiValueHeaders collection; the HTTP API v2 function instead collapses
+        // headers into the single-value Headers collection (see APIGatewayHttpApiV2ProxyFunction).
+        private class StandalonePreludeBuilder : APIGatewayProxyFunction
         {
             // Use the StartupMode.FirstRequest constructor so no host is started eagerly.
             public StandalonePreludeBuilder()

@@ -71,7 +71,10 @@ protected override APIGatewayHttpApiV2ProxyResponse MarshallResponse(
             }
         }
 
-        private class StandalonePreludeBuilder : APIGatewayHttpApiV2ProxyFunction
+        // Derives from the REST (v1) APIGatewayProxyFunction because the prelude property tests
+        // (Property3/Property4) assert on the multi-value MultiValueHeaders collection; the HTTP
+        // API v2 function instead collapses headers into the single-value Headers collection.
+        private class StandalonePreludeBuilder : APIGatewayProxyFunction
         {
             public StandalonePreludeBuilder() : base(StartupMode.FirstRequest) { }
 

@@ -2,6 +2,7 @@
 using System.IO;
 using System.Linq;
 using System.Net;
+using System.Net.Http;
 using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
@@ -299,15 +300,36 @@ public async Task TestSnapStartInitialization()
                     new DefaultLambdaJsonSerializer())
                 .ConfigureOptions(opt => opt.RuntimeApiEndpoint = "localhost:123")
                 .Build();
-
-            _ = bootstrap.RunAsync(cts.Token);
+
+            // Keep the task so we can tear the bootstrap down before the test returns. Previously this
+            // was fire-and-forget (_ = bootstrap.RunAsync(...)); the polling loop (against a dead
+            // endpoint) then kept running after the assert, and under parallel CI load the leaked loop
+            // hung the whole test process until the credentials expired ~2h later.
+            var bootstrapTask = bootstrap.RunAsync(cts.Token);
 
             // allow some time for Bootstrap to initialize in background
-            await Task.Delay(100, cts.Token);
+            await Task.Delay(100);
+
+            // Assert what the test is actually about (the SnapStart before-snapshot hook ran) before
+            // tearing down, so a teardown exception can't mask the real result.
+            Assert.True(SnapStartController.Invoked);
 
             await cts.CancelAsync();
 
-            Assert.True(SnapStartController.Invoked);
+            // Wait for the polling loop to actually stop so no background work outlives the test. The
+            // bootstrap runs against a dead endpoint (localhost:123), so its loop either observes the
+            // cancellation (OperationCanceledException) or throws the connection failure it was
+            // retrying (HttpRequestException); both are expected teardown noise, not test failures.
+            try
+            {
+                await bootstrapTask;
+            }
+            catch (OperationCanceledException)
+            {
+            }
+            catch (HttpRequestException)
+            {
+            }
         }
 
         private async Task<APIGatewayHttpApiV2ProxyResponse> InvokeAPIGatewayRequest(string fileName, bool configureApiToReturnExceptionDetail = false)

@@ -54,16 +54,32 @@ public T ExecuteRequest<T>(string eventFilePath)
                 processStartInfo.FileName = GetSystemShell();
                 processStartInfo.Arguments = $"{comamndArgument} dotnet run \"{requestFilePath}\" \"{responseFilePath}\"";
                 processStartInfo.WorkingDirectory = GetTestAppDirectory();
+                // Capture the child process output so a launch/build failure (e.g. the exit code 129
+                // seen intermittently in CI) surfaces a diagnosable message instead of a bare exit code.
+                processStartInfo.RedirectStandardOutput = true;
+                processStartInfo.RedirectStandardError = true;
+                processStartInfo.UseShellExecute = false;
 
 
                 lock (lock_process)
                 {
                     using var process = Process.Start(processStartInfo);
-                    process.WaitForExit(15000);
+                    var stdout = process.StandardOutput.ReadToEndAsync();
+                    var stderr = process.StandardError.ReadToEndAsync();
+
+                    // WaitForExit(timeout) returns false when the process is still running at the
+                    // timeout; kill it so it does not leak, then fail with the captured output.
+                    if (!process.WaitForExit(60000))
+                    {
+                        process.Kill(entireProcessTree: true);
+                        throw new Exception(
+                            $"Process timed out after 60s.{Environment.NewLine}STDOUT:{Environment.NewLine}{stdout.Result}{Environment.NewLine}STDERR:{Environment.NewLine}{stderr.Result}");
+                    }
 
                     if (process.ExitCode != 0)
                     {
-                        throw new Exception("Process failed with exit code: " + process.ExitCode);
+                        throw new Exception(
+                            $"Process failed with exit code: {process.ExitCode}.{Environment.NewLine}STDOUT:{Environment.NewLine}{stdout.Result}{Environment.NewLine}STDERR:{Environment.NewLine}{stderr.Result}");
                     }
 
                     if(!File.Exists(responseFilePath))

@@ -0,0 +1,13 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+using Xunit;
+
+// Tests in this assembly share process-global state: SetCustomerLoggerLogAction points a static
+// field on the loaded Amazon.Lambda.Core assembly at a per-invocation StringWriter, and the handler
+// tests drive LambdaBootstrap whose background RunAsync work writes through that static action. When
+// test collections run in parallel, one test's logging can be captured by another's writer (observed
+// intermittently under load as "Can't find method name in console text" in PositiveHandlerTestsAsync).
+// Some classes already share a [Collection] to serialize relative to each other, but that does not
+// cover the rest of the assembly, so disable in-assembly parallelization outright.
+[assembly: CollectionBehavior(DisableTestParallelization = true)]
@@ -257,11 +257,15 @@ private async Task<Exception> TestHandlerFailAsync(string handler, string expect
 
             using (var cancellationTokenSource = new CancellationTokenSource())
             {
-                var exceptionWaiterTask = Task.Run(() =>
+                var exceptionWaiterTask = Task.Run(async () =>
                 {
                     _output.WriteLine($"Waiting for an exception.");
+                    // Yield between checks instead of a tight spin. A busy-wait pins a CPU core and,
+                    // under load (e.g. when several test projects run in parallel), starves the
+                    // bootstrap thread that sets LastRecordedException, causing the test to flake.
                     while (testRuntimeApiClient.LastRecordedException == null)
                     {
+                        await Task.Delay(1);
                     }
                     _output.WriteLine($"Exception available.");
                     cancellationTokenSource.Cancel();
@@ -292,8 +296,12 @@ private async Task<string> InvokeAsync(LambdaBootstrap bootstrap, string dataIn,
                 var exceptionWaiterTask = Task.Run(async () =>
                 {
                     _output.WriteLine($"Waiting for an output.");
+                    // Yield between checks instead of a tight spin. A busy-wait pins a CPU core and,
+                    // under load (e.g. when several test projects run in parallel), starves the
+                    // bootstrap thread that sets LastOutputStream, causing the handler test to flake.
                     while (testRuntimeApiClient.LastOutputStream == null)
                     {
+                        await Task.Delay(1);
                     }
                     _output.WriteLine($"Output available.");
                     cancellationTokenSource.Cancel();

@@ -214,10 +214,10 @@
         IgnoreStandardErrorWarningFormat="true"/>
     </Target>
     <Target Name="run-unit-tests">
-        <Exec Command="dotnet test -c $(Configuration)" WorkingDirectory="..\Libraries\test\SnapshotRestore.Registry.Tests"/>
-        <Exec Command="dotnet test -c $(Configuration)" WorkingDirectory="..\Libraries\test\Amazon.Lambda.RuntimeSupport.Tests\Amazon.Lambda.RuntimeSupport.UnitTests"/>
-        <Exec Command="dotnet test -c $(Configuration)" WorkingDirectory="..\Libraries\test\Amazon.Lambda.Annotations.SourceGenerators.Tests"/>
-        <Exec Command="dotnet test -c $(Configuration)" WorkingDirectory="..\Libraries\test\Amazon.Lambda.DurableExecution.Tests"/>
+        <!-- Unit-test projects are discovered by convention (any *.csproj referencing the test SDK,
+             excluding *.IntegrationTests.csproj) so newly added test projects are picked up
+             automatically instead of having to be appended to a hardcoded list here. -->
+        <Exec Command="pwsh -NoProfile -File &quot;$(MSBuildThisFileDirectory)run-unit-tests.ps1&quot; -Configuration $(Configuration)"/>
     </Target>
     <Target Name="run-integ-tests">
         <!-- Each *.IntegrationTests.csproj deploys its own isolated CloudFormation stack, so the

@@ -5,6 +5,12 @@ phases:
     runtime-versions:
       dotnet: 8.x
     commands:
+      # Raise the inotify limits. The AspNetCoreServer tests each start an ASP.NET Core host, and the
+      # file-watching config providers allocate inotify instances; the container default (128) is
+      # exhausted once all unit-test projects run, surfacing as
+      # "The configured user limit (128) on the number of inotify instances has been reached".
+      - sysctl -w fs.inotify.max_user_instances=1024 || true
+      - sysctl -w fs.inotify.max_user_watches=524288 || true
       # Find and delete the global.json files that were added by CodeBuild. This causes issues when multiple SDKs are installed.
       - find / -type f -name 'global.json' -delete
       # The tests need .NET 3.1, 6, 8, 9 and 10. .NET6 is installed by default. .NET8 is added in the runtime-versions. .NET 3.1, 9 and 10 are installed manually.

@@ -0,0 +1,148 @@
+#!/usr/bin/env pwsh
+# Discovers and runs every unit-test project under Libraries/test in parallel, so new test projects
+# are picked up automatically instead of having to be added to a hardcoded list in build.proj (which
+# is how projects such as Amazon.Lambda.AspNetCoreServer.Test, Amazon.Lambda.Core.Tests and
+# EventsTests.NET8 silently stopped being run in CI).
+#
+# A project is treated as a unit-test project when its .csproj references Microsoft.NET.Test.Sdk (or
+# sets <IsTestProject>true</IsTestProject>). Naming is NOT used because the test projects in this repo
+# follow no single convention (*.Test, *.Tests, EventsTests.NET8, PowerShellTests, ...). Non-test
+# fixture projects (TestServerlessApp, TestWebApp, HandlerTest, ...) don't reference the test SDK, so
+# this content-based filter cleanly excludes them.
+#
+# Integration test projects (*.IntegrationTests.csproj) are excluded here because they run in a
+# separate phase via run-integ-tests-parallel.ps1 (they deploy real AWS resources).
+#
+# Mirrors run-integ-tests-parallel.ps1: each project's output is streamed live, prefixed with the
+# project name so interleaved logs stay attributable, and failed projects get their full output
+# reprinted as one clean block at the end. All projects run (it does not short-circuit) so every
+# failure is surfaced in a single pass, then it exits non-zero listing which projects failed.
+
+param(
+    [string]$Configuration = "Release",
+    # Directory to search for unit-test projects (defaults to the Libraries/test tree).
+    [string]$TestRoot = (Join-Path $PSScriptRoot ".." "Libraries" "test"),
+    # Upper bound on how many projects run at once.
+    [int]$ThrottleLimit = 5
+)
+
+$ErrorActionPreference = 'Stop'
+
+# Projects that must NOT run concurrently with the parallel pool. Amazon.Lambda.AspNetCoreServer.Test
+# drives SnapStart initialization that invokes before-snapshot hooks accumulated in a process-global
+# registry, each running in-process ASP.NET requests; under the CPU/host contention of the parallel
+# pool that work deadlocked and hung CI until the credentials expired. It passes fine on its own, so
+# it is run serially, after the parallel pool. Matched on the file name.
+$serialProjectNames = @(
+    'Amazon.Lambda.AspNetCoreServer.Test.csproj'
+)
+
+# Discover unit-test projects: reference the test SDK / IsTestProject, but are not integration tests.
+$allProjects = Get-ChildItem -Path $TestRoot -Recurse -Filter "*.csproj" |
+    Where-Object { $_.Name -notlike "*.IntegrationTests.csproj" } |
+    Where-Object {
+        $content = Get-Content -Raw -LiteralPath $_.FullName
+        ($content -match 'Microsoft\.NET\.Test\.Sdk') -or
+        ($content -match '<IsTestProject>\s*true\s*</IsTestProject>')
+    } |
+    Select-Object -ExpandProperty FullName |
+    Sort-Object
+
+if (-not $allProjects)
+{
+    Write-Host "No unit-test projects found under '$TestRoot'."
+    exit 0
+}
+
+$serialProjects = $allProjects | Where-Object { $serialProjectNames -contains [System.IO.Path]::GetFileName($_) }
+$parallelProjects = $allProjects | Where-Object { $serialProjectNames -notcontains [System.IO.Path]::GetFileName($_) }
+
+Write-Host "Discovered $($allProjects.Count) unit-test project(s):"
+Write-Host "  Parallel (throttle limit $ThrottleLimit):"
+$parallelProjects | ForEach-Object { Write-Host "    - $_" }
+if ($serialProjects)
+{
+    Write-Host "  Serial (run one at a time, after the parallel pool):"
+    $serialProjects | ForEach-Object { Write-Host "    - $_" }
+}
+
+# Build all projects ONCE, up front, before the parallel phase. The test projects share
+# ProjectReferences (e.g. TestWebApp, TestUtilities); running `dotnet test` on them concurrently
+# each rebuilt those shared projects, racing on their build output (e.g. a .deps.json "being used by
+# another process" / GenerateDepsFile failure). Building once here lets the runs use --no-build so
+# they only execute tests, never rebuild shared output.
+Write-Host "Building all unit-test projects once before running..."
+foreach ($project in $allProjects)
+{
+    dotnet build -c $Configuration $project
+    if ($LASTEXITCODE -ne 0)
+    {
+        Write-Host "Build failed for $project (exit $LASTEXITCODE)."
+        exit 1
+    }
+}
+
+$results = @()
+
+if ($parallelProjects)
+{
+    $results += $parallelProjects | ForEach-Object -ThrottleLimit $ThrottleLimit -Parallel {
+        $project = $_
+        $name = [System.IO.Path]::GetFileNameWithoutExtension($project)
+        $lines = [System.Collections.Generic.List[string]]::new()
+        # --no-build: everything was built serially above, so the runs only execute tests and never
+        # rebuild shared project output. 2>&1 folds stderr into the stream; each line is emitted as it
+        # arrives, prefixed with the project name, so progress is visible.
+        dotnet test -c $using:Configuration --no-build --logger "console;verbosity=detailed" $project 2>&1 |
+            ForEach-Object {
+                $line = $_.ToString()
+                $lines.Add($line)
+                Write-Host "[$name] $line"
+            }
+        [PSCustomObject]@{
+            Name     = $name
+            Project  = $project
+            ExitCode = $LASTEXITCODE
+            Output   = ($lines -join [System.Environment]::NewLine)
+        }
+    }
+}
+
+# Run the serial projects one at a time, after the parallel pool has finished, so their process-global
+# behavior can't interfere with (or be starved by) the concurrent runs.
+foreach ($project in $serialProjects)
+{
+    $name = [System.IO.Path]::GetFileNameWithoutExtension($project)
+    Write-Host ""
+    Write-Host "==================== $name (serial) ===================="
+    dotnet test -c $Configuration --no-build --logger "console;verbosity=detailed" $project
+    $results += [PSCustomObject]@{
+        Name     = $name
+        Project  = $project
+        ExitCode = $LASTEXITCODE
+        Output   = ""
+    }
+}
+
+# Reprint each failed project's output as one clean, un-interleaved block for easier diagnosis.
+$failed = $results | Where-Object { $_.ExitCode -ne 0 }
+foreach ($result in $failed)
+{
+    if ($result.Output)
+    {
+        Write-Host ""
+        Write-Host "==================== FAILED: $($result.Name) (exit $($result.ExitCode)) ===================="
+        Write-Host $result.Output
+    }
+}
+
+if ($failed)
+{
+    Write-Host ""
+    Write-Host "The following unit-test project(s) failed:"
+    $failed | ForEach-Object { Write-Host "  - $($_.Name)" }
+    exit 1
+}
+
+Write-Host ""
+Write-Host "All unit-test projects passed."