diff --git a/src/frontend/config/sidebar/docs.topics.ts b/src/frontend/config/sidebar/docs.topics.ts index 0d2791c6e..5f36a74cd 100644 --- a/src/frontend/config/sidebar/docs.topics.ts +++ b/src/frontend/config/sidebar/docs.topics.ts @@ -1011,6 +1011,52 @@ export const docsTopics: StarlightSidebarTopicsUserConfig = { }, slug: 'testing/accessing-resources', }, + { + label: 'Advanced testing scenarios', + translations: { + da: 'Avancerede testscenarier', + de: 'Erweiterte Testszenarien', + en: 'Advanced testing scenarios', + es: 'Escenarios de pruebas avanzadas', + fr: 'Scénarios de test avancés', + hi: 'उन्नत परीक्षण परिदृश्य', + id: 'Skenario pengujian lanjutan', + it: 'Scenario di test avanzati', + ja: '高度なテストシナリオ', + ko: '고급 테스트 시나리오', + pt: 'Cenários de teste avançados', + 'pt-BR': 'Cenários de teste avançados', + 'pt-PT': 'Cenários de teste avançados', + ru: 'Расширенные сценарии тестирования', + tr: 'Gelişmiş test senaryoları', + uk: 'Розширені сценарії тестування', + 'zh-CN': '高级测试场景', + }, + slug: 'testing/advanced-scenarios', + }, + { + label: 'Testing in CI/CD pipelines', + translations: { + da: 'Test i CI/CD-pipelines', + de: 'Tests in CI/CD-Pipelines', + en: 'Testing in CI/CD pipelines', + es: 'Pruebas en canalizaciones CI/CD', + fr: 'Tests dans les pipelines CI/CD', + hi: 'CI/CD पाइपलाइनों में परीक्षण', + id: 'Pengujian di pipeline CI/CD', + it: 'Test nelle pipeline CI/CD', + ja: 'CI/CD パイプラインでのテスト', + ko: 'CI/CD 파이프라인에서의 테스트', + pt: 'Testes em pipelines de CI/CD', + 'pt-BR': 'Testes em pipelines de CI/CD', + 'pt-PT': 'Testes em pipelines de CI/CD', + ru: 'Тестирование в конвейерах CI/CD', + tr: 'CI/CD ardışık düzenlerinde test', + uk: 'Тестування в конвеєрах CI/CD', + 'zh-CN': '在 CI/CD 流水线中的测试', + }, + slug: 'testing/testing-in-ci', + }, ], }, { diff --git a/src/frontend/src/content/docs/testing/advanced-scenarios.mdx b/src/frontend/src/content/docs/testing/advanced-scenarios.mdx new file mode 100644 index 000000000..0d88c6423 --- /dev/null +++ b/src/frontend/src/content/docs/testing/advanced-scenarios.mdx @@ -0,0 +1,191 @@ +--- +title: Advanced testing scenarios +description: Learn advanced patterns for using DistributedApplicationTestingBuilder, including selectively disabling resources, overriding environment variables, and customizing the test AppHost. +--- + +import { Aside } from '@astrojs/starlight/components'; +import LearnMore from '@components/LearnMore.astro'; + +This article covers advanced patterns for testing Aspire applications, including selectively disabling resources during tests, overriding environment variables, and customizing the AppHost for specific test scenarios. + +## Selectively disable resources in tests + +When running integration tests, you might want to exclude certain resources to reduce test complexity or cost—for example, disabling a monitoring dashboard or skipping a sidecar resource that isn't relevant to a particular test. + +### Use `WithExplicitStart` to control resource startup + +The recommended way to make a resource optional is to use `WithExplicitStart` in the AppHost, and then let tests choose whether to start that resource. Resources marked with `WithExplicitStart` are created but don't start automatically with the rest of the application. + +In your AppHost, mark the resource as explicitly started: + +```csharp title="C# — AppHost.cs" +var builder = DistributedApplication.CreateBuilder(args); + +var api = builder.AddProject("api"); + +// This resource is optional and won't start automatically +builder.AddContainer("monitoring", "grafana/grafana") + .WithExplicitStart(); + +builder.Build().Run(); +``` + +In your test, you can start the resource manually if needed, or leave it stopped: + +```csharp title="C# — IntegrationTest.cs" +[Fact] +public async Task ApiWorksWithoutMonitoring() +{ + var appHost = await DistributedApplicationTestingBuilder + .CreateAsync(); + + await using var app = await appHost.BuildAsync(); + await app.StartAsync(); + + // The "monitoring" resource is not started—only "api" is running + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await app.ResourceNotifications.WaitForResourceHealthyAsync("api", cts.Token); + + using var httpClient = app.CreateHttpClient("api"); + using var response = await httpClient.GetAsync("/health"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); +} +``` + +### Conditionally add resources based on configuration + +Another approach is to use configuration in your AppHost to conditionally add resources. This gives tests control over which resources are included: + +```csharp title="C# — AppHost.cs" +var builder = DistributedApplication.CreateBuilder(args); + +var api = builder.AddProject("api"); + +// Only add the database resource if not explicitly disabled +if (builder.Configuration.GetValue("AddDatabase", true)) +{ + var db = builder.AddPostgres("postgres").AddDatabase("mydb"); + api.WithReference(db); +} + +builder.Build().Run(); +``` + +In tests, pass the configuration argument to skip the database: + +```csharp title="C# — IntegrationTest.cs" +[Fact] +public async Task ApiStartsWithoutDatabase() +{ + var appHost = await DistributedApplicationTestingBuilder + .CreateAsync(["AddDatabase=false"]); + + // Assert that the "postgres" resource doesn't exist + Assert.DoesNotContain(appHost.Resources, r => r.Name == "postgres"); + + await using var app = await appHost.BuildAsync(); + await app.StartAsync(); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await app.ResourceNotifications.WaitForResourceAsync( + "api", KnownResourceStates.Running, cts.Token); +} +``` + +### Remove wait annotations in tests + +Resources in Aspire can have wait dependencies (via `WaitFor` or `WaitForCompletion`). In some tests, you may want to remove these wait annotations to speed up test startup or to test behavior when dependent resources are unavailable. + +You can remove `WaitAnnotation` instances after building the testing builder, before calling `BuildAsync`: + +```csharp title="C# — IntegrationTest.cs" +using Aspire.Hosting.ApplicationModel; + +[Fact] +public async Task ApiStartsWithoutWaitingForDatabase() +{ + var appHost = await DistributedApplicationTestingBuilder + .CreateAsync(); + + // Remove wait annotations from all resources so they start immediately + foreach (var resource in appHost.Resources) + { + var waitAnnotations = resource.Annotations.OfType().ToList(); + foreach (var annotation in waitAnnotations) + { + resource.Annotations.Remove(annotation); + } + } + + await using var app = await appHost.BuildAsync(); + await app.StartAsync(); + + // Resources start without waiting for dependencies +} +``` + + + +## Override environment variables in tests + +Because Aspire tests run services in separate processes, you can't inject services directly through dependency injection. However, you can influence application behavior through environment variables or configuration. + +### Override environment variables via the AppHost builder + +Use `WithEnvironment` on the resource builder after creating the testing builder to set environment variables for specific resources. You access the resource builder through the `CreateResourceBuilder` method: + +```csharp title="C# — IntegrationTest.cs" +[Fact] +public async Task ApiUsesTestFeatureFlag() +{ + var appHost = await DistributedApplicationTestingBuilder + .CreateAsync(); + + // Override an environment variable for the "api" resource + appHost.CreateResourceBuilder("api") + .WithEnvironment("FeatureFlags__NewCheckout", "true"); + + await using var app = await appHost.BuildAsync(); + await app.StartAsync(); + + // The "api" service now runs with the overridden feature flag +} +``` + +### Override configuration with AppHost arguments + +You can also pass arguments to the AppHost to override configuration values. Arguments are passed to the .NET configuration system, so they can override values from `appsettings.json`. For more information, see [Pass arguments to your AppHost](/testing/manage-app-host/#pass-arguments-to-your-apphost). + +### Override AppHost configuration before resources are created + +For more control over the AppHost configuration before any resources are created, use the `DistributedApplicationFactory` class and override the `OnBuilderCreating` lifecycle method. For more information, see [Use the `DistributedApplicationFactory` class](/testing/manage-app-host/#use-the-distributedapplicationfactory-class). + + +For full documentation on argument passing and the `DistributedApplicationFactory` lifecycle, see [Manage the AppHost in tests](/testing/manage-app-host/). + + +## File-based AppHost limitations + +The Aspire CLI supports **file-based AppHosts** — a lightweight `apphost.cs` file with no `.csproj` project file. These are created with `aspire init` or the `aspire-apphost-singlefile` template and are run directly with `aspire run`. + + + +The `DistributedApplicationTestingBuilder.CreateAsync()` method requires a type from the `Projects` namespace: + +```csharp title="C# — IntegrationTest.cs" +// This requires Projects.MyAppHost to exist — only available via ProjectReference +var appHost = await DistributedApplicationTestingBuilder + .CreateAsync(); +``` + +The `Projects` namespace entries are only generated for `` entries in your test project. A file-based `apphost.cs` produces no such entry. + +## See also + +- [Manage the AppHost in tests](/testing/manage-app-host/) +- [Access resources in tests](/testing/accessing-resources/) +- [Testing overview](/testing/overview/) diff --git a/src/frontend/src/content/docs/testing/testing-in-ci.mdx b/src/frontend/src/content/docs/testing/testing-in-ci.mdx new file mode 100644 index 000000000..ba9f7793a --- /dev/null +++ b/src/frontend/src/content/docs/testing/testing-in-ci.mdx @@ -0,0 +1,320 @@ +--- +title: Testing in CI/CD pipelines +description: Learn how to run Aspire integration tests reliably in CI/CD environments, such as GitHub Actions and Azure DevOps, covering timeout configuration, Azure authentication, and container requirements. +--- + +import { Aside, Tabs, TabItem } from '@astrojs/starlight/components'; +import LearnMore from '@components/LearnMore.astro'; + +This article covers how to run Aspire integration tests reliably in continuous integration (CI) and continuous deployment (CD) environments, addressing common challenges such as test timeouts, Azure authentication, and container requirements. + +## Container requirements + +Aspire integration tests typically start containers (databases, caches, and other services) as part of the test run. Your CI environment must have a container runtime available. + +- **GitHub Actions**: For Linux-based Aspire test containers, use `ubuntu-*` runners, which have Docker available by default. GitHub-hosted `windows-*` and `macos-*` runners do not provide a Docker Engine suitable for running Linux containers; use self-hosted runners with a configured container runtime if you must target those operating systems. +- **Azure DevOps**: Docker is available on Microsoft-hosted agents (`ubuntu-latest`, `windows-latest`). Ensure the agent pool supports Docker. +- **Self-hosted runners**: Install and start Docker or another compatible container runtime (such as Podman) before running tests. + + + +## Configure timeouts to prevent hanging tests + +One of the most common issues when running Aspire tests in CI is tests hanging indefinitely because resources never reach their expected state. Always configure explicit timeouts for resource waiting and test execution. + +### Set timeouts on resource waiting + +The `WaitForResourceAsync` and `WaitForResourceHealthyAsync` methods accept a `CancellationToken`. Always pass a token with a timeout to avoid indefinite waits: + +```csharp title="C# — IntegrationTest.cs" +using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + +await app.ResourceNotifications.WaitForResourceHealthyAsync( + "webfrontend", + cts.Token); +``` + +### Set timeouts on `BuildAsync` and `StartAsync` + +Use `WaitAsync` to apply a timeout to the build and startup operations: + +```csharp title="C# — IntegrationTest.cs" +var timeout = TimeSpan.FromMinutes(5); +var cancellationToken = CancellationToken.None; + +await using var app = await appHost.BuildAsync(cancellationToken) + .WaitAsync(timeout, cancellationToken); + +await app.StartAsync(cancellationToken) + .WaitAsync(timeout, cancellationToken); +``` + +### Use longer timeouts in CI + +CI environments are often slower than local development machines due to network latency when pulling container images, limited CPU and memory, and parallel test execution. Consider using environment-aware timeouts: + +```csharp title="C# — IntegrationTest.cs" +private static readonly TimeSpan DefaultTimeout = + Environment.GetEnvironmentVariable("CI") is not null + ? TimeSpan.FromMinutes(5) // Longer timeout in CI + : TimeSpan.FromSeconds(30); // Shorter timeout locally + +[Fact] +public async Task GetWebResourceRootReturnsOkStatusCode() +{ + var appHost = await DistributedApplicationTestingBuilder + .CreateAsync(); + + await using var app = await appHost.BuildAsync() + .WaitAsync(DefaultTimeout); + await app.StartAsync().WaitAsync(DefaultTimeout); + + using var cts = new CancellationTokenSource(DefaultTimeout); + await app.ResourceNotifications.WaitForResourceHealthyAsync( + "webfrontend", cts.Token); + + using var httpClient = app.CreateHttpClient("webfrontend"); + using var response = await httpClient.GetAsync("/"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); +} +``` + +## Configure Azure authentication in CI + +When running tests that involve Azure resources (such as Azure Cosmos DB, Azure Service Bus, or Azure Storage), you must configure Azure authentication appropriately for your CI environment. + +### Local vs. CI authentication + +In local development, Aspire uses your developer identity (via `DefaultAzureCredential`, which tries Visual Studio, Azure CLI, and other sources). In CI, no developer identity is available, so you must configure a **service principal** or **managed identity**. + +A common symptom of misconfigured CI authentication is errors like: + +``` +The principal type 'User' is not allowed. Expected 'ServicePrincipal'. +``` + +This occurs when a role assignment in your Aspire AppHost is configured with `principalType: "User"` but the CI pipeline is running as a service principal. + +### Use `DefaultAzureCredential` with environment variables + +The recommended approach is to configure a service principal and set the standard Azure SDK environment variables in your CI pipeline. `DefaultAzureCredential` automatically picks up these environment variables: + +| Environment variable | Description | +|----------------------|-------------| +| `AZURE_CLIENT_ID` | The application (client) ID of your service principal | +| `AZURE_TENANT_ID` | Your Azure Active Directory tenant ID | +| `AZURE_CLIENT_SECRET` | The client secret of your service principal | + + + + + + +```yaml title="YAML — .github/workflows/tests.yml" +- name: Run Aspire integration tests + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + run: dotnet test --no-build +``` + + + + +```yaml title="YAML — azure-pipelines.yml" +- task: DotNetCoreCLI@2 + displayName: Run Aspire integration tests + inputs: + command: test + projects: '**/*.Tests.csproj' + env: + AZURE_CLIENT_ID: $(AZURE_CLIENT_ID) + AZURE_TENANT_ID: $(AZURE_TENANT_ID) + AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET) + AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID) +``` + + + + +### Configure Azure credentials in the AppHost factory + +You can also set Azure credentials programmatically in your test setup using the `DistributedApplicationFactory`: + +```csharp title="C# — CiAppHostFactory.cs" +public class CiAppHostFactory() + : DistributedApplicationFactory(typeof(Projects.MyAppHost)) +{ + protected override void OnBuilderCreating( + DistributedApplicationOptions applicationOptions, + HostApplicationBuilderSettings hostOptions) + { + hostOptions.Configuration ??= new(); + + // Read credentials from environment and forward to AppHost configuration + if (Environment.GetEnvironmentVariable("AZURE_CLIENT_ID") is { } clientId) + { + hostOptions.Configuration["AZURE_CLIENT_ID"] = clientId; + } + + if (Environment.GetEnvironmentVariable("AZURE_TENANT_ID") is { } tenantId) + { + hostOptions.Configuration["AZURE_TENANT_ID"] = tenantId; + } + + if (Environment.GetEnvironmentVariable("AZURE_CLIENT_SECRET") is { } clientSecret) + { + hostOptions.Configuration["AZURE_CLIENT_SECRET"] = clientSecret; + } + + if (Environment.GetEnvironmentVariable("AZURE_SUBSCRIPTION_ID") is { } subscriptionId) + { + hostOptions.Configuration["AZURE_SUBSCRIPTION_ID"] = subscriptionId; + } + } +} +``` + +### Skip Azure tests when credentials are unavailable + +When Azure credentials aren't configured, you may want to skip tests that require Azure resources rather than fail them. Use a guard at the start of your test or in a base class: + +```csharp title="C# — IntegrationTest.cs" +[Fact] +public async Task TestWithAzureCosmosDb() +{ + if (Environment.GetEnvironmentVariable("AZURE_CLIENT_ID") is null + || Environment.GetEnvironmentVariable("AZURE_SUBSCRIPTION_ID") is null) + { + // Skip when Azure credentials are not available + throw new Xunit.Sdk.SkipException(""" + Azure credentials not configured; skipping test. + """); + } + + var appHost = await DistributedApplicationTestingBuilder + .CreateAsync(); + + // ... rest of test +} +``` + + + +## Run tests in parallel + +Aspire tests use random port assignment by default, which allows multiple test instances to run concurrently without port conflicts. This is controlled by the `DcpPublisher:RandomizePorts` setting, which is enabled by default in the testing builder. + +For CI environments running multiple test classes in parallel, random ports help prevent failures caused by port collisions. If you've disabled random ports (for example to match a specific port in a health check URL), re-enable them for CI: + +```csharp title="C# — IntegrationTest.cs" +var appHost = await DistributedApplicationTestingBuilder + .CreateAsync( + [ + "DcpPublisher:RandomizePorts=true" + ]); +``` + + +For information about disabling port randomization, see [Testing overview: Disable port randomization](/testing/overview/#disable-port-randomization). + + +## Example CI workflow + +The following is a complete GitHub Actions workflow for running Aspire integration tests: + + + +```yaml title="YAML — .github/workflows/integration-tests.yml" +name: Integration Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.x' + + - name: Restore dependencies + run: dotnet restore + + - name: Build + run: dotnet build --no-restore + + - name: Run integration tests + env: + # Azure credentials (optional—skip Azure tests if not set) + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + run: dotnet test --no-build --verbosity normal + timeout-minutes: 30 +``` + + + +## Troubleshooting common CI failures + +### Tests hang or time out + +**Cause**: Resources fail to start or become healthy within the expected time. + +**Solutions**: +- Increase the timeout values in your tests. +- Add logging to capture resource startup output. +- Check that Docker is available and running on the CI agent. +- Verify that container images can be pulled (network access, image name, and tags). + +### Container pull failures + +**Cause**: The CI environment can't pull required container images. + +**Solutions**: +- Check that the CI runner has internet access to Docker Hub or your container registry. +- Pre-pull commonly used images as a build step. +- Use a private registry mirror if Docker Hub rate limits are an issue. + +### Azure authentication errors + +**Cause**: Missing or incorrect Azure service principal credentials. + +**Solutions**: +- Ensure `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, and `AZURE_CLIENT_SECRET` are set as CI secrets. +- Verify the service principal has the required role assignments in Azure. +- Check that `AZURE_SUBSCRIPTION_ID` is set when provisioning Azure resources. + +### Port conflicts + +**Cause**: Multiple test instances using the same fixed ports. + +**Solution**: Ensure `DcpPublisher:RandomizePorts` isn't explicitly set to `false` when running tests in parallel. + +## See also + +- [Testing overview](/testing/overview/) +- [Manage the AppHost in tests](/testing/manage-app-host/) +- [Advanced testing scenarios](/testing/advanced-scenarios/)