From b9c1276474a6fa07d1ddabb177f2382d70c9ff14 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 11:42:38 +0200 Subject: [PATCH 1/9] acceptance: shard acceptance tests across 4 parallel CI jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds SHARD_INDEX / SHARD_TOTAL env-var support to getTests() so each CI job runs only its share of the 854 acceptance tests (~214 each). The sorted test list ensures the split is deterministic and stable. The CI matrix gains a shard_index dimension [0,1,2,3], turning 6 test jobs into 24 (and 2 → 8 in the merge queue). The test-result aggregator and testmask gating are unaffected — GitHub Actions waits for all matrix combinations automatically. Co-authored-by: Isaac --- .github/workflows/push.yml | 8 ++++++-- acceptance/acceptance_test.go | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0bab4ba3d76..c5ce427fa20 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -77,7 +77,7 @@ jobs: # Only run if the target is in the list of targets from testmask if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }} - name: "task test (${{matrix.os.name}}, ${{matrix.deployment}})" + name: "task test (${{matrix.os.name}}, ${{matrix.deployment}}, shard ${{matrix.shard_index}})" runs-on: ${{ matrix.os.runner }} defaults: @@ -115,6 +115,8 @@ jobs: - "terraform" - "direct" + shard_index: [0, 1, 2, 3] + # Include "event_name" in the matrix so we can include/exclude based on it. event: - ${{ github.event_name }} @@ -140,6 +142,8 @@ jobs: - name: Run tests env: ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} + SHARD_INDEX: ${{ matrix.shard_index }} + SHARD_TOTAL: 4 run: go tool -modfile=tools/task/go.mod task test - name: Upload gotestsum JSON output @@ -147,7 +151,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-output-${{ matrix.os.name }}-${{ matrix.deployment }} + name: test-output-${{ matrix.os.name }}-${{ matrix.deployment }}-shard${{ matrix.shard_index }} path: test-output.json if-no-files-found: warn retention-days: 7 diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index f6ec0805fb2..52bf8b8bf21 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -510,6 +510,20 @@ func getTests(t *testing.T) []string { require.NoError(t, err) slices.Sort(testDirs) + + // Shard the test list when SHARD_TOTAL > 1. Tests are sorted above so the + // split is deterministic and stable across runs. + if total, _ := strconv.Atoi(os.Getenv("SHARD_TOTAL")); total > 1 { + index, _ := strconv.Atoi(os.Getenv("SHARD_INDEX")) + sharded := testDirs[:0] + for i, d := range testDirs { + if i%total == index { + sharded = append(sharded, d) + } + } + testDirs = sharded + } + return testDirs } From 7397c340e783af551582e8dbd1cbd9b0db4e709e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 11:54:40 +0200 Subject: [PATCH 2/9] ci: split unit tests into their own job; centralize cache save MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sharded acceptance matrix re-ran the full unit suite on all 24 jobs (4 shards x 3 OS x 2 deployment), and all of them tried to save the Go cache under the same key on main — only the first writer wins, so the rest wasted time. Split into: - test-unit: one job per OS (no deployment/shard dimension), runs `task test-unit`, and is the sole writer of the shared "test" cache. - test (acc): runs `task test-acc` only, restores the "test" cache (save-cache=false so the many shard/deployment instances don't collide on the key). setup-build-environment gains a save-cache input (default true, so the test-exp-* / test-pipelines jobs with unique keys keep saving as before) that gates the on-main cache save. test-result now also waits on test-unit. Co-authored-by: Isaac --- .../setup-build-environment/action.yml | 21 +++-- .github/workflows/push.yml | 88 ++++++++++++++++++- 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 3e7bde9704e..d7dae7b63cf 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -5,6 +5,10 @@ inputs: cache-key: description: 'Cache key identifier for Go cache' required: true + save-cache: + description: 'Whether this job may save the Go cache (only effective on main). Set to false on jobs that share a cache-key across many matrix instances so only one designated job writes the key.' + required: false + default: 'true' runs: using: 'composite' @@ -38,9 +42,12 @@ runs: # On runs against main (push + the scheduled wipe-and-repopulate # cron added in #2092): restore now, save at job end via the # unified action's post-step (which fires at the calling job's - # end, even when invoked from a composite). + # end, even when invoked from a composite). Gated on save-cache so + # that when many matrix instances share one cache-key, only the + # designated job writes it (concurrent same-key saves all fail but + # the first, so the extra writers just waste time). - name: Restore and save Go cache (main) - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && inputs.save-cache == 'true' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: | @@ -50,12 +57,12 @@ runs: restore-keys: | setup-go-${{ inputs.cache-key }}-${{ runner.os }}-go${{ steps.setup-go.outputs.go-version }}- - # On every other ref (PR / merge_group): restore only. Prefix - # fallback via restore-keys means runs whose go.sum differs from - # main still restore main's most recent cache and rebuild only - # the delta. + # On every other ref (PR / merge_group) or when this job is not the + # designated cache writer: restore only. Prefix fallback via + # restore-keys means runs whose go.sum differs from main still + # restore main's most recent cache and rebuild only the delta. - name: Restore Go cache (non-main) - if: github.ref != 'refs/heads/main' + if: github.ref != 'refs/heads/main' || inputs.save-cache != 'true' uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: | diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c5ce427fa20..ec98366a3c6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -70,6 +70,81 @@ jobs: # Always run all tests echo "targets=[\"test\"]" >> $GITHUB_OUTPUT + test-unit: + needs: + - cleanups + - testmask + + # Only run if the target is in the list of targets from testmask + if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }} + name: "task test-unit (${{matrix.os.name}})" + runs-on: ${{ matrix.os.runner }} + + defaults: + run: + shell: bash + + permissions: + id-token: write + contents: read + + env: + TASK_CONCURRENCY: ${{ matrix.os.name == 'windows' && '1' || '' }} + + strategy: + fail-fast: false + matrix: + os: + - name: linux + runner: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + - name: windows + runner: + group: databricks-protected-runner-group-large + labels: windows-server-latest-large + + - name: macos + runner: + labels: macos-latest + + # Include "event_name" in the matrix so we can include/exclude based on it. + event: + - ${{ github.event_name }} + + # Run on Linux only in merge queue to reduce time to merge. + exclude: + - event: merge_group + os: + name: windows + - event: merge_group + os: + name: macos + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-build-environment + with: + # Sole writer of the shared "test" cache (test-acc shards restore it). + cache-key: test + + - name: Run tests + run: go tool -modfile=tools/task/go.mod task test-unit + + - name: Upload gotestsum JSON output + # Always upload so we can inspect timing even if tests fail. + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-output-unit-${{ matrix.os.name }} + path: test-output-unit.json + if-no-files-found: warn + retention-days: 7 + test: needs: - cleanups @@ -77,7 +152,7 @@ jobs: # Only run if the target is in the list of targets from testmask if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }} - name: "task test (${{matrix.os.name}}, ${{matrix.deployment}}, shard ${{matrix.shard_index}})" + name: "task test-acc (${{matrix.os.name}}, ${{matrix.deployment}}, shard ${{matrix.shard_index}})" runs-on: ${{ matrix.os.runner }} defaults: @@ -137,14 +212,18 @@ jobs: - name: Setup build environment uses: ./.github/actions/setup-build-environment with: - cache-key: test-${{ matrix.deployment }} + # Shares the cache-key with test-unit so these shards restore the + # cache it saves. save-cache is false because many shard/deployment + # instances share this key; test-unit is the sole writer. + cache-key: test + save-cache: false - name: Run tests env: ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} SHARD_INDEX: ${{ matrix.shard_index }} SHARD_TOTAL: 4 - run: go tool -modfile=tools/task/go.mod task test + run: go tool -modfile=tools/task/go.mod task test-acc - name: Upload gotestsum JSON output # Always upload so we can inspect timing even if tests fail. @@ -152,7 +231,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-output-${{ matrix.os.name }}-${{ matrix.deployment }}-shard${{ matrix.shard_index }} - path: test-output.json + path: test-output-acc.json if-no-files-found: warn retention-days: 7 @@ -333,6 +412,7 @@ jobs: # Reference: https://github.com/orgs/community/discussions/25970 test-result: needs: + - test-unit - test - test-exp-aitools - test-exp-ssh From 0e3b1e3057bed8e419efd3ea99222b9a9c6fee2c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 12:09:59 +0200 Subject: [PATCH 3/9] acceptance: apply sharding only to the full TestAccept run TestInprocessMode calls testAccept with a specific singleTest ("selftest/basic"). The shard filter lived in getTests(), so it ran before singleTest selection and could strip the requested test out of the shard, failing with "did not match any tests" on every shard that didn't own selftest/basic. Move the shard filter into a shardTests helper applied in testAccept only when singleTest == "", leaving named-test selection unsharded. Co-authored-by: Isaac --- acceptance/acceptance_test.go | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 52bf8b8bf21..fe8877c7489 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -350,6 +350,10 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { return n != singleTest }) require.NotEmpty(t, testDirs, "singleTest=%#v did not match any tests\n%#v", singleTest, testDirs) + } else { + // Sharding applies only to the full run. A specific singleTest (e.g. + // TestInprocessMode) must never be filtered out by the shard split. + testDirs = shardTests(testDirs) } skippedDirs := 0 @@ -510,21 +514,25 @@ func getTests(t *testing.T) []string { require.NoError(t, err) slices.Sort(testDirs) + return testDirs +} - // Shard the test list when SHARD_TOTAL > 1. Tests are sorted above so the - // split is deterministic and stable across runs. - if total, _ := strconv.Atoi(os.Getenv("SHARD_TOTAL")); total > 1 { - index, _ := strconv.Atoi(os.Getenv("SHARD_INDEX")) - sharded := testDirs[:0] - for i, d := range testDirs { - if i%total == index { - sharded = append(sharded, d) - } +// shardTests returns the subset of testDirs assigned to this CI shard when +// SHARD_TOTAL > 1, or testDirs unchanged otherwise. testDirs must be sorted so +// the split is deterministic and stable across runs. +func shardTests(testDirs []string) []string { + total, _ := strconv.Atoi(os.Getenv("SHARD_TOTAL")) + if total <= 1 { + return testDirs + } + index, _ := strconv.Atoi(os.Getenv("SHARD_INDEX")) + sharded := testDirs[:0] + for i, d := range testDirs { + if i%total == index { + sharded = append(sharded, d) } - testDirs = sharded } - - return testDirs + return sharded } func validateTestPhase(phase int) error { From c1d55a7bb81c3d783f4683473e828809614018cc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 13:07:34 +0200 Subject: [PATCH 4/9] ci: empty commit to trigger another run Co-authored-by: Isaac From e6a073e2fde7d6c2788f4adc5828938ee45c7926 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 14:47:33 +0200 Subject: [PATCH 5/9] ci: per-(os,engine) acceptance shard counts via generated matrix A static cross-product matrix forces one shard count for every (os, engine). Windows is the long pole (TASK_CONCURRENCY=1 serializes within a job) while the direct engine is fast, so a uniform count over- or under-shards most combinations. Generate the acceptance shard matrix in testmask as an explicit include-list and consume it via fromJSON. Shard counts: windows/terraform: 8 windows/direct: 8 linux/terraform: 4 linux/direct: 2 macos/terraform: 4 macos/direct: 2 merge_group still runs Linux only (6 jobs). PR/push runs 28 acc jobs. Co-authored-by: Isaac --- .github/workflows/push.yml | 91 ++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ec98366a3c6..302603aa860 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -35,6 +35,7 @@ jobs: runs-on: ubuntu-latest outputs: targets: ${{ steps.mask1.outputs.targets || steps.mask2.outputs.targets || steps.mask3.outputs.targets }} + acc_matrix: ${{ steps.accmatrix.outputs.matrix }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -70,6 +71,52 @@ jobs: # Always run all tests echo "targets=[\"test\"]" >> $GITHUB_OUTPUT + # Build the acceptance-test shard matrix. Shard counts vary per + # (os, engine), which a static cross-product matrix can't express, so we + # emit an explicit include-list consumed via fromJSON in the test job. + - name: Build acceptance test shard matrix + id: accmatrix + env: + EVENT_NAME: ${{ github.event_name }} + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json, os + + event = os.environ["EVENT_NAME"] + runners = { + "linux": {"group": "databricks-protected-runner-group-large", "labels": "linux-ubuntu-latest-large"}, + "windows": {"group": "databricks-protected-runner-group-large", "labels": "windows-server-latest-large"}, + "macos": {"labels": "macos-latest"}, + } + # (os, engine) -> shard count. Windows gets more shards because + # TASK_CONCURRENCY=1 serializes tests within each job, so the only + # way to cut its wall time is more parallel jobs. direct is faster + # than terraform and needs fewer shards. + shard_counts = { + ("linux", "terraform"): 4, + ("linux", "direct"): 2, + ("macos", "terraform"): 4, + ("macos", "direct"): 2, + ("windows", "terraform"): 8, + ("windows", "direct"): 8, + } + + include = [] + for (osname, engine), total in shard_counts.items(): + # Run on Linux only in merge queue to reduce time to merge. + if event == "merge_group" and osname != "linux": + continue + for index in range(total): + include.append({ + "os": {"name": osname, "runner": runners[osname]}, + "deployment": engine, + "shard_index": index, + "shard_total": total, + }) + + print("matrix=" + json.dumps({"include": include})) + PY + test-unit: needs: - cleanups @@ -152,7 +199,7 @@ jobs: # Only run if the target is in the list of targets from testmask if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }} - name: "task test-acc (${{matrix.os.name}}, ${{matrix.deployment}}, shard ${{matrix.shard_index}})" + name: "task test-acc (${{matrix.os.name}}, ${{matrix.deployment}}, shard ${{matrix.shard_index}}/${{matrix.shard_total}})" runs-on: ${{ matrix.os.runner }} defaults: @@ -168,42 +215,10 @@ jobs: strategy: fail-fast: false - matrix: - # Use separate fields for the OS name and runner configuration. - # When combined in a single object, "runs-on" errors with "Unexpected value 'name'". - os: - - name: linux - runner: - group: databricks-protected-runner-group-large - labels: linux-ubuntu-latest-large - - - name: windows - runner: - group: databricks-protected-runner-group-large - labels: windows-server-latest-large - - - name: macos - runner: - labels: macos-latest - - deployment: - - "terraform" - - "direct" - - shard_index: [0, 1, 2, 3] - - # Include "event_name" in the matrix so we can include/exclude based on it. - event: - - ${{ github.event_name }} - - # Run on Linux only in merge queue to reduce time to merge. - exclude: - - event: merge_group - os: - name: windows - - event: merge_group - os: - name: macos + # Generated by testmask: an include-list with per-(os, engine) shard + # counts. Each entry carries os{name,runner}, deployment, shard_index, + # and shard_total. + matrix: ${{ fromJSON(needs.testmask.outputs.acc_matrix) }} steps: - name: Checkout repository and submodules @@ -222,7 +237,7 @@ jobs: env: ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} SHARD_INDEX: ${{ matrix.shard_index }} - SHARD_TOTAL: 4 + SHARD_TOTAL: ${{ matrix.shard_total }} run: go tool -modfile=tools/task/go.mod task test-acc - name: Upload gotestsum JSON output From 7ad2d3a0e4a6eb6ce92b36100289d85813ecb7a2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 16:56:48 +0200 Subject: [PATCH 6/9] ci: don't fail test jobs on flaky gotestsum-artifact upload The gotestsum JSON upload is debug-only timing telemetry, yet a transient GitHub artifact-service error during finalization failed an otherwise- passing windows test-acc shard. Mark both upload steps continue-on-error so infra hiccups on a debug artifact never block the merge. Co-authored-by: Isaac --- .github/workflows/push.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 302603aa860..f9928964872 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -184,7 +184,10 @@ jobs: - name: Upload gotestsum JSON output # Always upload so we can inspect timing even if tests fail. + # This is debug-only telemetry; a flaky artifact upload must not fail + # an otherwise-passing job. if: ${{ always() }} + continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-output-unit-${{ matrix.os.name }} @@ -242,7 +245,10 @@ jobs: - name: Upload gotestsum JSON output # Always upload so we can inspect timing even if tests fail. + # This is debug-only telemetry; a flaky artifact upload must not fail + # an otherwise-passing job. if: ${{ always() }} + continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-output-${{ matrix.os.name }}-${{ matrix.deployment }}-shard${{ matrix.shard_index }} From 8504f841e5477c520bb728d520db4603d8102b64 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 17:25:31 +0200 Subject: [PATCH 7/9] ci: reduce windows shards to 4 for comparison run Co-authored-by: Isaac --- .github/workflows/push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f9928964872..b233e0ede76 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -97,8 +97,8 @@ jobs: ("linux", "direct"): 2, ("macos", "terraform"): 4, ("macos", "direct"): 2, - ("windows", "terraform"): 8, - ("windows", "direct"): 8, + ("windows", "terraform"): 4, + ("windows", "direct"): 4, } include = [] From d83464ecc7da6adc761bb5076a0d12775e2e4a12 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 17:55:14 +0200 Subject: [PATCH 8/9] ci: set all shard counts to 2 for data collection Co-authored-by: Isaac --- .github/workflows/push.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b233e0ede76..1f7cd8e3521 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -93,12 +93,12 @@ jobs: # way to cut its wall time is more parallel jobs. direct is faster # than terraform and needs fewer shards. shard_counts = { - ("linux", "terraform"): 4, + ("linux", "terraform"): 2, ("linux", "direct"): 2, - ("macos", "terraform"): 4, + ("macos", "terraform"): 2, ("macos", "direct"): 2, - ("windows", "terraform"): 4, - ("windows", "direct"): 4, + ("windows", "terraform"): 2, + ("windows", "direct"): 2, } include = [] From 6562479318cede7f1a6fe85cf525b2498ad891a9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 19:46:27 +0200 Subject: [PATCH 9/9] ci: finalize per-(os,engine) shard counts based on timing data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Data collected across 5 CI runs at ×1/×2/×4/×8: linux/terraform ×4 (~6m vs 13m unsharded) linux/direct ×2 (~5m vs 6m unsharded) ×4 saves only 30s more macos/terraform ×4 (~9m vs 17m unsharded) macos/direct ×2 (~7.5m vs 9m unsharded) ×4 saves only 8s more windows/terraform ×8 (~15m vs 35m unsharded) keeps improving to ×8 windows/direct ×4 (~14m vs 23m unsharded) ×8 saves only 12s (overhead-bound) Total acc jobs: 22 (PR/push), 6 (merge queue, linux only). Co-authored-by: Isaac --- .github/workflows/push.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1f7cd8e3521..8aaa12926a7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -93,12 +93,12 @@ jobs: # way to cut its wall time is more parallel jobs. direct is faster # than terraform and needs fewer shards. shard_counts = { - ("linux", "terraform"): 2, + ("linux", "terraform"): 4, ("linux", "direct"): 2, - ("macos", "terraform"): 2, + ("macos", "terraform"): 4, ("macos", "direct"): 2, - ("windows", "terraform"): 2, - ("windows", "direct"): 2, + ("windows", "terraform"): 8, + ("windows", "direct"): 4, } include = []