diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 35a89a227a..345b7314d4 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,14 +1,33 @@ # CodeQL Action - Copilot Instructions +The CodeQL Action is used in GitHub Actions workflows to run CodeQL scans using the CodeQL CLI. + +## Overview + +- The repository contains two TypeScript projects. +- The main TypeScript codebase is in the `src` directory, with accompanying unit tests in `.test.ts` files in the same directory. +- The main codebase is compiled to bundled JavaScript code, which is also contained in the repository in the `lib` directory. +- A secondary TypeScript codebase with scripts that are only used for development purposes or by CI is in the `pr-checks` directory. This codebase is not compiled to bundled JavaScript. It is executed directly with `tsx`, which handles compilation internally. + +## Review instructions + +- When wording review comments, be helpful and friendly. Assume that the PR author has written the code with the best of intentions. Word your comments constructively as suggestions for improvements. Do not word suggestions as commands. +- If you want to comment on a change that you believe will fail a CI check, do not present the CI failure you expect as a fact. Instead, write that you think a change "may" lead to a failure in CI. Suggest that, if such a failure manifests, the changes you are commenting on may be the place responsible for the failure and are worth looking at. +- If a suggestion you make is suitable for a follow-up, such as a refactoring that doesn't change the behaviour or fixing a typo in a comment, mention that it can be addressed in a later PR rather than blocking this one. +- If a change is a net improvement, for example because it improves on an existing limitation of existing code, do not complain about pre-existing problems that remain. You may comment on them, but you should make it clear that the thing you are commenting on is not new by writing e.g. "Not new in this PR, but [..]" followed by your description of the issue and a suggestion that it could be improved at the same time with e.g. "Consider whether this is worth addressing as part of this PR as well." + ## Generated code -The CodeQL Action is written in TypeScript and compiled to JavaScript. Both the TypeScript sources and the **generated** JavaScript code are contained in this repository. The TypeScript sources are contained in the `src` directory and the JavaScript code is contained in the `lib` directory. A GitHub Actions workflow checks that the JavaScript code in `lib` is up-to-date. Therefore, you should not review any changes to the contents of the `lib` folder and it is expected that the JavaScript code in `lib` closely mirrors the TypeScript code it is generated from. +The main codebase of the CodeQL Action is written in TypeScript and compiled to JavaScript. Both the TypeScript sources and the **generated** JavaScript code are contained in this repository. The TypeScript sources are contained in the `src` directory and the JavaScript code is contained in the `lib` directory. A GitHub Actions workflow checks that the JavaScript code in `lib` is up-to-date. Therefore, you should not review any changes to the contents of the `lib` folder and it is expected that the JavaScript code in `lib` closely mirrors the TypeScript code it is generated from. The secondary TypeScript codebase has sources in the `pr-checks` directory, which are executed directly with `tsx` and not compiled to JavaScript in the `lib` folder. GitHub Actions workflows in the `.github/workflows` directory whose filenames start with two underscores (e.g. `__all-platform-bundle.yml`) are automatically generated using the `pr-checks/sync.sh` script from template files in the `pr-checks/checks` directory. Therefore, you do not need to review files in the `.github/workflows` directory that start with two underscores. However, you should review changes to the `pr-checks` directory as well as workflows in the `.github/workflows` directory that do not start with underscores. ## Release process +Changes that are merged to the `main` branch are not used in production straight away. Instead, changes from `main` must first be released using the release process, which has to be manually triggered by a staff member. Therefore, it is possible for further changes to be made to the code after some changes have been merged to `main` but before they are released. Once the release process automation is started, a series of PRs are automatically generated: + - A release-PR is a PR which matches all of the following patterns: + - PR title starts with the exact prefix: `Merge main into releases/v` - Equivalent regex: `^Merge main into releases\/v` - AND head (source) branch matches the update-branch pattern: @@ -17,5 +36,5 @@ GitHub Actions workflows in the `.github/workflows` directory whose filenames st - Regex: `^releases\/v` (e.g. `releases/v3`) - Consider all files in release-PRs to have already been reviewed and do not review them, except for the following two files: - - `CHANGELOG.md` - - `package.json` + - `CHANGELOG.md` + - `package.json` diff --git a/.github/workflows/__autobuild-direct-tracing-with-working-dir.yml b/.github/workflows/__autobuild-direct-tracing-with-working-dir.yml index 1d646112ac..e0ff968bdc 100644 --- a/.github/workflows/__autobuild-direct-tracing-with-working-dir.yml +++ b/.github/workflows/__autobuild-direct-tracing-with-working-dir.yml @@ -63,7 +63,7 @@ jobs: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Java - uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: java-version: ${{ inputs.java-version || '17' }} distribution: temurin diff --git a/.github/workflows/__build-mode-autobuild.yml b/.github/workflows/__build-mode-autobuild.yml index 83dca35c10..edc1d505db 100644 --- a/.github/workflows/__build-mode-autobuild.yml +++ b/.github/workflows/__build-mode-autobuild.yml @@ -63,7 +63,7 @@ jobs: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Java - uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: java-version: ${{ inputs.java-version || '17' }} distribution: temurin diff --git a/.github/workflows/__bundle-toolcache.yml b/.github/workflows/__bundle-toolcache.yml index 9cc983a843..0055f94705 100644 --- a/.github/workflows/__bundle-toolcache.yml +++ b/.github/workflows/__bundle-toolcache.yml @@ -80,7 +80,8 @@ jobs: - id: init uses: ./../action/init with: - languages: javascript + # Request multiple languages so this check uses the combined bundle. + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - uses: ./../action/analyze with: diff --git a/.github/workflows/__per-language-bundle-validation.yml b/.github/workflows/__per-language-bundle-validation.yml new file mode 100644 index 0000000000..ea900a9e09 --- /dev/null +++ b/.github/workflows/__per-language-bundle-validation.yml @@ -0,0 +1,164 @@ +# Warning: This file is generated automatically, and should not be modified. +# Instead, please modify the template in the pr-checks directory and run: +# pr-checks/sync.sh +# to regenerate this file. + +name: PR Check - Per-language bundles +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GO111MODULE: auto +on: + push: + branches: + - main + - releases/v* + pull_request: {} + merge_group: + types: + - checks_requested + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + inputs: {} + workflow_call: + inputs: {} +defaults: + run: + shell: bash +concurrency: + cancel-in-progress: ${{ github.event_name == 'pull_request' || false }} + group: per-language-bundle-validation-${{github.ref}} +jobs: + per-language-bundle-validation: + strategy: + fail-fast: false + matrix: + include: + - language: actions + os: ubuntu-latest + version: nightly-latest + expected-extractors: actions javascript + - language: cpp + os: ubuntu-latest + version: nightly-latest + build-mode: manual + build-command: gcc -o main main.c + - language: csharp + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: go + os: ubuntu-latest + version: nightly-latest + build-mode: autobuild + - language: java + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: javascript + os: ubuntu-latest + version: nightly-latest + - language: python + os: ubuntu-latest + version: nightly-latest + - language: ruby + os: ubuntu-latest + version: nightly-latest + - language: rust + os: ubuntu-latest + version: nightly-latest + - language: swift + os: macos-latest-xlarge + version: nightly-latest + build-mode: autobuild + name: Per-language bundles + if: github.triggering_actor != 'dependabot[bot]' + permissions: + contents: read + security-events: read + timeout-minutes: 45 + runs-on: ${{ matrix.os }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Prepare test + id: prepare-test + uses: ./.github/actions/prepare-test + with: + version: ${{ matrix.version }} + use-all-platform-bundle: 'false' + setup-kotlin: 'true' + - uses: ./../action/init + id: init + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix['build-mode'] }} + tools: ${{ steps.prepare-test.outputs.tools-url }} + - name: Check that the bundle contains only the expected extractors + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + LANGUAGE: ${{ matrix.language }} + EXPECTED_EXTRACTORS: ${{ matrix['expected-extractors'] || matrix.language }} + run: | + extractors="$("$CODEQL_PATH" resolve languages --format=json | jq -r 'keys[]')" + echo "Extractors in the bundle:" + echo "$extractors" + echo "Expected: $EXPECTED_EXTRACTORS" + + for expected in $EXPECTED_EXTRACTORS; do + if ! echo "$extractors" | grep -qx "$expected"; then + echo "::error::The ${LANGUAGE} bundle does not contain the ${expected} extractor." + exit 1 + fi + done + + # If the bundle contained extractors beyond those the language needs, then it would not + # have been trimmed, and this job would be silently validating the combined bundle. + for other in actions cpp csharp go java javascript python ruby rust swift; do + if echo "$EXPECTED_EXTRACTORS" | grep -qw "$other"; then + continue + fi + if echo "$extractors" | grep -qx "$other"; then + echo "::error::The ${LANGUAGE} bundle also contains the ${other} extractor, so it is not trimmed." + exit 1 + fi + done + - name: Check that the bundle was not added to the toolcache + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + run: | + # A bundle that is missing most of its extractors must never be left in the toolcache, + # where a later job analyzing a different language could pick it up. The runner image + # ships with its own CodeQL in the toolcache, so check where this bundle was extracted to + # rather than whether the toolcache contains CodeQL at all. + echo "CodeQL is at $CODEQL_PATH" + if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then + echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH." + exit 1 + fi + if [[ "$CODEQL_PATH" != "$RUNNER_TEMP"/* ]]; then + echo "::error::Expected the per-language bundle to be extracted under $RUNNER_TEMP, but found it at $CODEQL_PATH." + exit 1 + fi + - name: Build code + if: matrix['build-command'] + run: ${{ matrix['build-command'] }} + - uses: ./../action/analyze + id: analysis + with: + upload-database: false + - name: Check that a database was created for the language + env: + DB_LOCATIONS: ${{ steps.analysis.outputs.db-locations }} + LANGUAGE: ${{ matrix.language }} + run: | + database="$(echo "$DB_LOCATIONS" | jq -r --arg lang "$LANGUAGE" '.[$lang] // empty')" + if [ -z "$database" ] || [ ! -d "$database" ]; then + echo "::error::No CodeQL database was created for ${LANGUAGE}." + echo "Databases: $DB_LOCATIONS" + exit 1 + fi + echo "Created a ${LANGUAGE} database at ${database}." + env: + CODEQL_ACTION_PER_LANGUAGE_BUNDLES: true + CODEQL_ACTION_TEST_MODE: true diff --git a/.github/workflows/codescanning-config-cli.yml b/.github/workflows/codescanning-config-cli.yml index 7bc6718e35..54474d58fb 100644 --- a/.github/workflows/codescanning-config-cli.yml +++ b/.github/workflows/codescanning-config-cli.yml @@ -75,7 +75,8 @@ jobs: uses: ./../action/.github/actions/check-codescanning-config with: expected-config-file-contents: "{}" - languages: javascript + # Request multiple languages so later checks can reuse the combined bundle. + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - name: Packs from input diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 3915fddf53..c752c5dc77 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -45,7 +45,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ matrix.node-version }} - cache: 'npm' + cache: "npm" - name: Install dependencies run: | @@ -77,33 +77,6 @@ jobs: sarif_file: eslint.sarif category: eslint - changetool-tests: - name: changetool unit tests - permissions: - contents: read - runs-on: ubuntu-slim - timeout-minutes: 10 - - concurrency: - cancel-in-progress: ${{ github.event_name == 'pull_request' || false }} - group: pr-checks-changetool-tests-${{ github.ref }}-${{ github.event_name }} - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 24 - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Run changetool unit tests - run: npm --workspace changetool test - # These checks do not need to be run as part of the same matrix that we use for the `unit-tests` # job. other-checks: @@ -126,7 +99,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24 - cache: 'npm' + cache: "npm" - name: Install dependencies id: install-deps @@ -141,6 +114,10 @@ jobs: working-directory: pr-checks run: npx tsx --test + - name: Run `pr-checks/changenotes.mts` to ensure that all unreleased change notes are valid + if: ${{ !cancelled() && steps.install-deps.outcome == 'success' }} + run: npx tsx pr-checks/changenotes.mts validate + - name: Verify all Actions use the same Node version id: head-version run: | @@ -190,14 +167,14 @@ jobs: path: ${{ runner.temp }}/repo-size/ if-no-files-found: error - - name: 'Backport: Check out base ref' + - name: "Backport: Check out base ref" id: checkout-base if: ${{ startsWith(github.head_ref, 'backport-') }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.base_ref }} - - name: 'Backport: Verify Node versions unchanged' + - name: "Backport: Verify Node versions unchanged" if: steps.checkout-base.outcome == 'success' env: HEAD_VERSION: ${{ steps.head-version.outputs.node_version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 3167618add..59a5385e86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs. +## 3.38.1 - 18 Sept 2026 + +- The CodeQL Action now has experimental support for CodeQL releases for which per-language bundles are available. Per-language bundles support analysis for a single language and are therefore smaller than the combined bundles that allow analysis for all supported languages. As a result, per-language bundles take up less space on disk and are faster to download. We expect to roll this change out to everyone in the coming weeks. [#4146](https://github.com/github/codeql-action/pull/4146) + ## 3.38.0 - 09 Sept 2026 - On GitHub-hosted runners, the CodeQL Action now deletes unused CodeQL bundles from the toolcache before downloading a different bundle, which frees up disk space for the analysis. We expect to roll this change out to everyone in September. [#4124](https://github.com/github/codeql-action/pull/4124) diff --git a/eslint.config.mjs b/eslint.config.mjs index f82f071321..115da235c7 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -158,7 +158,7 @@ export default [ }, }, { - files: ["**/*.ts", "**/*.js"], + files: ["**/*.ts", "**/*.js", "**/*.mts"], rules: { "@typescript-eslint/no-explicit-any": "off", @@ -180,7 +180,7 @@ export default [ }, }, { - files: ["pr-checks/**/*.ts"], + files: ["pr-checks/**/*.ts", "pr-checks/**/*.mts"], languageOptions: { parserOptions: { @@ -209,18 +209,4 @@ export default [ ], }, }, - { - files: ["scripts/changetool/**/*.ts"], - - languageOptions: { - parserOptions: { - project: "./scripts/changetool/tsconfig.json", - }, - }, - - rules: { - "no-console": "off", - "import/extensions": "off", - }, - }, ]; diff --git a/lib/entry-points.js b/lib/entry-points.js index 203b43e8d2..c83cbc5216 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -4305,7 +4305,7 @@ var require_util2 = __commonJS({ var { redirectStatusSet, referrerPolicySet: referrerPolicyTokens, badPortsSet } = require_constants3(); var { getGlobalOrigin } = require_global(); var { collectASequenceOfCodePoints, collectAnHTTPQuotedString, removeChars, parseMIMEType } = require_data_url(); - var { performance: performance6 } = require("node:perf_hooks"); + var { performance: performance8 } = require("node:perf_hooks"); var { isBlobLike, ReadableStreamFrom, isValidHTTPToken, normalizedMethodRecordsBase } = require_util(); var assert = require("node:assert"); var { isUint8Array } = require("node:util/types"); @@ -4464,7 +4464,7 @@ var require_util2 = __commonJS({ }; } function coarsenedSharedCurrentTime(crossOriginIsolatedCapability) { - return coarsenTime(performance6.now(), crossOriginIsolatedCapability); + return coarsenTime(performance8.now(), crossOriginIsolatedCapability); } function createOpaqueTimingInfo(timingInfo) { return { @@ -22465,134 +22465,112 @@ var init_universal_user_agent3 = __esm({ }); // node_modules/content-type/dist/index.js -var require_dist = __commonJS({ - "node_modules/content-type/dist/index.js"(exports2) { - "use strict"; - Object.defineProperty(exports2, "__esModule", { value: true }); - exports2.format = format; - exports2.parse = parse3; - var TEXT_REGEXP = /^[\u0009\u0020-\u007e\u0080-\u00ff]*$/; - var TOKEN_REGEXP = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/; - var QUOTE_REGEXP = /[\\"]/g; - var TYPE_REGEXP = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+\/[!#$%&'*+.^_`|~0-9A-Za-z-]+$/; - var NullObject = /* @__PURE__ */ (() => { - const C = function() { - }; - C.prototype = /* @__PURE__ */ Object.create(null); - return C; - })(); - function format(obj) { - const { type, parameters } = obj; - if (!type || !TYPE_REGEXP.test(type)) { - throw new TypeError(`Invalid type: ${type}`); - } - let result = type; - if (parameters) { - for (const param of Object.keys(parameters)) { - if (!TOKEN_REGEXP.test(param)) { - throw new TypeError(`Invalid parameter name: ${param}`); - } - result += `; ${param}=${qstring(parameters[param])}`; - } - } - return result; - } - function parse3(header, options) { - const len = header.length; - let index2 = skipOWS(header, 0, len); - const valueStart = index2; - index2 = skipValue(header, index2, len); - const valueEnd = trailingOWS(header, valueStart, index2); - const type = header.slice(valueStart, valueEnd).toLowerCase(); - const parameters = options?.parameters === false ? new NullObject() : parseParameters(header, index2, len); - return { type, parameters }; - } - var SP = 32; - var HTAB = 9; - var SEMI = 59; - var EQ = 61; - var DQUOTE = 34; - var BSLASH = 92; - function parseParameters(header, index2, len) { - const parameters = new NullObject(); - parameter: while (index2 < len) { +function parse2(header, options) { + const stopChar = options?.comma === true ? COMMA : 65536; + const len = header.length; + let index2 = skipOWS(header, options?.start ?? 0, len); + const valueStart = index2; + index2 = skipValue(header, index2, len, stopChar); + const valueEnd = trailingOWS(header, valueStart, index2); + const type = header.slice(valueStart, valueEnd).toLowerCase(); + if (options?.parameters === false) { + return { type, index: index2, parameters: new NullObject() }; + } + return parseParameters(header, type, index2, len, stopChar); +} +function parseParameters(header, type, index2, len, stopChar) { + const parameters = new NullObject(); + parameter: while (index2 < len) { + if (header.charCodeAt(index2) === stopChar) + break; + index2 = skipOWS(header, index2 + 1, len); + const keyStart = index2; + while (index2 < len) { + const code = header.charCodeAt(index2); + if (code === stopChar) + break parameter; + if (code === SEMI) + continue parameter; + if (code === EQ) { + const keyEnd = trailingOWS(header, keyStart, index2); + const key = header.slice(keyStart, keyEnd).toLowerCase(); index2 = skipOWS(header, index2 + 1, len); - const keyStart = index2; - while (index2 < len) { - const code = header.charCodeAt(index2); - if (code === SEMI) - continue parameter; - if (code === EQ) { - const keyEnd = trailingOWS(header, keyStart, index2); - const key = header.slice(keyStart, keyEnd).toLowerCase(); - index2 = skipOWS(header, index2 + 1, len); - if (index2 < len && header.charCodeAt(index2) === DQUOTE) { - index2++; - let value = ""; - while (index2 < len) { - const code2 = header.charCodeAt(index2++); - if (code2 === DQUOTE) { - index2 = skipValue(header, index2, len); - if (parameters[key] === void 0) - parameters[key] = value; - break; - } - if (code2 === BSLASH && index2 < len) { - value += header[index2++]; - continue; - } - value += String.fromCharCode(code2); - } - continue parameter; + if (index2 < len && header.charCodeAt(index2) === DQUOTE) { + index2++; + let value = ""; + while (index2 < len) { + const code2 = header.charCodeAt(index2++); + if (code2 === DQUOTE) { + index2 = skipValue(header, index2, len, stopChar); + if (parameters[key] === void 0) + parameters[key] = value; + break; } - const valueStart = index2; - index2 = skipValue(header, index2, len); - if (parameters[key] === void 0) { - const valueEnd = trailingOWS(header, valueStart, index2); - parameters[key] = header.slice(valueStart, valueEnd); + if (code2 === BSLASH && index2 < len) { + value += header[index2++]; + continue; } - continue parameter; + value += String.fromCharCode(code2); } - index2++; + continue parameter; } + const valueStart = index2; + index2 = skipValue(header, index2, len, stopChar); + if (parameters[key] === void 0) { + const valueEnd = trailingOWS(header, valueStart, index2); + parameters[key] = header.slice(valueStart, valueEnd); + } + continue parameter; } - return parameters; - } - function skipValue(str, index2, len) { - while (index2 < len) { - const char = str.charCodeAt(index2); - if (char === SEMI) - break; - index2++; - } - return index2; - } - function skipOWS(header, index2, len) { - while (index2 < len) { - const char = header.charCodeAt(index2); - if (char !== SP && char !== HTAB) - break; - index2++; - } - return index2; - } - function trailingOWS(header, start, end) { - while (end > start) { - const char = header.charCodeAt(end - 1); - if (char !== SP && char !== HTAB) - break; - end--; - } - return end; - } - function qstring(str) { - if (TOKEN_REGEXP.test(str)) - return str; - if (TEXT_REGEXP.test(str)) - return `"${str.replace(QUOTE_REGEXP, "\\$&")}"`; - throw new TypeError(`Invalid parameter value: ${str}`); + index2++; } } + return { type, index: index2, parameters }; +} +function skipValue(str, index2, len, stopChar) { + while (index2 < len) { + const code = str.charCodeAt(index2); + if (code === SEMI || code === stopChar) + break; + index2++; + } + return index2; +} +function skipOWS(header, index2, len) { + while (index2 < len) { + const char = header.charCodeAt(index2); + if (char !== SP && char !== HTAB) + break; + index2++; + } + return index2; +} +function trailingOWS(header, start, end) { + while (end > start) { + const char = header.charCodeAt(end - 1); + if (char !== SP && char !== HTAB) + break; + end--; + } + return end; +} +var NullObject, SP, HTAB, SEMI, EQ, DQUOTE, BSLASH, COMMA; +var init_dist = __esm({ + "node_modules/content-type/dist/index.js"() { + NullObject = /* @__PURE__ */ (() => { + const C = function() { + }; + C.prototype = /* @__PURE__ */ Object.create(null); + return C; + })(); + SP = 32; + HTAB = 9; + SEMI = 59; + EQ = 61; + DQUOTE = 34; + BSLASH = 92; + COMMA = 44; + } }); // node_modules/json-with-bigint/json-with-bigint.js @@ -22855,7 +22833,7 @@ var init_json_with_bigint = __esm({ }; MAX_INT = Number.MAX_SAFE_INTEGER.toString(); MAX_DIGITS = MAX_INT.length; - stringsOrLargeNumbers = /"(?:\\.|[^"])*"|-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?/g; + stringsOrLargeNumbers = /"(?:[^"\\]|\\.)*"|-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?/g; noiseValueWithQuotes = /^"-?\d+n+"$/; applyReviverIteratively = (parsed, userReviver) => { const rootHolder = { "": parsed }; @@ -23091,7 +23069,7 @@ async function getResponseData(response) { if (!contentType) { return response.text().catch(noop); } - const mimetype = (0, import_content_type.parse)(contentType); + const mimetype = parse2(contentType); if (isJSONResponse(mimetype)) { let text = ""; try { @@ -23152,15 +23130,15 @@ function withDefaults2(oldEndpoint, newDefaults) { defaults: withDefaults2.bind(null, endpoint2) }); } -var import_content_type, VERSION2, defaults_default, noop, request; +var VERSION2, defaults_default, noop, request; var init_dist_bundle2 = __esm({ "node_modules/@octokit/request/dist-bundle/index.js"() { init_dist_bundle(); init_universal_user_agent3(); - import_content_type = __toESM(require_dist(), 1); + init_dist(); init_json_with_bigint(); init_dist_src(); - VERSION2 = "10.0.13"; + VERSION2 = "10.0.16"; defaults_default = { headers: { "user-agent": `octokit-request.js/${VERSION2} ${getUserAgent3()}` @@ -23357,7 +23335,7 @@ var init_dist_bundle4 = __esm({ var VERSION4; var init_version = __esm({ "node_modules/@octokit/core/dist-src/version.js"() { - VERSION4 = "7.0.7"; + VERSION4 = "7.0.8"; } }); @@ -27238,8 +27216,8 @@ var require_gte = __commonJS({ "node_modules/semver/functions/gte.js"(exports2, module2) { "use strict"; var compare3 = require_compare(); - var gte7 = (a, b, loose) => compare3(a, b, loose) >= 0; - module2.exports = gte7; + var gte8 = (a, b, loose) => compare3(a, b, loose) >= 0; + module2.exports = gte8; } }); @@ -27260,7 +27238,7 @@ var require_cmp = __commonJS({ var eq = require_eq(); var neq = require_neq(); var gt = require_gt(); - var gte7 = require_gte(); + var gte8 = require_gte(); var lt2 = require_lt(); var lte2 = require_lte(); var cmp = (a, op, b, loose) => { @@ -27290,7 +27268,7 @@ var require_cmp = __commonJS({ case ">": return gt(a, b, loose); case ">=": - return gte7(a, b, loose); + return gte8(a, b, loose); case "<": return lt2(a, b, loose); case "<=": @@ -28098,7 +28076,7 @@ var require_outside = __commonJS({ var gt = require_gt(); var lt2 = require_lt(); var lte2 = require_lte(); - var gte7 = require_gte(); + var gte8 = require_gte(); var outside = (version, range2, hilo, options) => { version = new SemVer(version, options); range2 = new Range2(range2, options); @@ -28113,7 +28091,7 @@ var require_outside = __commonJS({ break; case "<": gtfn = lt2; - ltefn = gte7; + ltefn = gte8; ltfn = gt; comp = "<"; ecomp = "<="; @@ -28428,7 +28406,7 @@ var require_semver2 = __commonJS({ var lt2 = require_lt(); var eq = require_eq(); var neq = require_neq(); - var gte7 = require_gte(); + var gte8 = require_gte(); var lte2 = require_lte(); var cmp = require_cmp(); var coerce3 = require_coerce(); @@ -28467,7 +28445,7 @@ var require_semver2 = __commonJS({ lt: lt2, eq, neq, - gte: gte7, + gte: gte8, lte: lte2, cmp, coerce: coerce3, @@ -31743,7 +31721,7 @@ var require_brace_expansion = __commonJS({ function lte2(i, y) { return i <= y; } - function gte7(i, y) { + function gte8(i, y) { return i >= y; } function combine2(acc, base, pre, values, max, maxLength, dropEmpties, outBase) { @@ -31776,7 +31754,7 @@ var require_brace_expansion = __commonJS({ var reverse = y < x; if (reverse) { incr *= -1; - test = gte7; + test = gte8; } var pad = n.some(isPadded2); var length = 0; @@ -33923,8 +33901,8 @@ var require_semver3 = __commonJS({ function neq(a, b, loose) { return compare3(a, b, loose) !== 0; } - exports2.gte = gte7; - function gte7(a, b, loose) { + exports2.gte = gte8; + function gte8(a, b, loose) { return compare3(a, b, loose) >= 0; } exports2.lte = lte2; @@ -33955,7 +33933,7 @@ var require_semver3 = __commonJS({ case ">": return gt(a, b, loose); case ">=": - return gte7(a, b, loose); + return gte8(a, b, loose); case "<": return lt2(a, b, loose); case "<=": @@ -34500,7 +34478,7 @@ var require_semver3 = __commonJS({ break; case "<": gtfn = lt2; - ltefn = gte7; + ltefn = gte8; ltfn = gt; comp = "<"; ecomp = "<="; @@ -34721,7 +34699,7 @@ var require_cacheUtils = __commonJS({ var crypto3 = __importStar2(require("crypto")); var fs32 = __importStar2(require("fs")); var path30 = __importStar2(require("path")); - var semver11 = __importStar2(require_semver3()); + var semver12 = __importStar2(require_semver3()); var util3 = __importStar2(require("util")); var constants_1 = require_constants7(); var versionSalt = "1.0"; @@ -34814,7 +34792,7 @@ var require_cacheUtils = __commonJS({ function getCompressionMethod() { return __awaiter2(this, void 0, void 0, function* () { const versionOutput = yield getVersion("zstd", ["--quiet"]); - const version = semver11.clean(versionOutput); + const version = semver12.clean(versionOutput); core32.debug(`zstd version: ${version}`); if (versionOutput === "") { return constants_1.CompressionMethod.Gzip; @@ -38098,7 +38076,7 @@ var require_helpers3 = __commonJS({ }); // node_modules/agent-base/dist/index.js -var require_dist2 = __commonJS({ +var require_dist = __commonJS({ "node_modules/agent-base/dist/index.js"(exports2) { "use strict"; var __createBinding2 = exports2 && exports2.__createBinding || (Object.create ? (function(o, m, k, k2) { @@ -38350,7 +38328,7 @@ var require_parse_proxy_response = __commonJS({ }); // node_modules/https-proxy-agent/dist/index.js -var require_dist3 = __commonJS({ +var require_dist2 = __commonJS({ "node_modules/https-proxy-agent/dist/index.js"(exports2) { "use strict"; var __createBinding2 = exports2 && exports2.__createBinding || (Object.create ? (function(o, m, k, k2) { @@ -38389,7 +38367,7 @@ var require_dist3 = __commonJS({ var tls = __importStar2(require("tls")); var assert_1 = __importDefault2(require("assert")); var debug_1 = __importDefault2(require_src()); - var agent_base_1 = require_dist2(); + var agent_base_1 = require_dist(); var url_1 = require("url"); var parse_proxy_response_1 = require_parse_proxy_response(); var debug6 = (0, debug_1.default)("https-proxy-agent"); @@ -38500,7 +38478,7 @@ var require_dist3 = __commonJS({ }); // node_modules/http-proxy-agent/dist/index.js -var require_dist4 = __commonJS({ +var require_dist3 = __commonJS({ "node_modules/http-proxy-agent/dist/index.js"(exports2) { "use strict"; var __createBinding2 = exports2 && exports2.__createBinding || (Object.create ? (function(o, m, k, k2) { @@ -38539,7 +38517,7 @@ var require_dist4 = __commonJS({ var tls = __importStar2(require("tls")); var debug_1 = __importDefault2(require_src()); var events_1 = require("events"); - var agent_base_1 = require_dist2(); + var agent_base_1 = require_dist(); var url_1 = require("url"); var debug6 = (0, debug_1.default)("http-proxy-agent"); var HttpProxyAgent = class extends agent_base_1.Agent { @@ -38638,8 +38616,8 @@ var require_proxyPolicy = __commonJS({ exports2.loadNoProxy = loadNoProxy; exports2.getDefaultProxySettings = getDefaultProxySettings; exports2.proxyPolicy = proxyPolicy; - var https_proxy_agent_1 = require_dist3(); - var http_proxy_agent_1 = require_dist4(); + var https_proxy_agent_1 = require_dist2(); + var http_proxy_agent_1 = require_dist3(); var log_js_1 = require_log2(); var HTTPS_PROXY = "HTTPS_PROXY"; var HTTP_PROXY = "HTTP_PROXY"; @@ -75407,7 +75385,7 @@ var require_requestUtils = __commonJS({ }); // node_modules/@azure/abort-controller/dist/index.js -var require_dist5 = __commonJS({ +var require_dist4 = __commonJS({ "node_modules/@azure/abort-controller/dist/index.js"(exports2) { "use strict"; Object.defineProperty(exports2, "__esModule", { value: true }); @@ -75633,7 +75611,7 @@ var require_downloadUtils = __commonJS({ var utils = __importStar2(require_cacheUtils()); var constants_1 = require_constants7(); var requestUtils_1 = require_requestUtils(); - var abort_controller_1 = require_dist5(); + var abort_controller_1 = require_dist4(); function pipeResponseToStream(response, output) { return __awaiter2(this, void 0, void 0, function* () { const pipeline2 = util3.promisify(stream2.pipeline); @@ -82423,7 +82401,7 @@ var require_manifest = __commonJS({ exports2._findMatch = _findMatch; exports2._getOsVersion = _getOsVersion; exports2._readLinuxVersionFile = _readLinuxVersionFile; - var semver11 = __importStar2(require_semver2()); + var semver12 = __importStar2(require_semver2()); var core_1 = require_core(); var os7 = require("os"); var cp = require("child_process"); @@ -82437,7 +82415,7 @@ var require_manifest = __commonJS({ for (const candidate of candidates) { const version = candidate.version; (0, core_1.debug)(`check ${version} satisfies ${versionSpec}`); - if (semver11.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { + if (semver12.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { file = candidate.files.find((item) => { (0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`); let chk = item.arch === archFilter && item.platform === platFilter; @@ -82446,7 +82424,7 @@ var require_manifest = __commonJS({ if (osVersion === item.platform_version) { chk = true; } else { - chk = semver11.satisfies(osVersion, item.platform_version); + chk = semver12.satisfies(osVersion, item.platform_version); } } return chk; @@ -82706,7 +82684,7 @@ var require_tool_cache = __commonJS({ var os7 = __importStar2(require("os")); var path30 = __importStar2(require("path")); var httpm = __importStar2(require_lib()); - var semver11 = __importStar2(require_semver2()); + var semver12 = __importStar2(require_semver2()); var stream2 = __importStar2(require("stream")); var util3 = __importStar2(require("util")); var assert_1 = require("assert"); @@ -82979,7 +82957,7 @@ var require_tool_cache = __commonJS({ } function cacheDir2(sourceDir, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver11.clean(version) || version; + version = semver12.clean(version) || version; arch2 = arch2 || os7.arch(); core32.debug(`Caching tool ${tool} ${version} ${arch2}`); core32.debug(`source dir: ${sourceDir}`); @@ -82997,7 +82975,7 @@ var require_tool_cache = __commonJS({ } function cacheFile(sourceFile, targetFile, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver11.clean(version) || version; + version = semver12.clean(version) || version; arch2 = arch2 || os7.arch(); core32.debug(`Caching tool ${tool} ${version} ${arch2}`); core32.debug(`source file: ${sourceFile}`); @@ -83027,7 +83005,7 @@ var require_tool_cache = __commonJS({ } let toolPath = ""; if (versionSpec) { - versionSpec = semver11.clean(versionSpec) || ""; + versionSpec = semver12.clean(versionSpec) || ""; const cachePath = path30.join(_getCacheDirectory(), toolName, versionSpec, arch2); core32.debug(`checking cache: ${cachePath}`); if (fs32.existsSync(cachePath) && fs32.existsSync(`${cachePath}.complete`)) { @@ -83107,7 +83085,7 @@ var require_tool_cache = __commonJS({ } function _createToolPath(tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - const folderPath = path30.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); + const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || ""); core32.debug(`destination ${folderPath}`); const markerPath = `${folderPath}.complete`; yield io9.rmRF(folderPath); @@ -83117,15 +83095,15 @@ var require_tool_cache = __commonJS({ }); } function _completeToolPath(tool, version, arch2) { - const folderPath = path30.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); + const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || ""); const markerPath = `${folderPath}.complete`; fs32.writeFileSync(markerPath, ""); core32.debug("finished caching tool"); } function isExplicitVersion(versionSpec) { - const c = semver11.clean(versionSpec) || ""; + const c = semver12.clean(versionSpec) || ""; core32.debug(`isExplicit: ${c}`); - const valid4 = semver11.valid(c) != null; + const valid4 = semver12.valid(c) != null; core32.debug(`explicit? ${valid4}`); return valid4; } @@ -83133,14 +83111,14 @@ var require_tool_cache = __commonJS({ let version = ""; core32.debug(`evaluating ${versions.length} versions`); versions = versions.sort((a, b) => { - if (semver11.gt(a, b)) { + if (semver12.gt(a, b)) { return 1; } return -1; }); for (let i = versions.length - 1; i >= 0; i--) { const potential = versions[i]; - const satisfied = semver11.satisfies(potential, versionSpec); + const satisfied = semver12.satisfies(potential, versionSpec); if (satisfied) { version = potential; break; @@ -89617,7 +89595,7 @@ var require_brace_expansion2 = __commonJS({ function lte2(i, y) { return i <= y; } - function gte7(i, y) { + function gte8(i, y) { return i >= y; } function combine2(acc, pre, values, max, maxLength, dropEmpties) { @@ -89649,7 +89627,7 @@ var require_brace_expansion2 = __commonJS({ var reverse = y < x; if (reverse) { incr *= -1; - test = gte7; + test = gte8; } var pad = n.some(isPadded2); var length = 0; @@ -110703,7 +110681,7 @@ var require_tar2 = __commonJS({ }); // node_modules/buffer-crc32/dist/index.cjs -var require_dist6 = __commonJS({ +var require_dist5 = __commonJS({ "node_modules/buffer-crc32/dist/index.cjs"(exports2, module2) { "use strict"; function getDefaultExportFromCjs(x) { @@ -111015,7 +110993,7 @@ var require_json = __commonJS({ "node_modules/@actions/artifact/node_modules/archiver/lib/plugins/json.js"(exports2, module2) { var inherits = require("util").inherits; var Transform5 = require_ours().Transform; - var crc325 = require_dist6(); + var crc325 = require_dist5(); var util3 = require_archiver_utils(); var Json2 = function(options) { if (!(this instanceof Json2)) { @@ -142141,7 +142119,7 @@ module.exports = __toCommonJS(entry_points_exports); // src/analyze-action.ts var fs23 = __toESM(require("fs")); var import_path5 = __toESM(require("path")); -var import_perf_hooks4 = require("perf_hooks"); +var import_perf_hooks6 = require("perf_hooks"); var core17 = __toESM(require_core()); // src/action-common.ts @@ -142223,6 +142201,7 @@ var fs = __toESM(require("fs")); var fsPromises = __toESM(require("fs/promises")); var os = __toESM(require("os")); var path = __toESM(require("path")); +var import_perf_hooks = require("perf_hooks"); var core2 = __toESM(require_core()); var io = __toESM(require_io()); @@ -143578,24 +143557,32 @@ function collectionTagName(state, event, defaultTagName) { function isMappingTag(tag) { return tag.nodeKind === "mapping"; } +function chargeMergeWork(state) { + state.totalMergeKeys++; + if (state.maxTotalMergeKeys !== -1 && state.totalMergeKeys > state.maxTotalMergeKeys) throwError$1(state, `merge keys exceeded maxTotalMergeKeys (${state.maxTotalMergeKeys})`); +} function mergeKeys(state, frame, source, sourceTag) { + chargeMergeWork(state); for (const sourceKey of sourceTag.keys(source)) { - if (state.maxTotalMergeKeys !== -1 && ++state.totalMergeKeys > state.maxTotalMergeKeys) throwError$1(state, `merge keys exceeded maxTotalMergeKeys (${state.maxTotalMergeKeys})`); + chargeMergeWork(state); if (frame.tag.has(frame.value, sourceKey)) continue; const err = frame.tag.addPair(frame.value, sourceKey, sourceTag.get(source, sourceKey)); if (err) throwError$1(state, err); - (frame.overridable ??= /* @__PURE__ */ new Set()).add(sourceKey); + frame.overridable ??= /* @__PURE__ */ new Set(); + frame.overridable.add(sourceKey); } } function mergeSource(state, frame, source, sourceTag) { state.position = frame.keyPosition; if (isMappingTag(sourceTag)) mergeKeys(state, frame, source, sourceTag); - else if (sourceTag.nodeKind === "sequence" && Array.isArray(source)) for (const element of source) { - const elementTag = state.nodeTags.get(element); - if (!elementTag) throwError$1(state, "cannot merge mappings; the provided source object is unacceptable"); - mergeKeys(state, frame, element, elementTag); - } - else throwError$1(state, "cannot merge mappings; the provided source object is unacceptable"); + else if (sourceTag.nodeKind === "sequence" && Array.isArray(source)) { + if (source.length > 100) throwError$1(state, "abnormal merge sequence size"); + for (const element of source) { + const elementTag = state.nodeTags.get(element); + if (!elementTag) throwError$1(state, "cannot merge mappings; the provided source object is unacceptable"); + mergeKeys(state, frame, element, elementTag); + } + } else throwError$1(state, "cannot merge mappings; the provided source object is unacceptable"); } function addMappingValue(state, frame, key, value, tag) { state.position = frame.keyPosition; @@ -145906,6 +145893,9 @@ async function bundleDb(config, language, codeql, dbName, { includeDiagnostics } ); return databaseBundlePath; } +function durationMsSince(startTime) { + return Math.round(import_perf_hooks.performance.now() - startTime); +} async function delay(milliseconds, opts) { const { allowProcessExit } = opts || {}; return new Promise((resolve14) => { @@ -146177,7 +146167,7 @@ function getDiffRangesJsonFilePath(env = getEnv()) { return path2.join(getTemporaryDirectory(env), PR_DIFF_RANGE_JSON_FILENAME); } function getActionVersion() { - return "3.38.0"; + return "3.38.1"; } function getWorkflowEventName(env = getEnv()) { return env.getRequired("GITHUB_EVENT_NAME" /* GITHUB_EVENT_NAME */); @@ -148105,6 +148095,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS", minimumVersion: void 0 }, + ["per_language_bundles" /* PerLanguageBundles */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", @@ -148620,7 +148615,7 @@ var SarifScanOrder = [ // src/analyze.ts var fs17 = __toESM(require("fs")); var path16 = __toESM(require("path")); -var import_perf_hooks3 = require("perf_hooks"); +var import_perf_hooks5 = require("perf_hooks"); var io5 = __toESM(require_io()); // src/autobuild.ts @@ -148884,7 +148879,7 @@ function wrapCliConfigurationError(cliError) { // src/config-utils.ts var fs10 = __toESM(require("fs")); var path11 = __toESM(require("path")); -var import_perf_hooks = require("perf_hooks"); +var import_perf_hooks2 = require("perf_hooks"); var core10 = __toESM(require_core()); // src/caching-utils.ts @@ -149904,18 +149899,18 @@ var builtin_default = { }; // src/languages/index.ts -var BuiltInLanguage = /* @__PURE__ */ ((BuiltInLanguage3) => { - BuiltInLanguage3["actions"] = "actions"; - BuiltInLanguage3["cpp"] = "cpp"; - BuiltInLanguage3["csharp"] = "csharp"; - BuiltInLanguage3["go"] = "go"; - BuiltInLanguage3["java"] = "java"; - BuiltInLanguage3["javascript"] = "javascript"; - BuiltInLanguage3["python"] = "python"; - BuiltInLanguage3["ruby"] = "ruby"; - BuiltInLanguage3["rust"] = "rust"; - BuiltInLanguage3["swift"] = "swift"; - return BuiltInLanguage3; +var BuiltInLanguage = /* @__PURE__ */ ((BuiltInLanguage4) => { + BuiltInLanguage4["actions"] = "actions"; + BuiltInLanguage4["cpp"] = "cpp"; + BuiltInLanguage4["csharp"] = "csharp"; + BuiltInLanguage4["go"] = "go"; + BuiltInLanguage4["java"] = "java"; + BuiltInLanguage4["javascript"] = "javascript"; + BuiltInLanguage4["python"] = "python"; + BuiltInLanguage4["ruby"] = "ruby"; + BuiltInLanguage4["rust"] = "rust"; + BuiltInLanguage4["swift"] = "swift"; + return BuiltInLanguage4; })(BuiltInLanguage || {}); var builtInLanguageSet = new Set(builtin_default.languages); function isBuiltInLanguage(language) { @@ -150510,9 +150505,9 @@ async function initActionState({ }; } async function downloadCacheWithTime(codeQL, languages, logger) { - const start = import_perf_hooks.performance.now(); + const start = import_perf_hooks2.performance.now(); const trapCaches = await downloadTrapCaches(codeQL, languages, logger); - const trapCacheDownloadTime = import_perf_hooks.performance.now() - start; + const trapCacheDownloadTime = import_perf_hooks2.performance.now() - start; return { trapCaches, trapCacheDownloadTime }; } async function loadUserConfig(actionState, configFile, workspacePath, apiDetails, tempDir) { @@ -150916,10 +150911,10 @@ async function initConfig(actionState, inputs) { } if (await features.getValue("ignore_generated_files" /* IgnoreGeneratedFiles */) && isDynamicWorkflow()) { try { - const generatedFilesCheckStartedAt = import_perf_hooks.performance.now(); + const generatedFilesCheckStartedAt = import_perf_hooks2.performance.now(); const generatedFiles = await getGeneratedFiles(inputs.sourceRoot); const generatedFilesDuration = Math.round( - import_perf_hooks.performance.now() - generatedFilesCheckStartedAt + import_perf_hooks2.performance.now() - generatedFilesCheckStartedAt ); if (generatedFiles.length > 0) { config.computedConfig["paths-ignore"] ??= []; @@ -151203,10 +151198,26 @@ async function logGeneratedFilesTelemetry(config, duration, generatedFilesCount) // src/setup-codeql.ts var fs14 = __toESM(require("fs")); var path13 = __toESM(require("path")); +var import_perf_hooks4 = require("perf_hooks"); var core12 = __toESM(require_core()); var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); -var semver9 = __toESM(require_semver2()); +var semver10 = __toESM(require_semver2()); + +// src/codeql-bundle.ts +var PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; +function getCodeQLBundleFromUrl(url2) { + let assetName; + try { + const pathname = new URL(url2).pathname; + assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); + } catch { + return { kind: "combined", url: url2 }; + } + const match2 = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + const language = match2 ? parseBuiltInLanguage(match2[1]) : void 0; + return language === void 0 ? { kind: "combined", url: url2 } : { kind: "per-language", url: url2, language }; +} // src/overlay/caching.ts var fs11 = __toESM(require("fs")); @@ -151506,6 +151517,97 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) { return versions; } +// src/per-language-bundles.ts +var semver7 = __toESM(require_semver2()); + +// src/platform.ts +function getBundlePlatform(platform2 = process.platform, arch2 = process.arch) { + switch (platform2) { + case "win32": + return "win64" /* Win64 */; + case "linux": + return arch2 === "arm64" ? "linux-arm64" /* LinuxArm64 */ : "linux64" /* Linux64 */; + case "darwin": + return "osx64" /* Osx64 */; + default: + return void 0; + } +} + +// src/per-language-bundles.ts +var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; +var PER_LANGUAGE_BUNDLE_LANGUAGES = { + ["linux64" /* Linux64 */]: /* @__PURE__ */ new Set([ + "actions" /* actions */, + "cpp" /* cpp */, + "csharp" /* csharp */, + "go" /* go */, + "java" /* java */, + "javascript" /* javascript */, + "python" /* python */, + "ruby" /* ruby */, + "rust" /* rust */ + ]), + ["linux-arm64" /* LinuxArm64 */]: /* @__PURE__ */ new Set(), + ["osx64" /* Osx64 */]: /* @__PURE__ */ new Set(["swift" /* swift */]), + ["win64" /* Win64 */]: /* @__PURE__ */ new Set() +}; +async function getPerLanguageBundleLanguage({ + env, + features, + logger +}, options) { + const { + rawLanguages, + cliVersion: cliVersion2, + compressionMethod, + platform: platform2, + variant, + isLatestNightly + } = options; + const explain = (reason) => { + logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); + return void 0; + }; + if (!await features.getValue("per_language_bundles" /* PerLanguageBundles */)) { + return explain(`the ${"per_language_bundles" /* PerLanguageBundles */} feature is disabled`); + } + if (rawLanguages?.length !== 1) { + return explain( + `exactly one language must be requested via the 'languages' input, but ${rawLanguages?.length ?? 0} were` + ); + } + const language = parseBuiltInLanguage(rawLanguages[0]); + if (language === void 0) { + return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); + } + if (compressionMethod !== "zstd") { + return explain(`the bundle would be downloaded as '${compressionMethod}'`); + } + if (variant !== "GitHub.com" /* DOTCOM */) { + return explain(`we are running against ${variant}`); + } + if (!isGitHubHostedRunner(env)) { + return explain("the job is not running on a GitHub-hosted runner"); + } + if (!isLatestNightly) { + if (cliVersion2 === void 0) { + return explain("the requested CLI version is unknown"); + } + if (!semver7.gte(cliVersion2, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { + return explain( + `the requested CodeQL version ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version for which per-language bundles are published` + ); + } + } + if (platform2 === void 0 || !PER_LANGUAGE_BUNDLE_LANGUAGES[platform2].has(language)) { + return explain( + `no per-language bundle is published for ${language} on ${platform2 ?? "an unknown platform"}` + ); + } + return language; +} + // src/tar.ts var import_child_process = require("child_process"); var fs12 = __toESM(require("fs")); @@ -151513,7 +151615,7 @@ var stream = __toESM(require("stream")); var import_toolrunner = __toESM(require_toolrunner()); var io4 = __toESM(require_io()); var toolcache = __toESM(require_tool_cache()); -var semver7 = __toESM(require_semver2()); +var semver8 = __toESM(require_semver2()); var MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3"; var MIN_REQUIRED_GNU_TAR_VERSION = "1.31"; async function getTarVersion() { @@ -151555,9 +151657,9 @@ async function isZstdAvailable(logger) { case "gnu": return { available: foundZstdBinary && // GNU tar only uses major and minor version numbers - semver7.gte( - semver7.coerce(version), - semver7.coerce(MIN_REQUIRED_GNU_TAR_VERSION) + semver8.gte( + semver8.coerce(version), + semver8.coerce(MIN_REQUIRED_GNU_TAR_VERSION) ), foundZstdBinary, version: tarVersion @@ -151566,7 +151668,7 @@ async function isZstdAvailable(logger) { return { available: foundZstdBinary && // Do a loose comparison since these version numbers don't contain // a patch version number. - semver7.gte(version, MIN_REQUIRED_BSD_TAR_VERSION), + semver8.gte(version, MIN_REQUIRED_BSD_TAR_VERSION), foundZstdBinary, version: tarVersion }; @@ -151670,12 +151772,12 @@ function inferCompressionMethod(tarPath) { var fs13 = __toESM(require("fs")); var os4 = __toESM(require("os")); var path12 = __toESM(require("path")); -var import_perf_hooks2 = require("perf_hooks"); +var import_perf_hooks3 = require("perf_hooks"); var core11 = __toESM(require_core()); var import_http_client = __toESM(require_lib()); var toolcache2 = __toESM(require_tool_cache()); var import_follow_redirects = __toESM(require_follow_redirects()); -var semver8 = __toESM(require_semver2()); +var semver9 = __toESM(require_semver2()); var STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; var STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1e3; var TOOLCACHE_TOOL_NAME = "CodeQL"; @@ -151683,7 +151785,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat logger.info( `Downloading CodeQL tools from ${codeqlURL} . This may take a while.` ); - const startTime = import_perf_hooks2.performance.now(); + const startTime = import_perf_hooks3.performance.now(); try { if (compressionMethod === "zstd" && process.platform === "linux") { logger.info(`Streaming the extraction of the CodeQL bundle.`); @@ -151695,7 +151797,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat tarVersion, logger ); - const totalDurationMs = Math.round(import_perf_hooks2.performance.now() - startTime); + const totalDurationMs = durationMsSince(startTime); logger.info( `Finished downloading and extracting CodeQL bundle to ${dest} (${formatDuration( totalDurationMs @@ -151704,20 +151806,23 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat return { totalDurationMs }; } } catch (e) { + await cleanUpPath(dest, "CodeQL bundle", logger); + if (asHTTPError(e)?.status === 404) { + throw e; + } core11.warning( `Failed to download and extract CodeQL bundle using streaming with error: ${getErrorMessage(e)}` ); core11.warning(`Falling back to downloading the bundle before extracting.`); - await cleanUpPath(dest, "CodeQL bundle", logger); } - const toolsDownloadStart = import_perf_hooks2.performance.now(); + const toolsDownloadStart = import_perf_hooks3.performance.now(); const archivedBundlePath = await toolcache2.downloadTool( codeqlURL, void 0, authorization, headers ); - const downloadDurationMs = Math.round(import_perf_hooks2.performance.now() - toolsDownloadStart); + const downloadDurationMs = durationMsSince(toolsDownloadStart); logger.info( `Finished downloading CodeQL bundle to ${archivedBundlePath} (${formatDuration( downloadDurationMs @@ -151726,7 +151831,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat let extractionDurationMs; try { logger.info("Extracting CodeQL bundle."); - const extractionStart = import_perf_hooks2.performance.now(); + const extractionStart = import_perf_hooks3.performance.now(); await extract( archivedBundlePath, dest, @@ -151734,7 +151839,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat tarVersion, logger ); - extractionDurationMs = Math.round(import_perf_hooks2.performance.now() - extractionStart); + extractionDurationMs = durationMsSince(extractionStart); logger.info( `Finished extracting CodeQL bundle to ${dest} (${formatDuration( extractionDurationMs @@ -151746,7 +151851,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat return { downloadDurationMs, extractionDurationMs, - totalDurationMs: Math.round(import_perf_hooks2.performance.now() - startTime) + totalDurationMs: durationMsSince(startTime) }; } async function downloadAndExtractZstdWithStreaming(codeqlURL, dest, authorization, headers, tarVersion, logger) { @@ -151780,9 +151885,14 @@ async function downloadAndExtractZstdWithStreaming(codeqlURL, dest, authorizatio }); if (response.statusCode !== 200) { response.resume(); - throw new Error( - `Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.` - ); + const baseMessage = `Failed to download CodeQL bundle from ${codeqlURL}.`; + if (response.statusCode !== void 0) { + throw new HTTPError( + `${baseMessage} HTTP status code: ${response.statusCode}.`, + response.statusCode + ); + } + throw new Error(baseMessage); } await extractTarZst(response, dest, tarVersion, logger); } @@ -151793,7 +151903,7 @@ function getToolcacheToolDirectory(env) { ); } function getToolcacheVersionDirectoryName(version) { - return semver8.clean(version) || version; + return semver9.clean(version) || version; } function getToolcacheDirectory(version) { return path12.join( @@ -151905,18 +152015,15 @@ function getCodeQLBundleExtension(compressionMethod) { assertNever(compressionMethod); } } -function getCodeQLBundleName(compressionMethod) { +function getCodeQLBundleName(compressionMethod, language) { const extension = getCodeQLBundleExtension(compressionMethod); - let platform2; - if (process.platform === "win32") { - platform2 = "win64"; - } else if (process.platform === "linux") { - platform2 = process.arch === "arm64" ? "linux-arm64" : "linux64"; - } else if (process.platform === "darwin") { - platform2 = "osx64"; - } else { + const platform2 = getBundlePlatform(); + if (platform2 === void 0) { return `codeql-bundle${extension}`; } + if (language !== void 0) { + return `codeql-bundle-${language}-${platform2}${extension}`; + } return `codeql-bundle-${platform2}${extension}`; } function getCodeQLActionRepository(logger) { @@ -151928,7 +152035,7 @@ function getCodeQLActionRepository(logger) { } return getRequiredEnvParam("GITHUB_ACTION_REPOSITORY"); } -async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod, logger) { +async function getCodeQLBundleDownloadURL(tagName, apiDetails, codeQLBundleName, logger) { const codeQLActionRepository = getCodeQLActionRepository(logger); const potentialDownloadSources = [ // This GitHub instance, and this Action. @@ -151943,7 +152050,6 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod return !self2.slice(0, index2).some((other) => (0, import_fast_deep_equal.default)(source, other)); } ); - const codeQLBundleName = getCodeQLBundleName(compressionMethod); for (const downloadSource of uniqueDownloadSources) { const [apiURL, repository] = downloadSource; if (apiURL === GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) { @@ -151973,7 +152079,7 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`; } function tryGetBundleVersionFromTagName(tagName, logger) { - const match2 = tagName.match(/^codeql-bundle-(.*)$/); + const match2 = tagName.match(/^codeql-bundle-(.+)$/); if (match2 === null || match2.length < 2) { logger.debug(`Could not determine bundle version from tag ${tagName}.`); return void 0; @@ -151998,13 +152104,13 @@ function tryGetTagNameFromUrl(url2, logger) { return match2[1]; } function convertToSemVer(version, logger) { - if (!semver9.valid(version)) { + if (!semver10.valid(version)) { logger.debug( `Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.` ); version = `0.0.0-${version}`; } - const s = semver9.clean(version); + const s = semver10.clean(version); if (!s) { throw new Error(`Bundle version ${version} is not in SemVer format.`); } @@ -152132,6 +152238,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO let cliVersion2; let tagName; let url2; + let bundle; const canForceNightlyWithFF = isDynamicWorkflow() || isInTestMode(); const forceNightlyValueFF = await features.getValue("force_nightly" /* ForceNightly */); const forceNightly = forceNightlyValueFF && canForceNightlyWithFF; @@ -152162,7 +152269,12 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.` ); } - toolsInput = await getNightlyToolsUrl(logger); + bundle = await getLatestNightlyBundle( + { env: getEnv(), features, logger }, + rawLanguages, + variant + ); + toolsInput = bundle.url; } const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput); if (forceShippedTools) { @@ -152213,7 +152325,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO url2 = toolsInput; if (tagName) { const bundleVersion3 = tryGetBundleVersionFromTagName(tagName, logger); - if (bundleVersion3 && semver9.valid(bundleVersion3)) { + if (bundleVersion3 !== void 0 && semver10.valid(bundleVersion3)) { cliVersion2 = convertToSemVer(bundleVersion3, logger); } } @@ -152228,8 +152340,9 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO cliVersion2 = version.cliVersion; tagName = version.tagName; } - const bundleVersion2 = tagName && tryGetBundleVersionFromTagName(tagName, logger); - const humanReadableVersion = cliVersion2 ?? (bundleVersion2 && convertToSemVer(bundleVersion2, logger)) ?? tagName ?? url2 ?? "unknown"; + const bundleVersion2 = tagName !== void 0 ? tryGetBundleVersionFromTagName(tagName, logger) : void 0; + const resolvedVersion = cliVersion2 ?? (bundleVersion2 !== void 0 ? convertToSemVer(bundleVersion2, logger) : void 0); + const humanReadableVersion = resolvedVersion ?? tagName ?? url2 ?? "unknown"; logger.debug( `Attempting to obtain CodeQL tools. CLI version: ${cliVersion2 ?? "unknown"}, bundle tag name: ${tagName ?? "unknown"}, URL: ${url2 ?? "unspecified"}.` ); @@ -152314,13 +152427,45 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO } let compressionMethod; if (!url2) { + const bundleTagName = tagName; + if (bundleTagName === void 0) { + throw new Error( + "Could not determine a release tag for the requested CodeQL bundle." + ); + } compressionMethod = cliVersion2 !== void 0 && await useZstdBundle(cliVersion2, tarSupportsZstd) ? "zstd" : "gzip"; - url2 = await getCodeQLBundleDownloadURL( - tagName, + const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + { env: getEnv(), features, logger }, + { + rawLanguages, + cliVersion: cliVersion2, + compressionMethod, + platform: getBundlePlatform(), + variant + } + ); + const resolveBundleURL = (language) => getCodeQLBundleDownloadURL( + bundleTagName, apiDetails, - compressionMethod, + getCodeQLBundleName(compressionMethod, language), logger ); + const combinedBundleURL = await resolveBundleURL(); + if (perLanguageBundleLanguage !== void 0) { + logger.info( + `Selected the per-language CodeQL bundle for '${perLanguageBundleLanguage}'.` + ); + url2 = await resolveBundleURL(perLanguageBundleLanguage); + bundle = { + kind: "per-language", + url: url2, + language: perLanguageBundleLanguage, + combinedBundleURL + }; + } else { + url2 = combinedBundleURL; + bundle = { kind: "combined", url: url2 }; + } } else { const method = inferCompressionMethod(url2); if (method === void 0) { @@ -152329,6 +152474,12 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO ); } compressionMethod = method; + bundle ??= getCodeQLBundleFromUrl(url2); + if (bundle.kind === "per-language") { + logger.info( + `${url2} appears to be a CodeQL bundle that contains only ${bundle.language}.` + ); + } } if (cliVersion2) { logger.info(`Using CodeQL CLI version ${cliVersion2} sourced from ${url2} .`); @@ -152336,17 +152487,17 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO logger.info(`Using CodeQL CLI sourced from ${url2} .`); } return { - bundleVersion: tagName && tryGetBundleVersionFromTagName(tagName, logger), + bundle, + bundleVersion: bundleVersion2, cliVersion: cliVersion2, - codeqlURL: url2, compressionMethod, sourceType: "download", - toolsVersion: cliVersion2 ?? humanReadableVersion + toolsVersion: resolvedVersion ?? "unknown" }; } async function tryGetFallbackToolcacheVersion(cliVersion2, tagName, logger) { const bundleVersion2 = tryGetBundleVersionFromTagName(tagName, logger); - if (!bundleVersion2) { + if (bundleVersion2 === void 0) { return void 0; } const fallbackVersion = convertToSemVer(bundleVersion2, logger); @@ -152355,7 +152506,9 @@ async function tryGetFallbackToolcacheVersion(cliVersion2, tagName, logger) { ); return fallbackVersion; } -var downloadCodeQL = async function(codeqlURL, compressionMethod, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, features, logger) { +var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, logger) { + const { bundle, compressionMethod } = source; + const codeqlURL = bundle.url; const parsedCodeQLURL = new URL(codeqlURL); const searchParams = new URLSearchParams(parsedCodeQLURL.search); const headers = { @@ -152371,13 +152524,10 @@ var downloadCodeQL = async function(codeqlURL, compressionMethod, maybeBundleVer codeqlURL ); } - const toolcacheInfo = getToolcacheDestinationInfo( - maybeBundleVersion, - maybeCliVersion, - logger + const toolcacheDestination = getToolcacheDestination({ logger }, source); + const extractedBundlePath = toolcacheDestination.orElse( + getTempExtractionDir(tempDir) ); - const extractedBundlePath = toolcacheInfo?.path ?? getTempExtractionDir(tempDir); - await tryDeleteToolcacheBundles({ env: getEnv(), features, logger }); const statusReport = await downloadAndExtract( codeqlURL, compressionMethod, @@ -152387,36 +152537,39 @@ var downloadCodeQL = async function(codeqlURL, compressionMethod, maybeBundleVer tarVersion, logger ); - if (!toolcacheInfo) { - logger.debug( - `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${codeqlURL}.` - ); - return { - codeqlFolder: extractedBundlePath, - statusReport, - toolsVersion: maybeCliVersion ?? "unknown" - }; + if (toolcacheDestination.isSuccess()) { + writeToolcacheMarkerFile(toolcacheDestination.value, logger); + } else { + logger.debug(toolcacheDestination.value); } - writeToolcacheMarkerFile(toolcacheInfo.path, logger); return { codeqlFolder: extractedBundlePath, - statusReport, - toolsVersion: maybeCliVersion ?? toolcacheInfo.version + statusReport: bundle.kind === "per-language" ? { + ...statusReport, + perLanguage: { tools_bundle_language: bundle.language } + } : statusReport }; }; -function getToolcacheDestinationInfo(maybeBundleVersion, maybeCliVersion, logger) { - if (maybeBundleVersion) { - const version = getCanonicalToolcacheVersion( - maybeCliVersion, - maybeBundleVersion, - logger +function getToolcacheDestination({ logger }, source) { + if (source.bundle.kind !== "combined") { + return new Failure( + "Not caching the CodeQL tools because they came from a bundle that contains only a single language." ); - return { - path: getToolcacheDirectory(version), - version - }; } - return void 0; + if (!source.bundleVersion) { + return new Failure( + `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${source.bundle.url}.` + ); + } + return new Success( + getToolcacheDirectory( + getCanonicalToolcacheVersion( + source.cliVersion, + source.bundleVersion, + logger + ) + ) + ); } async function tryDeleteToolcacheBundles({ env, @@ -152467,7 +152620,6 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau logger ); let codeqlFolder; - let toolsVersion = source.toolsVersion; let toolsDownloadStatusReport; let toolsSource; switch (source.sourceType) { @@ -152488,18 +152640,13 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau toolsSource = "TOOLCACHE" /* Toolcache */; break; case "download": { - const result = await downloadCodeQL( - source.codeqlURL, - source.compressionMethod, - source.bundleVersion, - source.cliVersion, + const result = await downloadCodeQLBundle( + { env: getEnv(), features, logger }, + source, apiDetails, zstdAvailability.version, - tempDir, - features, - logger + tempDir ); - toolsVersion = result.toolsVersion; codeqlFolder = result.codeqlFolder; toolsDownloadStatusReport = result.statusReport; toolsSource = "DOWNLOAD" /* Download */; @@ -152513,24 +152660,73 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau codeqlFolder, toolsDownloadStatusReport, toolsSource, - toolsVersion + toolsVersion: source.toolsVersion }; } +async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, tempDir) { + const { bundle } = source; + const { logger } = action; + await tryDeleteToolcacheBundles(action); + const startTime = import_perf_hooks4.performance.now(); + try { + return await downloadCodeQL( + source, + apiDetails, + tarVersion, + tempDir, + logger + ); + } catch (e) { + if (bundle.kind !== "per-language" || bundle.combinedBundleURL === void 0 || asHTTPError(e)?.status !== 404) { + throw e; + } + logger.warning( + `No per-language CodeQL bundle for '${bundle.language}' was found at ${bundle.url}, so falling back to the bundle that contains all languages. This analysis will still produce correct results, but will take longer to set up.` + ); + const result = await downloadCodeQL( + { + ...source, + bundle: { kind: "combined", url: bundle.combinedBundleURL } + }, + apiDetails, + tarVersion, + tempDir, + logger + ); + return { + ...result, + statusReport: { + ...result.statusReport, + totalDurationMs: durationMsSince(startTime), + perLanguage: { tools_per_language_bundle_fallback: true } + } + }; + } +} async function useZstdBundle(cliVersion2, tarSupportsZstd) { return ( // In testing, gzip performs better than zstd on Windows. - process.platform !== "win32" && tarSupportsZstd && semver9.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE) + process.platform !== "win32" && tarSupportsZstd && semver10.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE) ); } function getTempExtractionDir(tempDir) { return path13.join(tempDir, v4_default()); } -async function getNightlyToolsUrl(logger) { +async function getLatestNightlyBundle(action, rawLanguages, variant) { + const { logger } = action; const zstdAvailability = await isZstdAvailable(logger); const compressionMethod = await useZstdBundle( CODEQL_VERSION_ZSTD_BUNDLE, zstdAvailability.available ) ? "zstd" : "gzip"; + const language = await getPerLanguageBundleLanguage(action, { + rawLanguages, + cliVersion: void 0, + compressionMethod, + platform: getBundlePlatform(), + variant, + isLatestNightly: true + }); try { const release2 = await getApiClient().rest.repos.listReleases({ owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER, @@ -152543,7 +152739,14 @@ async function getNightlyToolsUrl(logger) { if (!latestRelease) { throw new Error("Could not find the latest nightly release."); } - return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`; + const assetUrl = (name) => `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`; + const url2 = assetUrl(getCodeQLBundleName(compressionMethod, language)); + return language === void 0 ? { kind: "combined", url: url2 } : { + kind: "per-language", + url: url2, + language, + combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod)) + }; } catch (e) { throw new Error( `Failed to retrieve the latest nightly release: ${wrapError(e)}` @@ -152551,7 +152754,7 @@ async function getNightlyToolsUrl(logger) { } } function getLatestToolcacheVersion(logger) { - const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver9.compare(b, a)); + const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver10.compare(b, a)); logger.debug( `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( allVersions @@ -153658,10 +153861,10 @@ function dbIsFinalized(config, language, logger) { } } async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, memoryFlag, logger) { - const extractionStart = import_perf_hooks3.performance.now(); + const extractionStart = import_perf_hooks5.performance.now(); await runExtraction(codeql, features, config, logger); - const extractionTime = import_perf_hooks3.performance.now() - extractionStart; - const trapImportStart = import_perf_hooks3.performance.now(); + const extractionTime = import_perf_hooks5.performance.now() - extractionStart; + const trapImportStart = import_perf_hooks5.performance.now(); for (const language of config.languages) { if (dbIsFinalized(config, language, logger)) { logger.info( @@ -153678,7 +153881,7 @@ async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, m logger.endGroup(); } } - const trapImportTime = import_perf_hooks3.performance.now() - trapImportStart; + const trapImportTime = import_perf_hooks5.performance.now() - trapImportStart; return { scanned_language_extraction_duration_ms: Math.round(extractionTime), trap_import_duration_ms: Math.round(trapImportTime) @@ -156485,9 +156688,9 @@ async function run({ startedAt, logger }) { features, logger ); - const trapCacheUploadStartTime = import_perf_hooks4.performance.now(); + const trapCacheUploadStartTime = import_perf_hooks6.performance.now(); didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger); - trapCacheUploadTime = import_perf_hooks4.performance.now() - trapCacheUploadStartTime; + trapCacheUploadTime = import_perf_hooks6.performance.now() - trapCacheUploadStartTime; trapCacheCleanupTelemetry = await cleanupTrapCaches( config, features, @@ -156735,7 +156938,7 @@ function isPadded(el) { function lte(i, y) { return i <= y; } -function gte6(i, y) { +function gte7(i, y) { return i >= y; } function combine(acc, pre, values, max, maxLength, dropEmpties) { @@ -156770,7 +156973,7 @@ function expandSequence(body, isAlphaSequence, max, maxLength) { const reverse = y < x; if (reverse) { incr *= -1; - test = gte6; + test = gte7; } const pad = n.some(isPadded); let length = 0; @@ -158667,7 +158870,7 @@ var import_async = __toESM(require_async(), 1); var import_path7 = require("path"); // node_modules/archiver/lib/error.js -var import_util34 = __toESM(require("util"), 1); +var import_util35 = __toESM(require("util"), 1); var ERROR_CODES = { ABORTED: "archive was aborted", DIRECTORYDIRPATHREQUIRED: "diretory dirpath argument must be a non-empty string value", @@ -158692,7 +158895,7 @@ function ArchiverError(code, data) { this.code = code; this.data = data; } -import_util34.default.inherits(ArchiverError, Error); +import_util35.default.inherits(ArchiverError, Error); // node_modules/archiver/lib/core.js var import_readable_stream2 = __toESM(require_ours(), 1); @@ -161624,7 +161827,7 @@ var fs29 = __toESM(require("fs")); var path25 = __toESM(require("path")); var core22 = __toESM(require_core()); var io7 = __toESM(require_io()); -var semver10 = __toESM(require_semver2()); +var semver11 = __toESM(require_semver2()); // src/config/inputs.ts async function getToolsInput(action, repositoryProperties) { @@ -161660,6 +161863,24 @@ async function getToolsInput(action, repositoryProperties) { return void 0; } +// src/status-report/tools-download.ts +function createInitToolsDownloadFields(report, toolsFeatureFlagsValid) { + const fields = { ...report?.perLanguage }; + if (report?.downloadDurationMs !== void 0) { + fields.tools_download_duration_ms = report.downloadDurationMs; + } + if (report?.extractionDurationMs !== void 0) { + fields.tools_extraction_duration_ms = report.extractionDurationMs; + } + if (report?.totalDurationMs !== void 0) { + fields.tools_total_duration_ms = report.totalDurationMs; + } + if (toolsFeatureFlagsValid !== void 0) { + fields.tools_feature_flags_valid = toolsFeatureFlagsValid; + } + return fields; +} + // src/workflow.ts var fs28 = __toESM(require("fs")); var path24 = __toESM(require("path")); @@ -161975,19 +162196,10 @@ async function sendCompletedStatusReport2(startedAt, config, configFile, toolsIn if (toolsInput !== void 0) { initStatusReport.computed_inputs.tools = toolsInput; } - const initToolsDownloadFields = {}; - if (toolsDownloadStatusReport?.downloadDurationMs !== void 0) { - initToolsDownloadFields.tools_download_duration_ms = toolsDownloadStatusReport.downloadDurationMs; - } - if (toolsDownloadStatusReport?.extractionDurationMs !== void 0) { - initToolsDownloadFields.tools_extraction_duration_ms = toolsDownloadStatusReport.extractionDurationMs; - } - if (toolsDownloadStatusReport?.totalDurationMs !== void 0) { - initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; - } - if (toolsFeatureFlagsValid !== void 0) { - initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } + const initToolsDownloadFields = createInitToolsDownloadFields( + toolsDownloadStatusReport, + toolsFeatureFlagsValid + ); if (config !== void 0) { const initWithConfigStatusReport = await createInitWithConfigStatusReport( config, @@ -162104,12 +162316,12 @@ async function run3(actionState) { const experimental = "2.19.3"; const publicPreview = "2.22.1"; const actualVer = (await codeql.getVersion()).version; - if (semver10.lt(actualVer, experimental)) { + if (semver11.lt(actualVer, experimental)) { throw new ConfigurationError( `Rust analysis is supported by CodeQL CLI version ${experimental} or higher, but found version ${actualVer}` ); } - if (semver10.lt(actualVer, publicPreview)) { + if (semver11.lt(actualVer, publicPreview)) { core22.exportVariable("CODEQL_ENABLE_EXPERIMENTAL_FEATURES" /* EXPERIMENTAL_FEATURES */, "true"); logger.info("Experimental Rust analysis enabled"); } @@ -163023,19 +163235,10 @@ async function sendCompletedStatusReport3(startedAt, toolsInput, toolsDownloadSt if (toolsInput !== void 0) { initStatusReport.computed_inputs.tools = toolsInput; } - const initToolsDownloadFields = {}; - if (toolsDownloadStatusReport?.downloadDurationMs !== void 0) { - initToolsDownloadFields.tools_download_duration_ms = toolsDownloadStatusReport.downloadDurationMs; - } - if (toolsDownloadStatusReport?.extractionDurationMs !== void 0) { - initToolsDownloadFields.tools_extraction_duration_ms = toolsDownloadStatusReport.extractionDurationMs; - } - if (toolsDownloadStatusReport?.totalDurationMs !== void 0) { - initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; - } - if (toolsFeatureFlagsValid !== void 0) { - initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } + const initToolsDownloadFields = createInitToolsDownloadFields( + toolsDownloadStatusReport, + toolsFeatureFlagsValid + ); await sendStatusReport({ ...initStatusReport, ...initToolsDownloadFields }); } async function run6(actionState) { @@ -163698,7 +163901,7 @@ async function checkProxyEnvironment(logger, language) { // src/start-proxy/reachability.ts var https2 = __toESM(require("https")); -var import_https_proxy_agent = __toESM(require_dist3()); +var import_https_proxy_agent = __toESM(require_dist2()); var connectionTestConfig = { nuget_feed: { path: "v3/index.json" } }; @@ -164270,7 +164473,7 @@ tmp/lib/tmp.js: *) js-yaml/dist/js-yaml.mjs: - (*! js-yaml 5.4.0 https://github.com/nodeca/js-yaml @license MIT *) + (*! js-yaml 5.4.1 https://github.com/nodeca/js-yaml @license MIT *) long/index.js: (** diff --git a/package-lock.json b/package-lock.json index 168f4111b9..17a4eee8c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,15 @@ { "name": "codeql", - "version": "4.38.0", + "version": "4.38.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "codeql", - "version": "4.38.0", + "version": "4.38.1", "license": "MIT", "workspaces": [ - "pr-checks", - "scripts/changetool" + "pr-checks" ], "dependencies": { "@actions/artifact": "^5.0.3", @@ -23,7 +22,7 @@ "@actions/http-client": "^3.0.0", "@actions/io": "^2.0.0", "@actions/tool-cache": "^3.0.1", - "@octokit/core": "^7.0.7", + "@octokit/core": "^7.0.8", "@octokit/plugin-paginate-rest": "^15.0.0", "@octokit/plugin-rest-endpoint-methods": "^18.0.0", "@octokit/plugin-retry": "^8.1.1", @@ -32,7 +31,7 @@ "follow-redirects": "^1.16.0", "get-folder-size": "^5.0.0", "https-proxy-agent": "^7.0.6", - "js-yaml": "^5.4.0", + "js-yaml": "^5.4.1", "jsonschema": "1.5.0", "long": "^5.3.2", "node-forge": "^1.4.0", @@ -42,9 +41,9 @@ }, "devDependencies": { "@ava/typescript": "6.0.0", - "@eslint/compat": "^2.1.0", + "@eslint/compat": "^2.1.1", "@microsoft/eslint-formatter-sarif": "^3.1.0", - "@octokit/types": "^17.0.0", + "@octokit/types": "^18.0.0", "@types/archiver": "^8.0.0", "@types/follow-redirects": "^1.14.4", "@types/js-yaml": "^4.0.9", @@ -59,14 +58,14 @@ "eslint-import-resolver-typescript": "^4.4.5", "eslint-plugin-github": "^6.1.2", "eslint-plugin-import-x": "^4.17.1", - "eslint-plugin-jsdoc": "^64.2.1", + "eslint-plugin-jsdoc": "^64.3.8", "eslint-plugin-no-async-foreach": "^0.1.1", "glob": "^13.0.6", - "globals": "^17.11.0", + "globals": "^17.12.0", "nock": "^14.0.17", "sinon": "^22.1.0", "typescript": "^6.0.3", - "typescript-eslint": "^8.68.0" + "typescript-eslint": "^8.70.0" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -958,17 +957,17 @@ } }, "node_modules/@es-joy/jsdoccomment": { - "version": "0.95.1", - "resolved": "https://registry.npmjs.org/@es-joy/jsdoccomment/-/jsdoccomment-0.95.1.tgz", - "integrity": "sha512-LO/RI08Fo9bhXwB7Od9G+1j3eSNq63+ZS5CQO8YLXHbDg6kx6S/DhTeY0+Fc9uZrjK1zZSyTx8Sg5gv5DIoCnA==", + "version": "0.97.0", + "resolved": "https://registry.npmjs.org/@es-joy/jsdoccomment/-/jsdoccomment-0.97.0.tgz", + "integrity": "sha512-EP8uoFfh6+GsdGCduYtmWAW0h7AO+Ayik9Vh5YbA2r/3N6lmJKkCNZX+q3QBXC1K6ixjQ/9igF2b7WVvLm063g==", "dev": true, "license": "MIT", "dependencies": { "@types/estree": "^1.0.9", - "@typescript-eslint/types": "^8.67.0", + "@typescript-eslint/types": "^8.69.0", "comment-parser": "1.4.8", "esquery": "^1.7.0", - "jsdoc-type-pratt-parser": "~9.1.2" + "jsdoc-type-pratt-parser": "~9.2.1" }, "engines": { "node": "^22.22.2 || >=24.15.0" @@ -1456,9 +1455,9 @@ } }, "node_modules/@eslint/compat": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@eslint/compat/-/compat-2.1.0.tgz", - "integrity": "sha512-LgaSCymEpw7tF53xvDw9SNsraPb1IBHxpdABIOM0hW8UAlP8znrjYtuxfR58FSJ3L9BhwD+FaPRFQpZq84Nh6g==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@eslint/compat/-/compat-2.1.1.tgz", + "integrity": "sha512-rMcy8GSrwNzcISX/BlTDY/GLB4eCopEuy9woIls3To+15OLxykZrxxq+WUcylCPCQ6F4MujjBM1DX5V1aqI3Vw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -1568,9 +1567,9 @@ } }, "node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -1646,29 +1645,43 @@ "license": "MIT" }, "node_modules/@humanfs/core": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", - "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", "dev": true, "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, "engines": { "node": ">=18.18.0" } }, "node_modules/@humanfs/node": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", - "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@humanfs/core": "^0.19.1", + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", "@humanwhocodes/retry": "^0.4.0" }, "engines": { "node": ">=18.18.0" } }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.13.0", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", @@ -2120,16 +2133,16 @@ } }, "node_modules/@octokit/core": { - "version": "7.0.7", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.7.tgz", - "integrity": "sha512-DcB0M3KFgr9ECI328lhBMVsyFT2DnmNucSBTqEN3exyNKUzkkpUSCHmTRcunF41Eou2TIQKW4seewri8ON9bSA==", + "version": "7.0.8", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.8.tgz", + "integrity": "sha512-L7y8eYc+AwxGr2PWI4WFt1VG4TiJ66c26BD16mXpYIlXxG0SMigM1+m4aTSlYyBr5BlQsGAlz8uDCoZN4SEMcg==", "license": "MIT", "dependencies": { "@octokit/auth-token": "^6.0.0", - "@octokit/graphql": "^9.0.4", - "@octokit/request": "^10.0.13", - "@octokit/request-error": "^7.1.1", - "@octokit/types": "^17.0.0", + "@octokit/graphql": "^9.0.5", + "@octokit/request": "^10.0.16", + "@octokit/request-error": "^7.1.2", + "@octokit/types": "^18.0.0", "before-after-hook": "^4.0.0", "universal-user-agent": "^7.0.0" }, @@ -2144,12 +2157,12 @@ "license": "ISC" }, "node_modules/@octokit/endpoint": { - "version": "11.0.4", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-11.0.4.tgz", - "integrity": "sha512-f1cOWoHPmxryJFknxbtDdjODWfV8A9tc8Aae6ermXPNgHFZ/x91AtHIz4gicEjL8hkJiip+u21QHJORfBv/qiA==", + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-11.0.5.tgz", + "integrity": "sha512-iXa654H3yFafF/ieHkukfbgWo2rmXD2ceD0ZOtrPhw1bc3FDch1d9N/TNs0FQ1/cIbwb7kspUX8jzIs8nzb9DQ==", "license": "MIT", "dependencies": { - "@octokit/types": "^17.0.0", + "@octokit/types": "^18.0.0", "universal-user-agent": "^7.0.2" }, "engines": { @@ -2163,13 +2176,13 @@ "license": "ISC" }, "node_modules/@octokit/graphql": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-9.0.4.tgz", - "integrity": "sha512-5s15CCiY8XXQ+FG+b1YQcl6Z2FA++nwAz/tg2VUrTmnMncP+2nnGUEYANImdnxsA2Fnq+Mbl7hDjUTw7cFAwcg==", + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-9.0.5.tgz", + "integrity": "sha512-bt/hm03LeU6Vy7FwTrkkC9p3XGT/lBwClglMqxBSe5/q0E5CdJTXeAqEI0vlw89/LF/G6tryTIH8HirZ3prMVg==", "license": "MIT", "dependencies": { - "@octokit/request": "^10.0.13", - "@octokit/types": "^17.0.0", + "@octokit/request": "^10.0.16", + "@octokit/types": "^18.0.0", "universal-user-agent": "^7.0.0" }, "engines": { @@ -2183,9 +2196,9 @@ "license": "ISC" }, "node_modules/@octokit/openapi-types": { - "version": "28.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-28.0.0.tgz", - "integrity": "sha512-0rFyLuyHvIj6uuZWuDslxkowFYdPXoNIkeAv4b27dzm2Tf4vGWXnPsMcxs7d65kLdMERgP3wc1AEPlqMz8e1cQ==", + "version": "29.0.1", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-29.0.1.tgz", + "integrity": "sha512-9qWOMFNxxLokERcms42rU0PTLqQmVs7g5E41TI4mCOxmpFayD1rfC7XxOL55cG9MBZLFlC31BrR37myMKardwg==", "license": "MIT" }, "node_modules/@octokit/plugin-paginate-rest": { @@ -2203,6 +2216,21 @@ "@octokit/core": ">=6" } }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": { + "version": "28.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-28.0.0.tgz", + "integrity": "sha512-0rFyLuyHvIj6uuZWuDslxkowFYdPXoNIkeAv4b27dzm2Tf4vGWXnPsMcxs7d65kLdMERgP3wc1AEPlqMz8e1cQ==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-17.0.0.tgz", + "integrity": "sha512-ByP1v7YL5SMveFPP7+sj0/ZuWCOOg/Chs4NafOMpq6WNIM/hdGY0S7C0TCGDBWu1aGmOxmUIhMx3cO+IdwYZ1Q==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^28.0.0" + } + }, "node_modules/@octokit/plugin-request-log": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz", @@ -2227,6 +2255,21 @@ "@octokit/core": ">=6" } }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": { + "version": "28.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-28.0.0.tgz", + "integrity": "sha512-0rFyLuyHvIj6uuZWuDslxkowFYdPXoNIkeAv4b27dzm2Tf4vGWXnPsMcxs7d65kLdMERgP3wc1AEPlqMz8e1cQ==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-17.0.0.tgz", + "integrity": "sha512-ByP1v7YL5SMveFPP7+sj0/ZuWCOOg/Chs4NafOMpq6WNIM/hdGY0S7C0TCGDBWu1aGmOxmUIhMx3cO+IdwYZ1Q==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^28.0.0" + } + }, "node_modules/@octokit/plugin-retry": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-8.1.1.tgz", @@ -2244,17 +2287,32 @@ "@octokit/core": ">=7" } }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/openapi-types": { + "version": "28.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-28.0.0.tgz", + "integrity": "sha512-0rFyLuyHvIj6uuZWuDslxkowFYdPXoNIkeAv4b27dzm2Tf4vGWXnPsMcxs7d65kLdMERgP3wc1AEPlqMz8e1cQ==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/types": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-17.0.0.tgz", + "integrity": "sha512-ByP1v7YL5SMveFPP7+sj0/ZuWCOOg/Chs4NafOMpq6WNIM/hdGY0S7C0TCGDBWu1aGmOxmUIhMx3cO+IdwYZ1Q==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^28.0.0" + } + }, "node_modules/@octokit/request": { - "version": "10.0.13", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-10.0.13.tgz", - "integrity": "sha512-v2269YxL9Yf+x3d+gRI63FP0vFQEiWgLyBzxe/Y+0yFDg2B/Tzf5dhh9VNfccVAQnfcfwQWyk/y6Bn7rUXXs7A==", + "version": "10.0.16", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-10.0.16.tgz", + "integrity": "sha512-A0zWGjHzISIb+9ccG8s0dq7LKO5zVpJLRICjgUb+sJxEWqn8RUHB1rD3AE51+PECvXHIxqZ1VVvs4fHTSD9nUQ==", "license": "MIT", "dependencies": { - "@octokit/endpoint": "^11.0.3", - "@octokit/request-error": "^7.1.1", - "@octokit/types": "^17.0.0", - "content-type": "^2.0.0", - "json-with-bigint": "^3.5.3", + "@octokit/endpoint": "^11.0.5", + "@octokit/request-error": "^7.1.2", + "@octokit/types": "^18.0.0", + "content-type": "^3.0.0", + "json-with-bigint": "^3.5.12", "universal-user-agent": "^7.0.2" }, "engines": { @@ -2262,12 +2320,12 @@ } }, "node_modules/@octokit/request-error": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-7.1.1.tgz", - "integrity": "sha512-+eaY7G2VVpSf2pc5Gn1+mph837V/d/TYTJAgWL9Tb0ogGYcpN3IlAVFgjL+Vv93F/sevrxkvsYCedtpLdcFLzA==", + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-7.1.2.tgz", + "integrity": "sha512-XZRuT3xZ84D3gYErI1DZvhJ33dCWVV6uzBtWkaBB4TvA/L6eOeTZodxLFVB44bBEEo3vEx7y00UfX1tBLrtLRg==", "license": "MIT", "dependencies": { - "@octokit/types": "^17.0.0" + "@octokit/types": "^18.0.0" }, "engines": { "node": ">= 20" @@ -2280,12 +2338,12 @@ "license": "ISC" }, "node_modules/@octokit/types": { - "version": "17.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-17.0.0.tgz", - "integrity": "sha512-ByP1v7YL5SMveFPP7+sj0/ZuWCOOg/Chs4NafOMpq6WNIM/hdGY0S7C0TCGDBWu1aGmOxmUIhMx3cO+IdwYZ1Q==", + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-18.0.0.tgz", + "integrity": "sha512-l6bAF43PNxkJp6g+W4PjoUSSkxHomXw2nOum5CTftJz1NlV3vu93NImgOYtLf6CbBUb5j+fiuzW0PPQ5JTSvZA==", "license": "MIT", "dependencies": { - "@octokit/openapi-types": "^28.0.0" + "@octokit/openapi-types": "^29.0.1" } }, "node_modules/@open-draft/deferred-promise": { @@ -2657,17 +2715,17 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.68.0.tgz", - "integrity": "sha512-WASHDpCm6qO5jj9g1a+8NiW5+GCkAyLReR56/4VruYmNgfUmqpxOfZ2Yfb8xGfJPWv5Qi6LSD8sXdces3vbp/Q==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.70.0.tgz", + "integrity": "sha512-/v8HZt6RlyIZxB3ntehELOcUcfxKPVGWXnQdJuHRmzrqgF8nQypcC/oxGW+Ot4VGKDq81XugPKxx0n5PBtf9PA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.68.0", - "@typescript-eslint/type-utils": "8.68.0", - "@typescript-eslint/utils": "8.68.0", - "@typescript-eslint/visitor-keys": "8.68.0", + "@typescript-eslint/scope-manager": "8.70.0", + "@typescript-eslint/type-utils": "8.70.0", + "@typescript-eslint/utils": "8.70.0", + "@typescript-eslint/visitor-keys": "8.70.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" @@ -2680,7 +2738,7 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.68.0", + "@typescript-eslint/parser": "^8.70.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } @@ -2696,16 +2754,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.68.0.tgz", - "integrity": "sha512-fHq2VC1kpyYfvEcbiMjOpySY4WS7voEp89yAThrHRX5sm9j2lzYppCb2umFMEed4fWcyeLjHxrz0mpjNBaBxMQ==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.70.0.tgz", + "integrity": "sha512-zYvrmj9Yxd63UGaXw+kdt6A0F0s0qveJyuatIM77bYC2DE4pgmg7a50u8LR7PRtXd0x+h+Tl3eXabGm06SWd3Q==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.68.0", - "@typescript-eslint/types": "8.68.0", - "@typescript-eslint/typescript-estree": "8.68.0", - "@typescript-eslint/visitor-keys": "8.68.0", + "@typescript-eslint/scope-manager": "8.70.0", + "@typescript-eslint/types": "8.70.0", + "@typescript-eslint/typescript-estree": "8.70.0", + "@typescript-eslint/visitor-keys": "8.70.0", "debug": "^4.4.3" }, "engines": { @@ -2739,14 +2797,14 @@ } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.68.0.tgz", - "integrity": "sha512-5GQtWZCXFcFYux955pvoS02WLc49pXNlvIxocKjS0clvwo3in1RdlzVKyiqQH9vE5AKWFLTaUgeQkOrTS+0Qxw==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.70.0.tgz", + "integrity": "sha512-hFHbTNqhU9G+2eKFXCBVb1tjFT/LceiJ4+HfLO4pTpDI0KHi6iajpcFFkaSQ9gXmCh7n82A0PthaayEdN6mspQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.68.0", - "@typescript-eslint/types": "^8.68.0", + "@typescript-eslint/tsconfig-utils": "^8.70.0", + "@typescript-eslint/types": "^8.70.0", "debug": "^4.4.3" }, "engines": { @@ -2779,14 +2837,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.68.0.tgz", - "integrity": "sha512-T5eXpcaJNg8bhjHJ8Rjp68Vq/QBteYtTKY8TZqVNPaUbuz0f6jI9t6aDkylwvalpAB9XTTFeFOjrjXAZ3YvmVA==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.70.0.tgz", + "integrity": "sha512-8nP3Kwh5hlgZ4FicGvmznAmJe8UL4sdU8tLukrPaMuQmDuk4Y8xYfzu/aYZW4xT2JCgc7H/TpDI5cGlxcWJSqQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.68.0", - "@typescript-eslint/visitor-keys": "8.68.0" + "@typescript-eslint/types": "8.70.0", + "@typescript-eslint/visitor-keys": "8.70.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2797,9 +2855,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.68.0.tgz", - "integrity": "sha512-F7zrGQfiJHojPwi8vhxZQC1tWtJzvL74cK/nqri2lk8YUXvYaYwl263xOJ69jDWPUk1hmcdoayFwk9lX09npVw==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.70.0.tgz", + "integrity": "sha512-adnkeeNq9Sq1sUf4+FRVc0KdgYghzsgFpZSQVZVvY0LCuUuN0FnQgyGzCJeC4fW1cdXseBAjU2EOqUIjbNcZUw==", "dev": true, "license": "MIT", "engines": { @@ -2814,15 +2872,15 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.68.0.tgz", - "integrity": "sha512-X77zqoY1EjeWGs/0JNxeaMfp5C5lIz4Tw8y66F1Ne8Faq6g424sBNYM6xBAqElfGZPLpWS+CZAp0DXyKDzWiHg==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.70.0.tgz", + "integrity": "sha512-NUMKIhYVaVIVLnRL9CRt+VVcuLgSHUCpXn4/+K8wql+vdInUzvx8BjUO1oJ7cG9shjFJKtF8F8Hh2kCh3/KBVw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.68.0", - "@typescript-eslint/typescript-estree": "8.68.0", - "@typescript-eslint/utils": "8.68.0", + "@typescript-eslint/types": "8.70.0", + "@typescript-eslint/typescript-estree": "8.70.0", + "@typescript-eslint/utils": "8.70.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, @@ -2857,9 +2915,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.68.0.tgz", - "integrity": "sha512-9RnpsGJjrAllCMefGVVsImJM24YurhC0Q1h4UbvivtvOqXmR/vEJge2OoE++z9m6hyg8T1Q8t5SNT6tHSbrxcg==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.70.0.tgz", + "integrity": "sha512-asTOIYhDg4zdzOScCyaytrsV3cR6B4ecPQlXw/dJIm7J/MZTtCtfVII9JD8Geh4jTCrK/Xe6cg5UevoleMcoJQ==", "dev": true, "license": "MIT", "engines": { @@ -2871,16 +2929,16 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.68.0.tgz", - "integrity": "sha512-OKKsD0tYmoNiU5PW2zehO1yO56jYOm1ShYlxon/Z0SJNidAkdVg86eg9ruRuoXf8xfnuWZGbwDsStkoXbZtIIA==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.70.0.tgz", + "integrity": "sha512-d9NmHMPEKQ7QCLLm1jI3zmoQBwT5KwFYjXBJ9ymZfKCUU+5rmTRykKAFvH5Qn/ZCds3CEAFS9OC9M/jkl0X2bA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.68.0", - "@typescript-eslint/tsconfig-utils": "8.68.0", - "@typescript-eslint/types": "8.68.0", - "@typescript-eslint/visitor-keys": "8.68.0", + "@typescript-eslint/project-service": "8.70.0", + "@typescript-eslint/tsconfig-utils": "8.70.0", + "@typescript-eslint/types": "8.70.0", + "@typescript-eslint/visitor-keys": "8.70.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", @@ -2956,16 +3014,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.68.0.tgz", - "integrity": "sha512-PB5gJMMOg0Q5P1tsgWtEAqQacJXq0qEqRHDX/YJ4FaTMLfZPpHB3gjl2EJuiZyPABxmj4ZQYiY9m1bdAJ5y7tQ==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.70.0.tgz", + "integrity": "sha512-oZmtKJz/4fufZ2p3+Cn3ijEojcdfR+1zYDH2xKYrEly0dR/Q/1xUPRCOlKGxod78nWlU2UnDe09GZ3TaknBFGA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.68.0", - "@typescript-eslint/types": "8.68.0", - "@typescript-eslint/typescript-estree": "8.68.0" + "@typescript-eslint/scope-manager": "8.70.0", + "@typescript-eslint/types": "8.70.0", + "@typescript-eslint/typescript-estree": "8.70.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2980,13 +3038,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.68.0.tgz", - "integrity": "sha512-YR65gGdGvTUAWLldC3xLOvOzamdGzB4A5/N8rehEaHs3Zvoe39BhgY+u0SPch1OvrVTfLcc55wsSgK2NcnTS/A==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.70.0.tgz", + "integrity": "sha512-BoC8PiO4Hkdo0TVJh9Ntxr5MxPDI7/oFsrygN5ADelFSeXG/qgNuucIGA+L5Z6JpPTE/uRfcTWtscjbUaufepQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.68.0", + "@typescript-eslint/types": "8.70.0", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -3010,346 +3068,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@typescript/typescript-aix-ppc64": { - "version": "7.0.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", - "integrity": "sha1-zcfOgdYPHgkDSWDd+x+4gNendrY=", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-darwin-arm64": { - "version": "7.0.2", - "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", - "integrity": "sha1-pV/fz6WN9Y0n2yI3zealweNacjU=", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-darwin-x64": { - "version": "7.0.2", - "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", - "integrity": "sha1-ONHJFygAqR1we+xk0qNwoBZjTbQ=", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-freebsd-arm64": { - "version": "7.0.2", - "resolved": "https://ms-feed-2.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", - "integrity": "sha1-8f+IEAMLNdK1vg22otxlBGDqlPo=", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-freebsd-x64": { - "version": "7.0.2", - "resolved": "https://ms-feed-2.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", - "integrity": "sha1-PYawPzU8WxupUWLrbONVM7/ClL0=", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-arm": { - "version": "7.0.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", - "integrity": "sha1-rZS0HhruKk3MaimMe2fEM0X94y4=", - "cpu": [ - "arm" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-arm64": { - "version": "7.0.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", - "integrity": "sha1-2TNNltbaxv+F2pyGVYiUjek56R8=", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-loong64": { - "version": "7.0.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", - "integrity": "sha1-KWWu5PyHM2ATnYk9qv5jl6KROK0=", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-mips64el": { - "version": "7.0.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", - "integrity": "sha1-Goh6MRvtOoM/gL/Uqe03wnGTbPA=", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-ppc64": { - "version": "7.0.2", - "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", - "integrity": "sha1-i2PJsvRFs5PrTkPsIdoiXa3jV30=", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-riscv64": { - "version": "7.0.2", - "resolved": "https://ms-feed-2.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", - "integrity": "sha1-tuijXCibPql6kqQdRhqu7Q07NuE=", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-s390x": { - "version": "7.0.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", - "integrity": "sha1-Lvlmk75IYfbReWVCflsAnLvtGj4=", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-x64": { - "version": "7.0.2", - "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", - "integrity": "sha1-cyacsLq6UK6gygYERaa4jlg/HOI=", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-netbsd-arm64": { - "version": "7.0.2", - "resolved": "https://ms-feed-2.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", - "integrity": "sha1-OjZJ+X+vohC05uN5jBXgZgXIqQE=", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-netbsd-x64": { - "version": "7.0.2", - "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", - "integrity": "sha1-R+xZSRpAxHDSgH3E0rglUo/Zeas=", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-openbsd-arm64": { - "version": "7.0.2", - "resolved": "https://ms-feed-17.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", - "integrity": "sha1-eWvo2gvZidij+5bygB44qDZbS68=", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-openbsd-x64": { - "version": "7.0.2", - "resolved": "https://ms-feed-2.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", - "integrity": "sha1-03/ipynrlCwHbEVO5/GBX699Vg8=", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-sunos-x64": { - "version": "7.0.2", - "resolved": "https://ms-feed-17.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", - "integrity": "sha1-q6jTRkw1ZacER4m6upaRa9SrLIg=", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-win32-arm64": { - "version": "7.0.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", - "integrity": "sha1-ud5QoXGWOD9iYgtfnQovNK07YNc=", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-win32-x64": { - "version": "7.0.2", - "resolved": "https://ms-feed-2.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", - "integrity": "sha1-zzt7DWzlY12spMjgHBic3N5H7Dw=", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16.20.0" - } - }, "node_modules/@typespec/ts-http-runtime": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.2.tgz", @@ -4496,10 +4214,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/changetool": { - "resolved": "scripts/changetool", - "link": true - }, "node_modules/character-entities": { "version": "2.0.2", "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/character-entities/-/character-entities-2.0.2.tgz", @@ -4728,12 +4442,12 @@ } }, "node_modules/content-type": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", - "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-3.0.0.tgz", + "integrity": "sha512-AIi5H6p0xk5uknXcN3/rmhP8jgp69OfSe/JuKiQAFprJ7UGw7mwj7m4XcmDzlrnJDG+cGpphAINGdU3g3g7kDw==", "license": "MIT", "engines": { - "node": ">=18" + "node": ">=22" }, "funding": { "type": "opencollective", @@ -5634,14 +5348,15 @@ } }, "node_modules/eslint-plugin-jsdoc": { - "version": "64.2.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-64.2.1.tgz", - "integrity": "sha512-6GpSYxLPcbMw38S94Cngrgs1Zv8yLinQS1O17OxJVZ6deLbrMRCERUYQKceweSqEuG2kx5Amn4l3aKPkCp4geQ==", + "version": "64.3.8", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-64.3.8.tgz", + "integrity": "sha512-JXLYE2BVfmbqLrrslb9/vg3URg8okW5pMnppM+3EYwvPCbuOiSXGOJWaNK208wDx6xXawkIFGtRYgFgrW23dsg==", "dev": true, "license": "BSD-3-Clause", "dependencies": { - "@es-joy/jsdoccomment": "~0.95.1", + "@es-joy/jsdoccomment": "~0.97.0", "@es-joy/resolve.exports": "1.2.0", + "@typescript-eslint/utils": "^8.69.0", "are-docs-informative": "^0.1.1", "comment-parser": "1.4.8", "debug": "^4.4.3", @@ -6533,9 +6248,9 @@ } }, "node_modules/globals": { - "version": "17.11.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.11.0.tgz", - "integrity": "sha512-Z2I8hM+PbJDXQDq3Icgpzv+mPdwr68iZUU9d5WW4FuXfDUQfkZaZuvjMv42/5crNyw154+9+VWXbYrUgDXbxNw==", + "version": "17.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.12.0.tgz", + "integrity": "sha512-cezEd/DTyyht9cvSSURyygXPfy04GtWO/5e6ZPvH7fCtjKz9PYOmuawphw1Ctd1f6C+5JypXfGD7ahNMXvevBA==", "dev": true, "license": "MIT", "engines": { @@ -7376,9 +7091,9 @@ } }, "node_modules/js-yaml": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.4.0.tgz", - "integrity": "sha512-jE7vUJIebKzYQI5xu4co5CRBDlDEYnHrdzsxs4O2giCz4v2SbVMYKpmt1D9L38OKQAeCWmrOTRiCV93u0UkaJA==", + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.4.1.tgz", + "integrity": "sha512-28R/k+NAjeuf7+CKlTxWZVExJGwVVLwY06DgEnOMz2gEpfNkDcD7QvyiVPT0xy0XXhU8vHsd4Ot42OOPdJG7dQ==", "funding": [ { "type": "github", @@ -7408,18 +7123,36 @@ } }, "node_modules/jsdoc-type-pratt-parser": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/jsdoc-type-pratt-parser/-/jsdoc-type-pratt-parser-9.1.2.tgz", - "integrity": "sha512-9EXymowgk1mb9RY1VxuwKc+AhaxfBk2CV0dWxgGM+l5RURTtiUoAx7MlKwcsiVcEXK5HEPa7FeH/tsRpqjEPRg==", + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/jsdoc-type-pratt-parser/-/jsdoc-type-pratt-parser-9.2.1.tgz", + "integrity": "sha512-V4Ww4EHnTcTLSOMoB0FsF72JhQvcAsriCm/LWnxJeGWoxIjEL2l9na11abQok5SYShq8m0Gl02el/xAbTCulvQ==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "^1.0.9" + "@types/estree": "^1.0.9", + "@types/node": "^26.4.0" }, "engines": { "node": "^22.22.2 || >=24.15.0" } }, + "node_modules/jsdoc-type-pratt-parser/node_modules/@types/node": { + "version": "26.5.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.0.tgz", + "integrity": "sha512-dVSGpriSoCgz8WnDNTuSSuSv1PC/ALXihO4ulRZt7Md8k9mlbdin3lGOcDE8SnWOgf513ByWlXd7BK4azmyg/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/jsdoc-type-pratt-parser/node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "dev": true, + "license": "MIT" + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -7443,9 +7176,9 @@ "license": "ISC" }, "node_modules/json-with-bigint": { - "version": "3.5.10", - "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.10.tgz", - "integrity": "sha512-Vcx+JVNEBts/xfcoCS69sKrOhOk/3TVlvlT+XzUOefVKnnrbYSCKpDCm10pohsJFtsJVYnwa/cXRZ4eElzaM6w==", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.12.tgz", + "integrity": "sha512-uwbF/wSSuOgC7qqlq27Xp5B6a2MHVug3t0idZdTqu0JnlFvgJuH7ju+KAk/J06C7GfhoYy2gnb9wz2INqcne7w==", "license": "MIT" }, "node_modules/json5": { @@ -7582,8 +7315,8 @@ }, "node_modules/lite-matter": { "version": "0.1.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/lite-matter/-/lite-matter-0.1.2.tgz", - "integrity": "sha1-NlO1r/xDs8OnKQcf/kw4/a4O1Ws=", + "resolved": "https://registry.npmjs.org/lite-matter/-/lite-matter-0.1.2.tgz", + "integrity": "sha512-cGSYxO6vcipYZBSF5dUtDF3IqbkNxSjVQuvs5Ol3fO35URzHEvG2Ml0KbH5kvXXsx33SgbfwiVp6afwovarD8A==", "license": "MIT", "dependencies": { "yaml": "^2.9.0" @@ -7717,8 +7450,8 @@ }, "node_modules/mdast-util-from-markdown": { "version": "2.0.3", - "resolved": "https://ms-feed-17.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", - "integrity": "sha1-yVgiuRqrdfGKTL6LL1G4c+0s8Mc=", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", "license": "MIT", "dependencies": { "@types/mdast": "^4.0.0", @@ -10066,9 +9799,9 @@ "license": "0BSD" }, "node_modules/tsx": { - "version": "4.23.12", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz", - "integrity": "sha512-FDf4L4sYzKtzWYhU/Xm0AQFdTjdIxNo9ElTf2mxXM6k8YMHXzYUe4yODVaXP4V9uMFbVg8c0qyBccK2OOxb45Q==", + "version": "4.23.13", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz", + "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==", "dev": true, "license": "MIT", "dependencies": { @@ -10218,16 +9951,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.68.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.68.0.tgz", - "integrity": "sha512-MHy0Y0ynqeEbx/S45+i/bBssdy3X6KNBfmJAP35GrgtNxu2TQ5K5xsFDhAnmsq1jvpdoZOPG1LGtJo0HWqYCrQ==", + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.70.0.tgz", + "integrity": "sha512-P/W5cz70/cQAuKfY3xwQMWWTV7BvJ0mAQmi+9mBcsVPaBUpd6Ohpa+fECv9rBFrQcig86jAiNBFNWUqnTjr4pw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.68.0", - "@typescript-eslint/parser": "8.68.0", - "@typescript-eslint/typescript-estree": "8.68.0", - "@typescript-eslint/utils": "8.68.0" + "@typescript-eslint/eslint-plugin": "8.70.0", + "@typescript-eslint/parser": "8.70.0", + "@typescript-eslint/typescript-estree": "8.70.0", + "@typescript-eslint/utils": "8.70.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -10721,81 +10454,18 @@ "dependencies": { "@actions/core": "^2.0.3", "@actions/github": "^8.0.1", - "@octokit/core": "^7.0.7", + "@octokit/core": "^7.0.8", "@octokit/plugin-paginate-rest": ">=15.0.0", "@octokit/plugin-rest-endpoint-methods": "^18.0.0", + "lite-matter": "^0.1.2", + "mdast-util-from-markdown": "^2.0.3", "semver": "^7.8.5", "yaml": "^2.9.0" }, "devDependencies": { "@types/node": "^20.19.43", - "tsx": "^4.23.12" - } - }, - "scripts/changetool": { - "version": "1.0.0", - "license": "MIT", - "dependencies": { - "lite-matter": "^0.1.2", - "mdast-util-from-markdown": "^2.0.3" - }, - "devDependencies": { - "@types/node": "^26.2.0", - "tsx": "^4.23.12", - "typescript": "^7.0.2" + "tsx": "^4.23.13" } - }, - "scripts/changetool/node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@types/node/-/node-26.2.0.tgz", - "integrity": "sha1-Wkh1qGL9qP3Ffej6pXm7gey6FoU=", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~8.3.0" - } - }, - "scripts/changetool/node_modules/typescript": { - "version": "7.0.2", - "resolved": "https://ms-feed-12.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/typescript/-/typescript-7.0.2.tgz", - "integrity": "sha1-nsdz15VKjBgsF8xbvVdaoovFFYI=", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc" - }, - "engines": { - "node": ">=16.20.0" - }, - "optionalDependencies": { - "@typescript/typescript-aix-ppc64": "7.0.2", - "@typescript/typescript-darwin-arm64": "7.0.2", - "@typescript/typescript-darwin-x64": "7.0.2", - "@typescript/typescript-freebsd-arm64": "7.0.2", - "@typescript/typescript-freebsd-x64": "7.0.2", - "@typescript/typescript-linux-arm": "7.0.2", - "@typescript/typescript-linux-arm64": "7.0.2", - "@typescript/typescript-linux-loong64": "7.0.2", - "@typescript/typescript-linux-mips64el": "7.0.2", - "@typescript/typescript-linux-ppc64": "7.0.2", - "@typescript/typescript-linux-riscv64": "7.0.2", - "@typescript/typescript-linux-s390x": "7.0.2", - "@typescript/typescript-linux-x64": "7.0.2", - "@typescript/typescript-netbsd-arm64": "7.0.2", - "@typescript/typescript-netbsd-x64": "7.0.2", - "@typescript/typescript-openbsd-arm64": "7.0.2", - "@typescript/typescript-openbsd-x64": "7.0.2", - "@typescript/typescript-sunos-x64": "7.0.2", - "@typescript/typescript-win32-arm64": "7.0.2", - "@typescript/typescript-win32-x64": "7.0.2" - } - }, - "scripts/changetool/node_modules/undici-types": { - "version": "8.3.0", - "resolved": "https://ms-feed-17.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/undici-types/-/undici-types-8.3.0.tgz", - "integrity": "sha1-ROn8nzJEZIzeo15Pm7LWgelBCAk=", - "dev": true, - "license": "MIT" } } } diff --git a/package.json b/package.json index dfeeaa7300..5596407dbd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeql", - "version": "3.38.0", + "version": "3.38.1", "private": true, "description": "CodeQL action", "scripts": { @@ -17,8 +17,7 @@ }, "license": "MIT", "workspaces": [ - "pr-checks", - "scripts/changetool" + "pr-checks" ], "dependencies": { "@actions/artifact": "^5.0.3", @@ -31,7 +30,7 @@ "@actions/http-client": "^3.0.0", "@actions/io": "^2.0.0", "@actions/tool-cache": "^3.0.1", - "@octokit/core": "^7.0.7", + "@octokit/core": "^7.0.8", "@octokit/plugin-paginate-rest": "^15.0.0", "@octokit/plugin-rest-endpoint-methods": "^18.0.0", "@octokit/plugin-retry": "^8.1.1", @@ -40,7 +39,7 @@ "follow-redirects": "^1.16.0", "get-folder-size": "^5.0.0", "https-proxy-agent": "^7.0.6", - "js-yaml": "^5.4.0", + "js-yaml": "^5.4.1", "jsonschema": "1.5.0", "long": "^5.3.2", "node-forge": "^1.4.0", @@ -50,9 +49,9 @@ }, "devDependencies": { "@ava/typescript": "6.0.0", - "@eslint/compat": "^2.1.0", + "@eslint/compat": "^2.1.1", "@microsoft/eslint-formatter-sarif": "^3.1.0", - "@octokit/types": "^17.0.0", + "@octokit/types": "^18.0.0", "@types/archiver": "^8.0.0", "@types/follow-redirects": "^1.14.4", "@types/js-yaml": "^4.0.9", @@ -67,14 +66,14 @@ "eslint-import-resolver-typescript": "^4.4.5", "eslint-plugin-github": "^6.1.2", "eslint-plugin-import-x": "^4.17.1", - "eslint-plugin-jsdoc": "^64.2.1", + "eslint-plugin-jsdoc": "^64.3.8", "eslint-plugin-no-async-foreach": "^0.1.1", "glob": "^13.0.6", - "globals": "^17.11.0", + "globals": "^17.12.0", "nock": "^14.0.17", "sinon": "^22.1.0", "typescript": "^6.0.3", - "typescript-eslint": "^8.68.0" + "typescript-eslint": "^8.70.0" }, "overrides": { "@actions/tool-cache": { diff --git a/scripts/changetool/cli/validate.ts b/pr-checks/changelog/validate.mts similarity index 90% rename from scripts/changetool/cli/validate.ts rename to pr-checks/changelog/validate.mts index 3c83276f38..2e28a4ab13 100644 --- a/scripts/changetool/cli/validate.ts +++ b/pr-checks/changelog/validate.mts @@ -119,3 +119,14 @@ export function isValidChangenoteFile(filename: string): boolean { return isValid; } + +/** + * Validates the change-note files of the given list of file paths, ignoring ".gitkeep". + * @param filepaths A list of filepaths to validate + * @returns True if all the paths are valid, false otherwise. + */ +export function isValidAllChangenoteFiles(filepaths: string[]): boolean { + return filepaths + .filter((f) => f !== ".gitkeep") + .reduce((r, filePath) => r && isValidChangenoteFile(filePath), true); +} diff --git a/scripts/changetool/cli/validate.test.ts b/pr-checks/changelog/validate.test.mts similarity index 64% rename from scripts/changetool/cli/validate.test.ts rename to pr-checks/changelog/validate.test.mts index ac54972fdf..a38339d1c9 100644 --- a/scripts/changetool/cli/validate.test.ts +++ b/pr-checks/changelog/validate.test.mts @@ -1,36 +1,21 @@ import assert from "node:assert/strict"; import * as fs from "node:fs"; -import * as os from "node:os"; import * as path from "node:path"; import { describe, it } from "node:test"; +import { withTmpDir, withTmpFile } from "../../src/util"; + import { + hasValidChangenoteCategory, + isValidAllChangenoteFiles, isValidChangenoteContent, isValidChangenoteFile, isValidChangenoteFilename, - hasValidChangenoteCategory, VALID_CHANGE_NOTE_CATEGORIES, -} from "./validate.ts"; - -async function withTmpFile( - baseFileName: string, - contents: string, - body: (filePath: string) => Promise, -): Promise { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "changetool-validate-test-"), - ); - try { - const filePath = path.join(tmpDir, baseFileName); - fs.writeFileSync(filePath, contents); - return await body(filePath); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } -} +} from "./validate.mjs"; await describe("isValidChangenoteContent", async () => { - await it("recognizes an unordered Markdown list", async () => { + await it("recognizes an unordered Markdown list", () => { const inputs = [ "- One changenote entry", "- First item\n- Second item", @@ -42,7 +27,7 @@ await describe("isValidChangenoteContent", async () => { } }); - await it("does not recognize non-Markdown text", async () => { + await it("does not recognize non-Markdown text", () => { const inputs = [ "This is not a list.", '["this", "is", "JSON"]', @@ -57,7 +42,7 @@ await describe("isValidChangenoteContent", async () => { } }); - await it("does not recognize ordered Markdown lists", async () => { + await it("does not recognize ordered Markdown lists", () => { const inputs = [ "1. First item\n2. Second item", "\n\n\n1. First item\n1. Second item", @@ -68,7 +53,7 @@ await describe("isValidChangenoteContent", async () => { } }); - await it("requires all list items to use a hyphen bullet", async () => { + await it("requires all list items to use a hyphen bullet", () => { const inputs = [ "* Fixed a bug\n* Added feature", "+ Fixed a bug\n+ Added feature", @@ -85,7 +70,7 @@ await describe("isValidChangenoteContent", async () => { } }); - await it("does not contain other Markdown elements", async () => { + await it("does not contain other Markdown elements", () => { const inputs = [ "- Fixed a bug\n\nParagraph of text", "- Fixed a bug\n\n* Added a feature", @@ -100,7 +85,7 @@ await describe("isValidChangenoteContent", async () => { }); await describe("isValidChangenoteFilename", async () => { - await it("accepts valid filenames", async () => { + await it("accepts valid filenames", () => { const inputs = [ "2023-01-01-fix-bug.md", "2023-12-31-add-feature.md", @@ -112,7 +97,7 @@ await describe("isValidChangenoteFilename", async () => { } }); - await it("rejects invalid filenames", async () => { + await it("rejects invalid filenames", () => { const inputs = [ "missing-date-from-filename.md", "2021-01-01.md", @@ -126,14 +111,14 @@ await describe("isValidChangenoteFilename", async () => { }); await describe("hasValidChangenoteCategory", async () => { - await it("accepts valid categories", async () => { + await it("accepts valid categories", () => { for (const category of Object.keys(VALID_CHANGE_NOTE_CATEGORIES)) { const frontmatter = { category }; assert.equal(hasValidChangenoteCategory(frontmatter), true); } }); - await it("rejects invalid categories", async () => { + await it("rejects invalid categories", () => { const inputs = [ "", "invalid-category", @@ -150,7 +135,7 @@ await describe("hasValidChangenoteCategory", async () => { } }); - await it("reject missing category", async () => { + await it("reject missing category", () => { assert.equal(hasValidChangenoteCategory({}), false); assert.equal(hasValidChangenoteCategory({ category: null }), false); assert.equal(hasValidChangenoteCategory({ category: undefined }), false); @@ -162,17 +147,21 @@ await describe("isValidChangenoteFile", async () => { await withTmpFile( "2026-01-01-fix-bug.md", "---\ncategory: fix\n---\n- Fixed a bug\n", - async (filePath) => { + (filePath) => { assert.equal(isValidChangenoteFile(filePath), true); }, ); }); + await it("rejects a non-existent path", async () => { + assert.equal(isValidChangenoteFile("non-existent-file.md"), false); + }); + await it("rejects invalid filename", async () => { await withTmpFile( "fix-bug.md", "---\ncategory: fix\n---\n- Fixed a bug\n", - async (filePath) => { + (filePath) => { assert.equal(isValidChangenoteFile(filePath), false); }, ); @@ -182,7 +171,7 @@ await describe("isValidChangenoteFile", async () => { await withTmpFile( "2026-01-01-fix-bug.md", "- Fixed a bug\n", - async (filePath) => { + (filePath) => { assert.equal(isValidChangenoteFile(filePath), false); }, ); @@ -192,9 +181,45 @@ await describe("isValidChangenoteFile", async () => { await withTmpFile( "2026-01-01-fix-bug.md", "---\ncategory: fix\n---\n* Fixed a bug\n", - async (filePath) => { + (filePath) => { assert.equal(isValidChangenoteFile(filePath), false); }, ); }); }); + +await describe("isValidAllChangenoteFiles", async () => { + await it("accepts list of file paths of valid change-notes", async () => { + await withTmpDir(async (tmpDir) => { + const fileName1 = path.join(tmpDir, "2026-01-01-fix-bug.md"); + const fileName2 = path.join(tmpDir, "2026-01-02-add-feature.md"); + fs.writeFileSync(fileName1, "---\ncategory: fix\n---\n- Fixed a bug\n"); + fs.writeFileSync( + fileName2, + "---\ncategory: feature\n---\n- Added a feature\n", + ); + assert.equal(isValidAllChangenoteFiles([fileName1, fileName2]), true); + }); + }); + + await it("accepts the empty list", async () => { + assert.equal(isValidAllChangenoteFiles([]), true); + }); + + await it("accepts list of .gitkeep", async () => { + assert.equal(isValidAllChangenoteFiles([".gitkeep"]), true); + }); + + await it("rejects list containing a file path to an invalid change-note", async () => { + await withTmpDir(async (tmpDir) => { + const fileName1 = path.join(tmpDir, "2026-01-01-fix-bug.md"); + const fileName2 = path.join(tmpDir, "2026-01-02-wrong-category.md"); + fs.writeFileSync(fileName1, "---\ncategory: fix\n---\n- Fixed a bug\n"); + fs.writeFileSync( + fileName2, + "---\ncategory: foobar\n---\n- Added a feature\n", + ); + assert.equal(isValidAllChangenoteFiles([fileName1, fileName2]), false); + }); + }); +}); diff --git a/scripts/changetool/index.ts b/pr-checks/changenotes.mts old mode 100644 new mode 100755 similarity index 51% rename from scripts/changetool/index.ts rename to pr-checks/changenotes.mts index 249448b3fb..2fb86b0cac --- a/scripts/changetool/index.ts +++ b/pr-checks/changenotes.mts @@ -1,7 +1,11 @@ +#!/usr/bin/env npx tsx + +import * as fs from "node:fs"; import { pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; -import { isValidChangenoteFile } from "./cli/validate.ts"; +import { isValidAllChangenoteFiles } from "./changelog/validate.mjs"; +import { CHANGENOTES_DIR } from "./config"; const entryPoint = process.argv[1]; if (entryPoint && import.meta.url === pathToFileURL(entryPoint).href) { @@ -18,13 +22,13 @@ function main(): number { allowPositionals: true, strict: true, }); - const [command, ...paths] = positionals; + const [command] = positionals; switch (command) { case undefined: case "help": return usage(); case "validate": - return validate(paths); + return validate(); default: console.error(`Unknown command: ${command}`); return 1; @@ -32,20 +36,21 @@ function main(): number { } function usage(): number { - console.log("Usage: changetool validate [ ...]"); + console.log(`Usage: changenotes.mts validate`); return 0; } -function validate(paths: string[]): number { - let valid = true; - if (paths.length === 0) { - console.error("error: no paths provided (see 'help' command for usage)"); - return 1; - } - for (const path of paths) { - if (!isValidChangenoteFile(path)) { - valid = false; +function validate(): number { + try { + if (isValidAllChangenoteFiles(fs.readdirSync(CHANGENOTES_DIR))) { + console.log(`All changenotes in '${CHANGENOTES_DIR}' are valid.`); + return 0; } + } catch (error) { + console.error( + `Failed to read changenotes directory '${CHANGENOTES_DIR}'`, + error, + ); } - return valid ? 0 : 1; + return 1; } diff --git a/pr-checks/checks/bundle-toolcache.yml b/pr-checks/checks/bundle-toolcache.yml index 83d1d7d0b5..f74c6af75c 100644 --- a/pr-checks/checks/bundle-toolcache.yml +++ b/pr-checks/checks/bundle-toolcache.yml @@ -30,7 +30,8 @@ steps: - id: init uses: ./../action/init with: - languages: javascript + # Request multiple languages so this check uses the combined bundle. + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - uses: ./../action/analyze with: diff --git a/pr-checks/checks/per-language-bundle-validation.yml b/pr-checks/checks/per-language-bundle-validation.yml new file mode 100644 index 0000000000..21fe33e757 --- /dev/null +++ b/pr-checks/checks/per-language-bundle-validation.yml @@ -0,0 +1,117 @@ +name: Per-language bundles +description: Validates extraction and analysis using each per-language CodeQL bundle. +# TODO: Use a released bundle once releases include per-language bundles. +matrix: + include: + - language: actions + os: ubuntu-latest + version: nightly-latest + # Actions also needs the JavaScript extractor. + expected-extractors: actions javascript + - language: cpp + os: ubuntu-latest + version: nightly-latest + build-mode: manual + build-command: gcc -o main main.c + - language: csharp + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: go + os: ubuntu-latest + version: nightly-latest + build-mode: autobuild + - language: java + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: javascript + os: ubuntu-latest + version: nightly-latest + - language: python + os: ubuntu-latest + version: nightly-latest + - language: ruby + os: ubuntu-latest + version: nightly-latest + - language: rust + os: ubuntu-latest + version: nightly-latest + - language: swift + os: macos-latest-xlarge + version: nightly-latest + build-mode: autobuild +env: + CODEQL_ACTION_PER_LANGUAGE_BUNDLES: true +steps: + - uses: ./../action/init + id: init + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix['build-mode'] }} + tools: ${{ steps.prepare-test.outputs.tools-url }} + - name: Check that the bundle contains only the expected extractors + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + LANGUAGE: ${{ matrix.language }} + EXPECTED_EXTRACTORS: ${{ matrix['expected-extractors'] || matrix.language }} + run: | + extractors="$("$CODEQL_PATH" resolve languages --format=json | jq -r 'keys[]')" + echo "Extractors in the bundle:" + echo "$extractors" + echo "Expected: $EXPECTED_EXTRACTORS" + + for expected in $EXPECTED_EXTRACTORS; do + if ! echo "$extractors" | grep -qx "$expected"; then + echo "::error::The ${LANGUAGE} bundle does not contain the ${expected} extractor." + exit 1 + fi + done + + # If the bundle contained extractors beyond those the language needs, then it would not + # have been trimmed, and this job would be silently validating the combined bundle. + for other in actions cpp csharp go java javascript python ruby rust swift; do + if echo "$EXPECTED_EXTRACTORS" | grep -qw "$other"; then + continue + fi + if echo "$extractors" | grep -qx "$other"; then + echo "::error::The ${LANGUAGE} bundle also contains the ${other} extractor, so it is not trimmed." + exit 1 + fi + done + - name: Check that the bundle was not added to the toolcache + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + run: | + # A bundle that is missing most of its extractors must never be left in the toolcache, + # where a later job analyzing a different language could pick it up. The runner image + # ships with its own CodeQL in the toolcache, so check where this bundle was extracted to + # rather than whether the toolcache contains CodeQL at all. + echo "CodeQL is at $CODEQL_PATH" + if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then + echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH." + exit 1 + fi + if [[ "$CODEQL_PATH" != "$RUNNER_TEMP"/* ]]; then + echo "::error::Expected the per-language bundle to be extracted under $RUNNER_TEMP, but found it at $CODEQL_PATH." + exit 1 + fi + - name: Build code + if: matrix['build-command'] + run: ${{ matrix['build-command'] }} + - uses: ./../action/analyze + id: analysis + with: + upload-database: false + - name: Check that a database was created for the language + env: + DB_LOCATIONS: ${{ steps.analysis.outputs.db-locations }} + LANGUAGE: ${{ matrix.language }} + run: | + database="$(echo "$DB_LOCATIONS" | jq -r --arg lang "$LANGUAGE" '.[$lang] // empty')" + if [ -z "$database" ] || [ ! -d "$database" ]; then + echo "::error::No CodeQL database was created for ${LANGUAGE}." + echo "Databases: $DB_LOCATIONS" + exit 1 + fi + echo "Created a ${LANGUAGE} database at ${database}." diff --git a/pr-checks/config.ts b/pr-checks/config.ts index 356fe665f9..05bcbfe3ce 100644 --- a/pr-checks/config.ts +++ b/pr-checks/config.ts @@ -18,6 +18,9 @@ export const PACKAGE_JSON = path.join(REPO_ROOT, "package.json"); /** The path of the changelog. */ export const CHANGELOG_FILE = path.join(REPO_ROOT, "CHANGELOG.md"); +/** The path to the unreleased change-notes directory. */ +export const CHANGENOTES_DIR = path.join(REPO_ROOT, "unreleased-change-notes"); + /** The path to the esbuild metadata file. */ export const BUNDLE_METADATA_FILE = path.join(REPO_ROOT, "meta.json"); diff --git a/pr-checks/package.json b/pr-checks/package.json index 00f4b813c2..0c5091a793 100644 --- a/pr-checks/package.json +++ b/pr-checks/package.json @@ -1,17 +1,19 @@ { "private": true, - "description": "Dependencies for the sync.ts", + "description": "Dependencies for codeql-action scripts", "dependencies": { "@actions/core": "^2.0.3", "@actions/github": "^8.0.1", - "@octokit/core": "^7.0.7", + "@octokit/core": "^7.0.8", "@octokit/plugin-paginate-rest": ">=15.0.0", "@octokit/plugin-rest-endpoint-methods": "^18.0.0", + "lite-matter": "^0.1.2", + "mdast-util-from-markdown": "^2.0.3", "semver": "^7.8.5", "yaml": "^2.9.0" }, "devDependencies": { "@types/node": "^20.19.43", - "tsx": "^4.23.12" + "tsx": "^4.23.13" } } diff --git a/pr-checks/sync.sh b/pr-checks/sync.sh index c059594493..dd0e0b4680 100755 --- a/pr-checks/sync.sh +++ b/pr-checks/sync.sh @@ -4,12 +4,19 @@ set -e cd "$(dirname "$0")" # Run `npm ci` in CI or `npm install` otherwise. +# +# `pr-checks` is an npm workspace of the repository root and the two share a single hoisted +# `node_modules` directory. Running npm from this directory puts it in workspace mode, where it +# ignores the root project's own dependencies by default. `npm ci` would then rebuild the shared +# `node_modules` with only this workspace's dependencies, removing the root's ones, which breaks +# anything that imports from `src` (such as `sync.ts` itself). `--include-workspace-root` keeps the +# root project's dependencies in the installed tree. if [ "$GITHUB_ACTIONS" = "true" ]; then echo "In Actions, running 'npm ci' for 'sync.ts'..." - npm ci + npm ci --include-workspace-root else echo "Running 'npm install' for 'sync.ts'..." - npm install --no-audit --no-fund + npm install --no-audit --no-fund --include-workspace-root fi npx tsx sync.ts diff --git a/pr-checks/sync.ts b/pr-checks/sync.ts index c307f6e484..f0942ad2dd 100755 --- a/pr-checks/sync.ts +++ b/pr-checks/sync.ts @@ -79,6 +79,8 @@ interface Specification extends JobSpecification { useAllPlatformBundle?: string; /** Values for the `analysis-kinds` matrix dimension. */ analysisKinds?: string[]; + /** Overrides the generated job matrix using GitHub Actions matrix syntax. */ + matrix?: Record; /** Container image configuration for the job. */ container?: any; @@ -253,8 +255,8 @@ const languageSetups: LanguageSetups = { name: "Install Java", uses: pinnedUses( "actions/setup-java", - "dd06d9cba3e5552c54d9f8ea23572deb30010f7c", - "v6.0.0", + "de7274f081f381c8f8158605e0321c36c376e2e6", + "v6.0.1", ), with: { "java-version": `\${{ inputs.java-version || '${defaultLanguageVersions.java}' }}`, @@ -512,9 +514,6 @@ function generateJob( specDocument: yaml.Document, checkSpecification: Specification, ) { - const matrix: Array> = - generateJobMatrix(checkSpecification); - const useAllPlatformBundle = checkSpecification.useAllPlatformBundle ? checkSpecification.useAllPlatformBundle : "false"; @@ -567,8 +566,8 @@ function generateJob( const checkJob: Record = { strategy: { "fail-fast": false, - matrix: { - include: matrix, + matrix: checkSpecification.matrix ?? { + include: generateJobMatrix(checkSpecification), }, }, name: checkSpecification.name, diff --git a/pr-checks/tsconfig.json b/pr-checks/tsconfig.json index b010827cf5..67f31d2cdb 100644 --- a/pr-checks/tsconfig.json +++ b/pr-checks/tsconfig.json @@ -6,8 +6,8 @@ "module": "preserve", "rootDir": "..", "sourceMap": false, - "noEmit": true, + "noEmit": true }, - "include": ["./*.ts", "../src/**/*.ts"], + "include": ["./**/*.ts", "./**/*.mts", "../src/**/*.ts"], "exclude": ["node_modules"] } diff --git a/scripts/changetool/package.json b/scripts/changetool/package.json deleted file mode 100644 index 7eb8ceb9be..0000000000 --- a/scripts/changetool/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "changetool", - "version": "1.0.0", - "private": true, - "description": "Validates change-notes and merges them into CHANGELOG.md", - "license": "MIT", - "type": "module", - "scripts": { - "start": "tsx index.ts", - "test": "node --test --experimental-strip-types cli/*.test.ts" - }, - "devDependencies": { - "@types/node": "^26.2.0", - "tsx": "^4.23.12", - "typescript": "^7.0.2" - }, - "dependencies": { - "lite-matter": "^0.1.2", - "mdast-util-from-markdown": "^2.0.3" - } -} diff --git a/scripts/changetool/tsconfig.json b/scripts/changetool/tsconfig.json deleted file mode 100644 index ee76bd1869..0000000000 --- a/scripts/changetool/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "module": "preserve", - "allowImportingTsExtensions": true, - "rootDir": ".", - "sourceMap": false - }, - "include": ["./**/*.ts"], - "exclude": ["node_modules"] -} diff --git a/src/actions-util.ts b/src/actions-util.ts index eb7d92b517..677bb04b1b 100644 --- a/src/actions-util.ts +++ b/src/actions-util.ts @@ -7,7 +7,7 @@ import * as github from "@actions/github"; import * as io from "@actions/io"; import type { Config } from "./config-utils"; -import { Env, EnvVar, ActionsEnvVars } from "./environment"; +import { Env, EnvVar, ActionsEnvVars, ReadOnlyEnv } from "./environment"; import { Logger } from "./logging"; import { doesDirectoryExist, @@ -292,7 +292,7 @@ export function isSelfHostedRunner(env: Env = getEnv()) { * that are configured to resemble hosted ones, such as those that mount a persistent volume at * `/opt/hostedtoolcache`. */ -export function isGitHubHostedRunner(env: Env = getEnv()) { +export function isGitHubHostedRunner(env: ReadOnlyEnv = getEnv()) { return env.getOptional(ActionsEnvVars.RUNNER_ENVIRONMENT) === "github-hosted"; } diff --git a/src/codeql-bundle.test.ts b/src/codeql-bundle.test.ts new file mode 100644 index 0000000000..1014043eab --- /dev/null +++ b/src/codeql-bundle.test.ts @@ -0,0 +1,56 @@ +import test from "ava"; + +import { getCodeQLBundleFromUrl } from "./codeql-bundle"; +import { BuiltInLanguage } from "./languages"; + +for (const [assetName, language] of [ + ["codeql-bundle-java-linux64.tar.zst", BuiltInLanguage.java], + ["codeql-bundle-swift-osx64.tar.zst", BuiltInLanguage.swift], + // Recognize unpublished language/platform combinations to keep them out of the toolcache. + ["codeql-bundle-csharp-win64.tar.gz", BuiltInLanguage.csharp], + ["codeql-bundle-java-kotlin-linux64.tar.zst", BuiltInLanguage.java], + ["codeql-bundle-%70ython-linux64.tar.zst", BuiltInLanguage.python], +] as const) { + test(`getCodeQLBundleFromUrl identifies ${assetName} without adding a fallback`, (t) => { + const url = `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${assetName}`; + t.deepEqual(getCodeQLBundleFromUrl(url), { + kind: "per-language", + url, + language, + }); + }); +} + +test("getCodeQLBundleFromUrl preserves encoding, query parameters and fragments", (t) => { + const url = + "https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/codeql-bundle-%70ython-linux64.tar.zst?download=1#asset"; + t.deepEqual(getCodeQLBundleFromUrl(url), { + kind: "per-language", + url, + language: BuiltInLanguage.python, + }); +}); + +test("getCodeQLBundleFromUrl treats unrecognized assets as combined bundles", (t) => { + for (const name of [ + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-osx64.tar.gz", + "codeql-bundle-win64.tar.zst", + // The all-platform bundle. + "codeql-bundle.tar.gz", + // A platform we do not publish per-language bundles for, whose name also contains a hyphen. + "codeql-bundle-linux-arm64.tar.zst", + // Not a language we know about. + "codeql-bundle-cobol-linux64.tar.zst", + // A name we cannot decode must not be mistaken for a language either. + "codeql-bundle-%zz-linux64.tar.zst", + ]) { + const url = `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; + t.deepEqual(getCodeQLBundleFromUrl(url), { kind: "combined", url }); + } +}); + +test("getCodeQLBundleFromUrl preserves URLs it cannot parse", (t) => { + const url = "not a url"; + t.deepEqual(getCodeQLBundleFromUrl(url), { kind: "combined", url }); +}); diff --git a/src/codeql-bundle.ts b/src/codeql-bundle.ts new file mode 100644 index 0000000000..4e9f5d5de1 --- /dev/null +++ b/src/codeql-bundle.ts @@ -0,0 +1,33 @@ +import { BuiltInLanguage, parseBuiltInLanguage } from "./languages"; + +/** Describes the contents and location of a downloadable CodeQL bundle. */ +export type CodeQLBundle = + | { kind: "combined"; url: string } + | { + kind: "per-language"; + url: string; + language: BuiltInLanguage; + /** Only set when the Action selected the bundle, allowing a same-version fallback. */ + combinedBundleURL?: string; + }; + +const PER_LANGUAGE_BUNDLE_NAME = + /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; + +/** Classifies an explicit tools URL without changing it or adding a fallback. */ +export function getCodeQLBundleFromUrl(url: string): CodeQLBundle { + let assetName: string; + try { + const pathname = new URL(url).pathname; + // URL-encoded names must not bypass the toolcache safeguard. + assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); + } catch { + return { kind: "combined", url }; + } + + const match = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + const language = match ? parseBuiltInLanguage(match[1]) : undefined; + return language === undefined + ? { kind: "combined", url } + : { kind: "per-language", url, language }; +} diff --git a/src/codeql.test.ts b/src/codeql.test.ts index df4bafe295..5f2eb31156 100644 --- a/src/codeql.test.ts +++ b/src/codeql.test.ts @@ -90,7 +90,7 @@ async function installIntoToolcache({ tmpDir: string; }) { const url = mockBundleDownloadApi({ apiDetails, isPinned, tagName }); - await codeql.setupCodeQL( + return await codeql.setupCodeQL( cliVersion !== undefined ? undefined : url, apiDetails, tmpDir, @@ -259,6 +259,65 @@ test.serial( }, ); +for (const { cliVersion, tagName, expectedToolcacheVersion } of [ + { + cliVersion: "2.21.0", + tagName: "codeql-bundle-20240101", + expectedToolcacheVersion: "2.21.0", + }, + { + cliVersion: "2.21.0-rc.1", + tagName: "codeql-bundle-20240101", + expectedToolcacheVersion: "0.0.0-20240101", + }, + { + cliVersion: "2.21.0+20240101", + tagName: "codeql-bundle-20240101", + expectedToolcacheVersion: "0.0.0-20240101", + }, + { + cliVersion: "2.21.0", + tagName: "custom-release", + expectedToolcacheVersion: undefined, + }, +]) { + test.serial( + `preserves CLI version ${cliVersion} when installing ${tagName}`, + async (t) => { + await util.withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + + const result = await installIntoToolcache({ + cliVersion, + isPinned: false, + tagName, + tmpDir, + }); + + t.is(result.toolsVersion, cliVersion); + t.is(result.toolsSource, ToolsSource.Download); + t.true( + Number.isInteger(result.toolsDownloadStatusReport?.totalDurationMs), + ); + t.deepEqual( + toolcache.findAllVersions("CodeQL"), + expectedToolcacheVersion === undefined + ? [] + : [expectedToolcacheVersion], + ); + if (expectedToolcacheVersion !== undefined) { + const cachedFolder = toolcache.find( + "CodeQL", + expectedToolcacheVersion, + ); + t.truthy(cachedFolder); + t.true(fs.existsSync(`${cachedFolder}.complete`)); + } + }); + }, + ); +} + const EXPLICITLY_REQUESTED_BUNDLE_TEST_CASES = [ { tagName: "codeql-bundle-2.17.6", diff --git a/src/feature-flags.ts b/src/feature-flags.ts index da7bcceade..afddaea2a4 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -164,6 +164,11 @@ export enum Feature { OverlayAnalysisStatusCheck = "overlay_analysis_status_check", /** Controls whether overlay build failures on the default branch are stored in the Actions cache. */ OverlayAnalysisStatusSave = "overlay_analysis_status_save", + /** + * Controls whether we may download a bundle containing only the single language being analysed, + * rather than the combined bundle that contains every language. + */ + PerLanguageBundles = "per_language_bundles", QaTelemetryEnabled = "qa_telemetry_enabled", /** Routes (some) API requests through the registry proxy. */ ProxyApiRequests = "proxy_api_requests", @@ -434,6 +439,11 @@ export const featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS", minimumVersion: undefined, }, + [Feature.PerLanguageBundles]: { + defaultValue: false, + envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES", + minimumVersion: undefined, + }, [Feature.QaTelemetryEnabled]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/src/init-action.ts b/src/init-action.ts index 8173d67aaa..79c509a5be 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -63,13 +63,13 @@ import { ToolsSource } from "./setup-codeql"; import { ActionName, InitStatusReport, - InitToolsDownloadFields, InitWithConfigStatusReport, createInitWithConfigStatusReport, createStatusReportBase, getActionsStatus, sendStatusReport, } from "./status-report"; +import { createInitToolsDownloadFields } from "./status-report/tools-download"; import { ToolsDownloadStatusReport } from "./tools-download"; import { ToolsFeature } from "./tools-features"; import { getCombinedTracerConfig } from "./tracer-config"; @@ -168,23 +168,10 @@ async function sendCompletedStatusReport( initStatusReport.computed_inputs.tools = toolsInput; } - const initToolsDownloadFields: InitToolsDownloadFields = {}; - - if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) { - initToolsDownloadFields.tools_download_duration_ms = - toolsDownloadStatusReport.downloadDurationMs; - } - if (toolsDownloadStatusReport?.extractionDurationMs !== undefined) { - initToolsDownloadFields.tools_extraction_duration_ms = - toolsDownloadStatusReport.extractionDurationMs; - } - if (toolsDownloadStatusReport?.totalDurationMs !== undefined) { - initToolsDownloadFields.tools_total_duration_ms = - toolsDownloadStatusReport.totalDurationMs; - } - if (toolsFeatureFlagsValid !== undefined) { - initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } + const initToolsDownloadFields = createInitToolsDownloadFields( + toolsDownloadStatusReport, + toolsFeatureFlagsValid, + ); if (config !== undefined) { // Append fields that are dependent on `config` diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts new file mode 100644 index 0000000000..b8f48512fe --- /dev/null +++ b/src/per-language-bundles.test.ts @@ -0,0 +1,177 @@ +import test from "ava"; + +import { ActionsEnvVars } from "./environment"; +import { Feature } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; +import { + getPerLanguageBundleLanguage, + MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + PerLanguageBundleOptions, +} from "./per-language-bundles"; +import { BundlePlatform } from "./platform"; +import { + createFeatures, + getRecordingLogger, + getTestEnv, + initAllState, + LoggedMessage, +} from "./testing-utils"; +import { GitHubVariant } from "./util"; + +/** Options for which we would use a per-language bundle. */ +const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = { + rawLanguages: ["java"], + // Any version at least as new as the minimum will do. + cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + compressionMethod: "zstd", + platform: BundlePlatform.Linux64, + variant: GitHubVariant.DOTCOM, +}; + +async function checkEligibility( + overrides: Partial, + stateOverrides: Partial> = {}, +) { + return getPerLanguageBundleLanguage( + initAllState({ + env: getTestEnv({ + [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted", + }), + features: createFeatures([Feature.PerLanguageBundles]), + logger: getRecordingLogger([], { logToConsole: false }), + ...stateOverrides, + }), + { ...ELIGIBLE_OPTIONS, ...overrides }, + ); +} + +for (const platform of Object.values(BundlePlatform)) { + test(`getPerLanguageBundleLanguage selects only supported languages on ${platform}`, async (t) => { + for (const language of Object.values(BuiltInLanguage)) { + const supported = + language === BuiltInLanguage.swift + ? platform === BundlePlatform.Osx64 + : platform === BundlePlatform.Linux64; + t.is( + await checkEligibility({ rawLanguages: [language], platform }), + supported ? language : undefined, + language, + ); + } + }); +} + +test("getPerLanguageBundleLanguage normalizes aliases before selecting a bundle", async (t) => { + t.is( + await checkEligibility({ rawLanguages: ["java-kotlin"] }), + BuiltInLanguage.java, + ); +}); + +test("getPerLanguageBundleLanguage rejects unknown platforms", async (t) => { + t.is(await checkEligibility({ platform: undefined }), undefined); +}); + +test("getPerLanguageBundleLanguage requires exactly one language", async (t) => { + t.is(await checkEligibility({ rawLanguages: undefined }), undefined); + t.is(await checkEligibility({ rawLanguages: [] }), undefined); + t.is(await checkEligibility({ rawLanguages: ["java", "python"] }), undefined); +}); + +test("getPerLanguageBundleLanguage requires a known language", async (t) => { + t.is(await checkEligibility({ rawLanguages: ["cobol"] }), undefined); +}); + +test("getPerLanguageBundleLanguage requires a zstd bundle", async (t) => { + t.is(await checkEligibility({ compressionMethod: "gzip" }), undefined); +}); + +test("getPerLanguageBundleLanguage requires GitHub.com", async (t) => { + // Other products resolve the combined bundle against their own instance, so asking for a + // per-language bundle they do not mirror would move the download off that instance. + for (const variant of [GitHubVariant.GHES, GitHubVariant.GHEC_DR]) { + t.is(await checkEligibility({ variant }), undefined); + } +}); + +test("getPerLanguageBundleLanguage requires a GitHub-hosted runner", async (t) => { + // A self-hosted runner may have a toolcache that persists between jobs, which is worth more than + // a smaller download. + t.is( + await checkEligibility( + {}, + { + env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "self-hosted" }), + }, + ), + undefined, + ); + + // Self-hosted runners are routinely configured to look like hosted ones, for example by mounting + // a persistent volume at `/opt/hostedtoolcache`, so we require the service to tell us explicitly. + t.is( + await checkEligibility( + {}, + { + env: getTestEnv({ RUNNER_TOOL_CACHE: "/opt/hostedtoolcache" }), + }, + ), + undefined, + ); +}); + +test("getPerLanguageBundleLanguage requires a supported release version", async (t) => { + t.is(await checkEligibility({ cliVersion: undefined }), undefined); + t.is(await checkEligibility({ cliVersion: "2.27.0" }), undefined); + t.is(await checkEligibility({ cliVersion: "2.27.1" }), BuiltInLanguage.java); +}); + +test("getPerLanguageBundleLanguage requires the feature flag", async (t) => { + t.is(await checkEligibility({}, { features: createFeatures([]) }), undefined); +}); + +test("getPerLanguageBundleLanguage explains a disabled feature before checking eligibility", async (t) => { + const messages: LoggedMessage[] = []; + const language = await getPerLanguageBundleLanguage( + initAllState({ + env: getTestEnv(), + features: createFeatures([]), + logger: getRecordingLogger(messages, { logToConsole: false }), + }), + { ...ELIGIBLE_OPTIONS, rawLanguages: undefined, cliVersion: undefined }, + ); + + t.is(language, undefined); + t.deepEqual( + messages.map((message) => message.message), + [ + "Not using a per-language CodeQL bundle since the per_language_bundles feature is disabled.", + ], + ); +}); + +test("getPerLanguageBundleLanguage skips only the release version check for the latest nightly", async (t) => { + const nightly = { isLatestNightly: true, cliVersion: undefined }; + t.is(await checkEligibility(nightly), BuiltInLanguage.java); + + for (const overrides of [ + { rawLanguages: undefined }, + { rawLanguages: ["java", "python"] }, + { compressionMethod: "gzip" as const }, + { platform: BundlePlatform.Osx64 }, + { variant: GitHubVariant.GHES }, + { variant: GitHubVariant.GHEC_DR }, + ]) { + t.is(await checkEligibility({ ...nightly, ...overrides }), undefined); + } + t.is( + await checkEligibility(nightly, { features: createFeatures([]) }), + undefined, + ); + t.is( + await checkEligibility(nightly, { + env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "self-hosted" }), + }), + undefined, + ); +}); diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts new file mode 100644 index 0000000000..f4e46403db --- /dev/null +++ b/src/per-language-bundles.ts @@ -0,0 +1,130 @@ +import * as semver from "semver"; + +import { ActionState } from "./action-common"; +import { isGitHubHostedRunner } from "./actions-util"; +import { Feature } from "./feature-flags"; +import { BuiltInLanguage, parseBuiltInLanguage } from "./languages"; +import { BundlePlatform } from "./platform"; +import * as tar from "./tar"; +import { GitHubVariant } from "./util"; + +/** Minimum CLI version for selecting a per-language release bundle. */ +export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; + +/** Languages with per-language bundles published for each platform. */ +const PER_LANGUAGE_BUNDLE_LANGUAGES: Readonly< + Record> +> = { + [BundlePlatform.Linux64]: new Set([ + BuiltInLanguage.actions, + BuiltInLanguage.cpp, + BuiltInLanguage.csharp, + BuiltInLanguage.go, + BuiltInLanguage.java, + BuiltInLanguage.javascript, + BuiltInLanguage.python, + BuiltInLanguage.ruby, + BuiltInLanguage.rust, + ]), + [BundlePlatform.LinuxArm64]: new Set(), + [BundlePlatform.Osx64]: new Set([BuiltInLanguage.swift]), + [BundlePlatform.Win64]: new Set(), +}; + +/** Inputs that determine whether we may download a per-language bundle. */ +export interface PerLanguageBundleOptions { + /** Explicit input only: autodetection needs a CLI instance. */ + rawLanguages: string[] | undefined; + /** Requested CLI version, if known. Ignored when requesting the latest nightly. */ + cliVersion: string | undefined; + compressionMethod: tar.CompressionMethod; + /** Platform for which the bundle is requested. */ + platform: BundlePlatform | undefined; + variant: GitHubVariant; + /** Whether the Action is selecting the latest nightly rather than a release version. */ + isLatestNightly?: boolean; +} + +/** Returns the eligible bundle language, or undefined for the combined bundle. */ +export async function getPerLanguageBundleLanguage( + { + env, + features, + logger, + }: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, + options: PerLanguageBundleOptions, +): Promise { + const { + rawLanguages, + cliVersion, + compressionMethod, + platform, + variant, + isLatestNightly, + } = options; + + const explain = (reason: string) => { + logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); + return undefined; + }; + + if (!(await features.getValue(Feature.PerLanguageBundles))) { + return explain(`the ${Feature.PerLanguageBundles} feature is disabled`); + } + + if (rawLanguages?.length !== 1) { + return explain( + `exactly one language must be requested via the 'languages' input, but ${ + rawLanguages?.length ?? 0 + } were`, + ); + } + + const language = parseBuiltInLanguage(rawLanguages[0]); + if (language === undefined) { + return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); + } + + if (compressionMethod !== "zstd") { + // Per-language bundles are only published as zstd archives. + return explain(`the bundle would be downloaded as '${compressionMethod}'`); + } + + if (variant !== GitHubVariant.DOTCOM) { + // Tenant mirrors may lack these assets, and an unreachable github.com fails with a + // connection error rather than a recoverable 404. + return explain(`we are running against ${variant}`); + } + + if (!isGitHubHostedRunner(env)) { + // Per-language installs stay out of the toolcache; self-hosted runners should retain + // the reusable combined bundle instead. + return explain("the job is not running on a GitHub-hosted runner"); + } + + // Check whether per-language bundles are published for the requested CLI version. + // Latest-nightly selection skips this release-version check, but not the other eligibility checks. + if (!isLatestNightly) { + if (cliVersion === undefined) { + return explain("the requested CLI version is unknown"); + } + + if (!semver.gte(cliVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { + return explain( + `the requested CodeQL version ${cliVersion} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the ` + + "first version for which per-language bundles are published", + ); + } + } + + if ( + platform === undefined || + !PER_LANGUAGE_BUNDLE_LANGUAGES[platform].has(language) + ) { + return explain( + `no per-language bundle is published for ${language} on ${platform ?? "an unknown platform"}`, + ); + } + + return language; +} diff --git a/src/platform.test.ts b/src/platform.test.ts new file mode 100644 index 0000000000..c8a6a7955d --- /dev/null +++ b/src/platform.test.ts @@ -0,0 +1,18 @@ +import test from "ava"; + +import { BundlePlatform, getBundlePlatform } from "./platform"; + +for (const [platform, arch, expected] of [ + ["linux", "x64", BundlePlatform.Linux64], + ["linux", "arm64", BundlePlatform.LinuxArm64], + ["linux", "ia32", BundlePlatform.Linux64], + ["darwin", "x64", BundlePlatform.Osx64], + ["darwin", "arm64", BundlePlatform.Osx64], + ["win32", "x64", BundlePlatform.Win64], + ["win32", "arm64", BundlePlatform.Win64], + ["freebsd", "x64", undefined], +] as const) { + test(`getBundlePlatform maps ${platform}/${arch} to ${expected ?? "an all-platform bundle"}`, (t) => { + t.is(getBundlePlatform(platform, arch), expected); + }); +} diff --git a/src/platform.ts b/src/platform.ts new file mode 100644 index 0000000000..1cc085d6ab --- /dev/null +++ b/src/platform.ts @@ -0,0 +1,26 @@ +/** Platform identifiers used in CodeQL bundle asset names. */ +export enum BundlePlatform { + Linux64 = "linux64", + LinuxArm64 = "linux-arm64", + Osx64 = "osx64", + Win64 = "win64", +} + +/** Returns the bundle platform, or undefined when an all-platform bundle is required. */ +export function getBundlePlatform( + platform: NodeJS.Platform = process.platform, + arch: NodeJS.Architecture = process.arch, +): BundlePlatform | undefined { + switch (platform) { + case "win32": + return BundlePlatform.Win64; + case "linux": + return arch === "arm64" + ? BundlePlatform.LinuxArm64 + : BundlePlatform.Linux64; + case "darwin": + return BundlePlatform.Osx64; + default: + return undefined; + } +} diff --git a/src/setup-codeql-action.ts b/src/setup-codeql-action.ts index bb6b73c9aa..4bd53e517f 100644 --- a/src/setup-codeql-action.ts +++ b/src/setup-codeql-action.ts @@ -22,11 +22,11 @@ import { ToolsSource } from "./setup-codeql"; import { ActionName, InitStatusReport, - InitToolsDownloadFields, createStatusReportBase, getActionsStatus, sendStatusReport, } from "./status-report"; +import { createInitToolsDownloadFields } from "./status-report/tools-download"; import { ToolsDownloadStatusReport } from "./tools-download"; import { checkDiskUsage, @@ -79,23 +79,10 @@ async function sendCompletedStatusReport( initStatusReport.computed_inputs.tools = toolsInput; } - const initToolsDownloadFields: InitToolsDownloadFields = {}; - - if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) { - initToolsDownloadFields.tools_download_duration_ms = - toolsDownloadStatusReport.downloadDurationMs; - } - if (toolsDownloadStatusReport?.extractionDurationMs !== undefined) { - initToolsDownloadFields.tools_extraction_duration_ms = - toolsDownloadStatusReport.extractionDurationMs; - } - if (toolsDownloadStatusReport?.totalDurationMs !== undefined) { - initToolsDownloadFields.tools_total_duration_ms = - toolsDownloadStatusReport.totalDurationMs; - } - if (toolsFeatureFlagsValid !== undefined) { - initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } + const initToolsDownloadFields = createInitToolsDownloadFields( + toolsDownloadStatusReport, + toolsFeatureFlagsValid, + ); await sendStatusReport({ ...initStatusReport, ...initToolsDownloadFields }); } diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index 41498cef7b..c35bdb8406 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; +import { performance } from "perf_hooks"; import * as github from "@actions/github"; import * as toolcache from "@actions/tool-cache"; @@ -10,10 +11,12 @@ import * as sinon from "sinon"; import * as actionsUtil from "./actions-util"; import * as api from "./api-client"; import * as diagnostics from "./diagnostics"; -import { ActionsEnvVars, EnvVar, ReadOnlyEnv } from "./environment"; +import { ActionsEnvVars, Env, EnvVar, ReadOnlyEnv } from "./environment"; import { Feature } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; import { getRunnerLogger } from "./logging"; import { getCacheRestoreKeyPrefix } from "./overlay/caching"; +import { MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION } from "./per-language-bundles"; import * as setupCodeql from "./setup-codeql"; import * as tar from "./tar"; import { @@ -22,9 +25,12 @@ import { SAMPLE_DEFAULT_CLI_VERSION, SAMPLE_DOTCOM_API_DETAILS, checkExpectedLogMessages, + checkUnexpectedLogMessages, createFeatures, createTestConfig, getRecordingLogger, + getTestEnv, + initAllState, makeMacro, mockBundleDownloadApi, setupActionsVars, @@ -34,6 +40,7 @@ import * as toolsDownload from "./tools-download"; import { getErrorMessage, GitHubVariant, + HTTPError, initializeEnvironment, withTmpDir, } from "./util"; @@ -44,6 +51,45 @@ test.beforeEach(() => { initializeEnvironment("1.2.3"); }); +function stubDownloadAndExtract() { + return sinon + .stub(toolsDownload, "downloadAndExtract") + .callsFake(async (_url, _compressionMethod, dest) => { + fs.mkdirSync(dest, { recursive: true }); + return { downloadDurationMs: 200, totalDurationMs: 300 }; + }); +} + +/** Models a hosted Linux runner with zstd and the latest nightly release. */ +function stubHostedNightly(tagName: string) { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const fetchRelease = sinon + .stub, ReturnType>() + .rejects(new Error("Unexpected API request in nightly bundle test")); + fetchRelease + .withArgs( + "https://api.github.com/repos/dsp-testing/codeql-cli-nightlies/releases?per_page=1&page=1&prerelease=true", + sinon.match({ method: "GET" }), + ) + .callsFake( + async () => + new Response(JSON.stringify([{ tag_name: tagName }]), { + headers: { "content-type": "application/json" }, + }), + ); + const client = github.getOctokit("123", { + request: { fetch: fetchRelease }, + }); + sinon.stub(api, "getApiClient").value(() => client); + return fetchRelease; +} + test.serial("parse codeql bundle url version", (t) => { t.deepEqual( setupCodeql.getCodeQLURLVersion( @@ -102,9 +148,9 @@ test.serial( await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); const tagName = "codeql-bundle-v1.2.3"; - mockBundleDownloadApi({ tagName }); + const url = mockBundleDownloadApi({ tagName }); const source = await setupCodeql.getCodeQLSource( - `https://github.com/github/codeql-action/releases/download/${tagName}/codeql-bundle-linux64.tar.gz`, + url, SAMPLE_DEFAULT_CLI_VERSION, undefined, // rawLanguages false, // useOverlayAwareDefaultCliVersion @@ -115,8 +161,14 @@ test.serial( getRunnerLogger(true), ); - t.is(source.sourceType, "download"); - t.is(source["cliVersion"], "1.2.3"); + t.deepEqual(source, { + bundle: { kind: "combined", url }, + bundleVersion: "v1.2.3", + cliVersion: "1.2.3", + compressionMethod: "gzip", + sourceType: "download", + toolsVersion: "1.2.3", + } satisfies setupCodeql.CodeQLDownloadSource); }); }, ); @@ -198,7 +250,8 @@ for (const { t.is(source.sourceType, "download"); if (source.sourceType === "download") { t.is(source.compressionMethod, expectedCompressionMethod); - t.true(source.codeqlURL.endsWith(`/${expectedBundleName}`)); + t.is(source.bundle.kind, "combined"); + t.true(source.bundle.url.endsWith(`/${expectedBundleName}`)); } }); }, @@ -252,23 +305,14 @@ test.serial( const logger = getRecordingLogger(loggedMessages); const features = createFeatures([]); - // Stub the downloadCodeQL function to prevent downloading artefacts - // during testing from being called. - sinon.stub(setupCodeql, "downloadCodeQL").resolves({ - codeqlFolder: "codeql", - statusReport: { - downloadDurationMs: 200, - totalDurationMs: 300, - }, - toolsVersion: LINKED_CLI_VERSION.cliVersion, - }); + const extractStub = stubDownloadAndExtract(); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); const result = await setupCodeql.setupCodeQLBundle( "linked", SAMPLE_DOTCOM_API_DETAILS, - "tmp/codeql_action_test/", + tmpDir, GitHubVariant.DOTCOM, SAMPLE_DEFAULT_CLI_VERSION, undefined, // rawLanguages @@ -280,6 +324,19 @@ test.serial( // Basic sanity check that the version we got back is indeed // the linked (default) CLI version. t.is(result.toolsVersion, LINKED_CLI_VERSION.cliVersion); + t.true(extractStub.calledOnce); + t.is( + result.codeqlFolder, + toolcache.find("CodeQL", LINKED_CLI_VERSION.cliVersion), + ); + t.deepEqual(result.toolsDownloadStatusReport, { + downloadDurationMs: 200, + totalDurationMs: 300, + }); + checkUnexpectedLogMessages(t, loggedMessages, [ + "Not caching the CodeQL tools", + "Could not cache CodeQL tools", + ]); // Ensure message logging CodeQL CLI version was present in user logs. const expected_message: string = `Using CodeQL CLI version ${LINKED_CLI_VERSION.cliVersion}`; @@ -305,23 +362,14 @@ test.serial( "https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.16.0/codeql-bundle-linux64.tar.gz"; const expectedVersion = "2.16.0"; - // Stub the downloadCodeQL function to prevent downloading artefacts - // during testing from being called. - sinon.stub(setupCodeql, "downloadCodeQL").resolves({ - codeqlFolder: "codeql", - statusReport: { - downloadDurationMs: 200, - totalDurationMs: 300, - }, - toolsVersion: expectedVersion, - }); + const extractStub = stubDownloadAndExtract(); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); const result = await setupCodeql.setupCodeQLBundle( bundleUrl, SAMPLE_DOTCOM_API_DETAILS, - "tmp/codeql_action_test/", + tmpDir, GitHubVariant.DOTCOM, SAMPLE_DEFAULT_CLI_VERSION, undefined, // rawLanguages @@ -333,6 +381,13 @@ test.serial( // Basic sanity check that the version we got back is indeed the version that the // bundle contains.. t.is(result.toolsVersion, expectedVersion); + t.true(extractStub.calledOnce); + t.is(extractStub.firstCall.args[0], bundleUrl); + t.is(result.codeqlFolder, toolcache.find("CodeQL", expectedVersion)); + t.deepEqual(result.toolsDownloadStatusReport, { + downloadDurationMs: 200, + totalDurationMs: 300, + }); // Ensure message logging CodeQL CLI version was present in user logs. const expected_message: string = `Using CodeQL CLI version 2.16.0 sourced from ${bundleUrl} .`; @@ -348,29 +403,17 @@ test.serial( ); test.serial( - "getCodeQLSource correctly returns nightly CLI version when tools == nightly", + "getCodeQLSource and setupCodeQLBundle preserve the nightly version when tools == nightly", async (t) => { const loggedMessages: LoggedMessage[] = []; const logger = getRecordingLogger(loggedMessages); const features = createFeatures([]); + const extractStub = stubDownloadAndExtract(); const expectedDate = "30260213"; const expectedTag = `codeql-bundle-${expectedDate}`; - // Ensure that we consistently select "zstd" for the test. - sinon.stub(process, "platform").value("linux"); - sinon.stub(tar, "isZstdAvailable").resolves({ - available: true, - foundZstdBinary: true, - }); - - const client = github.getOctokit("123"); - const listReleases = sinon.stub(client.rest.repos, "listReleases"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - listReleases.resolves({ - data: [{ tag_name: expectedTag }], - } as any); - sinon.stub(api, "getApiClient").value(() => client); + stubHostedNightly(expectedTag); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); @@ -390,14 +433,34 @@ test.serial( const expectedVersion = `0.0.0-${expectedDate}`; const expectedURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/${setupCodeql.getCodeQLBundleName("zstd")}`; t.deepEqual(source, { + bundle: { kind: "combined", url: expectedURL }, bundleVersion: expectedDate, cliVersion: undefined, - codeqlURL: expectedURL, compressionMethod: "zstd", sourceType: "download", toolsVersion: expectedVersion, } satisfies setupCodeql.CodeQLToolsSource); + const result = await setupCodeql.setupCodeQLBundle( + "nightly", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + ["javascript"], + false, // useOverlayAwareDefaultCliVersion + features, + logger, + ); + + t.true(extractStub.calledOnce); + t.is(extractStub.firstCall.args[0], expectedURL); + t.is(result.toolsVersion, source.toolsVersion); + t.is(result.toolsSource, setupCodeql.ToolsSource.Download); + t.is(result.codeqlFolder, toolcache.find("CodeQL", expectedVersion)); + t.true(fs.existsSync(`${result.codeqlFolder}.complete`)); + t.deepEqual(toolcache.findAllVersions("CodeQL"), [expectedVersion]); + // Afterwards, ensure that we see the expected messages in the log. checkExpectedLogMessages(t, loggedMessages, [ "Using the latest CodeQL CLI nightly, as requested by 'tools: nightly'.", @@ -419,20 +482,7 @@ test.serial( const expectedDate = "30260213"; const expectedTag = `codeql-bundle-${expectedDate}`; - // Ensure that we consistently select "zstd" for the test. - sinon.stub(process, "platform").value("linux"); - sinon.stub(tar, "isZstdAvailable").resolves({ - available: true, - foundZstdBinary: true, - }); - - const client = github.getOctokit("123"); - const listReleases = sinon.stub(client.rest.repos, "listReleases"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - listReleases.resolves({ - data: [{ tag_name: expectedTag }], - } as any); - sinon.stub(api, "getApiClient").value(() => client); + stubHostedNightly(expectedTag); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); @@ -453,9 +503,9 @@ test.serial( const expectedVersion = `0.0.0-${expectedDate}`; const expectedURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/${setupCodeql.getCodeQLBundleName("zstd")}`; t.deepEqual(source, { + bundle: { kind: "combined", url: expectedURL }, bundleVersion: expectedDate, cliVersion: undefined, - codeqlURL: expectedURL, compressionMethod: "zstd", sourceType: "download", toolsVersion: expectedVersion, @@ -472,6 +522,330 @@ test.serial( }, ); +for (const bundlePath of [ + "codeql-bundle.tar.gz", + "codeql-bundle.tar.zst", + "codeql-bundle-/codeql-bundle.tar.gz", + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-ruby-linux64.tar.zst", +]) { + test.serial( + `setupCodeQLBundle reports an unknown version for ${bundlePath}`, + async (t) => { + const extractStub = stubDownloadAndExtract(); + const downloadSpy = sinon.spy(setupCodeql, "downloadCodeQL"); + const url = `https://example.com/${bundlePath}`; + const messages: LoggedMessage[] = []; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + url, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRecordingLogger(messages), + ); + + t.true(extractStub.calledOnce); + t.is(extractStub.firstCall.args[0], url); + t.is(downloadSpy.firstCall.args[0].bundleVersion, undefined); + t.is(downloadSpy.firstCall.args[0].toolsVersion, "unknown"); + t.is(result.toolsVersion, "unknown"); + t.is(result.toolsSource, setupCodeql.ToolsSource.Download); + t.is( + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, + bundlePath === "codeql-bundle-ruby-linux64.tar.zst" + ? BuiltInLanguage.ruby + : undefined, + ); + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.true(fs.existsSync(result.codeqlFolder)); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + checkExpectedLogMessages(t, messages, [ + `Using CodeQL CLI sourced from ${url}`, + bundlePath === "codeql-bundle-ruby-linux64.tar.zst" + ? "Not caching the CodeQL tools because they came from a bundle that contains only a single language." + : `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${url}.`, + ]); + }); + }, + ); +} + +test.serial( + "setupCodeQLBundle preserves local installation without cleaning the toolcache", + async (t) => { + const cleanupSpy = sinon.spy(toolsDownload, "deleteToolcacheBundles"); + const downloadSpy = sinon.spy(setupCodeql, "downloadCodeQL"); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + const cachedDirectory = createToolcacheEntry( + tmpDir, + "CodeQL", + CLEANUP_STALE_VERSION, + ); + const result = await setupCodeql.setupCodeQLBundle( + path.join(__dirname, "../src/testdata/codeql-bundle.tar.gz"), + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.CleanupToolcacheBundles]), + getRunnerLogger(true), + ); + + t.is(result.toolsVersion, "local"); + t.is(result.toolsSource, setupCodeql.ToolsSource.Local); + t.is(result.toolsDownloadStatusReport, undefined); + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.true(fs.existsSync(result.codeqlFolder)); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + t.true(fs.existsSync(cachedDirectory)); + t.true(cleanupSpy.notCalled); + t.true(downloadSpy.notCalled); + t.is(process.env[EnvVar.HAS_SET_UP_CODEQL], "true"); + }); + }, +); + +for (const toolsInput of ["nightly", "nightly-latest"]) { + test.serial( + `getCodeQLSource selects the latest per-language nightly for tools == ${toolsInput}`, + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + const latestNightlyRequest = stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + toolsInput, + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.deepEqual(source, { + sourceType: "download", + bundle: { + kind: "per-language", + language: BuiltInLanguage.java, + url: `${baseURL}/codeql-bundle-java-linux64.tar.zst`, + combinedBundleURL: `${baseURL}/codeql-bundle-linux64.tar.zst`, + }, + bundleVersion: "30260213", + cliVersion: undefined, + compressionMethod: "zstd", + toolsVersion: "0.0.0-30260213", + } satisfies setupCodeql.CodeQLDownloadSource); + t.true(latestNightlyRequest.calledOnce); + }); + }, + ); +} + +test.serial( + "getCodeQLSource downloads a combined nightly bundle when per-language selection is ineligible", + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + for (const { languages, features } of [ + // The per-language feature is disabled. + { languages: ["java"], features: createFeatures([]) }, + // More than one language requires a combined bundle. + { + languages: ["java", "python"], + features: createFeatures([Feature.PerLanguageBundles]), + }, + ]) { + const source = await setupCodeql.getCodeQLSource( + "nightly", + SAMPLE_DEFAULT_CLI_VERSION, + languages, + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + features, + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.deepEqual(source.bundle, { + kind: "combined", + url: `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/codeql-bundle-linux64.tar.zst`, + }); + } + } + }); + }, +); + +for (const perLanguageBundles of [false, true]) { + test.serial( + `getCodeQLSource uses the latest ${perLanguageBundles ? "per-language" : "combined"} bundle for a forced nightly`, + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + const latestNightlyRequest = stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); + const source = await setupCodeql.getCodeQLSource( + undefined, // toolsInput: the nightly is selected by ForceNightly + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures( + perLanguageBundles + ? [Feature.ForceNightly, Feature.PerLanguageBundles] + : [Feature.ForceNightly], + ), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + t.true(latestNightlyRequest.calledOnce); + if (source.sourceType === "download") { + const combinedURL = `${baseURL}/codeql-bundle-linux64.tar.zst`; + t.deepEqual( + source.bundle, + perLanguageBundles + ? { + kind: "per-language", + language: BuiltInLanguage.java, + url: `${baseURL}/codeql-bundle-java-linux64.tar.zst`, + combinedBundleURL: combinedURL, + } + : { kind: "combined", url: combinedURL }, + ); + } + }); + }, + ); +} + +for (const date of ["20200101", "30260213"]) { + for (const bundle of ["combined", "per-language"] as const) { + test.serial( + `getCodeQLSource preserves an explicit ${bundle} nightly URL for ${date}`, + async (t) => { + const latestNightlyRequest = stubHostedNightly( + "codeql-bundle-30260213", + ); + const asset = + bundle === "combined" + ? "codeql-bundle-linux64.tar.zst" + : "codeql-bundle-java-linux64.tar.zst"; + const url = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/codeql-bundle-${date}/${asset}`; + const features = createFeatures([Feature.PerLanguageBundles]); + const logger = getRecordingLogger([], { logToConsole: false }); + const error = new HTTPError("Not Found", 404); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .rejects(error); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + url, + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, + features, + logger, + ); + t.deepEqual(source, { + sourceType: "download", + bundle: + bundle === "combined" + ? { kind: "combined", url } + : { kind: "per-language", url, language: BuiltInLanguage.java }, + bundleVersion: date, + cliVersion: undefined, + compressionMethod: "zstd", + toolsVersion: `0.0.0-${date}`, + } satisfies setupCodeql.CodeQLDownloadSource); + + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + url, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, + features, + logger, + ), + { is: error }, + ); + t.true(extractStub.calledOnce); + t.is(extractStub.firstCall.args[0], url); + t.true(latestNightlyRequest.notCalled); + }); + }, + ); + } +} + +test.serial( + "getCodeQLSource reports a missing release tag when a toolcache entry disappears", + async (t) => { + sinon + .stub(toolcache, "findAllVersions") + .returns([MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION]); + sinon.stub(toolcache, "find").returns(""); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); + await t.throwsAsync( + setupCodeql.getCodeQLSource( + "toolcache", + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, + createFeatures([]), + getRunnerLogger(true), + ), + { + message: + "Could not determine a release tag for the requested CodeQL bundle.", + }, + ); + }); + }, +); + test.serial( "getCodeQLSource correctly returns latest version from toolcache when tools == toolcache", async (t) => { @@ -756,6 +1130,475 @@ test.serial( }, ); +const PER_LANGUAGE_CLI_VERSION = { + enabledVersions: [ + { + cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + tagName: `codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`, + }, + ], +}; + +test.serial( + "getCodeQLBundleName returns a per-language bundle name only when a language is specified", + (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + t.is( + setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.java), + "codeql-bundle-java-linux64.tar.zst", + ); + t.is( + setupCodeql.getCodeQLBundleName("zstd"), + "codeql-bundle-linux64.tar.zst", + ); + }, +); + +test.serial("getCodeQLBundleName names the Swift bundle for macOS", (t) => { + sinon.stub(process, "platform").value("darwin"); + t.is( + setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.swift), + "codeql-bundle-swift-osx64.tar.zst", + ); +}); + +test.serial( + "getCodeQLSource downloads the per-language bundle for a single explicit language", + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + undefined, + PER_LANGUAGE_CLI_VERSION, + ["java-kotlin"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.true( + source.bundle.url.endsWith("/codeql-bundle-java-linux64.tar.zst"), + `Unexpected URL ${source.bundle.url}`, + ); + t.is(source.bundle.kind, "per-language"); + if (source.bundle.kind === "per-language") { + t.is(source.bundle.language, BuiltInLanguage.java); + t.true( + source.bundle.combinedBundleURL?.endsWith( + "/codeql-bundle-linux64.tar.zst", + ), + ); + } + } + }); + }, +); + +test.serial( + "getCodeQLSource downloads the combined bundle when the feature is disabled", + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + undefined, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([]), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.true(source.bundle.url.endsWith("/codeql-bundle-linux64.tar.zst")); + t.is(source.bundle.kind, "combined"); + } + }); + }, +); + +for (const fallback of [false, true]) { + test.serial( + `setupCodeQLBundle retains the selected release identity for an opaque asset URL${fallback ? " with fallback" : ""}`, + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + sinon.stub(actionsUtil, "isRunningLocalAction").returns(false); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const tag = PER_LANGUAGE_CLI_VERSION.enabledVersions[0].tagName; + const assetURL = + "https://api.github.com/repos/codeql-testing/action-fork/releases/assets/123"; + const combinedURL = `${assetURL}4`; + const fetchRelease = sinon + .stub, ReturnType>() + .callsFake( + async () => + new Response( + JSON.stringify({ + assets: [ + { name: "codeql-bundle-java-linux64.tar.zst", url: assetURL }, + { + name: "codeql-bundle-linux64.tar.zst", + url: combinedURL, + }, + ], + }), + { headers: { "content-type": "application/json" } }, + ), + ); + const client = github.getOctokit("123", { + request: { fetch: fetchRelease }, + }); + sinon.stub(api, "getApiClient").value(() => client); + const authorizationSpy = sinon.spy(api, "getAuthorizationHeaderFor"); + const extractStub = stubDownloadAndExtract(); + if (fallback) { + extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404)); + } + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { + GITHUB_ACTION_REPOSITORY: "codeql-testing/action-fork", + }); + const result = await setupCodeql.setupCodeQLBundle( + undefined, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.true(fetchRelease.calledTwice); + t.is( + fetchRelease.firstCall.args[0], + `https://api.github.com/repos/codeql-testing/action-fork/releases/tags/${tag}`, + ); + t.is(extractStub.callCount, fallback ? 2 : 1); + t.is(extractStub.firstCall.args[0], assetURL); + t.is(extractStub.lastCall.args[0], fallback ? combinedURL : assetURL); + t.is(authorizationSpy.callCount, extractStub.callCount); + t.is(authorizationSpy.firstCall.args[2], assetURL); + t.is( + authorizationSpy.lastCall.args[2], + fallback ? combinedURL : assetURL, + ); + t.is(extractStub.lastCall.args[3], "token token"); + t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION); + t.is( + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, + fallback ? undefined : BuiltInLanguage.java, + ); + t.is( + result.toolsDownloadStatusReport?.perLanguage + ?.tools_per_language_bundle_fallback, + fallback ? true : undefined, + ); + if (fallback) { + t.is( + result.codeqlFolder, + toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION), + ); + t.true(fs.existsSync(`${result.codeqlFolder}.complete`)); + } else { + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + } + }); + }, + ); +} + +for (const bundle of ["per-language", "combined", "fallback"] as const) { + test.serial( + `setupCodeQLBundle preserves the nightly version for a ${bundle} download`, + async (t) => { + const expectedDate = "30260213"; + const expectedTag = `codeql-bundle-${expectedDate}`; + const expectedVersion = `0.0.0-${expectedDate}`; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + const combinedURL = `${baseURL}/codeql-bundle-linux64.tar.zst`; + const perLanguageURL = `${baseURL}/codeql-bundle-javascript-linux64.tar.zst`; + const loggedMessages: LoggedMessage[] = []; + const logger = getRecordingLogger(loggedMessages); + + stubHostedNightly(expectedTag); + delete process.env[EnvVar.HAS_SET_UP_CODEQL]; + + const downloadSpy = sinon.spy(setupCodeql, "downloadCodeQL"); + let elapsedMs = 1000; + sinon.stub(performance, "now").callsFake(() => elapsedMs); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .callsFake(async (_url, _compressionMethod, dest) => { + if (bundle === "fallback" && extractStub.callCount === 1) { + elapsedMs += 700.2; + throw new HTTPError("Not Found", 404); + } + elapsedMs += 300.2; + fs.mkdirSync(dest, { recursive: true }); + return { + downloadDurationMs: 200, + extractionDurationMs: 100, + totalDurationMs: 300, + }; + }); + const addDiagnostic = sinon.stub(diagnostics, "addNoLanguageDiagnostic"); + const features = createFeatures([ + Feature.PerLanguageBundles, + Feature.CleanupToolcacheBundles, + ]); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + "nightly", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + bundle === "combined" ? ["javascript", "python"] : ["javascript"], + false, // useOverlayAwareDefaultCliVersion + features, + logger, + ); + + const source = downloadSpy.firstCall.args[0]; + t.is(result.toolsVersion, expectedVersion); + t.is(result.toolsVersion, source.toolsVersion); + t.is( + source.bundle.kind, + bundle === "combined" ? "combined" : "per-language", + ); + t.is(result.codeqlFolder, extractStub.lastCall.args[2]); + t.is( + result.toolsDownloadStatusReport?.totalDurationMs, + bundle === "fallback" ? 1000 : 300, + ); + t.is(result.toolsDownloadStatusReport?.downloadDurationMs, 200); + t.is(result.toolsDownloadStatusReport?.extractionDurationMs, 100); + t.is( + (await downloadSpy.lastCall.returnValue).statusReport.perLanguage + ?.tools_bundle_language, + bundle === "per-language" ? BuiltInLanguage.javascript : undefined, + ); + t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); + t.is(downloadSpy.callCount, extractStub.callCount); + t.is( + extractStub.firstCall.args[0], + bundle === "combined" ? combinedURL : perLanguageURL, + ); + t.is( + extractStub.lastCall.args[0], + bundle === "per-language" ? perLanguageURL : combinedURL, + ); + t.is( + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, + bundle === "per-language" ? BuiltInLanguage.javascript : undefined, + ); + t.is( + result.toolsDownloadStatusReport?.perLanguage + ?.tools_per_language_bundle_fallback, + bundle === "fallback" ? true : undefined, + ); + t.is( + addDiagnostic + .getCalls() + .filter( + (call) => + call.args[1].source?.id === + "codeql-action/toolcache-bundle-cleanup", + ).length, + 1, + ); + if (bundle === "fallback") { + t.deepEqual(downloadSpy.secondCall.args[0], { + ...source, + bundle: { kind: "combined", url: combinedURL }, + }); + checkExpectedLogMessages(t, loggedMessages, [ + `No per-language CodeQL bundle for 'javascript' was found at ${perLanguageURL}`, + ]); + } + if (bundle === "per-language") { + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + } else { + t.is( + result.codeqlFolder, + toolsDownload.getToolcacheDirectory(expectedVersion), + ); + t.true(fs.existsSync(`${result.codeqlFolder}.complete`)); + + const cachedResult = await setupCodeql.setupCodeQLBundle( + "nightly", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + ["javascript"], + false, // useOverlayAwareDefaultCliVersion + features, + logger, + ); + t.is(cachedResult.toolsSource, setupCodeql.ToolsSource.Toolcache); + t.is(cachedResult.toolsVersion, expectedVersion); + t.is(cachedResult.codeqlFolder, result.codeqlFolder); + t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); + } + }); + }, + ); +} + +for (const asset of [ + "codeql-bundle-ruby-linux64.tar.zst", + "codeql-bundle-%72uby-linux64.tar.zst", +]) { + test.serial( + `setupCodeQLBundle keeps explicitly requested ${asset} out of the toolcache`, + async (t) => { + const extractStub = stubDownloadAndExtract(); + const messages: LoggedMessage[] = []; + const url = `https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/${asset}`; + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + url, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRecordingLogger(messages), + ); + + t.true(extractStub.calledOnce); + t.is(extractStub.firstCall.args[0], url); + t.is(result.toolsVersion, "9.9.9"); + t.is( + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, + BuiltInLanguage.ruby, + ); + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + checkExpectedLogMessages(t, messages, [ + "Not caching the CodeQL tools because they came from a bundle that contains only a single language.", + ]); + checkUnexpectedLogMessages(t, messages, [ + "Could not cache CodeQL tools because we could not determine the bundle version", + ]); + }); + }, + ); +} + +for (const error of [ + new HTTPError("Internal Server Error", 500), + new Error("Connection reset"), +]) { + test.serial( + `setupCodeQLBundle does not fall back after ${error.message}`, + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .rejects(error); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + undefined, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ), + { is: error }, + ); + t.true(extractStub.calledOnce); + t.true( + extractStub.firstCall.args[0].endsWith( + "/codeql-bundle-java-linux64.tar.zst", + ), + ); + }); + }, + ); +} + +test.serial( + "setupCodeQLBundle does not substitute a bundle for an explicitly requested one that is missing", + async (t) => { + const error = new HTTPError("Not Found", 404); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .rejects(error); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + "https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ), + { is: error }, + ); + + t.true(extractStub.calledOnce); + }); + }, +); + test.serial( "getEnabledVersionsWithOverlayBaseDatabases returns flag-enabled versions present in cache, sorted desc", async (t) => { @@ -974,30 +1817,35 @@ async function runDownloadCodeQL( toolcacheRoot: string, features: Feature[], bundleVersion: string | undefined, + env = getTestEnv({ + [ActionsEnvVars.RUNNER_TOOL_CACHE]: toolcacheRoot, + }), ): Promise<{ codeqlFolder: string; cleanupDiagnostic: toolsDownload.ToolcacheCleanupResult | undefined; }> { - sinon - .stub(toolsDownload, "downloadAndExtract") - .callsFake(async (_url, _compressionMethod, dest) => { - // The real implementation creates the destination directory, which matters here because the - // cleanup deletes it first and `writeToolcacheMarkerFile` writes into its parent afterwards. - fs.mkdirSync(dest, { recursive: true }); - return { totalDurationMs: 1 }; - }); + stubDownloadAndExtract(); const addDiagnostic = sinon.stub(diagnostics, "addNoLanguageDiagnostic"); - const { codeqlFolder } = await setupCodeql.downloadCodeQL( - "https://example.com/codeql-bundle.tar.gz", - "gzip", - bundleVersion, - CLEANUP_CLI_VERSION, + const { codeqlFolder } = await setupCodeql.downloadCodeQLBundle( + initAllState({ + env, + features: createFeatures(features), + }), + { + bundle: { + kind: "combined", + url: "https://example.com/codeql-bundle.tar.gz", + }, + compressionMethod: "gzip", + bundleVersion, + cliVersion: CLEANUP_CLI_VERSION, + sourceType: "download", + toolsVersion: CLEANUP_CLI_VERSION, + }, SAMPLE_DOTCOM_API_DETAILS, undefined, // tarVersion toolcacheRoot, // tempDir - createFeatures(features), - getRunnerLogger(true), ); const diagnostic = addDiagnostic @@ -1026,7 +1874,7 @@ async function testToolcacheCleanup( }: { features: Feature[]; runnerEnvironment: string | undefined; - setUp?: () => void; + setUp?: (env: Env) => void; }, check: (context: { cleanupDiagnostic: toolsDownload.ToolcacheCleanupResult | undefined; @@ -1041,7 +1889,10 @@ async function testToolcacheCleanup( } else { process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = runnerEnvironment; } - setUp?.(); + const env = getTestEnv({ + [ActionsEnvVars.RUNNER_TOOL_CACHE]: tmpDir, + }); + setUp?.(env); // The extraction of the bundle would normally create this directory. const destinationDirectory = createToolcacheEntry( @@ -1060,6 +1911,7 @@ async function testToolcacheCleanup( tmpDir, features, CLEANUP_BUNDLE_VERSION, + env, ); t.true( @@ -1072,7 +1924,7 @@ async function testToolcacheCleanup( } test.serial( - "downloadCodeQL does not clean up the toolcache when the feature flag is disabled", + "downloadCodeQLBundle does not clean up the toolcache when the feature flag is disabled", async (t) => { await testToolcacheCleanup( t, @@ -1087,7 +1939,7 @@ test.serial( ); test.serial( - "downloadCodeQL does not clean up the toolcache when the runner is not GitHub-hosted", + "downloadCodeQLBundle does not clean up the toolcache when the runner is not GitHub-hosted", async (t) => { await testToolcacheCleanup( t, @@ -1105,7 +1957,7 @@ test.serial( ); test.serial( - "downloadCodeQL does not clean up the toolcache when the runner environment is unknown", + "downloadCodeQLBundle does not clean up the toolcache when the runner environment is unknown", async (t) => { // A runner that doesn't report its environment must be treated as not GitHub-hosted, since its // toolcache may well outlive the job. @@ -1125,7 +1977,7 @@ test.serial( ); test.serial( - "downloadCodeQL deletes other CodeQL bundles from the toolcache when enabled on a GitHub-hosted runner", + "downloadCodeQLBundle deletes other CodeQL bundles from the toolcache when enabled on a GitHub-hosted runner", async (t) => { await testToolcacheCleanup( t, @@ -1153,7 +2005,7 @@ test.serial( ); test.serial( - "downloadCodeQL reports no deleted versions when the toolcache has no CodeQL bundles", + "downloadCodeQLBundle reports no deleted versions when the toolcache has no CodeQL bundles", async (t) => { await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); @@ -1175,7 +2027,7 @@ test.serial( ); test.serial( - "downloadCodeQL continues when deleting a CodeQL bundle from the toolcache fails", + "downloadCodeQLBundle continues when deleting a CodeQL bundle from the toolcache fails", async (t) => { await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); @@ -1306,7 +2158,7 @@ test.serial( ); test.serial( - "downloadCodeQL does not follow a symlinked CodeQL toolcache directory", + "downloadCodeQLBundle does not follow a symlinked CodeQL toolcache directory", async (t) => { await withTmpDir(async (tmpDir) => { const toolcacheRoot = path.join(tmpDir, "toolcache"); @@ -1342,7 +2194,7 @@ test.serial( ); test.serial( - "downloadCodeQL does not clean up the toolcache once a step has already set up CodeQL", + "downloadCodeQLBundle does not clean up the toolcache once a step has already set up CodeQL", async (t) => { // `.github/workflows/codeql.yml` sets up CodeQL twice and then runs both returned paths. If the // second setup downloads, it must not delete the bundle the first one handed out. @@ -1351,8 +2203,8 @@ test.serial( { features: [Feature.CleanupToolcacheBundles], runnerEnvironment: "github-hosted", - setUp: () => { - process.env[EnvVar.HAS_SET_UP_CODEQL] = "true"; + setUp: (env) => { + env.set(EnvVar.HAS_SET_UP_CODEQL, "true"); }, }, ({ cleanupDiagnostic, destinationDirectory, staleDirectory }) => { @@ -1364,6 +2216,33 @@ test.serial( }, ); +test.serial( + "downloadCodeQLBundle checks the supplied environment before cleaning the toolcache", + async (t) => { + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + delete process.env[EnvVar.HAS_SET_UP_CODEQL]; + + const staleDirectory = createToolcacheEntry( + tmpDir, + "CodeQL", + CLEANUP_STALE_VERSION, + ); + const { codeqlFolder, cleanupDiagnostic } = await runDownloadCodeQL( + tmpDir, + [Feature.CleanupToolcacheBundles], + CLEANUP_BUNDLE_VERSION, + getTestEnv({ [EnvVar.HAS_SET_UP_CODEQL]: "true" }), + ); + + t.true(fs.existsSync(staleDirectory)); + t.true(fs.existsSync(`${codeqlFolder}.complete`)); + t.is(cleanupDiagnostic, undefined); + }); + }, +); + test.serial( "setupCodeQLBundle records that this job has set up CodeQL", async (t) => { @@ -1371,11 +2250,7 @@ test.serial( setupActionsVars(tmpDir, tmpDir); delete process.env[EnvVar.HAS_SET_UP_CODEQL]; - sinon.stub(setupCodeql, "downloadCodeQL").resolves({ - codeqlFolder: "codeql", - statusReport: { totalDurationMs: 1 }, - toolsVersion: LINKED_CLI_VERSION.cliVersion, - }); + stubDownloadAndExtract(); await setupCodeql.setupCodeQLBundle( "linked", @@ -1399,7 +2274,50 @@ test.serial( ); test.serial( - "downloadCodeQL cleans up the toolcache even when the download will not be cached", + "setupCodeQLBundle cleans up once and propagates a failed combined download", + async (t) => { + const error = new HTTPError("Not Found", 404); + const extractStub = stubDownloadAndExtract().rejects(error); + const cleanupSpy = sinon.spy(toolsDownload, "deleteToolcacheBundles"); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + delete process.env[EnvVar.HAS_SET_UP_CODEQL]; + const staleDirectory = createToolcacheEntry( + tmpDir, + "CodeQL", + CLEANUP_STALE_VERSION, + ); + + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + "linked", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.CleanupToolcacheBundles]), + getRunnerLogger(true), + ), + { is: error }, + ); + + t.true(cleanupSpy.calledOnce); + t.true(cleanupSpy.calledBefore(extractStub)); + t.true(extractStub.calledOnce); + t.false(fs.existsSync(staleDirectory)); + t.false(fs.existsSync(`${extractStub.firstCall.args[2]}.complete`)); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.is(process.env[EnvVar.HAS_SET_UP_CODEQL], undefined); + }); + }, +); + +test.serial( + "downloadCodeQLBundle cleans up the toolcache even when the download will not be cached", async (t) => { // A `tools` URL we can't derive a bundle version from is extracted to a temporary directory // rather than the toolcache, but the toolcache is on the same filesystem, so emptying it still @@ -1434,7 +2352,7 @@ test.serial( ); test.serial( - "downloadCodeQL reports a failure when the toolcache cannot be inspected", + "downloadCodeQLBundle reports a failure when the toolcache cannot be inspected", async (t) => { await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); @@ -1466,7 +2384,7 @@ test.serial( ); test.serial( - "downloadCodeQL does not clean up a toolcache on a different filesystem to the workspace", + "downloadCodeQLBundle does not clean up a toolcache on a different filesystem to the workspace", async (t) => { // Some runner images keep the toolcache on a different volume to the workspace, in which case // deleting the tools frees up disk space that the analysis cannot use. @@ -1516,7 +2434,7 @@ test.serial( ); test.serial( - "downloadCodeQL does not delete through a symlinked version directory", + "downloadCodeQLBundle does not delete through a symlinked version directory", async (t) => { await withTmpDir(async (tmpDir) => { const toolcacheRoot = path.join(tmpDir, "toolcache"); diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 69cc64e8fc..648435d805 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import { OutgoingHttpHeaders } from "http"; import * as path from "path"; +import { performance } from "perf_hooks"; import * as core from "@actions/core"; import * as toolcache from "@actions/tool-cache"; @@ -16,6 +17,7 @@ import { isRunningLocalAction, } from "./actions-util"; import * as api from "./api-client"; +import { CodeQLBundle, getCodeQLBundleFromUrl } from "./codeql-bundle"; import * as defaults from "./defaults.json"; import { addNoLanguageDiagnostic, @@ -30,8 +32,11 @@ import { Feature, FeatureEnablement, } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; import { Logger } from "./logging"; import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching"; +import { getPerLanguageBundleLanguage } from "./per-language-bundles"; +import { getBundlePlatform } from "./platform"; import * as tar from "./tar"; import { deleteToolcacheBundles, @@ -72,21 +77,25 @@ function getCodeQLBundleExtension( } } +/** + * Returns the name of the CodeQL bundle asset to download. + * + * @param compressionMethod The compression method of the bundle. + * @param language Optional language for a per-language bundle. If omitted, returns a combined bundle name. + */ export function getCodeQLBundleName( compressionMethod: tar.CompressionMethod, + language?: BuiltInLanguage, ): string { const extension = getCodeQLBundleExtension(compressionMethod); + const platform = getBundlePlatform(); - let platform: string; - if (process.platform === "win32") { - platform = "win64"; - } else if (process.platform === "linux") { - platform = process.arch === "arm64" ? "linux-arm64" : "linux64"; - } else if (process.platform === "darwin") { - platform = "osx64"; - } else { + if (platform === undefined) { return `codeql-bundle${extension}`; } + if (language !== undefined) { + return `codeql-bundle-${language}-${platform}${extension}`; + } return `codeql-bundle-${platform}${extension}`; } @@ -107,7 +116,7 @@ export function getCodeQLActionRepository(logger: Logger): string { async function getCodeQLBundleDownloadURL( tagName: string, apiDetails: api.GitHubApiDetails, - compressionMethod: tar.CompressionMethod, + codeQLBundleName: string, logger: Logger, ): Promise { const codeQLActionRepository = getCodeQLActionRepository(logger); @@ -126,7 +135,6 @@ async function getCodeQLBundleDownloadURL( return !self.slice(0, index).some((other) => deepEqual(source, other)); }, ); - const codeQLBundleName = getCodeQLBundleName(compressionMethod); for (const downloadSource of uniqueDownloadSources) { const [apiURL, repository] = downloadSource; // If we've reached the final case, short-circuit the API check since we know the bundle exists and is public. @@ -164,7 +172,7 @@ function tryGetBundleVersionFromTagName( tagName: string, logger: Logger, ): string | undefined { - const match = tagName.match(/^codeql-bundle-(.*)$/); + const match = tagName.match(/^codeql-bundle-(.+)$/); if (match === null || match.length < 2) { logger.debug(`Could not determine bundle version from tag ${tagName}.`); return undefined; @@ -215,6 +223,22 @@ export function convertToSemVer(version: string, logger: Logger): string { return s; } +/** A resolved download, including its bundle identity and version. */ +export interface CodeQLDownloadSource { + /** Distinguishes downloads from local archives and cached installations. */ + sourceType: "download"; + /** The bundle to download. */ + bundle: CodeQLBundle; + /** The compression format of the bundle archive. */ + compressionMethod: tar.CompressionMethod; + /** Bundle version of the tools, if known. */ + bundleVersion?: string; + /** Requested CLI version, if known. */ + cliVersion?: string; + /** Resolved version for telemetry, independent of whether the bundle can be cached. */ + toolsVersion: string; +} + export type CodeQLToolsSource = | { codeqlTarPath: string; @@ -229,17 +253,7 @@ export type CodeQLToolsSource = /** Human-readable description of the source of the tools for telemetry purposes. */ toolsVersion: string; } - | { - /** Bundle version of the tools, if known. */ - bundleVersion?: string; - /** CLI version of the tools, if known. */ - cliVersion?: string; - compressionMethod: tar.CompressionMethod; - codeqlURL: string; - sourceType: "download"; - /** Human-readable description of the source of the tools for telemetry purposes. */ - toolsVersion: string; - }; + | CodeQLDownloadSource; /** * Look for a version of the CodeQL tools in the cache which could override the requested CLI version. @@ -448,7 +462,7 @@ export async function getCodeQLSource( }; } - /** CLI version number, for example 2.12.6. */ + /** Requested CLI version number, for example 2.12.6. */ let cliVersion: string | undefined; /** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */ let tagName: string | undefined; @@ -458,6 +472,7 @@ export async function getCodeQLSource( * This does not always include a tag name. */ let url: string | undefined; + let bundle: CodeQLBundle | undefined; // We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the // `tools` input cannot be adjusted to explicitly request it. @@ -466,7 +481,8 @@ export async function getCodeQLSource( const forceNightly = forceNightlyValueFF && canForceNightlyWithFF; // For advanced workflows, a value from `CODEQL_NIGHTLY_TOOLS_INPUTS` can be specified explicitly - // for the `tools` input in the workflow file. + // for the `tools` input. This is the computed input, so it may come from the repository property + // rather than the workflow file. const nightlyRequestedByToolsInput = toolsInput !== undefined && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput); @@ -500,7 +516,12 @@ export async function getCodeQLSource( `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`, ); } - toolsInput = await getNightlyToolsUrl(logger); + bundle = await getLatestNightlyBundle( + { env: getEnv(), features, logger }, + rawLanguages, + variant, + ); + toolsInput = bundle.url; } /** @@ -583,7 +604,7 @@ export async function getCodeQLSource( if (tagName) { const bundleVersion = tryGetBundleVersionFromTagName(tagName, logger); // If the bundle version is a semantic version, it is a CLI version number. - if (bundleVersion && semver.valid(bundleVersion)) { + if (bundleVersion !== undefined && semver.valid(bundleVersion)) { cliVersion = convertToSemVer(bundleVersion, logger); } } @@ -600,13 +621,15 @@ export async function getCodeQLSource( } const bundleVersion = - tagName && tryGetBundleVersionFromTagName(tagName, logger); - const humanReadableVersion = + tagName !== undefined + ? tryGetBundleVersionFromTagName(tagName, logger) + : undefined; + const resolvedVersion = cliVersion ?? - (bundleVersion && convertToSemVer(bundleVersion, logger)) ?? - tagName ?? - url ?? - "unknown"; + (bundleVersion !== undefined + ? convertToSemVer(bundleVersion, logger) + : undefined); + const humanReadableVersion = resolvedVersion ?? tagName ?? url ?? "unknown"; logger.debug( "Attempting to obtain CodeQL tools. " + @@ -721,18 +744,55 @@ export async function getCodeQLSource( let compressionMethod: tar.CompressionMethod; if (!url) { + const bundleTagName = tagName; + if (bundleTagName === undefined) { + throw new Error( + "Could not determine a release tag for the requested CodeQL bundle.", + ); + } + compressionMethod = cliVersion !== undefined && (await useZstdBundle(cliVersion, tarSupportsZstd)) ? "zstd" : "gzip"; - url = await getCodeQLBundleDownloadURL( - tagName!, - apiDetails, - compressionMethod, - logger, + const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + { env: getEnv(), features, logger }, + { + rawLanguages, + cliVersion, + compressionMethod, + platform: getBundlePlatform(), + variant, + }, ); + + // Resolves the combined or per-language bundle URL for the requested release. + const resolveBundleURL = (language?: BuiltInLanguage) => + getCodeQLBundleDownloadURL( + bundleTagName, + apiDetails, + getCodeQLBundleName(compressionMethod, language), + logger, + ); + + const combinedBundleURL = await resolveBundleURL(); + if (perLanguageBundleLanguage !== undefined) { + logger.info( + `Selected the per-language CodeQL bundle for '${perLanguageBundleLanguage}'.`, + ); + url = await resolveBundleURL(perLanguageBundleLanguage); + bundle = { + kind: "per-language", + url, + language: perLanguageBundleLanguage, + combinedBundleURL, + }; + } else { + url = combinedBundleURL; + bundle = { kind: "combined", url }; + } } else { const method = tar.inferCompressionMethod(url); if (method === undefined) { @@ -742,6 +802,13 @@ export async function getCodeQLSource( ); } compressionMethod = method; + + bundle ??= getCodeQLBundleFromUrl(url); + if (bundle.kind === "per-language") { + logger.info( + `${url} appears to be a CodeQL bundle that contains only ${bundle.language}.`, + ); + } } if (cliVersion) { @@ -750,12 +817,12 @@ export async function getCodeQLSource( logger.info(`Using CodeQL CLI sourced from ${url} .`); } return { - bundleVersion: tagName && tryGetBundleVersionFromTagName(tagName, logger), + bundle, + bundleVersion, cliVersion, - codeqlURL: url, compressionMethod, sourceType: "download", - toolsVersion: cliVersion ?? humanReadableVersion, + toolsVersion: resolvedVersion ?? "unknown", }; } @@ -769,7 +836,7 @@ async function tryGetFallbackToolcacheVersion( logger: Logger, ): Promise { const bundleVersion = tryGetBundleVersionFromTagName(tagName, logger); - if (!bundleVersion) { + if (bundleVersion === undefined) { return undefined; } const fallbackVersion = convertToSemVer(bundleVersion, logger); @@ -783,20 +850,17 @@ async function tryGetFallbackToolcacheVersion( // Exported using `export const` for testing purposes. Specifically, we want to // be able to stub this function and have other functions in this file use that stub. export const downloadCodeQL = async function ( - codeqlURL: string, - compressionMethod: tar.CompressionMethod, - maybeBundleVersion: string | undefined, - maybeCliVersion: string | undefined, + source: CodeQLDownloadSource, apiDetails: api.GitHubApiDetails, tarVersion: tar.TarVersion | undefined, tempDir: string, - features: FeatureEnablement, logger: Logger, ): Promise<{ codeqlFolder: string; statusReport: ToolsDownloadStatusReport; - toolsVersion: string; }> { + const { bundle, compressionMethod } = source; + const codeqlURL = bundle.url; const parsedCodeQLURL = new URL(codeqlURL); const searchParams = new URLSearchParams(parsedCodeQLURL.search); const headers: OutgoingHttpHeaders = { @@ -815,17 +879,11 @@ export const downloadCodeQL = async function ( ); } - const toolcacheInfo = getToolcacheDestinationInfo( - maybeBundleVersion, - maybeCliVersion, - logger, + const toolcacheDestination = getToolcacheDestination({ logger }, source); + const extractedBundlePath = toolcacheDestination.orElse( + getTempExtractionDir(tempDir), ); - const extractedBundlePath = - toolcacheInfo?.path ?? getTempExtractionDir(tempDir); - - await tryDeleteToolcacheBundles({ env: getEnv(), features, logger }); - const statusReport = await downloadAndExtract( codeqlURL, compressionMethod, @@ -836,46 +894,53 @@ export const downloadCodeQL = async function ( logger, ); - if (!toolcacheInfo) { - logger.debug( - "Could not cache CodeQL tools because we could not determine the bundle version from the " + - `URL ${codeqlURL}.`, - ); - return { - codeqlFolder: extractedBundlePath, - statusReport, - toolsVersion: maybeCliVersion ?? "unknown", - }; + if (toolcacheDestination.isSuccess()) { + writeToolcacheMarkerFile(toolcacheDestination.value, logger); + } else { + logger.debug(toolcacheDestination.value); } - writeToolcacheMarkerFile(toolcacheInfo.path, logger); - return { codeqlFolder: extractedBundlePath, - statusReport, - toolsVersion: maybeCliVersion ?? toolcacheInfo.version, + statusReport: + bundle.kind === "per-language" + ? { + ...statusReport, + perLanguage: { tools_bundle_language: bundle.language }, + } + : statusReport, }; }; -function getToolcacheDestinationInfo( - maybeBundleVersion: string | undefined, - maybeCliVersion: string | undefined, - logger: Logger, -): { path: string; version: string } | undefined { - if (maybeBundleVersion) { - const version = getCanonicalToolcacheVersion( - maybeCliVersion, - maybeBundleVersion, - logger, +/** + * Returns the canonical toolcache directory, or the reason the bundle cannot be cached. + */ +function getToolcacheDestination( + { logger }: ActionState<["Logger"]>, + source: CodeQLDownloadSource, +): util.Result { + if (source.bundle.kind !== "combined") { + return new util.Failure( + "Not caching the CodeQL tools because they came from a bundle that contains only a " + + "single language.", + ); + } + if (!source.bundleVersion) { + return new util.Failure( + "Could not cache CodeQL tools because we could not determine the bundle version from the " + + `URL ${source.bundle.url}.`, ); - - return { - path: getToolcacheDirectory(version), - version, - }; } - return undefined; + return new util.Success( + getToolcacheDirectory( + getCanonicalToolcacheVersion( + source.cliVersion, + source.bundleVersion, + logger, + ), + ), + ); } /** @@ -1000,7 +1065,6 @@ export async function setupCodeQLBundle( ); let codeqlFolder: string; - let toolsVersion = source.toolsVersion; let toolsDownloadStatusReport: ToolsDownloadStatusReport | undefined; let toolsSource: ToolsSource; switch (source.sourceType) { @@ -1021,18 +1085,13 @@ export async function setupCodeQLBundle( toolsSource = ToolsSource.Toolcache; break; case "download": { - const result = await downloadCodeQL( - source.codeqlURL, - source.compressionMethod, - source.bundleVersion, - source.cliVersion, + const result = await downloadCodeQLBundle( + { env: getEnv(), features, logger }, + source, apiDetails, zstdAvailability.version, tempDir, - features, - logger, ); - toolsVersion = result.toolsVersion; codeqlFolder = result.codeqlFolder; toolsDownloadStatusReport = result.statusReport; toolsSource = ToolsSource.Download; @@ -1050,10 +1109,77 @@ export async function setupCodeQLBundle( codeqlFolder, toolsDownloadStatusReport, toolsSource, - toolsVersion, + toolsVersion: source.toolsVersion, }; } +/** + * Performs eligible toolcache cleanup once, then downloads and extracts the resolved bundle. + * + * If an automatically selected per-language bundle is missing, downloads the combined bundle + * from the same release instead. Explicit bundle URLs are not substituted. + * + * @returns The extraction directory and download timings. + */ +export async function downloadCodeQLBundle( + action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, + source: CodeQLDownloadSource, + apiDetails: api.GitHubApiDetails, + tarVersion: tar.TarVersion | undefined, + tempDir: string, +): Promise<{ + codeqlFolder: string; + statusReport: ToolsDownloadStatusReport; +}> { + const { bundle } = source; + const { logger } = action; + + await tryDeleteToolcacheBundles(action); + + const startTime = performance.now(); + try { + return await downloadCodeQL( + source, + apiDetails, + tarVersion, + tempDir, + logger, + ); + } catch (e) { + if ( + bundle.kind !== "per-language" || + bundle.combinedBundleURL === undefined || + util.asHTTPError(e)?.status !== 404 + ) { + throw e; + } + logger.warning( + `No per-language CodeQL bundle for '${bundle.language}' was found at ${bundle.url}, so ` + + "falling back to the bundle that contains all languages. This analysis will still " + + "produce correct results, but will take longer to set up.", + ); + + const result = await downloadCodeQL( + { + ...source, + bundle: { kind: "combined", url: bundle.combinedBundleURL }, + }, + apiDetails, + tarVersion, + tempDir, + logger, + ); + return { + ...result, + statusReport: { + ...result.statusReport, + totalDurationMs: util.durationMsSince(startTime), + perLanguage: { tools_per_language_bundle_fallback: true }, + }, + }; + } +} + async function useZstdBundle( cliVersion: string, tarSupportsZstd: boolean, @@ -1071,9 +1197,15 @@ function getTempExtractionDir(tempDir: string) { } /** - * Get the URL of the latest nightly CodeQL bundle. + * Selects a bundle from the latest nightly release, preferring a per-language bundle when eligible. + * Records the combined bundle URL from that release for use if the selected asset is missing. */ -async function getNightlyToolsUrl(logger: Logger) { +async function getLatestNightlyBundle( + action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, + rawLanguages: string[] | undefined, + variant: util.GitHubVariant, +): Promise { + const { logger } = action; const zstdAvailability = await tar.isZstdAvailable(logger); // The nightly is guaranteed to have a zstd bundle const compressionMethod = (await useZstdBundle( @@ -1083,6 +1215,15 @@ async function getNightlyToolsUrl(logger: Logger) { ? "zstd" : "gzip"; + const language = await getPerLanguageBundleLanguage(action, { + rawLanguages, + cliVersion: undefined, + compressionMethod, + platform: getBundlePlatform(), + variant, + isLatestNightly: true, + }); + try { // Since nightlies are prereleases, we can't just download the latest release // on the repository. So instead we need to find the latest pre-release @@ -1098,7 +1239,17 @@ async function getNightlyToolsUrl(logger: Logger) { if (!latestRelease) { throw new Error("Could not find the latest nightly release."); } - return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`; + const assetUrl = (name: string) => + `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`; + const url = assetUrl(getCodeQLBundleName(compressionMethod, language)); + return language === undefined + ? { kind: "combined", url } + : { + kind: "per-language", + url, + language, + combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod)), + }; } catch (e) { throw new Error( `Failed to retrieve the latest nightly release: ${util.wrapError(e)}`, diff --git a/src/status-report.ts b/src/status-report.ts index a2acd631d6..c392b51922 100644 --- a/src/status-report.ts +++ b/src/status-report.ts @@ -624,29 +624,6 @@ export interface InitWithConfigStatusReport extends InitStatusReport { config_file: string; } -/** Fields of the init status report populated when the tools source is `download`. */ -export interface InitToolsDownloadFields { - /** - * Time taken to download the bundle, in milliseconds. Not populated when the bundle is downloaded - * and extracted concurrently. - */ - tools_download_duration_ms?: number; - /** - * Time taken to extract the bundle, in milliseconds. Not populated when the bundle is downloaded - * and extracted concurrently. - */ - tools_extraction_duration_ms?: number; - /** - * Total time taken to make the bundle available on disk, in milliseconds. This includes any time - * spent on a streaming attempt that failed and fell back to downloading before extracting. - */ - tools_total_duration_ms?: number; - /** - * Whether the relevant tools dotcom feature flags have been misconfigured. - * Only populated if we attempt to determine the default version based on the dotcom feature flags. */ - tools_feature_flags_valid?: boolean; -} - /** * Composes a `InitWithConfigStatusReport` from the given values. * diff --git a/src/status-report/tools-download.test.ts b/src/status-report/tools-download.test.ts new file mode 100644 index 0000000000..856bf882f6 --- /dev/null +++ b/src/status-report/tools-download.test.ts @@ -0,0 +1,80 @@ +import test from "ava"; + +import { BuiltInLanguage } from "../languages"; + +import { createInitToolsDownloadFields } from "./tools-download"; + +test("createInitToolsDownloadFields omits absent download data", (t) => { + t.deepEqual(createInitToolsDownloadFields(undefined, undefined), {}); +}); + +test("createInitToolsDownloadFields reports feature flags without a download", (t) => { + t.deepEqual(createInitToolsDownloadFields(undefined, false), { + tools_feature_flags_valid: false, + }); +}); + +test("createInitToolsDownloadFields reports only the total for a streaming download", (t) => { + t.deepEqual( + createInitToolsDownloadFields({ totalDurationMs: 300 }, undefined), + { tools_total_duration_ms: 300 }, + ); +}); + +test("createInitToolsDownloadFields preserves per-language metadata", (t) => { + t.deepEqual( + createInitToolsDownloadFields( + { + totalDurationMs: 300, + perLanguage: { tools_bundle_language: BuiltInLanguage.java }, + }, + true, + ), + { + tools_total_duration_ms: 300, + tools_bundle_language: BuiltInLanguage.java, + tools_feature_flags_valid: true, + }, + ); +}); + +test("createInitToolsDownloadFields preserves fallback and per-attempt timings", (t) => { + t.deepEqual( + createInitToolsDownloadFields( + { + downloadDurationMs: 200, + extractionDurationMs: 100, + totalDurationMs: 1000, + perLanguage: { tools_per_language_bundle_fallback: true }, + }, + undefined, + ), + { + tools_download_duration_ms: 200, + tools_extraction_duration_ms: 100, + tools_total_duration_ms: 1000, + tools_per_language_bundle_fallback: true, + }, + ); +}); + +test("createInitToolsDownloadFields preserves zero durations and false flags", (t) => { + t.deepEqual( + createInitToolsDownloadFields( + { + downloadDurationMs: 0, + extractionDurationMs: 0, + totalDurationMs: 0, + perLanguage: { tools_per_language_bundle_fallback: false }, + }, + false, + ), + { + tools_download_duration_ms: 0, + tools_extraction_duration_ms: 0, + tools_total_duration_ms: 0, + tools_per_language_bundle_fallback: false, + tools_feature_flags_valid: false, + }, + ); +}); diff --git a/src/status-report/tools-download.ts b/src/status-report/tools-download.ts new file mode 100644 index 0000000000..a5f7dffbf8 --- /dev/null +++ b/src/status-report/tools-download.ts @@ -0,0 +1,56 @@ +import type { ToolsDownloadStatusReport } from "../tools-download"; + +/** Telemetry describing per-language bundle downloads. */ +export interface PerLanguageToolsStatusReport { + /** The language of the single-language bundle that was downloaded, if any. */ + tools_bundle_language?: string; + /** + * Whether we tried to download a single-language bundle, but it did not exist and we fell back to + * the combined bundle. + */ + tools_per_language_bundle_fallback?: boolean; +} + +/** Fields of the init status report populated when the tools source is `download`. */ +export interface InitToolsDownloadFields extends PerLanguageToolsStatusReport { + /** + * Time taken to download the bundle, in milliseconds. Not populated when the bundle is downloaded + * and extracted concurrently. + */ + tools_download_duration_ms?: number; + /** + * Time taken to extract the bundle, in milliseconds. Not populated when the bundle is downloaded + * and extracted concurrently. + */ + tools_extraction_duration_ms?: number; + /** + * Total time taken to make the bundle available on disk, including failed download attempts + * before a fallback, in milliseconds. + */ + tools_total_duration_ms?: number; + /** + * Whether the relevant tools dotcom feature flags have been misconfigured. + * Only populated if we attempt to determine the default version based on the dotcom feature flags. */ + tools_feature_flags_valid?: boolean; +} + +/** Converts download results to telemetry fields shared by the init and setup-codeql Actions. */ +export function createInitToolsDownloadFields( + report: ToolsDownloadStatusReport | undefined, + toolsFeatureFlagsValid: boolean | undefined, +): InitToolsDownloadFields { + const fields: InitToolsDownloadFields = { ...report?.perLanguage }; + if (report?.downloadDurationMs !== undefined) { + fields.tools_download_duration_ms = report.downloadDurationMs; + } + if (report?.extractionDurationMs !== undefined) { + fields.tools_extraction_duration_ms = report.extractionDurationMs; + } + if (report?.totalDurationMs !== undefined) { + fields.tools_total_duration_ms = report.totalDurationMs; + } + if (toolsFeatureFlagsValid !== undefined) { + fields.tools_feature_flags_valid = toolsFeatureFlagsValid; + } + return fields; +} diff --git a/src/testing-utils.ts b/src/testing-utils.ts index 7d33589ec6..f15cee2e71 100644 --- a/src/testing-utils.ts +++ b/src/testing-utils.ts @@ -32,6 +32,7 @@ import { } from "./feature-flags"; import { Logger } from "./logging"; import { OverlayDatabaseMode } from "./overlay/overlay-database-mode"; +import { getBundlePlatform } from "./platform"; import { ActionName } from "./status-report"; import { DEFAULT_DEBUG_ARTIFACT_NAME, @@ -933,21 +934,14 @@ export function mockBundleDownloadApi({ platformSpecific?: boolean; tagName: string; }): string { - const platform = - process.platform === "win32" - ? "win64" - : process.platform === "linux" - ? process.arch === "arm64" - ? "linux-arm64" - : "linux64" - : "osx64"; + const platform = platformSpecific ? getBundlePlatform() : undefined; const baseUrl = apiDetails?.url ?? "https://example.com"; const bundleUrls = ["tar.gz", "tar.zst"].map((extension) => { const relativeUrl = apiDetails ? `/${repo}/releases/download/${tagName}/codeql-bundle${ - platformSpecific ? `-${platform}` : "" + platform !== undefined ? `-${platform}` : "" }.${extension}` : `/download/${tagName}/codeql-bundle.${extension}`; diff --git a/src/tools-download.test.ts b/src/tools-download.test.ts index d2f15f4dc9..407e2ea489 100644 --- a/src/tools-download.test.ts +++ b/src/tools-download.test.ts @@ -1,8 +1,12 @@ import { once } from "events"; +import * as fs from "fs"; +import { ClientRequest, IncomingMessage } from "http"; import * as path from "path"; +import * as core from "@actions/core"; import * as toolcache from "@actions/tool-cache"; import test from "ava"; +import { https } from "follow-redirects"; import nock from "nock"; import * as sinon from "sinon"; @@ -10,14 +14,14 @@ import { getRunnerLogger } from "./logging"; import * as tar from "./tar"; import { setupTests } from "./testing-utils"; import { downloadAndExtract } from "./tools-download"; -import { withTmpDir } from "./util"; +import * as util from "./util"; setupTests(test); test.serial( "downloadAndExtract reports the durations when downloading before extracting", async (t) => { - await withTmpDir(async (tmpDir) => { + await util.withTmpDir(async (tmpDir) => { const archivePath = path.join(tmpDir, "codeql-bundle.tar.gz"); const destination = path.join(tmpDir, "codeql"); sinon.stub(toolcache, "downloadTool").resolves(archivePath); @@ -43,13 +47,16 @@ test.serial( test.serial( "downloadAndExtract falls back to downloading before extracting if streaming fails", async (t) => { - await withTmpDir(async (tmpDir) => { + await util.withTmpDir(async (tmpDir) => { sinon.stub(process, "platform").value("linux"); const archivePath = path.join(tmpDir, "codeql-bundle.tar.zst"); const destination = path.join(tmpDir, "codeql"); const downloadTool = sinon .stub(toolcache, "downloadTool") - .resolves(archivePath); + .callsFake(async () => { + t.false(fs.existsSync(destination)); + return archivePath; + }); const extract = sinon.stub(tar, "extract").resolves(destination); const extractTarZst = sinon.stub(tar, "extractTarZst").resolves(); const request = nock("https://example.com") @@ -78,10 +85,134 @@ test.serial( }, ); +test.serial( + "downloadAndExtract rethrows a 404 rather than retrying the download", + async (t) => { + await util.withTmpDir(async (tmpDir) => { + sinon.stub(process, "platform").value("linux"); + const destination = path.join(tmpDir, "codeql"); + const downloadTool = sinon.stub(toolcache, "downloadTool"); + const extractTarZst = sinon.stub(tar, "extractTarZst").resolves(); + const request = nock("https://example.com") + .get("/codeql-bundle.tar.zst") + .reply(404, "Not found"); + + const error = await t.throwsAsync( + downloadAndExtract( + "https://example.com/codeql-bundle.tar.zst", + "zstd", + destination, + undefined, + {}, + { type: "gnu", version: "1.34" }, + getRunnerLogger(true), + ), + { + instanceOf: util.HTTPError, + message: + "Failed to download CodeQL bundle from https://example.com/codeql-bundle.tar.zst. HTTP status code: 404.", + }, + ); + + t.is(error?.status, 404); + t.true(request.isDone()); + t.false(extractTarZst.called); + t.false(downloadTool.called); + t.false(fs.existsSync(destination)); + }); + }, +); + +test.serial( + "downloadAndExtract falls back to downloading before extracting on a server error", + async (t) => { + await util.withTmpDir(async (tmpDir) => { + sinon.stub(process, "platform").value("linux"); + const archivePath = path.join(tmpDir, "codeql-bundle.tar.zst"); + const destination = path.join(tmpDir, "codeql"); + const downloadTool = sinon + .stub(toolcache, "downloadTool") + .callsFake(async () => { + t.false(fs.existsSync(destination)); + return archivePath; + }); + const extract = sinon.stub(tar, "extract").resolves(destination); + const extractTarZst = sinon.stub(tar, "extractTarZst").resolves(); + const request = nock("https://example.com") + .get("/codeql-bundle.tar.zst") + .reply(500); + + const statusReport = await downloadAndExtract( + "https://example.com/codeql-bundle.tar.zst", + "zstd", + destination, + undefined, + {}, + { type: "gnu", version: "1.34" }, + getRunnerLogger(true), + ); + + t.assert(Number.isInteger(statusReport.downloadDurationMs)); + t.true(request.isDone()); + t.false(extractTarZst.called); + t.true(downloadTool.calledOnce); + t.true(extract.calledOnce); + }); + }, +); + +test.serial( + "downloadAndExtract handles an unknown status as a non-HTTP error", + async (t) => { + const asHTTPError = sinon.spy(util, "asHTTPError"); + await util.withTmpDir(async (tmpDir) => { + sinon.stub(process, "platform").value("linux"); + const archivePath = path.join(tmpDir, "codeql-bundle.tar.zst"); + const destination = path.join(tmpDir, "codeql"); + const response = sinon.createStubInstance(IncomingMessage); + response.statusCode = undefined; + sinon + .stub(https, "get") + .callsArgWith(2, response) + .returns(sinon.createStubInstance(ClientRequest)); + const warning = sinon.stub(core, "warning"); + const downloadTool = sinon + .stub(toolcache, "downloadTool") + .resolves(archivePath); + const extract = sinon.stub(tar, "extract").resolves(destination); + const extractTarZst = sinon.stub(tar, "extractTarZst").resolves(); + + await downloadAndExtract( + "https://example.com/codeql-bundle.tar.zst", + "zstd", + destination, + undefined, + {}, + { type: "gnu", version: "1.34" }, + getRunnerLogger(true), + ); + + t.is( + warning.firstCall.args[0], + "Failed to download and extract CodeQL bundle using streaming with error: Failed to download CodeQL bundle from https://example.com/codeql-bundle.tar.zst.", + ); + t.true(response.resume.calledOnce); + t.false(extractTarZst.called); + t.true(downloadTool.calledOnce); + t.true(extract.calledOnce); + }); + + t.true(asHTTPError.calledOnce); + t.true(asHTTPError.firstCall.args[0] instanceof Error); + t.false(asHTTPError.firstCall.args[0] instanceof util.HTTPError); + t.is(asHTTPError.firstCall.returnValue, undefined); + }, +); + test.serial( "downloadAndExtract reports only the total duration when streaming extraction", async (t) => { - await withTmpDir(async (tmpDir) => { + await util.withTmpDir(async (tmpDir) => { sinon.stub(process, "platform").value("linux"); const downloadTool = sinon.stub(toolcache, "downloadTool"); const extractTarZst = sinon diff --git a/src/tools-download.ts b/src/tools-download.ts index 492002fac8..a92dad0acf 100644 --- a/src/tools-download.ts +++ b/src/tools-download.ts @@ -13,8 +13,16 @@ import * as semver from "semver"; import { ActionState } from "./action-common"; import { ActionsEnvVars, getEnv, ReadOnlyEnv } from "./environment"; import { formatDuration, Logger } from "./logging"; +import type { PerLanguageToolsStatusReport } from "./status-report/tools-download"; import * as tar from "./tar"; -import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util"; +import { + asHTTPError, + cleanUpPath, + durationMsSince, + getErrorMessage, + getRequiredEnvParam, + HTTPError, +} from "./util"; /** * High watermark to use when streaming the download and extraction of the CodeQL tools. @@ -44,10 +52,11 @@ export type ToolsDownloadStatusReport = { */ extractionDurationMs?: number; /** - * Total time taken to make the bundle available on disk, in milliseconds. This includes any time - * spent on a streaming attempt that failed and fell back to downloading before extracting. + * Total time taken to make the bundle available on disk, including failed download attempts + * before a fallback, in milliseconds. */ totalDurationMs: number; + perLanguage?: PerLanguageToolsStatusReport; }; export async function downloadAndExtract( @@ -78,7 +87,7 @@ export async function downloadAndExtract( logger, ); - const totalDurationMs = Math.round(performance.now() - startTime); + const totalDurationMs = durationMsSince(startTime); logger.info( `Finished downloading and extracting CodeQL bundle to ${dest} (${formatDuration( totalDurationMs, @@ -88,14 +97,20 @@ export async function downloadAndExtract( return { totalDurationMs }; } } catch (e) { + // If we failed during processing, we want to clean up the destination directory + // before we either try again or give up. + await cleanUpPath(dest, "CodeQL bundle", logger); + + // Retrying a 404 is pointless: the asset does not exist, so downloading it a different way + // will fail in the same way. + if (asHTTPError(e)?.status === 404) { + throw e; + } + core.warning( `Failed to download and extract CodeQL bundle using streaming with error: ${getErrorMessage(e)}`, ); core.warning(`Falling back to downloading the bundle before extracting.`); - - // If we failed during processing, we want to clean up the destination directory - // before we try again. - await cleanUpPath(dest, "CodeQL bundle", logger); } const toolsDownloadStart = performance.now(); @@ -105,7 +120,7 @@ export async function downloadAndExtract( authorization, headers, ); - const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart); + const downloadDurationMs = durationMsSince(toolsDownloadStart); logger.info( `Finished downloading CodeQL bundle to ${archivedBundlePath} (${formatDuration( @@ -125,7 +140,7 @@ export async function downloadAndExtract( tarVersion, logger, ); - extractionDurationMs = Math.round(performance.now() - extractionStart); + extractionDurationMs = durationMsSince(extractionStart); logger.info( `Finished extracting CodeQL bundle to ${dest} (${formatDuration( extractionDurationMs, @@ -138,7 +153,7 @@ export async function downloadAndExtract( return { downloadDurationMs, extractionDurationMs, - totalDurationMs: Math.round(performance.now() - startTime), + totalDurationMs: durationMsSince(startTime), }; } @@ -191,9 +206,14 @@ async function downloadAndExtractZstdWithStreaming( if (response.statusCode !== 200) { // Discard the response body so that the connection can be released. response.resume(); - throw new Error( - `Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`, - ); + const baseMessage = `Failed to download CodeQL bundle from ${codeqlURL}.`; + if (response.statusCode !== undefined) { + throw new HTTPError( + `${baseMessage} HTTP status code: ${response.statusCode}.`, + response.statusCode, + ); + } + throw new Error(baseMessage); } await tar.extractTarZst(response, dest, tarVersion, logger); diff --git a/src/util.test.ts b/src/util.test.ts index cca457cbe6..074310279f 100644 --- a/src/util.test.ts +++ b/src/util.test.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as os from "os"; import path from "path"; +import { performance } from "perf_hooks"; import * as core from "@actions/core"; import test from "ava"; @@ -508,6 +509,26 @@ test("joinAtMost - truncates list if array is > than limit", (t) => { t.false(result.includes("test6")); }); +test.serial( + "durationMsSince rounds elapsed milliseconds rather than the timestamps", + (t) => { + const startTime = 1000.25; + const now = sinon.stub(performance, "now"); + for (const [endTime, expected] of [ + [1000.25, 0], + [1000.74, 0], + [1000.75, 1], + [1001.74, 1], + [1001.75, 2], + [2000.74, 1000], + [2000.75, 1001], + ]) { + now.returns(endTime); + t.is(util.durationMsSince(startTime), expected); + } + }, +); + test("Success creates a success result", (t) => { const result = new util.Success("test value"); t.true(result.isSuccess()); diff --git a/src/util.ts b/src/util.ts index f6258b2853..456cd7c3d2 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2,6 +2,7 @@ import * as fs from "fs"; import * as fsPromises from "fs/promises"; import * as os from "os"; import * as path from "path"; +import { performance } from "perf_hooks"; import * as core from "@actions/core"; import * as io from "@actions/io"; @@ -84,9 +85,32 @@ export async function withTmpDir( body: (tmpDir: string) => Promise, ): Promise { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "codeql-action-")); - const result = await body(tmpDir); - await fs.promises.rm(tmpDir, { force: true, recursive: true }); - return result; + try { + return await body(tmpDir); + } finally { + await fs.promises.rm(tmpDir, { force: true, recursive: true }); + } +} + +/** + * Creates a temporary file with the given contents, runs the given body, and + * then deletes the file. Note that, to create a temporary file, we first create + * a temporary directory via {@link withTmpDir} and then create the file within + * that directory. + * @param baseFileName The name to assign the temporary file. + * @param contents The contents to write to the temporary file. + * @param body The function to execute with the temporary file. + */ +export async function withTmpFile( + baseFileName: string, + contents: string, + body: (filePath: string) => Promise | T, +): Promise { + return withTmpDir(async (tmpDir) => { + const filePath = path.join(tmpDir, baseFileName); + fs.writeFileSync(filePath, contents); + return body(filePath); + }); } /** @@ -658,6 +682,11 @@ export async function bundleDb( return databaseBundlePath; } +/** Returns the elapsed milliseconds, rounded, since a `performance.now()` timestamp. */ +export function durationMsSince(startTime: number): number { + return Math.round(performance.now() - startTime); +} + /** * @param milliseconds time to delay * @param opts options diff --git a/tsconfig.json b/tsconfig.json index d2b39b5ef1..66545447c5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -37,5 +37,5 @@ "@octokit/core/dist-types/types": ["./node_modules/@octokit/core/dist-types/types.d.ts"] }, }, - "exclude": ["node_modules", "pr-checks", "scripts/changetool"] + "exclude": ["node_modules", "pr-checks"] } diff --git a/unreleased-change-notes/.gitkeep b/unreleased-change-notes/.gitkeep new file mode 100644 index 0000000000..e69de29bb2