Support cache-aware CodeQL bundles (codeql-bundle 0.5.0)

.github/actions/install-qlt-local/action.yml

@@ -60,7 +60,7 @@ runs:
         pip install -U pyinstaller 
 
         # run the packaging 
-        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/
+        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/
       env:
         GH_TOKEN: ${{ github.token }}
 

.github/workflows/internal-build-release-linux64.yml

@@ -50,7 +50,7 @@ jobs:
         pip install -U pyinstaller 
 
         # run the packaging 
-        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/
+        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/
       env:
         GH_TOKEN: ${{ github.token }}
 

.github/workflows/internal-build-release-macos64.yml

@@ -48,7 +48,7 @@ jobs:
         pip install -U pyinstaller 
 
         # run the packaging 
-        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/macos-arm64/tools/
+        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/macos-arm64/tools/
       env:
         GH_TOKEN: ${{ github.token }}
 

.github/workflows/internal-build-release-win64.yml

@@ -45,7 +45,7 @@ jobs:
         pip install -U pyinstaller 
 
         # run the packaging 
-        .\scripts\build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory .\src\CodeQLToolkit.Core\bin\Release\net6.0\publish\windows-x64\tools\
+        .\scripts\build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory .\src\CodeQLToolkit.Core\bin\Release\net6.0\publish\windows-x64\tools\
       env:
         GH_TOKEN: ${{ github.token }}
 

developer_guide.md

@@ -15,7 +15,7 @@ Note that we keep recent copies of tools (for local debugging purposes) in the `
 **CodeQL Bundle**
 
 ```
-./scripts/build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Debug/net6.0/tools
+./scripts/build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Debug/net6.0/tools
 ```
 
 

example/cpp/customizations/src/codeql-pack.lock.yml

@@ -1,18 +1,4 @@
 ---
 lockVersion: 1.0.0
-dependencies:
-  codeql/cpp-all:
-    version: 0.12.2
-  codeql/dataflow:
-    version: 0.1.5
-  codeql/rangeanalysis:
-    version: 0.0.4
-  codeql/ssa:
-    version: 0.2.5
-  codeql/tutorial:
-    version: 0.2.5
-  codeql/typetracking:
-    version: 0.2.5
-  codeql/util:
-    version: 0.2.5
+dependencies: {}
 compiled: false

example/cpp/customizations/src/qlpack.yml

@@ -2,4 +2,4 @@ library: true
 name: qlt/cpp-customizations
 version: 0.0.1
 dependencies:
-  "codeql/cpp-all": "0.12.2"
+  "codeql/cpp-all": "12.0.0"

example/cpp/customizations/test/codeql-pack.lock.yml

@@ -1,18 +1,4 @@
 ---
 lockVersion: 1.0.0
-dependencies:
-  codeql/cpp-all:
-    version: 0.12.2
-  codeql/dataflow:
-    version: 0.1.5
-  codeql/rangeanalysis:
-    version: 0.0.4
-  codeql/ssa:
-    version: 0.2.5
-  codeql/tutorial:
-    version: 0.2.5
-  codeql/typetracking:
-    version: 0.2.5
-  codeql/util:
-    version: 0.2.5
+dependencies: {}
 compiled: false

example/cpp/customizations/test/post-bundle/FooExternalSourceFunction/Foo.ql

@@ -1,5 +1,5 @@
 import cpp
-import qlt.Customizations
+import qlt.cpp_customizations.Customizations
 import semmle.code.cpp.security.FlowSources
 
 select any(RemoteFlowSource s)

example/cpp/stuff/src/codeql-pack.lock.yml

@@ -1,18 +1,4 @@
 ---
 lockVersion: 1.0.0
-dependencies:
-  codeql/cpp-all:
-    version: 0.12.2
-  codeql/dataflow:
-    version: 0.1.5
-  codeql/rangeanalysis:
-    version: 0.0.4
-  codeql/ssa:
-    version: 0.2.5
-  codeql/tutorial:
-    version: 0.2.5
-  codeql/typetracking:
-    version: 0.2.5
-  codeql/util:
-    version: 0.2.5
+dependencies: {}
 compiled: false

example/cpp/stuff/src/qlpack.yml

@@ -6,4 +6,4 @@ description: Default description
 suites: 
 license: 
 dependencies:
-  codeql/cpp-all: "0.12.2"
+  codeql/cpp-all: "12.0.0"

example/cpp/stuff/test/codeql-pack.lock.yml

@@ -1,18 +1,4 @@
 ---
 lockVersion: 1.0.0
-dependencies:
-  codeql/cpp-all:
-    version: 0.12.2
-  codeql/dataflow:
-    version: 0.1.5
-  codeql/rangeanalysis:
-    version: 0.0.4
-  codeql/ssa:
-    version: 0.2.5
-  codeql/tutorial:
-    version: 0.2.5
-  codeql/typetracking:
-    version: 0.2.5
-  codeql/util:
-    version: 0.2.5
+dependencies: {}
 compiled: false

example/cpp/stuff2/src/codeql-pack.lock.yml

@@ -1,18 +1,4 @@
 ---
 lockVersion: 1.0.0
-dependencies:
-  codeql/cpp-all:
-    version: 0.12.2
-  codeql/dataflow:
-    version: 0.1.5
-  codeql/rangeanalysis:
-    version: 0.0.4
-  codeql/ssa:
-    version: 0.2.5
-  codeql/tutorial:
-    version: 0.2.5
-  codeql/typetracking:
-    version: 0.2.5
-  codeql/util:
-    version: 0.2.5
+dependencies: {}
 compiled: false

example/cpp/stuff2/src/qlpack.yml

@@ -6,4 +6,4 @@ description: Default description
 suites: 
 license: 
 dependencies:
-  codeql/cpp-all: "0.12.2"
+  codeql/cpp-all: "12.0.0"

example/cpp/stuff2/test/codeql-pack.lock.yml

@@ -1,18 +1,4 @@
 ---
 lockVersion: 1.0.0
-dependencies:
-  codeql/cpp-all:
-    version: 0.12.2
-  codeql/dataflow:
-    version: 0.1.5
-  codeql/rangeanalysis:
-    version: 0.0.4
-  codeql/ssa:
-    version: 0.2.5
-  codeql/tutorial:
-    version: 0.2.5
-  codeql/typetracking:
-    version: 0.2.5
-  codeql/util:
-    version: 0.2.5
+dependencies: {}
 compiled: false

example/qlt.conf.json

@@ -1,9 +1,9 @@
 {
-  "CodeQLCLI": "2.15.5",
-  "CodeQLStandardLibrary": "codeql-cli/v2.15.5",
-  "CodeQLCLIBundle": "codeql-bundle-v2.15.5",
+  "CodeQLCLI": "2.26.1",
+  "CodeQLStandardLibrary": "codeql-cli/v2.26.1",
+  "CodeQLCLIBundle": "codeql-bundle-v2.26.1",
   "EnableCustomCodeQLBundles": true,
-  "CodeQLStandardLibraryIdent": "codeql-cli_v2.15.5",
+  "CodeQLStandardLibraryIdent": "codeql-cli_v2.26.1",
   "CodeQLPackConfiguration" : [
     {
       "Name": "qlt/cpp-customizations",
@@ -15,4 +15,4 @@
       "ReferencesBundle" : true 
     }
   ]
-}
+}

scripts/build_codeql_bundle_dist.ps1

@@ -54,8 +54,27 @@ if ($LASTEXITCODE -ne 0) {
 # Move into the cli directory
 Push-Location "codeql_bundle"
 
+# PyInstaller only freezes Python modules, so any package data files read at
+# runtime (via importlib.resources) have to be added explicitly. The separator
+# expected by --add-data is os.pathsep, which differs per platform, so it is
+# resolved here rather than hardcoded. Older codeql-bundle releases do not ship
+# these files, hence the Test-Path guard.
+$PathSeparator = [System.IO.Path]::PathSeparator
+$DataArgs = @()
+
+foreach ($DataFile in @("supported-codeql-bundles.json", "supported-codeql-bundles.schema.json")) {
+    if (Test-Path $DataFile) {
+        $DataArgs += "--add-data"
+        $DataArgs += "${DataFile}${PathSeparator}codeql_bundle"
+    }
+    else {
+        Write-Host "Note: data file '$DataFile' not present in this release, skipping."
+    }
+}
+
 # Build executable with pyinstaller
-pyinstaller -F -n codeql_bundle cli.py
+Write-Host "Running: pyinstaller -F -n codeql_bundle $($DataArgs -join ' ') cli.py"
+pyinstaller -F -n codeql_bundle @DataArgs cli.py
 if ($LASTEXITCODE -ne 0) {
     throw "PyInstaller build failed"
 }

src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs

@@ -42,14 +42,16 @@ public void Register(Command parentCommand)
             var customBundleOption = new Option<bool>("--custom-bundle", () => false, "Build a custom bundle and compile the bundle.") { IsRequired = true };
             var quickBundleOption = new Option<bool>("--quick-bundle", () => false, "Build a custom bundle and DO NOT compile the bundle.") { IsRequired = true };
             var packsOption = new Option<string[]>("--packs", "When creating bundles, this specifies the packs to include, Example `pack1 pack2 pack3`. You may specify also as `--pack pack1 --pack2 --pack3`") { IsRequired = false, AllowMultipleArgumentsPerToken = true };
+            var cacheDirOption = new Option<string>("--cache-dir", () => null, "Directory the bundle tool uses for downloaded bundles and compilation caches. Restore it across CI runs (e.g. with actions/cache) to avoid re-downloading and to reuse published compilation caches. Overrides the 'CacheDir' value in qlt.conf.json. If unset, the bundle tool's platform-specific default is used.") { IsRequired = false };
 
             installCommand.Add(customBundleOption);
             installCommand.Add(quickBundleOption);
             installCommand.Add(packsOption);
+            installCommand.Add(cacheDirOption);
 
             runCommand.Add(installCommand);
 
-            installCommand.SetHandler((basePath, automationType, customBundleOption, quickBundleOption, packs) =>
+            installCommand.SetHandler((basePath, automationType, customBundleOption, quickBundleOption, packs, cacheDir) =>
             {
                 Log<CodeQLCommandFeature>.G().LogInformation("Executing install command...");
 
@@ -59,11 +61,12 @@ public void Register(Command parentCommand)
                     AutomationTarget = automationType,
                     CustomBundles = customBundleOption,
                     QuickBundles = quickBundleOption,
-                    Packs = packs
+                    Packs = packs,
+                    CacheDir = cacheDir
                 }.Run();
 
 
-            }, Globals.BasePathOption, Globals.AutomationTypeOption, customBundleOption, quickBundleOption, packsOption);
+            }, Globals.BasePathOption, Globals.AutomationTypeOption, customBundleOption, quickBundleOption, packsOption, cacheDirOption);
         }
 
         public int Run()

src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs

@@ -15,6 +15,7 @@ public class InstallCommand : CommandTarget
         public bool CustomBundles { get; set; }
         public bool QuickBundles { get; set; }
         public string[] Packs { get; set; }
+        public string CacheDir { get; set; }
 
         void SetEnvironmentVariableMultiTarget(string name, string value)
         {
@@ -50,6 +51,15 @@ public override void Run()
 
             // First, check if CodeQL is installed.
             var installation = CodeQLInstallation.LoadFromConfig(Base);
+
+            // A cache directory supplied on the command line takes precedence
+            // over any value loaded from `qlt.conf.json`.
+            if (!string.IsNullOrEmpty(CacheDir))
+            {
+                Log<InstallCommand>.G().LogInformation($"Overriding bundle cache directory from the command line: {CacheDir}");
+                installation.CacheDir = CacheDir;
+            }
+
             if (CustomBundles || QuickBundles)
             {
                 installation.EnableCustomCodeQLBundles = true;