Use bqrs diff instead of sarif diff

Author: paldepind

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -203,6 +203,10 @@ interface BqrsDecodeOptions {
   entities?: string[];
 }
 
+interface BqrsDiffOptions {
+  retainResultSets?: string[];
+}
+
 type OnLineCallback = (line: string) => Promise<string | undefined>;
 
 type VersionChangedListener = (
@@ -1255,9 +1259,11 @@ export class CodeQLCliServer implements Disposable {
   async bqrsDiff(
     bqrsPath1: string,
     bqrsPath2: string,
+    options?: BqrsDiffOptions,
   ): Promise<{
     uniquePath1: string;
     uniquePath2: string;
+    path: string;
     cleanup: () => Promise<void>;
   }> {
     const { path, cleanup } = await dir({ unsafeCleanup: true });
@@ -1270,13 +1276,14 @@ export class CodeQLCliServer implements Disposable {
         uniquePath1,
         "--right",
         uniquePath2,
-        "--retain-result-sets=''",
+        "--retain-result-sets",
+        options?.retainResultSets?.join(",") ?? "",
         bqrsPath1,
         bqrsPath2,
       ],
       "Diffing bqrs files",
     );
-    return { uniquePath1, uniquePath2, cleanup };
+    return { uniquePath1, uniquePath2, path, cleanup };
   }
 
   private async runInterpretCommand(

extensions/ql-vscode/src/compare/compare-view.ts

@@ -2,7 +2,6 @@ import { ViewColumn } from "vscode";
 
 import type {
   FromCompareViewMessage,
-  InterpretedQueryCompareResult,
   QueryCompareResult,
   RawQueryCompareResult,
   ToCompareViewMessage,
@@ -14,7 +13,7 @@ import { extLogger } from "../common/logging/vscode";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
 import type { DatabaseManager } from "../databases/local-databases";
 import { jumpToLocation } from "../databases/local-databases/locations";
-import type { BqrsInfo } from "../common/bqrs-cli-types";
+import type { BqrsInfo, BqrsResultSetSchema } from "../common/bqrs-cli-types";
 import resultsDiff from "./resultsDiff";
 import type { CompletedLocalQueryInfo } from "../query-results";
 import { assertNever, getErrorMessage } from "../common/helpers-pure";
@@ -36,7 +35,7 @@ import { compareInterpretedResults } from "./interpreted-results";
 import { isCanary } from "../config";
 import { nanoid } from "nanoid";
 
-interface ComparePair {
+export interface ComparePair {
   from: CompletedLocalQueryInfo;
   fromInfo: CompareQueryInfo;
   to: CompletedLocalQueryInfo;
@@ -45,7 +44,7 @@ interface ComparePair {
   commonResultSetNames: readonly string[];
 }
 
-function findSchema(info: BqrsInfo, name: string) {
+export function findSchema(info: BqrsInfo, name: string) {
   const schema = info["result-sets"].find((schema) => schema.name === name);
   if (schema === undefined) {
     throw new Error(`Schema ${name} not found.`);
@@ -182,8 +181,7 @@ export class CompareView extends AbstractWebview<
           result = await compareInterpretedResults(
             this.databaseManager,
             this.cliServer,
-            this.comparePair.from,
-            this.comparePair.to,
+            this.comparePair,
           );
         } else {
           result = await this.compareResults(
@@ -373,18 +371,12 @@ export class CompareView extends AbstractWebview<
     const fromPath = from.completedQuery.query.resultsPath;
     const toPath = to.completedQuery.query.resultsPath;
 
-    const fromSchema = findSchema(fromInfo.schemas, fromResultSetName);
-    const toSchema = findSchema(toInfo.schemas, toResultSetName);
-
-    if (fromSchema.columns.length !== toSchema.columns.length) {
-      throw new Error("CodeQL Compare: Columns do not match.");
-    }
-    if (fromSchema.rows === 0) {
-      throw new Error("CodeQL Compare: Source query has no results.");
-    }
-    if (toSchema.rows === 0) {
-      throw new Error("CodeQL Compare: Target query has no results.");
-    }
+    const { fromSchema, toSchema } = getComparableSchemas(
+      fromInfo,
+      toInfo,
+      fromResultSetName,
+      toResultSetName,
+    );
 
     // If the result set names are the same, we use `bqrs diff`. This is more
     // efficient, but we can't use it in general as it does not support
@@ -393,13 +385,13 @@ export class CompareView extends AbstractWebview<
       const { uniquePath1, uniquePath2, cleanup } =
         await this.cliServer.bqrsDiff(fromPath, toPath);
       try {
-        const info1 = await this.cliServer.bqrsInfo(uniquePath1);
-        const info2 = await this.cliServer.bqrsInfo(uniquePath2);
+        const uniqueInfo1 = await this.cliServer.bqrsInfo(uniquePath1);
+        const uniqueInfo2 = await this.cliServer.bqrsInfo(uniquePath2);
 
         // We avoid decoding the results sets if there is no overlap
         if (
-          fromSchema.rows === findSchema(info1, fromResultSetName).rows &&
-          toSchema.rows === findSchema(info2, toResultSetName).rows
+          fromSchema.rows === findSchema(uniqueInfo1, fromResultSetName).rows &&
+          toSchema.rows === findSchema(uniqueInfo2, toResultSetName).rows
         ) {
           throw new Error(
             "CodeQL Compare: No overlap between the selected queries.",
@@ -440,3 +432,28 @@ export class CompareView extends AbstractWebview<
     }
   }
 }
+
+/**
+ * Check that `fromInfo` and `toInfo` have comparable schemas for the given
+ * result set names and get them if so.
+ */
+export function getComparableSchemas(
+  fromInfo: CompareQueryInfo,
+  toInfo: CompareQueryInfo,
+  fromResultSetName: string,
+  toResultSetName: string,
+): { fromSchema: BqrsResultSetSchema; toSchema: BqrsResultSetSchema } {
+  const fromSchema = findSchema(fromInfo.schemas, fromResultSetName);
+  const toSchema = findSchema(toInfo.schemas, toResultSetName);
+
+  if (fromSchema.columns.length !== toSchema.columns.length) {
+    throw new Error("CodeQL Compare: Columns do not match.");
+  }
+  if (fromSchema.rows === 0) {
+    throw new Error("CodeQL Compare: Source query has no results.");
+  }
+  if (toSchema.rows === 0) {
+    throw new Error("CodeQL Compare: Target query has no results.");
+  }
+  return { fromSchema, toSchema };
+}

extensions/ql-vscode/src/compare/interpreted-results.ts

@@ -1,30 +1,31 @@
 import { Uri } from "vscode";
-import type { Log } from "sarif";
-import { pathExists } from "fs-extra";
-import { sarifParser } from "../common/sarif-parser";
-import type { CompletedLocalQueryInfo } from "../query-results";
+import { join } from "path";
 import type { DatabaseManager } from "../databases/local-databases";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
-import type { InterpretedQueryCompareResult } from "../common/interface-types";
+import {
+  ALERTS_TABLE_NAME,
+  SELECT_TABLE_NAME,
+  type InterpretedQueryCompareResult,
+} from "../common/interface-types";
 
-import { sarifDiff } from "./sarif-diff";
-
-async function getInterpretedResults(
-  interpretedResultsPath: string,
-): Promise<Log | undefined> {
-  if (!(await pathExists(interpretedResultsPath))) {
-    return undefined;
-  }
-
-  return await sarifParser(interpretedResultsPath);
-}
+import { getComparableSchemas, ComparePair } from "./compare-view";
 
 export async function compareInterpretedResults(
   databaseManager: DatabaseManager,
   cliServer: CodeQLCliServer,
-  fromQuery: CompletedLocalQueryInfo,
-  toQuery: CompletedLocalQueryInfo,
+  comparePair: ComparePair,
 ): Promise<InterpretedQueryCompareResult> {
+  const { from: fromQuery, fromInfo, to: toQuery, toInfo } = comparePair;
+
+  // `ALERTS_TABLE_NAME` is inserted by `getResultSetNames` into the schema
+  // names even if it does not occur as a result set. Hence we check for
+  // `SELECT_TABLE_NAME` first, and use that if it exists.
+  const tableName = fromInfo.schemaNames.includes(SELECT_TABLE_NAME)
+    ? SELECT_TABLE_NAME
+    : ALERTS_TABLE_NAME;
+
+  getComparableSchemas(fromInfo, toInfo, tableName, tableName);
+
   const database = databaseManager.findDatabaseItem(
     Uri.parse(toQuery.initialInfo.databaseInfo.databaseUri),
   );
@@ -34,37 +35,46 @@ export async function compareInterpretedResults(
     );
   }
 
-  const [fromResultSet, toResultSet, sourceLocationPrefix] = await Promise.all([
-    getInterpretedResults(
-      fromQuery.completedQuery.query.interpretedResultsPath,
-    ),
-    getInterpretedResults(toQuery.completedQuery.query.interpretedResultsPath),
-    database.getSourceLocationPrefix(cliServer),
-  ]);
-
-  if (!fromResultSet || !toResultSet) {
-    throw new Error(
-      "Could not find interpreted results for one or both queries.",
-    );
-  }
+  const { uniquePath1, uniquePath2, path, cleanup } = await cliServer.bqrsDiff(
+    fromQuery.completedQuery.query.resultsPath,
+    toQuery.completedQuery.query.resultsPath,
+    { retainResultSets: ["nodes", "edges", "subpaths"] },
+  );
+  try {
+    const sarifOutput1 = join(path, "from.sarif");
+    const sarifOutput2 = join(path, "to.sarif");
 
-  const fromResults = fromResultSet.runs[0].results;
-  const toResults = toResultSet.runs[0].results;
+    const sourceLocationPrefix =
+      await database.getSourceLocationPrefix(cliServer);
+    const sourceArchiveUri = database.sourceArchive;
+    const sourceInfo =
+      sourceArchiveUri === undefined
+        ? undefined
+        : {
+            sourceArchive: sourceArchiveUri.fsPath,
+            sourceLocationPrefix,
+          };
 
-  if (!fromResults) {
-    throw new Error("No results found in the 'from' query.");
-  }
+    const fromResultSet = await cliServer.interpretBqrsSarif(
+      fromQuery.completedQuery.query.metadata!,
+      uniquePath1,
+      sarifOutput1,
+      sourceInfo,
+    );
+    const toResultSet = await cliServer.interpretBqrsSarif(
+      toQuery.completedQuery.query.metadata!,
+      uniquePath2,
+      sarifOutput2,
+      sourceInfo,
+    );
 
-  if (!toResults) {
-    throw new Error("No results found in the 'to' query.");
+    return {
+      kind: "interpreted",
+      sourceLocationPrefix,
+      from: fromResultSet.runs[0].results!,
+      to: toResultSet.runs[0].results!,
+    };
+  } finally {
+    await cleanup();
   }
-
-  const { from, to } = sarifDiff(fromResults, toResults);
-
-  return {
-    kind: "interpreted",
-    sourceLocationPrefix,
-    from,
-    to,
-  };
 }