Add merge-android-profiles script

Author: mstange

package.json

@@ -19,6 +19,8 @@
     "build-l10n-prod:quiet": "yarn build:clean && yarn build-photon && cross-env NODE_ENV=production L10N=1 webpack",
     "build-l10n-prod": "yarn build-l10n-prod:quiet --progress",
     "build-photon": "webpack --config res/photon/webpack.config.js",
+    "build-merge-android-profiles": "yarn build-merge-android-profiles:quiet --progress",
+    "build-merge-android-profiles:quiet": "yarn build:clean && cross-env NODE_ENV=production webpack --config src/merge-android-profiles/webpack.config.js",
     "build-symbolicator-cli": "yarn build-symbolicator-cli:quiet --progress",
     "build-symbolicator-cli:quiet": "yarn build:clean && cross-env NODE_ENV=production webpack --config src/symbolicator-cli/webpack.config.js",
     "lint": "node bin/output-fixing-commands.js run-p lint-js lint-css prettier-run",

src/merge-android-profiles/index.js

@@ -0,0 +1,212 @@
+// @flow
+
+/**
+ * Merge two existing profiles, taking the samples from the first profile and
+ * the markers from the second profile.
+ *
+ * This was useful during early 2025 when the Mozilla Performance team was
+ * doing a lot of Android startup profiling:
+ *
+ * - The "samples" profile would be collected using simpleperf and converted
+ *   with samply import.
+ * - The "markers" profile would be collected using the Gecko profiler.
+ *
+ * To use this script, it first needs to be built:
+ *   yarn build-merge-android-profiles
+ *
+ * Then it can be run from the `dist` directory:
+ *   node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json
+ *
+ * For example:
+ *   yarn build-merge-android-profiles && node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json
+ *
+ */
+
+const fs = require('fs');
+
+import {
+  unserializeProfileOfArbitraryFormat,
+  adjustMarkerTimestamps,
+} from '../profile-logic/process-profile';
+import { getProfileUrlForHash } from '../actions/receive-profile';
+import { computeStringIndexMarkerFieldsByDataType } from '../profile-logic/marker-schema';
+import { ensureExists } from '../utils/flow';
+import { StringTable } from '../utils/string-table';
+
+import type { Profile } from '../types/profile';
+
+interface CliOptions {
+  samplesHash: string;
+  markersHash: string;
+}
+
+async function fetchProfileWithHash(hash: string): Promise<Profile> {
+  const response = await fetch(getProfileUrlForHash(hash));
+  const serializedProfile = await response.json();
+  return unserializeProfileOfArbitraryFormat(serializedProfile);
+}
+
+export async function run(options: CliOptions) {
+  const profileWithSamples: Profile = await fetchProfileWithHash(
+    options.samplesHash
+  );
+  const profileWithMarkers: Profile = await fetchProfileWithHash(
+    options.markersHash
+  );
+
+  // const referenceSampleTime = 169912951.547432; // filteredThread.samples.time[0] after zooming in on samples in mozilla::dom::indexedDB::BackgroundTransactionChild::RecvComplete
+  // const referenceMarkerTime = 664.370158 ; // selectedMarker.start after selecting the marker for the "complete" DOMEvent
+
+  // console.log(profileWithSamples.meta);
+  // console.log(profileWithMarkers.meta);
+
+  let timeDelta =
+    profileWithMarkers.meta.startTime - profileWithSamples.meta.startTime;
+  if (
+    profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !==
+      undefined &&
+    profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !==
+      undefined
+  ) {
+    timeDelta =
+      (profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot -
+        profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot) /
+      1000000;
+  }
+
+  // console.log({ timeDelta });
+
+  const profile = profileWithSamples;
+  profile.meta.markerSchema = profileWithMarkers.meta.markerSchema;
+  profile.pages = profileWithMarkers.pages;
+
+  const markerProfileCategoryToCategory = new Map();
+  const markerProfileCategories = ensureExists(
+    profileWithMarkers.meta.categories
+  );
+  const profileCategories = ensureExists(profile.meta.categories);
+  for (
+    let markerCategoryIndex = 0;
+    markerCategoryIndex < markerProfileCategories.length;
+    markerCategoryIndex++
+  ) {
+    const category = markerProfileCategories[markerCategoryIndex];
+    let categoryIndex = profileCategories.findIndex(
+      (c) => c.name === category.name
+    );
+    if (categoryIndex === -1) {
+      categoryIndex = profileCategories.length;
+      profileCategories[categoryIndex] = {
+        name: category.name,
+        color: category.color,
+        subcategories: ['Other'],
+      };
+    }
+    markerProfileCategoryToCategory.set(markerCategoryIndex, categoryIndex);
+  }
+
+  const markerThreadsByTid = new Map(
+    profileWithMarkers.threads.map((thread) => ['' + thread.tid, thread])
+  );
+  // console.log([...markerThreadsByTid.keys()]);
+
+  // console.log(profile.threads.map((thread) => thread.tid));
+
+  const stringIndexMarkerFieldsByDataType =
+    computeStringIndexMarkerFieldsByDataType(profile.meta.markerSchema);
+
+  const sampleThreadTidsWithoutCorrespondingMarkerThreads = new Set();
+
+  for (const thread of profile.threads) {
+    const tid = thread.tid;
+    const markerThread = markerThreadsByTid.get(tid);
+    if (markerThread === undefined) {
+      sampleThreadTidsWithoutCorrespondingMarkerThreads.add(tid);
+      continue;
+    }
+    markerThreadsByTid.delete(tid);
+
+    const stringTable = StringTable.withBackingArray(thread.stringArray);
+    const markerStringArray = markerThread.stringArray;
+
+    thread.markers = adjustMarkerTimestamps(markerThread.markers, timeDelta);
+    for (let i = 0; i < thread.markers.length; i++) {
+      thread.markers.category[i] = ensureExists(
+        markerProfileCategoryToCategory.get(thread.markers.category[i])
+      );
+      thread.markers.name[i] = stringTable.indexForString(
+        markerStringArray[thread.markers.name[i]]
+      );
+      const data = thread.markers.data[i];
+      if (data !== null && data.type) {
+        const markerType = data.type;
+        const stringIndexMarkerFields =
+          stringIndexMarkerFieldsByDataType.get(markerType);
+        if (stringIndexMarkerFields !== undefined) {
+          for (const fieldKey of stringIndexMarkerFields) {
+            const stringIndex = data[fieldKey];
+            if (typeof stringIndex === 'number') {
+              const newStringIndex = stringTable.indexForString(
+                markerStringArray[stringIndex]
+              );
+              data[fieldKey] = newStringIndex;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // console.log(
+  //   `Have ${markerThreadsByTid.size} marker threads left over which weren't slurped up by sample threads:`,
+  //   [...markerThreadsByTid.keys()]
+  // );
+  // if (markerThreadsByTid.size !== 0) {
+  //   console.log(
+  //     `Have ${sampleThreadTidsWithoutCorrespondingMarkerThreads.size} sample threads which didn't find corresponding marker threads:`,
+  //     [...sampleThreadTidsWithoutCorrespondingMarkerThreads]
+  //   );
+  // }
+
+  fs.writeFileSync(options.outputFile, JSON.stringify(profile));
+}
+
+export function makeOptionsFromArgv(processArgv: string[]): CliOptions {
+  const argv = require('minimist')(processArgv.slice(2));
+
+  if (!('samples-hash' in argv && typeof argv['samples-hash'] === 'string')) {
+    throw new Error(
+      'Argument --samples-hash must be supplied with the path to a text file of profile hashes'
+    );
+  }
+
+  if (!('markers-hash' in argv && typeof argv['markers-hash'] === 'string')) {
+    throw new Error(
+      'Argument --markers-hash must be supplied with the path to a text file of profile hashes'
+    );
+  }
+
+  if (!('output-file' in argv && typeof argv['output-file'] === 'string')) {
+    throw new Error(
+      'Argument --output-file must be supplied with the path to a text file of profile hashes'
+    );
+  }
+
+  return {
+    samplesHash: argv['samples-hash'],
+    markersHash: argv['markers-hash'],
+    outputFile: argv['output-file'],
+  };
+}
+
+if (!module.parent) {
+  try {
+    const options = makeOptionsFromArgv(process.argv);
+    run(options).catch((err) => {
+      throw err;
+    });
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+}

src/merge-android-profiles/webpack.config.js

@@ -0,0 +1,32 @@
+// @noflow
+const path = require('path');
+const projectRoot = path.join(__dirname, '../..');
+const includes = [path.join(projectRoot, 'src')];
+
+module.exports = {
+  name: 'merge-android-profiles',
+  target: 'node',
+  mode: process.env.NODE_ENV,
+  output: {
+    path: path.resolve(projectRoot, 'dist'),
+    filename: 'merge-android-profiles.js',
+  },
+  entry: './src/merge-android-profiles/index.js',
+  module: {
+    rules: [
+      {
+        test: /\.js$/,
+        use: ['babel-loader'],
+        include: includes,
+      },
+      {
+        test: /\.svg$/,
+        type: 'asset/resource',
+      },
+    ],
+  },
+  experiments: {
+    // Make WebAssembly work just like in webpack v4
+    syncWebAssembly: true,
+  },
+};

src/profile-logic/marker-schema.js

@@ -679,7 +679,12 @@ export function computeStringIndexMarkerFieldsByDataType(
     const { name, fields } = schema;
     const stringIndexFields = [];
     for (const field of fields) {
-      if (field.format === 'unique-string' && field.key) {
+      if (
+        (field.format === 'unique-string' ||
+          field.format === 'flow-id' ||
+          field.format === 'terminating-flow-id') &&
+        field.key
+      ) {
         stringIndexFields.push(field.key);
       }
     }