feat(common-utils): add a compact form of the pattern predicate

.changeset/compact-pattern-predicate.md

@@ -0,0 +1,9 @@
+---
+'@hyperdx/common-utils': minor
+---
+
+feat: add `buildCompactPatternPredicate`
+
+It builds the same match as `buildPatternPredicate` in much shorter SQL, for
+pattern filters that are kept in a URL. `parsePatternPredicate` reads both
+forms.

packages/common-utils/src/__tests__/pattern-predicate.test.ts

@@ -1,5 +1,6 @@
 import { TemplateMinerConfig } from '@/drain/config';
 import {
+  buildCompactPatternPredicate,
   buildPatternPredicate,
   MAX_PREDICATE_TEMPLATE_LENGTH,
   parsePatternPredicate,
@@ -15,15 +16,57 @@ const withPunctuation = { extraDelimiters: PUNCTUATION };
 // RE2 spells code points \x{HHHH} and categories \pZ; JS (with the u flag)
 // spells them \u{HHHH} and \p{Z}, and has no POSIX classes. RE2's \s is
 // [\t\n\f\r ]; JS's also covers Unicode spaces and BOM.
+function toJsSource(re2: string): string {
+  return re2
+    .replace(/(?<!\\)\\s/g, '\\t\\n\\f\\r ')
+    .replace(/\\x\{([0-9A-Fa-f]+)\}/g, '\\u{$1}')
+    .replace(/(?<!\\)\\p([A-Z])/g, '\\p{$1}')
+    .replaceAll('[:punct:]', '!-\\/:-@\\[-`{-~');
+}
+
 function toJsRegex(re2: string): RegExp {
-  return RegExp(
-    re2
-      .replace(/(?<!\\)\\s/g, '\\t\\n\\f\\r ')
-      .replace(/\\x\{([0-9A-Fa-f]+)\}/g, '\\u{$1}')
-      .replace(/(?<!\\)\\p([A-Z])/g, '\\p{$1}')
-      .replaceAll('[:punct:]', '!-\\/:-@\\[-`{-~'),
-    'u',
-  );
+  return RegExp(toJsSource(re2), 'u');
+}
+
+const unescapeSql = (literal: string) =>
+  literal.replace(/''/g, "'").replace(/\\\\/g, '\\');
+
+// Evaluates a compact predicate the way ClickHouse would: replaceRegexpAll,
+// then match.
+function matchesCompact(predicate: string, value: string): boolean {
+  const call =
+    /^match\(replaceRegexpAll\(.+, '((?:[^'\\]|\\\\|'')*)', ' '\), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec(
+      predicate,
+    );
+  if (call == null) throw new Error(`not a compact predicate: ${predicate}`);
+  const separatorRun = RegExp(toJsSource(unescapeSql(call[1])), 'gu');
+  return toJsRegex(unescapeSql(call[2])).test(value.replace(separatorRun, ' '));
+}
+
+const MESSAGES = [
+  'Connection to db-1:5432 timed out after 30s (retry 1/3)',
+  'Connection to db-2:5432 timed out after 30s (retry 2/3)',
+  'user=alice action=login status=ok',
+  'user=bob action=login status=ok',
+  '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -',
+  '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -',
+  'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok',
+  'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok',
+];
+
+function mineClusters(): { template: string; members: string[] }[] {
+  const config = new TemplateMinerConfig();
+  config.drainExtraDelimiters = PUNCTUATION;
+  const miner = new TemplateMiner(config);
+  const members = new Map<number, string[]>();
+  for (const message of MESSAGES) {
+    const { clusterId } = miner.addLogMessage(message);
+    members.set(clusterId, [...(members.get(clusterId) ?? []), message]);
+  }
+  return [...members].map(([clusterId, clusterMembers]) => ({
+    template: miner.drain.getCluster(clusterId)!.getTemplate(),
+    members: clusterMembers,
+  }));
 }
 
 describe('templateToRegex', () => {
@@ -95,31 +138,125 @@ describe('templateToRegex', () => {
   });
 
   it('matches every member of every cluster mined with the same delimiters', () => {
-    const config = new TemplateMinerConfig();
-    config.drainExtraDelimiters = PUNCTUATION;
-    const miner = new TemplateMiner(config);
-    const messages = [
-      'Connection to db-1:5432 timed out after 30s (retry 1/3)',
-      'Connection to db-2:5432 timed out after 30s (retry 2/3)',
-      'user=alice action=login status=ok',
-      'user=bob action=login status=ok',
-      '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -',
-      '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -',
-      'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok',
-      'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok',
+    for (const { template, members } of mineClusters()) {
+      const regex = toJsRegex(templateToRegex(template, withPunctuation));
+      for (const message of MESSAGES) {
+        expect(regex.test(message)).toBe(members.includes(message));
+      }
+    }
+  });
+});
+
+const PARTIAL: PatternRegexOptions = {
+  extraDelimiters: ['/', ':', '-', ']', '\\', '^'],
+};
+
+// The option sets the equivalence and round-trip tests run under.
+const OPTION_SETS: (PatternRegexOptions | undefined)[] = [
+  withPunctuation,
+  undefined,
+  PARTIAL,
+  { extraDelimiters: PUNCTUATION, whitespace: 'python' },
+];
+
+describe('buildCompactPatternPredicate', () => {
+  it('matches exactly what the full predicate matches', () => {
+    const values = [
+      ...MESSAGES,
+      '  GET /api/v1/users/9 404 3ms.',
+      '',
+      ' \t ',
+      'GET api v1 users 9 404',
+      // Separators outside ASCII, and ones only one whitespace set has.
+      'GET\u00a0api\u2003v1\x0busers\u3000 9 404 3ms',
+      'GET api\ufeffv1 users 9 404 3ms',
+      'GET api v1 users 9\x1e404\x853ms',
     ];
-    const members = new Map<number, string[]>();
-    for (const message of messages) {
-      const { clusterId } = miner.addLogMessage(message);
-      members.set(clusterId, [...(members.get(clusterId) ?? []), message]);
+    const templates = [
+      ...mineClusters().map(c => c.template),
+      'GET api v1 users <*> <*> <*>',
+      '<*>',
+      '',
+    ];
+    for (const options of OPTION_SETS) {
+      for (const template of templates) {
+        const full = toJsRegex(templateToRegex(template, options));
+        const compact = buildCompactPatternPredicate(
+          'Body',
+          template,
+          options,
+        )!;
+        for (const value of values) {
+          expect([template, value, matchesCompact(compact, value)]).toEqual([
+            template,
+            value,
+            full.test(value),
+          ]);
+        }
+      }
     }
-    for (const [clusterId, clusterMembers] of members) {
-      const template = miner.drain.getCluster(clusterId)!.getTemplate();
-      const regex = toJsRegex(templateToRegex(template, withPunctuation));
-      for (const message of messages) {
-        expect(regex.test(message)).toBe(clusterMembers.includes(message));
+  });
+
+  it('is much shorter than the full predicate for long templates', () => {
+    const template = Array.from({ length: 40 }, (_, i) =>
+      i % 3 === 0 ? '<*>' : `word${i}`,
+    ).join(' ');
+    const full = buildPatternPredicate('Body', template, withPunctuation)!;
+    const compact = buildCompactPatternPredicate(
+      'Body',
+      template,
+      withPunctuation,
+    )!;
+    expect(compact.length * 4).toBeLessThan(full.length);
+  });
+
+  it('escapes regex and SQL syntax in tokens', () => {
+    // Mined without punctuation delimiters, tokens keep their punctuation.
+    const cases: [PatternRegexOptions | undefined, string][] = [
+      [undefined, 'GET /api/v1.2 (ok) $5 a|b'],
+      [undefined, "user=<*> it's C:\\tmp 3ms."],
+      [PARTIAL, 'GET api v1.2 (ok) $5 a|b'],
+    ];
+    const values = [
+      'GET /api/v1.2 (ok) $5 a|b',
+      'GET /api/v1x2 (ok) $5 a|b',
+      'GET api v1.2 (ok) $5 a|b',
+      'GET/api/v1.2 (ok) $5 a|b',
+      "user=<*> it's C:\\tmp 3ms.",
+      "user=<*> it's C:\\tmp 3msX",
+    ];
+    for (const [options, template] of cases) {
+      const full = toJsRegex(templateToRegex(template, options));
+      const compact = buildCompactPatternPredicate('Body', template, options)!;
+      for (const value of values) {
+        expect([template, value, matchesCompact(compact, value)]).toEqual([
+          template,
+          value,
+          full.test(value),
+        ]);
       }
+      expect(parsePatternPredicate(compact, options)?.template).toBe(template);
     }
+    // `.` is a literal, so only the template's own text matches.
+    const compact = buildCompactPatternPredicate(
+      'Body',
+      'GET /api/v1.2 (ok) $5 a|b',
+    )!;
+    expect(matchesCompact(compact, 'GET /api/v1.2 (ok) $5 a|b')).toBe(true);
+    expect(matchesCompact(compact, 'GET /api/v1x2 (ok) $5 a|b')).toBe(false);
+  });
+
+  it('builds no predicate for a token that holds a delimiter', () => {
+    // Mined without delimiters, then built with them: normalizing would split
+    // `db-1:5432`, so the compact form can't match what the full one does.
+    const template = 'Connection to db-1:5432 timed out';
+    expect(
+      buildCompactPatternPredicate('Body', template, withPunctuation),
+    ).toBeNull();
+    expect(buildPatternPredicate('Body', template, withPunctuation)).not.toBe(
+      null,
+    );
+    expect(buildCompactPatternPredicate('Body', template)).not.toBeNull();
   });
 });
 
@@ -136,12 +273,19 @@ describe('parsePatternPredicate', () => {
     ['Body', 'GET api v1 users <*> <*> <*>'],
     ["JSONExtractString(Body, 'message')", 'user <*> action login status ok'],
     ['Body', '<*>'],
-  ])('reads back what buildPatternPredicate built (%s)', (expr, template) => {
-    const predicate = buildPatternPredicate(expr, template, withPunctuation)!;
-    expect(parsePatternPredicate(predicate, withPunctuation)).toEqual({
-      valueExpression: expr,
-      template,
-    });
+  ])('reads back what either builder built (%s)', (expr, template) => {
+    for (const options of OPTION_SETS) {
+      for (const build of [
+        buildPatternPredicate,
+        buildCompactPatternPredicate,
+      ]) {
+        const predicate = build(expr, template, options)!;
+        expect(parsePatternPredicate(predicate, options)).toEqual({
+          valueExpression: expr,
+          template,
+        });
+      }
+    }
   });
 
   it('reads back what buildPatternPredicate built without options', () => {
@@ -167,6 +311,25 @@ describe('parsePatternPredicate', () => {
     expect(parsePatternPredicate("ServiceName IN ('api')")).toBeNull();
     expect(parsePatternPredicate("match(Body, 'timed out')")).toBeNull();
     // A predicate only reads back with the options it was built with.
+    expect(
+      parsePatternPredicate(
+        "match(replaceRegexpAll(Body, '[ ]+', ' '), '^ ?timed out ?$')",
+        withPunctuation,
+      ),
+    ).toBeNull();
+    // The real separator class with a hand-written regex: only the round trip
+    // through compactRegex rejects it.
+    const compact = buildCompactPatternPredicate(
+      'Body',
+      'timed out',
+      withPunctuation,
+    )!;
+    const handWritten = compact.replace(
+      "'^ ?timed out ?$'",
+      "'^ ?timed.* out ?$'",
+    );
+    expect(handWritten).not.toBe(compact);
+    expect(parsePatternPredicate(handWritten, withPunctuation)).toBeNull();
     expect(
       parsePatternPredicate(
         buildPatternPredicate('Body', 'Sending <*> rows')!,

packages/common-utils/src/drain/index.ts

@@ -21,6 +21,7 @@ export { buildNoisyBodyMatcher, selectNoisyPatterns } from './noisy-matcher';
 export type { PatternRegexOptions } from './pattern-predicate';
 export {
   ASCII_PUNCTUATION,
+  buildCompactPatternPredicate,
   buildPatternPredicate,
   MAX_PREDICATE_TEMPLATE_LENGTH,
   parsePatternPredicate,