feat(common-utils): share the Event Patterns drill-down predicate

.changeset/shared-pattern-predicate.md

@@ -0,0 +1,12 @@
+---
+'@hyperdx/common-utils': minor
+---
+
+feat: share the Event Patterns drill-down predicate
+
+`templateToRegex` and `buildPatternPredicate`, which the MCP server uses to
+build the `clickstack_event_patterns` drill-down filter, are now in
+`@hyperdx/common-utils/dist/drain`. New options match templates mined with extra
+delimiters (`extraDelimiters`) or by Python Drain3 (`whitespace: 'python'`), and
+`parsePatternPredicate` reads a predicate back into its value expression and
+template. Without options the output is the same as the MCP server's.

packages/common-utils/src/__tests__/pattern-predicate.test.ts

@@ -0,0 +1,182 @@
+import { TemplateMinerConfig } from '@/drain/config';
+import {
+  buildPatternPredicate,
+  MAX_PREDICATE_TEMPLATE_LENGTH,
+  parsePatternPredicate,
+  PatternRegexOptions,
+  templateToRegex,
+} from '@/drain/pattern-predicate';
+import { TemplateMiner } from '@/drain/template-miner';
+
+// Python's string.punctuation, the extra delimiters the app's miner uses.
+const PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split('');
+const withPunctuation = { extraDelimiters: PUNCTUATION };
+
+// RE2 spells code points \x{HHHH} and categories \pZ; JS (with the u flag)
+// spells them \u{HHHH} and \p{Z}, and has no POSIX classes. RE2's \s is
+// [\t\n\f\r ]; JS's also covers Unicode spaces and BOM.
+function toJsRegex(re2: string): RegExp {
+  return RegExp(
+    re2
+      .replace(/(?<!\\)\\s/g, '\\t\\n\\f\\r ')
+      .replace(/\\x\{([0-9A-Fa-f]+)\}/g, '\\u{$1}')
+      .replace(/(?<!\\)\\p([A-Z])/g, '\\p{$1}')
+      .replaceAll('[:punct:]', '!-\\/:-@\\[-`{-~'),
+    'u',
+  );
+}
+
+describe('templateToRegex', () => {
+  it('splits on whitespace only without extra delimiters', () => {
+    const ws = '[\\s\\x{0B}\\pZ\\x{FEFF}]';
+    expect(templateToRegex('Sending <*> rows')).toBe(
+      `^${ws}*Sending${ws}+[^\\s\\x{0B}\\pZ\\x{FEFF}]+${ws}+rows${ws}*$`,
+    );
+  });
+
+  it('spells all ASCII punctuation as [:punct:] and lists other delimiters', () => {
+    expect(templateToRegex('a', withPunctuation)).toBe(
+      '^[\\s\\x{0B}\\pZ\\x{FEFF}[:punct:]]*a[\\s\\x{0B}\\pZ\\x{FEFF}[:punct:]]*$',
+    );
+    const partial = { extraDelimiters: ['/', ':', '-', ']', '\\', '^'] };
+    const regex = toJsRegex(templateToRegex('a <*> c d e f g', partial));
+    expect(regex.test('a/x:c-d]e\\f^g')).toBe(true);
+    expect(regex.test('a/x:c-d]e\\f^g.h')).toBe(false);
+  });
+
+  it('splits on Python whitespace for templates mined by Drain3', () => {
+    const python: PatternRegexOptions = {
+      extraDelimiters: PUNCTUATION,
+      whitespace: 'python',
+    };
+    // Python's str.split() splits on the \x1C-\x1F separators and NEL but not
+    // on BOM; JS `\s` is the other way round.
+    const drain3 = toJsRegex(templateToRegex('a b c', python));
+    expect(drain3.test('a\x1eb\x85c')).toBe(true);
+    expect(drain3.test('a\ufeffb c')).toBe(false);
+    const ts = toJsRegex(templateToRegex('a b c', withPunctuation));
+    expect(ts.test('a\x1eb\x85c')).toBe(false);
+    expect(ts.test('a\ufeffb c')).toBe(true);
+    // So a Drain3 token can hold a BOM, as in a body that starts with one.
+    const bomTemplate = '\ufeffGET api <*>';
+    expect(
+      toJsRegex(templateToRegex(bomTemplate, python)).test('\ufeffGET /api/x'),
+    ).toBe(true);
+    expect(
+      parsePatternPredicate(
+        buildPatternPredicate('Body', bomTemplate, python)!,
+        python,
+      )?.template,
+    ).toBe(bomTemplate);
+
+    const predicate = buildPatternPredicate('Body', 'a b c', python)!;
+    expect(parsePatternPredicate(predicate, python)?.template).toBe('a b c');
+    expect(parsePatternPredicate(predicate, withPunctuation)).toBeNull();
+  });
+
+  it('rejects delimiters longer than one character', () => {
+    expect(() => templateToRegex('a', { extraDelimiters: ['::'] })).toThrow(
+      'single characters',
+    );
+  });
+
+  it('treats extra delimiters as separators', () => {
+    const regex = toJsRegex(
+      templateToRegex('GET api v1 users <*> <*> <*>', withPunctuation),
+    );
+    expect(regex.test('GET /api/v1/users/123 200 15ms')).toBe(true);
+    expect(regex.test('GET /api/v1/users/9 404 3ms')).toBe(true);
+    expect(regex.test('  GET /api/v1/users/9 404 3ms.')).toBe(true);
+
+    expect(regex.test('POST /api/v1/users/9 404 3ms')).toBe(false);
+    expect(regex.test('GET /api/v1/users/9/orders 404 3ms')).toBe(false);
+    expect(regex.test('GET /api/v1/users/9 404')).toBe(false);
+    expect(regex.test('GET /api/v2/users/9 404 3ms')).toBe(false);
+  });
+
+  it('matches every member of every cluster mined with the same delimiters', () => {
+    const config = new TemplateMinerConfig();
+    config.drainExtraDelimiters = PUNCTUATION;
+    const miner = new TemplateMiner(config);
+    const messages = [
+      'Connection to db-1:5432 timed out after 30s (retry 1/3)',
+      'Connection to db-2:5432 timed out after 30s (retry 2/3)',
+      'user=alice action=login status=ok',
+      'user=bob action=login status=ok',
+      '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -',
+      '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -',
+      'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok',
+      'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok',
+    ];
+    const members = new Map<number, string[]>();
+    for (const message of messages) {
+      const { clusterId } = miner.addLogMessage(message);
+      members.set(clusterId, [...(members.get(clusterId) ?? []), message]);
+    }
+    for (const [clusterId, clusterMembers] of members) {
+      const template = miner.drain.getCluster(clusterId)!.getTemplate();
+      const regex = toJsRegex(templateToRegex(template, withPunctuation));
+      for (const message of messages) {
+        expect(regex.test(message)).toBe(clusterMembers.includes(message));
+      }
+    }
+  });
+});
+
+describe('buildPatternPredicate', () => {
+  it('builds no predicate for a template over the length limit', () => {
+    const atLimit = 'x'.repeat(MAX_PREDICATE_TEMPLATE_LENGTH);
+    expect(buildPatternPredicate('Body', atLimit)).not.toBeNull();
+    expect(buildPatternPredicate('Body', `${atLimit}x`)).toBeNull();
+  });
+});
+
+describe('parsePatternPredicate', () => {
+  it.each([
+    ['Body', 'GET api v1 users <*> <*> <*>'],
+    ["JSONExtractString(Body, 'message')", 'user <*> action login status ok'],
+    ['Body', '<*>'],
+  ])('reads back what buildPatternPredicate built (%s)', (expr, template) => {
+    const predicate = buildPatternPredicate(expr, template, withPunctuation)!;
+    expect(parsePatternPredicate(predicate, withPunctuation)).toEqual({
+      valueExpression: expr,
+      template,
+    });
+  });
+
+  it('reads back what buildPatternPredicate built without options', () => {
+    const predicate = buildPatternPredicate('Body', 'Sending <*> rows')!;
+    expect(parsePatternPredicate(predicate)).toEqual({
+      valueExpression: 'Body',
+      template: 'Sending <*> rows',
+    });
+  });
+
+  it('reads back tokens that hold quotes, backslashes and regex syntax', () => {
+    // Without delimiters a token keeps its punctuation.
+    const expr = "JSONExtractString(Body, 'msg')";
+    const template = "it's C:\\tmp\\a.txt (ok) <*>";
+    const predicate = buildPatternPredicate(expr, template)!;
+    expect(parsePatternPredicate(predicate)).toEqual({
+      valueExpression: expr,
+      template,
+    });
+  });
+
+  it('rejects other conditions', () => {
+    expect(parsePatternPredicate("ServiceName IN ('api')")).toBeNull();
+    expect(parsePatternPredicate("match(Body, 'timed out')")).toBeNull();
+    // A predicate only reads back with the options it was built with.
+    expect(
+      parsePatternPredicate(
+        buildPatternPredicate('Body', 'Sending <*> rows')!,
+        withPunctuation,
+      ),
+    ).toBeNull();
+    expect(
+      parsePatternPredicate(
+        buildPatternPredicate('Body', 'Sending <*> rows', withPunctuation)!,
+      ),
+    ).toBeNull();
+  });
+});

packages/common-utils/src/drain/index.ts

@@ -18,5 +18,13 @@ export {
 } from './mine-patterns';
 export { Node } from './node';
 export { buildNoisyBodyMatcher, selectNoisyPatterns } from './noisy-matcher';
+export type { PatternRegexOptions } from './pattern-predicate';
+export {
+  ASCII_PUNCTUATION,
+  buildPatternPredicate,
+  MAX_PREDICATE_TEMPLATE_LENGTH,
+  parsePatternPredicate,
+  templateToRegex,
+} from './pattern-predicate';
 export type { AddLogMessageResult, ExtractedParameter } from './template-miner';
 export { TemplateMiner } from './template-miner';

packages/common-utils/src/drain/pattern-predicate.ts

@@ -0,0 +1,136 @@
+import { escapeSqlString } from '@/core/utils';
+
+const PATTERN_WILDCARD = '<*>';
+
+/**
+ * Templates longer than this get no predicate: the regex (and the SQL that
+ * embeds it) would be unwieldy, and very long templates are usually a
+ * single unparameterized payload rather than a reusable shape.
+ */
+export const MAX_PREDICATE_TEMPLATE_LENGTH = 4000;
+
+export type PatternRegexOptions = {
+  /**
+   * The extra delimiters the miner was configured with, one character each.
+   * Drain turns each one into a space before splitting, so they separate
+   * tokens like whitespace.
+   */
+  extraDelimiters?: string[];
+  /**
+   * The whitespace the miner split on: JavaScript's `\s` for the TypeScript
+   * Drain port (the default), or Python's `str.split()` for Drain3.
+   */
+  whitespace?: 'js' | 'python';
+};
+
+// The TypeScript Drain port tokenizes with JS `\s`, which includes Unicode
+// spaces and BOM; RE2's `\s` is ASCII-only. JS `\s` = RE2 `\s` + \v + \pZ
+// (Zs/Zl/Zp) + BOM. `\pZ` keeps the snippet short; spelling out the code
+// points repeats ~100 chars between every token.
+const JS_WHITESPACE = '\\s\\x{0B}\\pZ\\x{FEFF}';
+// Python's `str.split()`, which Drain3 uses: RE2 `\s` + \v + the separators
+// \x1C-\x1F + NEL + \pZ, and no BOM.
+const PYTHON_WHITESPACE = '\\s\\x{0B}\\x{1C}-\\x{1F}\\x{85}\\pZ';
+
+function escapeRe2Literal(token: string): string {
+  return token.replace(/[\\^$.|?*+()[\]{}]/g, '\\$&');
+}
+
+// The characters that are special inside a character class. `[` can open a
+// POSIX class such as `[:alpha:]` in RE2.
+function escapeRe2ClassChars(chars: string): string {
+  return chars.replace(/[\\\]^[-]/g, '\\$&');
+}
+
+/** The ASCII punctuation: exactly RE2's `[:punct:]`, and Python's `string.punctuation`. */
+export const ASCII_PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~';
+
+function separatorClass(options?: PatternRegexOptions): string {
+  const delimiters = options?.extraDelimiters ?? [];
+  // Drain splits on each delimiter as a whole string, which a character class
+  // can't express for one longer than a character.
+  if (delimiters.some(d => [...d].length !== 1)) {
+    throw new Error('extraDelimiters must be single characters');
+  }
+  const whitespace =
+    options?.whitespace === 'python' ? PYTHON_WHITESPACE : JS_WHITESPACE;
+  // The class repeats between every token, so spelling out all 32 ASCII
+  // punctuation characters (most of which need URL encoding) makes the
+  // filter too long to keep in a URL.
+  const allPunctuation = [...ASCII_PUNCTUATION].every(c =>
+    delimiters.includes(c),
+  );
+  const rest = allPunctuation
+    ? delimiters.filter(c => !ASCII_PUNCTUATION.includes(c))
+    : delimiters;
+  return (
+    whitespace +
+    (allPunctuation ? '[:punct:]' : '') +
+    escapeRe2ClassChars(rest.join(''))
+  );
+}
+
+/** Anchored RE2 regex that a value matches when it fits the Drain template. */
+export function templateToRegex(
+  template: string,
+  options?: PatternRegexOptions,
+): string {
+  const separators = separatorClass(options);
+  const separator = `[${separators}]`;
+  // Drain3 keeps a BOM inside a token, where JS `\s` would split.
+  const split = options?.whitespace === 'python' ? /[^\S\u{FEFF}]+/u : /\s+/;
+  const tokens = template.split(split).filter(t => t.length > 0);
+  const body = tokens
+    .map(t =>
+      t === PATTERN_WILDCARD ? `[^${separators}]+` : escapeRe2Literal(t),
+    )
+    .join(`${separator}+`);
+  return `^${separator}*${body}${separator}*$`;
+}
+
+/**
+ * SQL predicate selecting events whose `valueExpression` fits the template,
+ * or null when the template is too long to turn into a predicate.
+ */
+export function buildPatternPredicate(
+  valueExpression: string,
+  template: string,
+  options?: PatternRegexOptions,
+): string | null {
+  if (template.length > MAX_PREDICATE_TEMPLATE_LENGTH) return null;
+  return `match(${valueExpression}, '${escapeSqlString(templateToRegex(template, options))}')`;
+}
+
+/**
+ * Reverse of buildPatternPredicate: the value expression and template of a
+ * predicate it built with the same options, or null for any other condition.
+ */
+export function parsePatternPredicate(
+  condition: string,
+  options?: PatternRegexOptions,
+): { valueExpression: string; template: string } | null {
+  const call = /^match\((.+), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec(condition);
+  if (call == null) return null;
+  const [, valueExpression, literal] = call;
+  const regex = literal.replace(/''/g, "'").replace(/\\\\/g, '\\');
+
+  const separators = separatorClass(options);
+  const separator = `[${separators}]`;
+  const prefix = `^${separator}*`;
+  const suffix = `${separator}*$`;
+  if (!regex.startsWith(prefix) || !regex.endsWith(suffix)) return null;
+  const template = regex
+    .slice(prefix.length, regex.length - suffix.length)
+    .split(`${separator}+`)
+    .map(part =>
+      part === `[^${separators}]+`
+        ? PATTERN_WILDCARD
+        : part.replace(/\\(.)/g, '$1'),
+    )
+    .join(' ');
+  // Only what this module builds counts, so a hand-written match() isn't
+  // mistaken for a pattern.
+  return templateToRegex(template, options) === regex
+    ? { valueExpression, template }
+    : null;
+}