diff --git a/.changeset/compact-pattern-predicate.md b/.changeset/compact-pattern-predicate.md new file mode 100644 index 000000000..f5ded73f6 --- /dev/null +++ b/.changeset/compact-pattern-predicate.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': minor +--- + +feat: add `buildCompactPatternPredicate` + +It builds the same match as `buildPatternPredicate` in much shorter SQL, for +pattern filters that are kept in a URL. `parsePatternPredicate` reads both +forms. diff --git a/packages/common-utils/src/__tests__/pattern-predicate.test.ts b/packages/common-utils/src/__tests__/pattern-predicate.test.ts index 5a0f0eeb9..8791957ab 100644 --- a/packages/common-utils/src/__tests__/pattern-predicate.test.ts +++ b/packages/common-utils/src/__tests__/pattern-predicate.test.ts @@ -1,5 +1,6 @@ import { TemplateMinerConfig } from '@/drain/config'; import { + buildCompactPatternPredicate, buildPatternPredicate, MAX_PREDICATE_TEMPLATE_LENGTH, parsePatternPredicate, @@ -15,15 +16,57 @@ const withPunctuation = { extraDelimiters: PUNCTUATION }; // RE2 spells code points \x{HHHH} and categories \pZ; JS (with the u flag) // spells them \u{HHHH} and \p{Z}, and has no POSIX classes. RE2's \s is // [\t\n\f\r ]; JS's also covers Unicode spaces and BOM. +function toJsSource(re2: string): string { + return re2 + .replace(/(? + literal.replace(/''/g, "'").replace(/\\\\/g, '\\'); + +// Evaluates a compact predicate the way ClickHouse would: replaceRegexpAll, +// then match. +function matchesCompact(predicate: string, value: string): boolean { + const call = + /^match\(replaceRegexpAll\(.+, '((?:[^'\\]|\\\\|'')*)', ' '\), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec( + predicate, + ); + if (call == null) throw new Error(`not a compact predicate: ${predicate}`); + const separatorRun = RegExp(toJsSource(unescapeSql(call[1])), 'gu'); + return toJsRegex(unescapeSql(call[2])).test(value.replace(separatorRun, ' ')); +} + +const MESSAGES = [ + 'Connection to db-1:5432 timed out after 30s (retry 1/3)', + 'Connection to db-2:5432 timed out after 30s (retry 2/3)', + 'user=alice action=login status=ok', + 'user=bob action=login status=ok', + '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -', + '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -', + 'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok', + 'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok', +]; + +function mineClusters(): { template: string; members: string[] }[] { + const config = new TemplateMinerConfig(); + config.drainExtraDelimiters = PUNCTUATION; + const miner = new TemplateMiner(config); + const members = new Map(); + for (const message of MESSAGES) { + const { clusterId } = miner.addLogMessage(message); + members.set(clusterId, [...(members.get(clusterId) ?? []), message]); + } + return [...members].map(([clusterId, clusterMembers]) => ({ + template: miner.drain.getCluster(clusterId)!.getTemplate(), + members: clusterMembers, + })); } describe('templateToRegex', () => { @@ -95,31 +138,125 @@ describe('templateToRegex', () => { }); it('matches every member of every cluster mined with the same delimiters', () => { - const config = new TemplateMinerConfig(); - config.drainExtraDelimiters = PUNCTUATION; - const miner = new TemplateMiner(config); - const messages = [ - 'Connection to db-1:5432 timed out after 30s (retry 1/3)', - 'Connection to db-2:5432 timed out after 30s (retry 2/3)', - 'user=alice action=login status=ok', - 'user=bob action=login status=ok', - '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -', - '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -', - 'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok', - 'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok', + for (const { template, members } of mineClusters()) { + const regex = toJsRegex(templateToRegex(template, withPunctuation)); + for (const message of MESSAGES) { + expect(regex.test(message)).toBe(members.includes(message)); + } + } + }); +}); + +const PARTIAL: PatternRegexOptions = { + extraDelimiters: ['/', ':', '-', ']', '\\', '^'], +}; + +// The option sets the equivalence and round-trip tests run under. +const OPTION_SETS: (PatternRegexOptions | undefined)[] = [ + withPunctuation, + undefined, + PARTIAL, + { extraDelimiters: PUNCTUATION, whitespace: 'python' }, +]; + +describe('buildCompactPatternPredicate', () => { + it('matches exactly what the full predicate matches', () => { + const values = [ + ...MESSAGES, + ' GET /api/v1/users/9 404 3ms.', + '', + ' \t ', + 'GET api v1 users 9 404', + // Separators outside ASCII, and ones only one whitespace set has. + 'GET\u00a0api\u2003v1\x0busers\u3000 9 404 3ms', + 'GET api\ufeffv1 users 9 404 3ms', + 'GET api v1 users 9\x1e404\x853ms', ]; - const members = new Map(); - for (const message of messages) { - const { clusterId } = miner.addLogMessage(message); - members.set(clusterId, [...(members.get(clusterId) ?? []), message]); + const templates = [ + ...mineClusters().map(c => c.template), + 'GET api v1 users <*> <*> <*>', + '<*>', + '', + ]; + for (const options of OPTION_SETS) { + for (const template of templates) { + const full = toJsRegex(templateToRegex(template, options)); + const compact = buildCompactPatternPredicate( + 'Body', + template, + options, + )!; + for (const value of values) { + expect([template, value, matchesCompact(compact, value)]).toEqual([ + template, + value, + full.test(value), + ]); + } + } } - for (const [clusterId, clusterMembers] of members) { - const template = miner.drain.getCluster(clusterId)!.getTemplate(); - const regex = toJsRegex(templateToRegex(template, withPunctuation)); - for (const message of messages) { - expect(regex.test(message)).toBe(clusterMembers.includes(message)); + }); + + it('is much shorter than the full predicate for long templates', () => { + const template = Array.from({ length: 40 }, (_, i) => + i % 3 === 0 ? '<*>' : `word${i}`, + ).join(' '); + const full = buildPatternPredicate('Body', template, withPunctuation)!; + const compact = buildCompactPatternPredicate( + 'Body', + template, + withPunctuation, + )!; + expect(compact.length * 4).toBeLessThan(full.length); + }); + + it('escapes regex and SQL syntax in tokens', () => { + // Mined without punctuation delimiters, tokens keep their punctuation. + const cases: [PatternRegexOptions | undefined, string][] = [ + [undefined, 'GET /api/v1.2 (ok) $5 a|b'], + [undefined, "user=<*> it's C:\\tmp 3ms."], + [PARTIAL, 'GET api v1.2 (ok) $5 a|b'], + ]; + const values = [ + 'GET /api/v1.2 (ok) $5 a|b', + 'GET /api/v1x2 (ok) $5 a|b', + 'GET api v1.2 (ok) $5 a|b', + 'GET/api/v1.2 (ok) $5 a|b', + "user=<*> it's C:\\tmp 3ms.", + "user=<*> it's C:\\tmp 3msX", + ]; + for (const [options, template] of cases) { + const full = toJsRegex(templateToRegex(template, options)); + const compact = buildCompactPatternPredicate('Body', template, options)!; + for (const value of values) { + expect([template, value, matchesCompact(compact, value)]).toEqual([ + template, + value, + full.test(value), + ]); } + expect(parsePatternPredicate(compact, options)?.template).toBe(template); } + // `.` is a literal, so only the template's own text matches. + const compact = buildCompactPatternPredicate( + 'Body', + 'GET /api/v1.2 (ok) $5 a|b', + )!; + expect(matchesCompact(compact, 'GET /api/v1.2 (ok) $5 a|b')).toBe(true); + expect(matchesCompact(compact, 'GET /api/v1x2 (ok) $5 a|b')).toBe(false); + }); + + it('builds no predicate for a token that holds a delimiter', () => { + // Mined without delimiters, then built with them: normalizing would split + // `db-1:5432`, so the compact form can't match what the full one does. + const template = 'Connection to db-1:5432 timed out'; + expect( + buildCompactPatternPredicate('Body', template, withPunctuation), + ).toBeNull(); + expect(buildPatternPredicate('Body', template, withPunctuation)).not.toBe( + null, + ); + expect(buildCompactPatternPredicate('Body', template)).not.toBeNull(); }); }); @@ -136,12 +273,19 @@ describe('parsePatternPredicate', () => { ['Body', 'GET api v1 users <*> <*> <*>'], ["JSONExtractString(Body, 'message')", 'user <*> action login status ok'], ['Body', '<*>'], - ])('reads back what buildPatternPredicate built (%s)', (expr, template) => { - const predicate = buildPatternPredicate(expr, template, withPunctuation)!; - expect(parsePatternPredicate(predicate, withPunctuation)).toEqual({ - valueExpression: expr, - template, - }); + ])('reads back what either builder built (%s)', (expr, template) => { + for (const options of OPTION_SETS) { + for (const build of [ + buildPatternPredicate, + buildCompactPatternPredicate, + ]) { + const predicate = build(expr, template, options)!; + expect(parsePatternPredicate(predicate, options)).toEqual({ + valueExpression: expr, + template, + }); + } + } }); it('reads back what buildPatternPredicate built without options', () => { @@ -167,6 +311,25 @@ describe('parsePatternPredicate', () => { expect(parsePatternPredicate("ServiceName IN ('api')")).toBeNull(); expect(parsePatternPredicate("match(Body, 'timed out')")).toBeNull(); // A predicate only reads back with the options it was built with. + expect( + parsePatternPredicate( + "match(replaceRegexpAll(Body, '[ ]+', ' '), '^ ?timed out ?$')", + withPunctuation, + ), + ).toBeNull(); + // The real separator class with a hand-written regex: only the round trip + // through compactRegex rejects it. + const compact = buildCompactPatternPredicate( + 'Body', + 'timed out', + withPunctuation, + )!; + const handWritten = compact.replace( + "'^ ?timed out ?$'", + "'^ ?timed.* out ?$'", + ); + expect(handWritten).not.toBe(compact); + expect(parsePatternPredicate(handWritten, withPunctuation)).toBeNull(); expect( parsePatternPredicate( buildPatternPredicate('Body', 'Sending <*> rows')!, diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index 65c433ea1..b0b64d73c 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -21,6 +21,7 @@ export { buildNoisyBodyMatcher, selectNoisyPatterns } from './noisy-matcher'; export type { PatternRegexOptions } from './pattern-predicate'; export { ASCII_PUNCTUATION, + buildCompactPatternPredicate, buildPatternPredicate, MAX_PREDICATE_TEMPLATE_LENGTH, parsePatternPredicate, diff --git a/packages/common-utils/src/drain/pattern-predicate.ts b/packages/common-utils/src/drain/pattern-predicate.ts index d7de608bf..8b5ed5623 100644 --- a/packages/common-utils/src/drain/pattern-predicate.ts +++ b/packages/common-utils/src/drain/pattern-predicate.ts @@ -70,6 +70,13 @@ function separatorClass(options?: PatternRegexOptions): string { ); } +// A template is its tokens joined by spaces. Drain3 keeps a BOM inside a +// token, where JS `\s` would split. +function tokensOf(template: string, options?: PatternRegexOptions): string[] { + const split = options?.whitespace === 'python' ? /[^\S\u{FEFF}]+/u : /\s+/; + return template.split(split).filter(t => t.length > 0); +} + /** Anchored RE2 regex that a value matches when it fits the Drain template. */ export function templateToRegex( template: string, @@ -77,10 +84,7 @@ export function templateToRegex( ): string { const separators = separatorClass(options); const separator = `[${separators}]`; - // Drain3 keeps a BOM inside a token, where JS `\s` would split. - const split = options?.whitespace === 'python' ? /[^\S\u{FEFF}]+/u : /\s+/; - const tokens = template.split(split).filter(t => t.length > 0); - const body = tokens + const body = tokensOf(template, options) .map(t => t === PATTERN_WILDCARD ? `[^${separators}]+` : escapeRe2Literal(t), ) @@ -101,18 +105,99 @@ export function buildPatternPredicate( return `match(${valueExpression}, '${escapeSqlString(templateToRegex(template, options))}')`; } +// The compact regex: tokens joined by single spaces, at most one space at +// each end, and one run of non-spaces per wildcard. +const COMPACT_START = '^ ?'; +const COMPACT_END = ' ?$'; +const COMPACT_WILDCARD = '[^ ]+'; + +// Regex over a value whose separator runs were replaced by single spaces. +function compactRegex(template: string, options?: PatternRegexOptions): string { + const body = tokensOf(template, options) + .map(t => (t === PATTERN_WILDCARD ? COMPACT_WILDCARD : escapeRe2Literal(t))) + .join(' '); + return `${COMPACT_START}${body}${COMPACT_END}`; +} + +/** + * The same match as buildPatternPredicate in much shorter SQL: separator runs + * in the value become single spaces first, so the separator class appears + * once instead of between every token. For conditions that are stored or + * shared, such as search filters kept in the URL. + * + * Null when a template token holds one of the extra delimiters: normalizing + * would split it, so the two forms would disagree. Templates mined with the + * same options never have one. As in buildPatternPredicate, `valueExpression` + * goes into the SQL as is: pass a column expression, never user input. + */ +export function buildCompactPatternPredicate( + valueExpression: string, + template: string, + options?: PatternRegexOptions, +): string | null { + if (template.length > MAX_PREDICATE_TEMPLATE_LENGTH) return null; + const delimiters = options?.extraDelimiters ?? []; + const splitsToken = tokensOf(template, options).some( + t => t !== PATTERN_WILDCARD && [...t].some(c => delimiters.includes(c)), + ); + if (splitsToken) return null; + const separatorRun = `[${separatorClass(options)}]+`; + const normalized = `replaceRegexpAll(${valueExpression}, '${escapeSqlString(separatorRun)}', ' ')`; + return `match(${normalized}, '${escapeSqlString(compactRegex(template, options))}')`; +} + +const unescapeSql = (literal: string) => + literal.replace(/''/g, "'").replace(/\\\\/g, '\\'); +const unescapeRe2Literal = (token: string) => token.replace(/\\(.)/g, '$1'); + +function parseCompactPredicate( + condition: string, + options?: PatternRegexOptions, +): { valueExpression: string; template: string } | null { + const call = + /^match\(replaceRegexpAll\((.+), '((?:[^'\\]|\\\\|'')*)', ' '\), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec( + condition, + ); + if (call == null) return null; + const [, valueExpression, separatorLiteral, regexLiteral] = call; + if (unescapeSql(separatorLiteral) !== `[${separatorClass(options)}]+`) { + return null; + } + const regex = unescapeSql(regexLiteral); + if (!regex.startsWith(COMPACT_START) || !regex.endsWith(COMPACT_END)) { + return null; + } + const inner = regex.slice( + COMPACT_START.length, + regex.length - COMPACT_END.length, + ); + // COMPACT_WILDCARD itself contains a space, so match tokens rather than + // splitting on spaces. + const template = (inner.match(/\[\^ \]\+|[^ ]+/g) ?? []) + .map(part => + part === COMPACT_WILDCARD ? PATTERN_WILDCARD : unescapeRe2Literal(part), + ) + .join(' '); + return compactRegex(template, options) === regex + ? { valueExpression, template } + : null; +} + /** - * Reverse of buildPatternPredicate: the value expression and template of a - * predicate it built with the same options, or null for any other condition. + * Reverse of buildPatternPredicate and buildCompactPatternPredicate: the + * value expression and template of a predicate either built with the same + * options, or null for any other condition. */ export function parsePatternPredicate( condition: string, options?: PatternRegexOptions, ): { valueExpression: string; template: string } | null { + const compact = parseCompactPredicate(condition, options); + if (compact != null) return compact; const call = /^match\((.+), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec(condition); if (call == null) return null; const [, valueExpression, literal] = call; - const regex = literal.replace(/''/g, "'").replace(/\\\\/g, '\\'); + const regex = unescapeSql(literal); const separators = separatorClass(options); const separator = `[${separators}]`; @@ -125,7 +210,7 @@ export function parsePatternPredicate( .map(part => part === `[^${separators}]+` ? PATTERN_WILDCARD - : part.replace(/\\(.)/g, '$1'), + : unescapeRe2Literal(part), ) .join(' '); // Only what this module builds counts, so a hand-written match() isn't