From 6a2dbd6d60e275f3a6fb7f3f1f14fd1c0063385c Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:35:12 +0000 Subject: [PATCH 1/3] feat(common-utils): share the Event Patterns drill-down predicate templateToRegex and buildPatternPredicate build the whereSnippet that the MCP clickstack_event_patterns tool returns for each pattern. They live in the MCP server today. This adds them to @hyperdx/common-utils/dist/drain so the app can build the same filter. The app mines patterns with all ASCII punctuation as extra delimiters, so the separator class gets an extraDelimiters option. When every ASCII punctuation character is a delimiter, the class uses RE2's [:punct:] instead of listing 32 characters between every token. parsePatternPredicate reads a predicate back into its value expression and template. It only accepts predicates these functions built, so a hand-written match() is not mistaken for one. Without extraDelimiters the output is byte-identical to the MCP copy. The next change switches MCP over and deletes its copy. Part of FR-385. Co-Authored-By: Claude Opus 5.5 --- .changeset/shared-pattern-predicate.md | 12 ++ .../src/__tests__/pattern-predicate.test.ts | 109 ++++++++++++++++ packages/common-utils/src/drain/index.ts | 7 ++ .../src/drain/pattern-predicate.ts | 118 ++++++++++++++++++ 4 files changed, 246 insertions(+) create mode 100644 .changeset/shared-pattern-predicate.md create mode 100644 packages/common-utils/src/__tests__/pattern-predicate.test.ts create mode 100644 packages/common-utils/src/drain/pattern-predicate.ts diff --git a/.changeset/shared-pattern-predicate.md b/.changeset/shared-pattern-predicate.md new file mode 100644 index 000000000..c9d5cf18e --- /dev/null +++ b/.changeset/shared-pattern-predicate.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/common-utils': minor +--- + +feat: share the Event Patterns drill-down predicate + +`templateToRegex` and `buildPatternPredicate`, which the MCP server uses to +build the `clickstack_event_patterns` drill-down filter, are now in +`@hyperdx/common-utils/dist/drain`. A new `extraDelimiters` option matches +templates mined with extra delimiters, and `parsePatternPredicate` reads a +predicate back into its value expression and template. Without the option the +output is the same as the MCP server's. diff --git a/packages/common-utils/src/__tests__/pattern-predicate.test.ts b/packages/common-utils/src/__tests__/pattern-predicate.test.ts new file mode 100644 index 000000000..85698f99a --- /dev/null +++ b/packages/common-utils/src/__tests__/pattern-predicate.test.ts @@ -0,0 +1,109 @@ +import { TemplateMinerConfig } from '@/drain/config'; +import { + buildPatternPredicate, + parsePatternPredicate, + templateToRegex, +} from '@/drain/pattern-predicate'; +import { TemplateMiner } from '@/drain/template-miner'; + +// Python's string.punctuation, the extra delimiters the app's miner uses. +const PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split(''); +const withPunctuation = { extraDelimiters: PUNCTUATION }; + +// RE2 spells code points \x{HHHH} and categories \pZ; JS (with the u flag) +// spells them \u{HHHH} and \p{Z}, and has no POSIX classes. +function toJsRegex(re2: string): RegExp { + return RegExp( + re2 + .replace(/\\x\{([0-9A-Fa-f]+)\}/g, '\\u{$1}') + .replace(/(? { + it('splits on whitespace only without extra delimiters', () => { + const ws = '[\\s\\x{0B}\\pZ\\x{FEFF}]'; + expect(templateToRegex('Sending <*> rows')).toBe( + `^${ws}*Sending${ws}+[^\\s\\x{0B}\\pZ\\x{FEFF}]+${ws}+rows${ws}*$`, + ); + }); + + it('spells all ASCII punctuation as [:punct:] and lists other delimiters', () => { + expect(templateToRegex('a', withPunctuation)).toBe( + '^[\\s\\x{0B}\\pZ\\x{FEFF}[:punct:]]*a[\\s\\x{0B}\\pZ\\x{FEFF}[:punct:]]*$', + ); + const partial = { extraDelimiters: ['/', ':', '-', ']', '\\', '^'] }; + const regex = toJsRegex(templateToRegex('a <*> c d e f g', partial)); + expect(regex.test('a/x:c-d]e\\f^g')).toBe(true); + expect(regex.test('a/x:c-d]e\\f^g.h')).toBe(false); + }); + + it('treats extra delimiters as separators', () => { + const regex = toJsRegex( + templateToRegex('GET api v1 users <*> <*> <*>', withPunctuation), + ); + expect(regex.test('GET /api/v1/users/123 200 15ms')).toBe(true); + expect(regex.test('GET /api/v1/users/9 404 3ms')).toBe(true); + expect(regex.test(' GET /api/v1/users/9 404 3ms.')).toBe(true); + + expect(regex.test('POST /api/v1/users/9 404 3ms')).toBe(false); + expect(regex.test('GET /api/v1/users/9/orders 404 3ms')).toBe(false); + expect(regex.test('GET /api/v1/users/9 404')).toBe(false); + expect(regex.test('GET /api/v2/users/9 404 3ms')).toBe(false); + }); + + it('matches every member of every cluster mined with the same delimiters', () => { + const config = new TemplateMinerConfig(); + config.drainExtraDelimiters = PUNCTUATION; + const miner = new TemplateMiner(config); + const messages = [ + 'Connection to db-1:5432 timed out after 30s (retry 1/3)', + 'Connection to db-2:5432 timed out after 30s (retry 2/3)', + 'user=alice action=login status=ok', + 'user=bob action=login status=ok', + '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -', + '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -', + 'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok', + 'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok', + ]; + const members = new Map(); + for (const message of messages) { + const { clusterId } = miner.addLogMessage(message); + members.set(clusterId, [...(members.get(clusterId) ?? []), message]); + } + for (const [clusterId, clusterMembers] of members) { + const template = miner.drain.getCluster(clusterId)!.getTemplate(); + const regex = toJsRegex(templateToRegex(template, withPunctuation)); + for (const message of messages) { + expect(regex.test(message)).toBe(clusterMembers.includes(message)); + } + } + }); +}); + +describe('parsePatternPredicate', () => { + it.each([ + ['Body', 'GET api v1 users <*> <*> <*>'], + ["JSONExtractString(Body, 'message')", 'user <*> action login status ok'], + ['Body', '<*>'], + ])('reads back what buildPatternPredicate built (%s)', (expr, template) => { + const predicate = buildPatternPredicate(expr, template, withPunctuation)!; + expect(parsePatternPredicate(predicate, withPunctuation)).toEqual({ + valueExpression: expr, + template, + }); + }); + + it('rejects other conditions', () => { + expect(parsePatternPredicate("ServiceName IN ('api')")).toBeNull(); + expect(parsePatternPredicate("match(Body, 'timed out')")).toBeNull(); + expect( + parsePatternPredicate( + buildPatternPredicate('Body', 'Sending <*> rows')!, + withPunctuation, + ), + ).toBeNull(); + }); +}); diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index 7615f1109..e457a7779 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -18,5 +18,12 @@ export { } from './mine-patterns'; export { Node } from './node'; export { buildNoisyBodyMatcher, selectNoisyPatterns } from './noisy-matcher'; +export type { PatternRegexOptions } from './pattern-predicate'; +export { + buildPatternPredicate, + MAX_PREDICATE_TEMPLATE_LENGTH, + parsePatternPredicate, + templateToRegex, +} from './pattern-predicate'; export type { AddLogMessageResult, ExtractedParameter } from './template-miner'; export { TemplateMiner } from './template-miner'; diff --git a/packages/common-utils/src/drain/pattern-predicate.ts b/packages/common-utils/src/drain/pattern-predicate.ts new file mode 100644 index 000000000..30b11db1e --- /dev/null +++ b/packages/common-utils/src/drain/pattern-predicate.ts @@ -0,0 +1,118 @@ +import { escapeSqlString } from '@/core/utils'; + +const PATTERN_WILDCARD = '<*>'; + +/** + * Templates longer than this get no predicate: the regex (and the SQL that + * embeds it) would be unwieldy, and very long templates are usually a + * single unparameterized payload rather than a reusable shape. + */ +export const MAX_PREDICATE_TEMPLATE_LENGTH = 4000; + +export type PatternRegexOptions = { + /** + * The extra delimiters the miner was configured with. Drain turns each one + * into a space before splitting, so they separate tokens like whitespace. + */ + extraDelimiters?: string[]; +}; + +// Drain tokenizes with JS `\s`, which includes Unicode spaces and BOM; RE2's +// `\s` is ASCII-only. JS `\s` = RE2 `\s` + \v + \pZ (Zs/Zl/Zp) + BOM. `\pZ` +// keeps the snippet short; spelling out the code points repeats ~100 chars +// between every token. +const JS_WHITESPACE = '\\s\\x{0B}\\pZ\\x{FEFF}'; + +function escapeRe2Literal(token: string): string { + return token.replace(/[\\^$.|?*+()[\]{}]/g, '\\$&'); +} + +// The characters that are special inside a character class. `[` can open a +// POSIX class such as `[:alpha:]` in RE2. +function escapeRe2ClassChars(chars: string): string { + return chars.replace(/[\\\]^[-]/g, '\\$&'); +} + +// Exactly RE2's `[:punct:]`, and Python's string.punctuation. +const ASCII_PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'; + +function separatorClass(options?: PatternRegexOptions): string { + const delimiters = options?.extraDelimiters ?? []; + // The class repeats between every token, so spelling out all 32 ASCII + // punctuation characters (most of which need URL encoding) makes the + // filter too long to keep in a URL. + const allPunctuation = [...ASCII_PUNCTUATION].every(c => + delimiters.includes(c), + ); + const rest = allPunctuation + ? delimiters.filter(c => !ASCII_PUNCTUATION.includes(c)) + : delimiters; + return ( + JS_WHITESPACE + + (allPunctuation ? '[:punct:]' : '') + + escapeRe2ClassChars(rest.join('')) + ); +} + +/** Anchored RE2 regex that a value matches when it fits the Drain template. */ +export function templateToRegex( + template: string, + options?: PatternRegexOptions, +): string { + const separators = separatorClass(options); + const separator = `[${separators}]`; + const tokens = template.split(/\s+/).filter(t => t.length > 0); + const body = tokens + .map(t => + t === PATTERN_WILDCARD ? `[^${separators}]+` : escapeRe2Literal(t), + ) + .join(`${separator}+`); + return `^${separator}*${body}${separator}*$`; +} + +/** + * SQL predicate selecting events whose `valueExpression` fits the template, + * or null when the template is too long to turn into a predicate. + */ +export function buildPatternPredicate( + valueExpression: string, + template: string, + options?: PatternRegexOptions, +): string | null { + if (template.length > MAX_PREDICATE_TEMPLATE_LENGTH) return null; + return `match(${valueExpression}, '${escapeSqlString(templateToRegex(template, options))}')`; +} + +/** + * Reverse of buildPatternPredicate: the value expression and template of a + * predicate it built with the same options, or null for any other condition. + */ +export function parsePatternPredicate( + condition: string, + options?: PatternRegexOptions, +): { valueExpression: string; template: string } | null { + const call = /^match\((.+), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec(condition); + if (call == null) return null; + const [, valueExpression, literal] = call; + const regex = literal.replace(/''/g, "'").replace(/\\\\/g, '\\'); + + const separators = separatorClass(options); + const separator = `[${separators}]`; + const prefix = `^${separator}*`; + const suffix = `${separator}*$`; + if (!regex.startsWith(prefix) || !regex.endsWith(suffix)) return null; + const template = regex + .slice(prefix.length, regex.length - suffix.length) + .split(`${separator}+`) + .map(part => + part === `[^${separators}]+` + ? PATTERN_WILDCARD + : part.replace(/\\(.)/g, '$1'), + ) + .join(' '); + // Only what this module builds counts, so a hand-written match() isn't + // mistaken for a pattern. + return templateToRegex(template, options) === regex + ? { valueExpression, template } + : null; +} From 04c8a768a2e501cf6ab5f83d0887da1e2ab9ba9c Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:11:31 +0000 Subject: [PATCH 2/3] fix(common-utils): address review on the shared pattern predicate - whitespace: 'python' matches templates mined by Drain3, whose str.split() also splits on \x1C-\x1F and NEL but not on BOM. - A delimiter longer than one character throws: Drain splits on it as a whole string, which a character class can't express. - Export ASCII_PUNCTUATION so the app builds its delimiter list from it. - Tests for the length limit, a round trip without options, and options that don't match in either direction. Co-Authored-By: Claude Opus 5.5 --- .changeset/shared-pattern-predicate.md | 8 +-- .../src/__tests__/pattern-predicate.test.ts | 53 ++++++++++++++++++- packages/common-utils/src/drain/index.ts | 1 + .../src/drain/pattern-predicate.ts | 34 ++++++++---- 4 files changed, 82 insertions(+), 14 deletions(-) diff --git a/.changeset/shared-pattern-predicate.md b/.changeset/shared-pattern-predicate.md index c9d5cf18e..99784c662 100644 --- a/.changeset/shared-pattern-predicate.md +++ b/.changeset/shared-pattern-predicate.md @@ -6,7 +6,7 @@ feat: share the Event Patterns drill-down predicate `templateToRegex` and `buildPatternPredicate`, which the MCP server uses to build the `clickstack_event_patterns` drill-down filter, are now in -`@hyperdx/common-utils/dist/drain`. A new `extraDelimiters` option matches -templates mined with extra delimiters, and `parsePatternPredicate` reads a -predicate back into its value expression and template. Without the option the -output is the same as the MCP server's. +`@hyperdx/common-utils/dist/drain`. New options match templates mined with extra +delimiters (`extraDelimiters`) or by Python Drain3 (`whitespace: 'python'`), and +`parsePatternPredicate` reads a predicate back into its value expression and +template. Without options the output is the same as the MCP server's. diff --git a/packages/common-utils/src/__tests__/pattern-predicate.test.ts b/packages/common-utils/src/__tests__/pattern-predicate.test.ts index 85698f99a..ea98c6231 100644 --- a/packages/common-utils/src/__tests__/pattern-predicate.test.ts +++ b/packages/common-utils/src/__tests__/pattern-predicate.test.ts @@ -1,7 +1,9 @@ import { TemplateMinerConfig } from '@/drain/config'; import { buildPatternPredicate, + MAX_PREDICATE_TEMPLATE_LENGTH, parsePatternPredicate, + PatternRegexOptions, templateToRegex, } from '@/drain/pattern-predicate'; import { TemplateMiner } from '@/drain/template-miner'; @@ -11,10 +13,12 @@ const PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split(''); const withPunctuation = { extraDelimiters: PUNCTUATION }; // RE2 spells code points \x{HHHH} and categories \pZ; JS (with the u flag) -// spells them \u{HHHH} and \p{Z}, and has no POSIX classes. +// spells them \u{HHHH} and \p{Z}, and has no POSIX classes. RE2's \s is +// [\t\n\f\r ]; JS's also covers Unicode spaces and BOM. function toJsRegex(re2: string): RegExp { return RegExp( re2 + .replace(/(? { expect(regex.test('a/x:c-d]e\\f^g.h')).toBe(false); }); + it('splits on Python whitespace for templates mined by Drain3', () => { + const python: PatternRegexOptions = { + extraDelimiters: PUNCTUATION, + whitespace: 'python', + }; + // Python's str.split() splits on the \x1C-\x1F separators and NEL but not + // on BOM; JS `\s` is the other way round. + const drain3 = toJsRegex(templateToRegex('a b c', python)); + expect(drain3.test('a\x1eb\x85c')).toBe(true); + expect(drain3.test('a\ufeffb c')).toBe(false); + const ts = toJsRegex(templateToRegex('a b c', withPunctuation)); + expect(ts.test('a\x1eb\x85c')).toBe(false); + expect(ts.test('a\ufeffb c')).toBe(true); + + const predicate = buildPatternPredicate('Body', 'a b c', python)!; + expect(parsePatternPredicate(predicate, python)?.template).toBe('a b c'); + expect(parsePatternPredicate(predicate, withPunctuation)).toBeNull(); + }); + + it('rejects delimiters longer than one character', () => { + expect(() => templateToRegex('a', { extraDelimiters: ['::'] })).toThrow( + 'single characters', + ); + }); + it('treats extra delimiters as separators', () => { const regex = toJsRegex( templateToRegex('GET api v1 users <*> <*> <*>', withPunctuation), @@ -83,6 +112,14 @@ describe('templateToRegex', () => { }); }); +describe('buildPatternPredicate', () => { + it('builds no predicate for a template over the length limit', () => { + const atLimit = 'x'.repeat(MAX_PREDICATE_TEMPLATE_LENGTH); + expect(buildPatternPredicate('Body', atLimit)).not.toBeNull(); + expect(buildPatternPredicate('Body', `${atLimit}x`)).toBeNull(); + }); +}); + describe('parsePatternPredicate', () => { it.each([ ['Body', 'GET api v1 users <*> <*> <*>'], @@ -96,14 +133,28 @@ describe('parsePatternPredicate', () => { }); }); + it('reads back what buildPatternPredicate built without options', () => { + const predicate = buildPatternPredicate('Body', 'Sending <*> rows')!; + expect(parsePatternPredicate(predicate)).toEqual({ + valueExpression: 'Body', + template: 'Sending <*> rows', + }); + }); + it('rejects other conditions', () => { expect(parsePatternPredicate("ServiceName IN ('api')")).toBeNull(); expect(parsePatternPredicate("match(Body, 'timed out')")).toBeNull(); + // A predicate only reads back with the options it was built with. expect( parsePatternPredicate( buildPatternPredicate('Body', 'Sending <*> rows')!, withPunctuation, ), ).toBeNull(); + expect( + parsePatternPredicate( + buildPatternPredicate('Body', 'Sending <*> rows', withPunctuation)!, + ), + ).toBeNull(); }); }); diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index e457a7779..65c433ea1 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -20,6 +20,7 @@ export { Node } from './node'; export { buildNoisyBodyMatcher, selectNoisyPatterns } from './noisy-matcher'; export type { PatternRegexOptions } from './pattern-predicate'; export { + ASCII_PUNCTUATION, buildPatternPredicate, MAX_PREDICATE_TEMPLATE_LENGTH, parsePatternPredicate, diff --git a/packages/common-utils/src/drain/pattern-predicate.ts b/packages/common-utils/src/drain/pattern-predicate.ts index 30b11db1e..20ad58d2c 100644 --- a/packages/common-utils/src/drain/pattern-predicate.ts +++ b/packages/common-utils/src/drain/pattern-predicate.ts @@ -11,17 +11,26 @@ export const MAX_PREDICATE_TEMPLATE_LENGTH = 4000; export type PatternRegexOptions = { /** - * The extra delimiters the miner was configured with. Drain turns each one - * into a space before splitting, so they separate tokens like whitespace. + * The extra delimiters the miner was configured with, one character each. + * Drain turns each one into a space before splitting, so they separate + * tokens like whitespace. */ extraDelimiters?: string[]; + /** + * The whitespace the miner split on: JavaScript's `\s` for the TypeScript + * Drain port (the default), or Python's `str.split()` for Drain3. + */ + whitespace?: 'js' | 'python'; }; -// Drain tokenizes with JS `\s`, which includes Unicode spaces and BOM; RE2's -// `\s` is ASCII-only. JS `\s` = RE2 `\s` + \v + \pZ (Zs/Zl/Zp) + BOM. `\pZ` -// keeps the snippet short; spelling out the code points repeats ~100 chars -// between every token. +// The TypeScript Drain port tokenizes with JS `\s`, which includes Unicode +// spaces and BOM; RE2's `\s` is ASCII-only. JS `\s` = RE2 `\s` + \v + \pZ +// (Zs/Zl/Zp) + BOM. `\pZ` keeps the snippet short; spelling out the code +// points repeats ~100 chars between every token. const JS_WHITESPACE = '\\s\\x{0B}\\pZ\\x{FEFF}'; +// Python's `str.split()`, which Drain3 uses: RE2 `\s` + \v + the separators +// \x1C-\x1F + NEL + \pZ, and no BOM. +const PYTHON_WHITESPACE = '\\s\\x{0B}\\x{1C}-\\x{1F}\\x{85}\\pZ'; function escapeRe2Literal(token: string): string { return token.replace(/[\\^$.|?*+()[\]{}]/g, '\\$&'); @@ -33,11 +42,18 @@ function escapeRe2ClassChars(chars: string): string { return chars.replace(/[\\\]^[-]/g, '\\$&'); } -// Exactly RE2's `[:punct:]`, and Python's string.punctuation. -const ASCII_PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'; +/** The ASCII punctuation: exactly RE2's `[:punct:]`, and Python's `string.punctuation`. */ +export const ASCII_PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'; function separatorClass(options?: PatternRegexOptions): string { const delimiters = options?.extraDelimiters ?? []; + // Drain splits on each delimiter as a whole string, which a character class + // can't express for one longer than a character. + if (delimiters.some(d => [...d].length !== 1)) { + throw new Error('extraDelimiters must be single characters'); + } + const whitespace = + options?.whitespace === 'python' ? PYTHON_WHITESPACE : JS_WHITESPACE; // The class repeats between every token, so spelling out all 32 ASCII // punctuation characters (most of which need URL encoding) makes the // filter too long to keep in a URL. @@ -48,7 +64,7 @@ function separatorClass(options?: PatternRegexOptions): string { ? delimiters.filter(c => !ASCII_PUNCTUATION.includes(c)) : delimiters; return ( - JS_WHITESPACE + + whitespace + (allPunctuation ? '[:punct:]' : '') + escapeRe2ClassChars(rest.join('')) ); From c2fd02324015b9ae0365c78c6e49dd32cc8d9ed8 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:38:00 +0000 Subject: [PATCH 3/3] fix(common-utils): split Drain3 templates on Python whitespace With whitespace: 'python' the template was still split on JS \s, which also splits on BOM. Drain3 keeps a BOM inside a token, so a template mined from a body that starts with one gave a regex its own events failed. Also test a round trip with quotes, backslashes and regex syntax in tokens. Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/pattern-predicate.test.ts | 22 +++++++++++++++++++ .../src/drain/pattern-predicate.ts | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/common-utils/src/__tests__/pattern-predicate.test.ts b/packages/common-utils/src/__tests__/pattern-predicate.test.ts index ea98c6231..5a0f0eeb9 100644 --- a/packages/common-utils/src/__tests__/pattern-predicate.test.ts +++ b/packages/common-utils/src/__tests__/pattern-predicate.test.ts @@ -57,6 +57,17 @@ describe('templateToRegex', () => { const ts = toJsRegex(templateToRegex('a b c', withPunctuation)); expect(ts.test('a\x1eb\x85c')).toBe(false); expect(ts.test('a\ufeffb c')).toBe(true); + // So a Drain3 token can hold a BOM, as in a body that starts with one. + const bomTemplate = '\ufeffGET api <*>'; + expect( + toJsRegex(templateToRegex(bomTemplate, python)).test('\ufeffGET /api/x'), + ).toBe(true); + expect( + parsePatternPredicate( + buildPatternPredicate('Body', bomTemplate, python)!, + python, + )?.template, + ).toBe(bomTemplate); const predicate = buildPatternPredicate('Body', 'a b c', python)!; expect(parsePatternPredicate(predicate, python)?.template).toBe('a b c'); @@ -141,6 +152,17 @@ describe('parsePatternPredicate', () => { }); }); + it('reads back tokens that hold quotes, backslashes and regex syntax', () => { + // Without delimiters a token keeps its punctuation. + const expr = "JSONExtractString(Body, 'msg')"; + const template = "it's C:\\tmp\\a.txt (ok) <*>"; + const predicate = buildPatternPredicate(expr, template)!; + expect(parsePatternPredicate(predicate)).toEqual({ + valueExpression: expr, + template, + }); + }); + it('rejects other conditions', () => { expect(parsePatternPredicate("ServiceName IN ('api')")).toBeNull(); expect(parsePatternPredicate("match(Body, 'timed out')")).toBeNull(); diff --git a/packages/common-utils/src/drain/pattern-predicate.ts b/packages/common-utils/src/drain/pattern-predicate.ts index 20ad58d2c..d7de608bf 100644 --- a/packages/common-utils/src/drain/pattern-predicate.ts +++ b/packages/common-utils/src/drain/pattern-predicate.ts @@ -77,7 +77,9 @@ export function templateToRegex( ): string { const separators = separatorClass(options); const separator = `[${separators}]`; - const tokens = template.split(/\s+/).filter(t => t.length > 0); + // Drain3 keeps a BOM inside a token, where JS `\s` would split. + const split = options?.whitespace === 'python' ? /[^\S\u{FEFF}]+/u : /\s+/; + const tokens = template.split(split).filter(t => t.length > 0); const body = tokens .map(t => t === PATTERN_WILDCARD ? `[^${separators}]+` : escapeRe2Literal(t),