diff --git a/.changeset/adhoc-filters-external-api.md b/.changeset/adhoc-filters-external-api.md new file mode 100644 index 0000000000..9edb98ee56 --- /dev/null +++ b/.changeset/adhoc-filters-external-api.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/api': minor +--- + +feat: Create, read, and update ad hoc dashboard filters through the external API + +`/api/v2/dashboards` now accepts and returns `ADHOC` filters, with SQL and +PromQL source types, and their `adhoc` entries in `savedFilterValues`. Each +filter's sources must exist and be a kind its `sourceType` supports. MCP still +hides ad hoc filters. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index a2c7b3faea..55f996c55a 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -959,13 +959,16 @@ "description": "Query language for the where clause." }, "SavedFilterValue": { - "description": "A single saved dashboard filter selection. Either a rendered SQL condition, or a selection addressed by the name of the dashboard variable it belongs to.\n", + "description": "A single saved dashboard filter selection. Either a rendered SQL condition, a selection addressed by the name of the dashboard variable it belongs to, or the conditions of an ADHOC filter.\n", "oneOf": [ { "$ref": "#/components/schemas/SqlSavedFilterValue" }, { "$ref": "#/components/schemas/VariableSavedFilterValue" + }, + { + "$ref": "#/components/schemas/AdhocSavedFilterValue" } ] }, @@ -1029,6 +1032,78 @@ } } }, + "AdhocSavedFilterValue": { + "type": "object", + "required": [ + "type", + "name", + "conditions" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "adhoc" + ], + "description": "Filter type.", + "example": "adhoc" + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "description": "The variableName of the ADHOC filter these conditions belong to.", + "example": "conditions" + }, + "conditions": { + "type": "array", + "maxItems": 100, + "description": "The filter's conditions.", + "items": { + "$ref": "#/components/schemas/AdhocFilterCondition" + } + } + } + }, + "AdhocFilterCondition": { + "type": "object", + "description": "One \"key operator value\" condition. Conditions are AND'd, except that IN (sql) and =| (promql) conditions sharing a key are OR'd into one.\n", + "required": [ + "key", + "operator", + "value" + ], + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 10000, + "description": "A SQL expression (column or map access) for sql filters, or a label name for promql filters.", + "example": "ServiceName" + }, + "operator": { + "type": "string", + "enum": [ + "=", + "!=", + "=~", + "!~", + "LIKE", + "NOT LIKE", + "IN", + "=|" + ], + "description": "Comparison operator. =~ and !~ match a regular expression. LIKE, NOT LIKE, and IN apply to sql filters only, and =| to promql filters only; a condition its filter's sourceType can't express is not applied.\n", + "example": "=" + }, + "value": { + "type": "string", + "maxLength": 10000, + "description": "Value, pattern, or regular expression to compare against.", + "example": "api" + } + } + }, "MetricDataType": { "type": "string", "enum": [ @@ -3951,6 +4026,9 @@ }, { "$ref": "#/components/schemas/PrometheusLabelFilterInput" + }, + { + "$ref": "#/components/schemas/AdhocFilterInput" } ], "discriminator": { @@ -3958,7 +4036,8 @@ "mapping": { "QUERY_EXPRESSION": "#/components/schemas/QueryExpressionFilterInput", "STATIC_LIST": "#/components/schemas/StaticListFilterInput", - "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput" + "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput", + "ADHOC": "#/components/schemas/AdhocFilterInput" } } }, @@ -4262,6 +4341,101 @@ } } }, + "AdhocFilterInput": { + "type": "object", + "description": "A filter holding free-form \"key operator value\" conditions, with keys\nand values suggested from its sources: columns and map keys for sql\nsources, labels for promql sources. Its conditions are always\nbroadcast to the tiles it applies to: as WHERE conditions on builder\ntiles and raw sql tiles using the $__filters macro, or as label\nmatchers on PromQL tiles. Its conditions are saved in\nsavedFilterValues as an \"adhoc\" entry. Ad hoc filters are a preview\nfeature.\n", + "required": [ + "type", + "name", + "sourceType", + "sources" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "ADHOC" + ], + "description": "Discriminator. Must be \"ADHOC\".", + "example": "ADHOC" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for the dashboard filter", + "example": "Conditions" + }, + "sourceType": { + "type": "string", + "enum": [ + "sql", + "promql" + ], + "description": "The kind of sources the filter reads keys from and applies to: Log,\nTrace, or Session sources for sql, PromQL sources for promql.\n", + "example": "sql" + }, + "sources": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + }, + "description": "IDs of the sources whose keys and values the filter suggests, all of sourceType.", + "example": [ + "65f5e4a3b9e77c001a111111" + ] + }, + "where": { + "type": "string", + "description": "Optional condition narrowing the rows keys and values are suggested\nfrom. sql filters only; never applied to tiles. May reference the\ndashboard's variables.\n", + "example": "ServiceName:api" + }, + "whereLanguage": { + "type": "string", + "enum": [ + "sql", + "lucene" + ], + "description": "Language of the where condition", + "example": "lucene" + }, + "appliesToSourceIds": { + "type": "array", + "items": { + "type": "string" + }, + "description": "IDs of the sources whose tiles the conditions apply to, all of\nsourceType. Omit or send an empty array to apply to the tiles of\nthe filter's sources.\n", + "example": [ + "65f5e4a3b9e77c001a111111" + ] + }, + "isBroadcastEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true or omitted.", + "example": true + }, + "isVariableEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true or omitted.", + "example": true + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Unique name the filter's conditions are saved under, in\nsavedFilterValues and the dashboard URL. Tiles can't reference it\nas a variable. Defaults to the display name with whitespace\nreplaced by underscores and remaining illegal characters removed.\nMust be unique across a dashboard's variable-enabled filters.\n", + "example": "conditions" + } + } + }, "Filter": { "allOf": [ { diff --git a/packages/api/src/mcp/tools/dashboards/getDashboard.ts b/packages/api/src/mcp/tools/dashboards/getDashboard.ts index a18f09952b..2133ec09b3 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboard.ts @@ -63,6 +63,7 @@ export function registerGetDashboard({ } const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); return { content: [ diff --git a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts index f9a4d2632f..7d331d965c 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts @@ -51,6 +51,7 @@ export function registerGetDashboardTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const tile = externalDashboard.tiles.find(t => t.id === tileId); if (!tile) { diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts index aeea54d7bd..4488ca1c7b 100644 --- a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -15,7 +15,7 @@ import { isConfigTile, validateDashboardTiles, } from '@/routers/external-api/v2/utils/dashboards'; -import { getExternalCompatibleFilterValues } from '@/utils/externalApi'; +import { getMcpCompatibleFilterValues } from '@/utils/externalApi'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; @@ -127,7 +127,7 @@ export function registerPatchDashboard({ // Convert only for the error message (read-only, no write-back). const externalDashboard = convertToExternalDashboard( existingDashboard, - { hidePromql: !promqlEnabled }, + { hidePromql: !promqlEnabled, hideAdhoc: true }, ); return mcpUserError( `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => `${t.id} (${t.name})`).join(', ')}`, @@ -271,7 +271,7 @@ export function registerPatchDashboard({ tags: updatedDashboard.tags, ...(savedFilterValues !== undefined ? { - savedFilterValues: getExternalCompatibleFilterValues( + savedFilterValues: getMcpCompatibleFilterValues( updatedDashboard.savedFilterValues, ), } diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index b6cd385b57..0f9df6b116 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -78,6 +78,7 @@ export function registerQueryTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const tile = externalDashboard.tiles.find(t => t.id === tileId); if (!tile) { diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 44af35c950..9860ad768c 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -184,6 +184,7 @@ export function registerQueryTiles({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const allTiles = externalDashboard.tiles; diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index ba0c0d21e3..bde01373e8 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -236,6 +236,7 @@ async function createDashboard({ const externalDashboard = convertToExternalDashboard(newDashboard, { hidePromql: !promqlEnabled, + hideAdhoc: true, }); return { content: [ @@ -452,6 +453,7 @@ async function updateDashboard({ const externalDashboard = convertToExternalDashboard(updatedDashboard, { hidePromql: !promqlEnabled, + hideAdhoc: true, }); return { content: [ diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index 987ab6ad2f..58d82da8de 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -18,6 +18,7 @@ import { TIME_RANGE_MACROS, } from '@hyperdx/common-utils/dist/macros'; import { + DashboardFilterValue, DisplayType, QueryExpressionDashboardFilter, SavedChartConfig, @@ -36,8 +37,6 @@ import { } from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; -import type { McpSavedFilterValues } from './schemas'; - /** * Returns one entry per raw SQL tile that uses a source-dependent macro but * does not set a `sourceId`, recording which macro(s) triggered it so callers @@ -253,7 +252,7 @@ export function getFilterVariableWarnings( * applies. Warn so the agent can fix the name or add the filter. */ export function getSavedFilterValueWarnings( - savedFilterValues: McpSavedFilterValues | undefined, + savedFilterValues: DashboardFilterValue[] | undefined, filters: FilterForVariableDeclaration[] | undefined, ): string[] { if (!savedFilterValues?.length) return []; diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 93fea0e4d5..28eb69c859 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -8456,9 +8456,16 @@ describe('External API v2 Dashboards - new format', () => { conditions: [{ key: 'ServiceName', operator: '=', value: 'api' }], }; - it('hides ad hoc filters and their saved values from GET', async () => { + const promqlAdhocFilter = () => ({ + ...adhocFilter(), + sourceType: 'promql' as const, + sources: [promqlSource._id.toString()], + }); + + it('returns ad hoc filters and their saved values from GET', async () => { + const filter = adhocFilter(); const dashboard = await createTestDashboard({ - filters: [adhocFilter()], + filters: [filter], savedFilterValues: [adhocValue], }); @@ -8467,25 +8474,163 @@ describe('External API v2 Dashboards - new format', () => { `${BASE_URL}/${dashboard._id}`, ).expect(200); - expect(response.body.data.filters).toEqual([]); - expect(response.body.data.savedFilterValues).toEqual([]); + expect(response.body.data.filters).toEqual([filter]); + expect(response.body.data.savedFilterValues).toEqual([adhocValue]); + }); + + it('creates SQL and PromQL ad hoc filters with saved conditions', async () => { + const promqlValue = { + type: 'adhoc' as const, + name: 'prom', + conditions: [{ key: 'job', operator: '=|', value: 'api' }], + }; + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [ + omit( + { ...adhocFilter(), appliesToSourceIds: [traceSource._id] }, + 'id', + ), + // The mode flags default when omitted. + omit( + { ...promqlAdhocFilter(), name: 'Prom', variableName: 'prom' }, + 'id', + 'isBroadcastEnabled', + 'isVariableEnabled', + ), + ], + savedFilterValues: [adhocValue, promqlValue], + }), + ) + .expect(200); + + expect(response.body.data.filters).toEqual([ + { + ...adhocFilter(), + id: expect.any(String), + appliesToSourceIds: [traceSource._id.toString()], + }, + { + ...promqlAdhocFilter(), + id: expect.any(String), + name: 'Prom', + variableName: 'prom', + }, + ]); + expect(response.body.data.savedFilterValues).toEqual([ + adhocValue, + promqlValue, + ]); + + const stored = await Dashboard.findById(response.body.data.id).lean(); + expect(stored?.filters?.[1]).toMatchObject({ + type: 'ADHOC', + sourceType: 'promql', + sources: [promqlSource._id.toString()], + isBroadcastEnabled: true, + }); + }); + + it('round-trips a GET response through PUT', async () => { + const dashboard = await createTestDashboard({ + filters: [adhocFilter()], + savedFilterValues: [adhocValue], + }); + const { data } = ( + await authRequest('get', `${BASE_URL}/${dashboard._id}`).expect(200) + ).body; + + const response = await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send(omit(data, 'id')) + .expect(200); + + expect(response.body.data.filters).toEqual(data.filters); + expect(response.body.data.savedFilterValues).toEqual( + data.savedFilterValues, + ); }); - it('rejects an ad hoc filter in a POST', async () => { + it.each([ + ['broadcast disabled', { isBroadcastEnabled: false }], + ['variables disabled', { isVariableEnabled: false }], + ['no sources', { sources: [] }], + ['an invalid source ID', { sources: ['not-an-id'] }], + ['an unknown source type', { sourceType: 'lucene' }], + ['a selection limit', { maxSelections: 1 }], + ['the external sourceId field', { sourceId: new ObjectId().toString() }], + ])('rejects an ad hoc filter with %s', async (_label, overrides) => { await authRequest('post', BASE_URL) .send( createMockDashboard(traceSource._id.toString(), { - filters: [omit(adhocFilter(), 'id')], + filters: [omit({ ...adhocFilter(), ...overrides }, 'id')], }), ) .expect(400); }); - it('rejects ad hoc saved values in a POST', async () => { + it('rejects an ad hoc filter over a missing source', async () => { + const missing = new ObjectId().toString(); + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [omit({ ...adhocFilter(), sources: [missing] }, 'id')], + }), + ) + .expect(400); + + expect(response.body.message).toContain(missing); + }); + + it.each([ + ['a SQL filter over a PromQL source', () => adhocFilter(), 'promql'], + ['a SQL filter over a Metric source', () => adhocFilter(), 'metric'], + ['a PromQL filter over a Trace source', promqlAdhocFilter, 'trace'], + ] as const)('rejects %s', async (_label, makeFilter, kind) => { + const sourceId = { + promql: promqlSource, + metric: metricSource, + trace: traceSource, + }[kind]._id.toString(); + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [omit({ ...makeFilter(), sources: [sourceId] }, 'id')], + }), + ) + .expect(400); + + expect(response.body.message).toContain(sourceId); + }); + + it('rejects an ad hoc filter applying to a source of the wrong kind', async () => { + await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [ + omit( + { + ...adhocFilter(), + appliesToSourceIds: [promqlSource._id.toString()], + }, + 'id', + ), + ], + }), + ) + .expect(400); + }); + + it('rejects ad hoc saved values with an unknown operator', async () => { await authRequest('post', BASE_URL) .send( createMockDashboard(traceSource._id.toString(), { - savedFilterValues: [adhocValue], + savedFilterValues: [ + { + ...adhocValue, + conditions: [{ key: 'a', operator: '>', value: '1' }], + }, + ], }), ) .expect(400); diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 77d3b31361..5a37bc7fb1 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -57,11 +57,12 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * SavedFilterValue: * description: > * A single saved dashboard filter selection. Either a rendered SQL - * condition, or a selection addressed by the name of the dashboard - * variable it belongs to. + * condition, a selection addressed by the name of the dashboard + * variable it belongs to, or the conditions of an ADHOC filter. * oneOf: * - $ref: '#/components/schemas/SqlSavedFilterValue' * - $ref: '#/components/schemas/VariableSavedFilterValue' + * - $ref: '#/components/schemas/AdhocSavedFilterValue' * SqlSavedFilterValue: * type: object * required: [condition] @@ -102,6 +103,54 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: string * maxLength: 10000 * example: ["hdx-oss-dev-api"] + * AdhocSavedFilterValue: + * type: object + * required: [type, name, conditions] + * properties: + * type: + * type: string + * enum: [adhoc] + * description: Filter type. + * example: "adhoc" + * name: + * type: string + * minLength: 1 + * maxLength: 1024 + * description: The variableName of the ADHOC filter these conditions belong to. + * example: "conditions" + * conditions: + * type: array + * maxItems: 100 + * description: The filter's conditions. + * items: + * $ref: '#/components/schemas/AdhocFilterCondition' + * AdhocFilterCondition: + * type: object + * description: > + * One "key operator value" condition. Conditions are AND'd, except that + * IN (sql) and =| (promql) conditions sharing a key are OR'd into one. + * required: [key, operator, value] + * properties: + * key: + * type: string + * minLength: 1 + * maxLength: 10000 + * description: A SQL expression (column or map access) for sql filters, or a label name for promql filters. + * example: "ServiceName" + * operator: + * type: string + * enum: ["=", "!=", "=~", "!~", "LIKE", "NOT LIKE", "IN", "=|"] + * description: > + * Comparison operator. =~ and !~ match a regular expression. LIKE, + * NOT LIKE, and IN apply to sql filters only, and =| to promql + * filters only; a condition its filter's sourceType can't express + * is not applied. + * example: "=" + * value: + * type: string + * maxLength: 10000 + * description: Value, pattern, or regular expression to compare against. + * example: "api" * MetricDataType: * type: string * enum: [sum, gauge, histogram, summary, exponential histogram] @@ -2368,12 +2417,14 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * - $ref: '#/components/schemas/QueryExpressionFilterInput' * - $ref: '#/components/schemas/StaticListFilterInput' * - $ref: '#/components/schemas/PrometheusLabelFilterInput' + * - $ref: '#/components/schemas/AdhocFilterInput' * discriminator: * propertyName: type * mapping: * QUERY_EXPRESSION: '#/components/schemas/QueryExpressionFilterInput' * STATIC_LIST: '#/components/schemas/StaticListFilterInput' * PROMETHEUS_LABEL: '#/components/schemas/PrometheusLabelFilterInput' + * ADHOC: '#/components/schemas/AdhocFilterInput' * * QueryExpressionFilterInput: * type: object @@ -2684,6 +2735,92 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * behavior. * example: 1 * + * AdhocFilterInput: + * type: object + * description: | + * A filter holding free-form "key operator value" conditions, with keys + * and values suggested from its sources: columns and map keys for sql + * sources, labels for promql sources. Its conditions are always + * broadcast to the tiles it applies to: as WHERE conditions on builder + * tiles and raw sql tiles using the $__filters macro, or as label + * matchers on PromQL tiles. Its conditions are saved in + * savedFilterValues as an "adhoc" entry. Ad hoc filters are a preview + * feature. + * required: + * - type + * - name + * - sourceType + * - sources + * properties: + * type: + * type: string + * enum: [ADHOC] + * description: Discriminator. Must be "ADHOC". + * example: "ADHOC" + * name: + * type: string + * minLength: 1 + * description: Display name for the dashboard filter + * example: "Conditions" + * sourceType: + * type: string + * enum: [sql, promql] + * description: | + * The kind of sources the filter reads keys from and applies to: Log, + * Trace, or Session sources for sql, PromQL sources for promql. + * example: "sql" + * sources: + * type: array + * minItems: 1 + * items: + * type: string + * description: IDs of the sources whose keys and values the filter suggests, all of sourceType. + * example: ["65f5e4a3b9e77c001a111111"] + * where: + * type: string + * description: | + * Optional condition narrowing the rows keys and values are suggested + * from. sql filters only; never applied to tiles. May reference the + * dashboard's variables. + * example: "ServiceName:api" + * whereLanguage: + * type: string + * enum: [sql, lucene] + * description: Language of the where condition + * example: "lucene" + * appliesToSourceIds: + * type: array + * items: + * type: string + * description: | + * IDs of the sources whose tiles the conditions apply to, all of + * sourceType. Omit or send an empty array to apply to the tiles of + * the filter's sources. + * example: ["65f5e4a3b9e77c001a111111"] + * isBroadcastEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true or omitted. + * example: true + * isVariableEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true or omitted. + * example: true + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: | + * Unique name the filter's conditions are saved under, in + * savedFilterValues and the dashboard URL. Tiles can't reference it + * as a variable. Defaults to the display name with whitespace + * replaced by underscores and remaining illegal characters removed. + * Must be unique across a dashboard's variable-enabled filters. + * example: "conditions" + * * Filter: * allOf: * - $ref: '#/components/schemas/FilterInput' diff --git a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts index dc8882183e..dec4319981 100644 --- a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts @@ -508,6 +508,43 @@ describe('convertToExternalDashboard orphan-ref heal', () => { expect(hidden.filters?.map(f => f.id)).toEqual(['static-filter']); }); + it('hides ad hoc filters and their saved conditions only when asked', () => { + const adhoc = { + id: 'adhoc-filter', + type: 'ADHOC' as const, + name: 'Conditions', + sourceType: 'sql' as const, + sources: [new mongoose.Types.ObjectId().toString()], + isBroadcastEnabled: true as const, + isVariableEnabled: true as const, + variableName: 'conds', + }; + const variableValue = { + type: 'variable' as const, + name: 'env', + values: ['prod'], + }; + const adhocValue = { + type: 'adhoc' as const, + name: 'conds', + conditions: [ + { key: 'ServiceName', operator: '=' as const, value: 'api' }, + ], + }; + const doc = makeDoc({ + filters: [adhoc], + savedFilterValues: [variableValue, adhocValue], + }); + + const shown = convertToExternalDashboard(doc); + expect(shown.filters).toEqual([adhoc]); + expect(shown.savedFilterValues).toEqual([variableValue, adhocValue]); + + const hidden = convertToExternalDashboard(doc, { hideAdhoc: true }); + expect(hidden.filters).toEqual([]); + expect(hidden.savedFilterValues).toEqual([variableValue]); + }); + // seriesLimit is three-state (matching the internal schema): omitted = // default cap, 0 = unlimited, positive N = top-N. All three must survive a // GET->PUT round-trip: 0 and N pass through, null maps to absent (the diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 69e143e950..7e84a6b664 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -14,6 +14,8 @@ import { validateDashboardTileContainerRefs, } from '@hyperdx/common-utils/dist/dashboardValidation'; import { + getAdhocFilterSourceKinds, + isAdhocFilter, isPrometheusLabelFilter, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; @@ -55,8 +57,8 @@ import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import Dashboard, { DashboardDocument } from '@/models/dashboard'; import { - getExternalCompatibleFilters, - getExternalCompatibleFilterValues, + getMcpCompatibleFilters, + getMcpCompatibleFilterValues, translateExternalChartToTileConfig, translateExternalFilterToFilter, translateFilterToExternalFilter, @@ -773,10 +775,13 @@ export function convertToExternalDashboard( dashboard: DashboardDocument, { hidePromql = false, + hideAdhoc = false, }: { // MCP hides PromQL tiles and PROMETHEUS_LABEL filters while PromQL is // disabled on the deployment. hidePromql?: boolean; + // MCP doesn't expose ad hoc filters or their saved conditions yet. + hideAdhoc?: boolean; } = {}, ): ExternalDashboard { const containers = dashboard.containers ?? []; @@ -798,14 +803,17 @@ export function convertToExternalDashboard( convertTileToExternalChart(tile, containerById, dashboardId), ), tags: dashboard.tags || [], - filters: getExternalCompatibleFilters(dashboard.filters) + filters: (hideAdhoc + ? getMcpCompatibleFilters(dashboard.filters) + : (dashboard.filters ?? []) + ) .filter(filter => !(hidePromql && isPrometheusLabelFilter(filter))) .map(translateFilterToExternalFilter), savedQuery: dashboard.savedQuery ?? null, savedQueryLanguage: dashboard.savedQueryLanguage ?? null, - savedFilterValues: getExternalCompatibleFilterValues( - dashboard.savedFilterValues, - ), + savedFilterValues: hideAdhoc + ? getMcpCompatibleFilterValues(dashboard.savedFilterValues) + : (dashboard.savedFilterValues ?? []), // Mongoose persists missing arrays as []. Only emit containers when // the user actually saved one or more, so dashboards without the // organization layer round-trip with the field absent. @@ -1279,6 +1287,39 @@ function getTileOnClick(tile: ExternalDashboardTileWithId) { return tile.config.onClick; } +/** Source IDs a filter reads from or applies to. */ +function getFilterSourceIds( + filter: ExternalDashboardFilter | ExternalDashboardFilterWithId, +): string[] { + if (isStaticListFilter(filter)) return []; + if (isAdhocFilter(filter)) { + return [...filter.sources, ...(filter.appliesToSourceIds ?? [])]; + } + return [filter.sourceId]; +} + +/** + * Returns an error message if an ad hoc filter reads from or applies to a + * source of a kind its `sourceType` can't query, or null if none does. + */ +function getAdhocFilterSourceKindError( + sources: SourceForValidation[], + filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[], +): string | null { + const kindById = new Map(sources.map(s => [s._id.toString(), s.kind])); + const invalid = new Set( + filters.filter(isAdhocFilter).flatMap(filter => { + const kinds = getAdhocFilterSourceKinds(filter.sourceType); + return getFilterSourceIds(filter).filter(id => { + const kind = kindById.get(id); + return kind !== undefined && !kinds.includes(kind); + }); + }), + ); + if (invalid.size === 0) return null; + return `ADHOC filters with sourceType "sql" require Log, Trace, or Session sources, and with sourceType "promql" require PromQL sources. The following source IDs are not a supported kind: ${[...invalid].join(', ')}`; +} + /** Returns source IDs referenced in tiles/filters that do not exist for the team */ function getMissingSources( sources: SourceForValidation[], @@ -1307,13 +1348,8 @@ function getMissingSources( } } - if (filters?.length) { - for (const filter of filters) { - // Every variant but the static one names a source. - if (!isStaticListFilter(filter)) { - sourceIds.add(filter.sourceId); - } - } + for (const filter of filters ?? []) { + for (const sourceId of getFilterSourceIds(filter)) sourceIds.add(sourceId); } const existingSourceIds = new Set( @@ -1832,6 +1868,12 @@ export async function validateDashboardTiles( ); if (promqlLabelFilterError != null) return promqlLabelFilterError; + const adhocFilterError = getAdhocFilterSourceKindError( + sources, + filters ?? [], + ); + if (adhocFilterError != null) return adhocFilterError; + if (missingOnClickDashboards.length > 0) { return `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join(', ')}`; } diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 187abd7a24..b0e2ed35d7 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -240,32 +240,22 @@ export function translateExternalChartToTileConfig( }; } -/** Filter types the external API and MCP don't expose yet. */ -type InternalOnlyDashboardFilter = Extract; -export type ExternalCompatibleDashboardFilter = Exclude< - DashboardFilter, - InternalOnlyDashboardFilter ->; - -/** The filters the external API and MCP may read. */ -export function getExternalCompatibleFilters( +/** The filters MCP may read; it doesn't expose ad hoc filters yet. */ +export function getMcpCompatibleFilters( filters: DashboardFilter[] | undefined, -): ExternalCompatibleDashboardFilter[] { - return (filters ?? []).filter( - (filter): filter is ExternalCompatibleDashboardFilter => - !isAdhocFilter(filter), - ); +): DashboardFilter[] { + return (filters ?? []).filter(filter => !isAdhocFilter(filter)); } -/** Saved filter values the external API and MCP may read. */ -export function getExternalCompatibleFilterValues( +/** Saved filter values MCP may read; it doesn't expose ad hoc filters yet. */ +export function getMcpCompatibleFilterValues( values: IDashboard['savedFilterValues'], ): NonNullable { return (values ?? []).filter(value => !isAdhocFilterValue(value)); } export function translateFilterToExternalFilter( - filter: ExternalCompatibleDashboardFilter, + filter: DashboardFilter, ): ExternalDashboardFilterWithId { // Omitted from responses unless the filter is required, so a GET response // can be PUT back verbatim. @@ -277,6 +267,9 @@ export function translateFilterToExternalFilter( case 'STATIC_LIST': return omit(filter, ...omittedKeys); + case 'ADHOC': + return filter; + case 'PROMETHEUS_LABEL': return { ...omit(filter, 'source', ...omittedKeys), @@ -309,6 +302,7 @@ export function translateExternalFilterToFilter( ): DashboardFilter { switch (filter.type) { case 'STATIC_LIST': + case 'ADHOC': return filter; case 'PROMETHEUS_LABEL': diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index d14d34849d..5a27dccf2b 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1,6 +1,8 @@ import { validateChartConfigFormulas } from '@hyperdx/common-utils/dist/dashboardValidation'; import { addDuplicateTileIdIssues, + AdhocDashboardFilterSchema, + AdhocFilterValueSchema, AggregateFunctionSchema, AlertChartConfigSchema, alertDisplayNameSchema, @@ -189,12 +191,21 @@ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ isVariableEnabled: z.literal(true).default(true), }); +// Field names match the stored filter, so it needs no translation. +const externalAdhocFilterShape = AdhocDashboardFilterSchema.extend({ + sources: z.array(objectIdSchema).min(1), + appliesToSourceIds: z.array(objectIdSchema).optional(), + isBroadcastEnabled: z.literal(true).default(true), + isVariableEnabled: z.literal(true).default(true), +}); + export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( 'type', [ externalQueryExpressionFilterShape.strict(), externalStaticListFilterShape.strict(), externalPromqlLabelFilterShape.strict(), + externalAdhocFilterShape.strict(), ], ); @@ -211,6 +222,7 @@ export const externalDashboardFilterSchema = z.discriminatedUnion('type', [ externalQueryExpressionFilterShape.omit({ id: true }).strict(), externalStaticListFilterShape.omit({ id: true }).strict(), externalPromqlLabelFilterShape.omit({ id: true }).strict(), + externalAdhocFilterShape.omit({ id: true }).strict(), ]); export type ExternalDashboardFilter = z.infer< @@ -218,8 +230,9 @@ export type ExternalDashboardFilter = z.infer< >; /** - * One entry in a dashboard's `savedFilterValues`: either a rendered SQL - * predicate, or a selection addressed by the dashboard variable it belongs to. + * One entry in a dashboard's `savedFilterValues`: a rendered SQL predicate, a + * selection addressed by the dashboard variable it belongs to, or an ad hoc + * filter's conditions addressed by its variable name. */ export const externalSqlSavedFilterValueSchema = z.object({ type: z.literal('sql').optional().default('sql'), @@ -232,6 +245,7 @@ export const externalVariableSavedFilterValueSchema = export const externalDashboardSavedFilterValueSchema = z.union([ externalSqlSavedFilterValueSchema, externalVariableSavedFilterValueSchema, + AdhocFilterValueSchema.strict(), ]); // ================================