perf: read a configurable set of parts for text index key and value lookups

.changeset/map-key-index-median-parts.md

@@ -0,0 +1,15 @@
+---
+'@hyperdx/common-utils': patch
+'@hyperdx/api': patch
+'@hyperdx/app': patch
+---
+
+perf: read only 20 median-sized parts for text index key and value lookups
+
+Map key discovery and "all values" lookups on text indexes read the index
+dictionary of every part in the time range, which is slow on tables with many
+parts. They now read the 20 parts closest to the median size. Log and trace
+sources can change the count with the new "Map key index parts" setting
+(`mapKeyIndexMaxParts`), and which parts are read with "Map key index part size"
+(`mapKeyIndexPartSelection`: median, smallest, or largest). "Load more" on a
+single filter key still reads every part in the time range.

packages/api/openapi.json

@@ -5203,6 +5203,24 @@
             "nullable": true,
             "example": "auto"
           },
+          "mapKeyIndexMaxParts": {
+            "type": "integer",
+            "minimum": 1,
+            "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.",
+            "nullable": true,
+            "example": 20
+          },
+          "mapKeyIndexPartSelection": {
+            "type": "string",
+            "enum": [
+              "median",
+              "smallest",
+              "largest"
+            ],
+            "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.",
+            "nullable": true,
+            "example": "median"
+          },
           "highlightedTraceAttributeExpressions": {
             "type": "array",
             "description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.",
@@ -5457,6 +5475,24 @@
             "nullable": true,
             "example": "auto"
           },
+          "mapKeyIndexMaxParts": {
+            "type": "integer",
+            "minimum": 1,
+            "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.",
+            "nullable": true,
+            "example": 20
+          },
+          "mapKeyIndexPartSelection": {
+            "type": "string",
+            "enum": [
+              "median",
+              "smallest",
+              "largest"
+            ],
+            "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.",
+            "nullable": true,
+            "example": "median"
+          },
           "highlightedTraceAttributeExpressions": {
             "type": "array",
             "description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.",

packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts

@@ -12,6 +12,8 @@ jest.mock('@/controllers/connection', () => ({
 jest.mock('@/clickhouse', () => ({ ClickhouseClient: jest.fn() }));
 jest.mock('@hyperdx/common-utils/dist/core/metadata', () => ({
   getMetadata: () => mockMetadata,
+  tcFromSource: jest.requireActual('@hyperdx/common-utils/dist/core/metadata')
+    .tcFromSource,
 }));
 jest.mock('@/controllers/promql', () => ({
   getPromqlLabelCatalog: jest.fn(),

packages/api/src/mcp/tools/sources/describeSource.ts

@@ -3,7 +3,10 @@ import {
   filterColumnMetaByType,
   JSDataType,
 } from '@hyperdx/common-utils/dist/clickhouse';
-import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
+import {
+  getMetadata,
+  tcFromSource,
+} from '@hyperdx/common-utils/dist/core/metadata';
 import { type MetricTable, SourceKind } from '@hyperdx/common-utils/dist/types';
 import { trace } from '@opentelemetry/api';
 import SqlString from 'sqlstring';
@@ -224,10 +227,8 @@ async function describeSourceSchema(
 
   // Shared by stages 2–4 so map-key discovery can use rollup tables
   // instead of falling back to expensive main-table scans.
-  const metadataMVs =
-    'metadataMaterializedViews' in source
-      ? source.metadataMaterializedViews
-      : undefined;
+  const { metadataMVs, mapKeyIndexMaxParts, mapKeyIndexPartSelection } =
+    tcFromSource(source);
 
   const now = new Date();
   const dateRange: [Date, Date] = [
@@ -306,6 +307,8 @@ async function describeSourceSchema(
             maxKeys: 50,
             connectionId,
             metadataMVs,
+            mapKeyIndexMaxParts,
+            mapKeyIndexPartSelection,
             dateRange,
             timestampValueExpression,
             signal,
@@ -352,6 +355,8 @@ async function describeSourceSchema(
         maxValuesPerKey: MAX_LC_VALUES,
         connectionId,
         metadataMVs,
+        mapKeyIndexMaxParts,
+        mapKeyIndexPartSelection,
         dateRange,
         timestampValueExpression,
         signal,
@@ -399,6 +404,8 @@ async function describeSourceSchema(
         maxValuesPerKey: MAX_MAP_KEY_VALUES,
         connectionId,
         metadataMVs,
+        mapKeyIndexMaxParts,
+        mapKeyIndexPartSelection,
         dateRange,
         timestampValueExpression,
         signal,

packages/api/src/mcp/tools/sources/schemas.ts

@@ -466,6 +466,26 @@ const mcpSaveSourceSchema = z.object({
       'Log and trace sources. Rollup tables used for fast field discovery and ' +
         'value autocomplete in filters and search suggestions.',
     ),
+  mapKeyIndexMaxParts: z
+    .number()
+    .int()
+    .min(1)
+    .optional()
+    .describe(
+      'Log and trace sources. How many data parts are read from text indexes ' +
+        'to discover attribute keys and filter values, chosen by ' +
+        'mapKeyIndexPartSelection. Raise it if keys or values are missing from ' +
+        'filters; lower it if they load slowly. Omit for the default of 20.',
+    ),
+  mapKeyIndexPartSelection: z
+    .enum(['median', 'smallest', 'largest'])
+    .optional()
+    .describe(
+      'Log and trace sources. Which data parts, by row count, are read from ' +
+        'text indexes for attribute keys and filter values: "median" (used when unset) ' +
+        'balances coverage and cost, "smallest" reads the newest unmerged ' +
+        'parts, "largest" reads the most merged data.',
+    ),
 });
 
 export type McpSaveSourceInput = z.infer<typeof mcpSaveSourceSchema>;

packages/api/src/models/source.ts

@@ -1,6 +1,7 @@
 import {
   BaseSourceSchema,
   LogSourceSchema,
+  MapKeyIndexPartSelection,
   MetricsDataType,
   MetricSourceSchema,
   PROMQL_SOURCE_BACKENDS,
@@ -189,6 +190,11 @@ export const LogSource = Source.discriminator<ILogSource>(
       type: MetadataMaterializedViewsMongoSchema,
       default: undefined,
     },
+    mapKeyIndexMaxParts: Number,
+    mapKeyIndexPartSelection: {
+      type: String,
+      enum: Object.values(MapKeyIndexPartSelection),
+    },
     orderByExpression: String,
   }),
 );
@@ -240,6 +246,11 @@ export const TraceSource = Source.discriminator<ITraceSource>(
       type: MetadataMaterializedViewsMongoSchema,
       default: undefined,
     },
+    mapKeyIndexMaxParts: Number,
+    mapKeyIndexPartSelection: {
+      type: String,
+      enum: Object.values(MapKeyIndexPartSelection),
+    },
     orderByExpression: String,
   }),
 );

packages/api/src/routers/api/__tests__/sources.int.test.ts

@@ -1672,4 +1672,79 @@ describe('sources router', () => {
       expect(sources).toHaveLength(0);
     });
   });
+
+  describe('mapKeyIndexMaxParts field', () => {
+    it('POST / - persists mapKeyIndexMaxParts and PUT /:id updates it', async () => {
+      const { agent } = await getLoggedInAgent(server);
+
+      const created = await agent
+        .post('/sources')
+        .send({ ...MOCK_SOURCE, mapKeyIndexMaxParts: 50 })
+        .expect(200);
+      expect(created.body.mapKeyIndexMaxParts).toBe(50);
+
+      await agent
+        .put(`/sources/${created.body.id}`)
+        .send({ ...MOCK_SOURCE, id: created.body.id, mapKeyIndexMaxParts: 5 })
+        .expect(200);
+
+      const updated = await Source.findById(created.body.id).lean();
+      if (updated?.kind !== SourceKind.Log) {
+        throw new Error(`expected Log source, got ${updated?.kind}`);
+      }
+      expect(updated.mapKeyIndexMaxParts).toBe(5);
+    });
+
+    it.each([0, 2.5, -1])(
+      'POST / - rejects mapKeyIndexMaxParts %s',
+      async mapKeyIndexMaxParts => {
+        const { agent } = await getLoggedInAgent(server);
+
+        await agent
+          .post('/sources')
+          .send({ ...MOCK_SOURCE, mapKeyIndexMaxParts })
+          .expect(400);
+
+        expect(await Source.find({}).lean()).toHaveLength(0);
+      },
+    );
+  });
+
+  describe('mapKeyIndexPartSelection field', () => {
+    it('POST / - persists mapKeyIndexPartSelection and PUT /:id updates it', async () => {
+      const { agent } = await getLoggedInAgent(server);
+
+      const created = await agent
+        .post('/sources')
+        .send({ ...MOCK_SOURCE, mapKeyIndexPartSelection: 'largest' })
+        .expect(200);
+      expect(created.body.mapKeyIndexPartSelection).toBe('largest');
+
+      await agent
+        .put(`/sources/${created.body.id}`)
+        .send({
+          ...MOCK_SOURCE,
+          id: created.body.id,
+          mapKeyIndexPartSelection: 'smallest',
+        })
+        .expect(200);
+
+      const updated = await Source.findById(created.body.id).lean();
+      if (updated?.kind !== SourceKind.Log) {
+        throw new Error(`expected Log source, got ${updated?.kind}`);
+      }
+      expect(updated.mapKeyIndexPartSelection).toBe('smallest');
+    });
+
+    it('POST / - rejects an unknown mapKeyIndexPartSelection', async () => {
+      const { agent } = await getLoggedInAgent(server);
+
+      await agent
+        .post('/sources')
+        .send({ ...MOCK_SOURCE, mapKeyIndexPartSelection: 'random' })
+        .expect(400);
+
+      expect(await Source.find({}).lean()).toHaveLength(0);
+    });
+  });
 });

packages/api/src/routers/external-api/v2/sources.ts

@@ -412,6 +412,18 @@ function formatExternalSource(source: SourceDocument) {
  *           description: Controls whether lucene rendering uses ClickHouse text indices via hasAllTokens() against the implicit column. "auto" detects a covering index at query time, "enabled" forces text index usage, "disabled" forces a LIKE/hasToken fallback.
  *           nullable: true
  *           example: auto
+ *         mapKeyIndexMaxParts:
+ *           type: integer
+ *           minimum: 1
+ *           description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.
+ *           nullable: true
+ *           example: 20
+ *         mapKeyIndexPartSelection:
+ *           type: string
+ *           enum: [median, smallest, largest]
+ *           description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data.
+ *           nullable: true
+ *           example: median
  *         highlightedTraceAttributeExpressions:
  *           type: array
  *           description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.
@@ -613,6 +625,18 @@ function formatExternalSource(source: SourceDocument) {
  *           description: Controls whether lucene rendering uses ClickHouse text indices via hasAllTokens() against the implicit column. "auto" detects a covering index at query time, "enabled" forces text index usage, "disabled" forces a LIKE/hasToken fallback.
  *           nullable: true
  *           example: auto
+ *         mapKeyIndexMaxParts:
+ *           type: integer
+ *           minimum: 1
+ *           description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.
+ *           nullable: true
+ *           example: 20
+ *         mapKeyIndexPartSelection:
+ *           type: string
+ *           enum: [median, smallest, largest]
+ *           description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data.
+ *           nullable: true
+ *           example: median
  *         highlightedTraceAttributeExpressions:
  *           type: array
  *           description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.

packages/app/src/components/DBSearchPageFilters/hooks.test.tsx

@@ -807,6 +807,9 @@ describe('useFetchFacets', () => {
       });
 
       expect(getAllKeyValues).toHaveBeenCalledTimes(1);
+      expect(getAllKeyValues).toHaveBeenCalledWith(
+        expect.objectContaining({ mapKeyIndexMaxParts: null }),
+      );
       expect(getKeyValuesWithMVs).not.toHaveBeenCalled();
       expect(result.current.data.keyValues).toEqual([
         { key: 'ServiceName', value: ['api', 'web'] },

packages/app/src/components/DBSearchPageFilters/hooks.ts

@@ -255,6 +255,8 @@ function useFacets({
           tableName: tableConnection.tableName,
           connectionId: tableConnection.connectionId,
           metadataMVs: tableConnection.metadataMVs,
+          // "Load more" wants every value in the time range, not a sample of parts.
+          mapKeyIndexMaxParts: null,
           keyExpressions: [sqlKey],
           maxValuesPerKey: SINGLE_KEY_VALUES_LIMIT,
           dateRange,

packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx

@@ -20,6 +20,10 @@ import {
 import { ExpressionFormRow } from './ExpressionFormRow';
 import { FormRow } from './FormRow';
 import { HighlightedAttributeExpressionsFormRow } from './HighlightedAttributes';
+import {
+  MapKeyIndexMaxPartsFormRow,
+  MapKeyIndexPartSelectionFormRow,
+} from './MapKeyIndexMaxPartsFormRow';
 import {
   MaterializedViewsFormSection,
   MetadataMaterializedViewsFormSection,
@@ -286,6 +290,8 @@ export function LogTableModelForm(props: TableModelProps) {
           />
         </FormRow>
         <UseTextIndexFormRow control={control} />
+        <MapKeyIndexMaxPartsFormRow control={control} />
+        <MapKeyIndexPartSelectionFormRow control={control} />
         <Separator size="xs" />
         <HighlightedAttributeExpressionsFormRow
           {...props}