perf: show the most frequent text index values, in batches of 10 keys

.changeset/text-index-dictionary-block-size.md

@@ -0,0 +1,21 @@
+---
+'@hyperdx/otel-collector': patch
+---
+
+perf: use 8,192-token text index dictionary blocks on the seeded tables
+
+On ClickHouse 26.6 and later, the collector's schema migration now sets
+`text_index_dictionary_block_size = 8192` (the default is 512) on `otel_logs`,
+`otel_traces` and `hyperdx_sessions`, with `ALTER TABLE … MODIFY SETTING` after
+the seed. Each part's text index sparse index is 16x smaller. Reading filter
+values with `mergeTreeTextIndex()` loads that index for every part on every
+read, and on a 537M-row table 4,096-token blocks already made those reads about
+10x cheaper per part, made searches for a rare attribute value 3-7x faster, and
+made searches for a common value 10-30% faster.
+
+Because it's an `ALTER`, existing installs get it too: new parts use 8,192-token
+blocks, and existing parts keep theirs until they're merged. A table that
+already sets `text_index_dictionary_block_size` is left alone, and operators who
+manage these tables themselves can turn the change off with
+`HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE=false`. ClickHouse 26.2–26.5
+doesn't have the setting, so those servers keep 512-token blocks.

.changeset/topk-text-index-values.md

@@ -0,0 +1,27 @@
+---
+'@hyperdx/common-utils': patch
+---
+
+perf: read large-block Map text index values per key, most frequent first
+
+"All values" lookups on Map key=value text indexes with large dictionary blocks
+(2048 or more, e.g. `text_index_dictionary_block_size = 8192`, which the
+collector's migration now sets on ClickHouse 26.6+)
+now read each key separately, in parallel queries of at most 10 keys, and stop
+after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs)
+can't hold up the others. Values come from `topKWeighted(limit)` weighted by
+each token's row count, so a capped list holds the most common values, most
+common first. These reads no longer apply `max_rows_to_read`: the index is read
+in sorted token order, so the cap left every key that sorts after a
+high-cardinality key with no values.
+
+Indexes with smaller blocks, including ClickHouse's default of 512, keep the
+previous query shape (one read for all of a query's keys, the alphabetically
+first values, and the `max_rows_to_read` cap), now in parallel queries of at
+most 10 keys, so the cap's cut-off affects at most 9 other keys. Every read loads
+each part's sparse index, which is large with small blocks, so one read per key
+would multiply that cost. The block size comes from the index definition, then
+the table's `text_index_dictionary_block_size`, then the server's.
+
+Text indexes on plain columns now return the most frequent values with
+`topKWeighted`, in parallel queries of at most 10 columns, without the row cap.

docker-compose.dev.yml

@@ -45,6 +45,8 @@ services:
       #   HYPERDX_OTEL_EXPORTER_LOGS_TTL / _TRACES_TTL / _METRICS_TTL / _SESSIONS_TTL
       # Apply TTL changes to already-existing tables too (default false):
       #   HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: 'true'
+      # Leave text index dictionary block sizes alone (default: set 8192 on 26.6+):
+      #   HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE: 'false'
       ENABLE_PROMQL: 'true'
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml

docker/clickhouse/local/init-db-e2e.sh

@@ -53,7 +53,7 @@ ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
 ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)
 TTL toDateTime(Timestamp) + toIntervalDay(30)
-SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, text_index_dictionary_block_size = 8192;
 
 -- Distributed table over the e2e logs table whose DDL declares a column
 -- (\`ColumnMissingFromShard\`) that its underlying MergeTree table does NOT have.
@@ -129,7 +129,7 @@ ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
 ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
 TTL toDate(Timestamp) + toIntervalDay(30)
-SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, text_index_dictionary_block_size = 8192;
 
 CREATE TABLE ${DATABASE}.e2e_hyperdx_sessions
 (

packages/app/src/components/DBRowTable.tsx

@@ -28,7 +28,10 @@ import {
   isJSDataTypeJSONStringifiable,
   JSDataType,
 } from '@hyperdx/common-utils/dist/clickhouse';
-import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
+import {
+  getEngineSetting,
+  splitAndTrimWithBracket,
+} from '@hyperdx/common-utils/dist/core/utils';
 import {
   BuilderChartConfigWithDateRange,
   SelectList,
@@ -1541,14 +1544,6 @@ const BLOCK_COLUMN_SETTINGS = [
   { column: '_block_offset', setting: 'enable_block_offset_column' },
 ];
 
-// engine_full ends with the SETTINGS clause.
-function getEngineSetting(engineFull: string, name: string) {
-  const settingsClause = engineFull.match(/ SETTINGS (.*)$/)?.[1] ?? '';
-  return splitAndTrimWithBracket(settingsClause)
-    .map(setting => setting.split(' = '))
-    .find(([key]) => key === name)?.[1];
-}
-
 function omitsBlockColumnSetting(engineFull: string) {
   return BLOCK_COLUMN_SETTINGS.some(
     ({ setting }) => getEngineSetting(engineFull, setting) == null,

packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts

@@ -70,7 +70,7 @@ describe('KV items rewrite integration', () => {
         LogAttributeItems Array(String) MATERIALIZED
           arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
           CODEC(ZSTD(1)),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
       )
       ENGINE = MergeTree
       ORDER BY (Timestamp)`,

packages/common-utils/src/__tests__/metadata.int.test.ts

@@ -697,7 +697,7 @@ describe('Metadata Integration Tests', () => {
             ResourceAttributeItems Array(String) MATERIALIZED
               arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))'))
               CODEC(ZSTD(1)),
-            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
@@ -712,7 +712,7 @@ describe('Metadata Integration Tests', () => {
             LogAttributeItems Array(String) MATERIALIZED
               arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
               CODEC(ZSTD(1)),
-            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
@@ -907,6 +907,41 @@ describe('Metadata Integration Tests', () => {
       );
     });
 
+    it('keeps the most frequent text index values when maxValuesPerKey truncates the list', async () => {
+      if (!textIndexSupported) {
+        console.warn(
+          'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()',
+        );
+        return;
+      }
+      // The index dictionary is sorted, so 'a-rare' is read first.
+      const freqKey = `freq.${tag}`;
+      const timestamp = testTime.toISOString().replace('T', ' ').slice(0, 23);
+      await client.command({
+        query: `INSERT INTO default.otel_logs (Timestamp, ResourceAttributes)
+          SELECT '${timestamp}', map('${freqKey}', if(number = 0, 'a-rare', 'z-common'))
+          FROM numbers(4)`,
+      });
+
+      const result = await metadata.getAllKeyValues({
+        ...commonArgs,
+        keyExpressions: [
+          `ResourceAttributes['${freqKey}']`,
+          `ResourceAttributes['${mapKey}']`,
+        ],
+        maxValuesPerKey: 1,
+      });
+
+      // UNION ALL branches can arrive in any order.
+      expect(result).toHaveLength(2);
+      expect(result).toEqual(
+        expect.arrayContaining([
+          { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
+          { key: `ResourceAttributes['${mapKey}']`, value: [podNameA] },
+        ]),
+      );
+    });
+
     it('routes TraceId through getTextIndexKeyValues', async () => {
       if (!textIndexSupported) {
         console.warn(
@@ -1622,14 +1657,14 @@ describe('Metadata Integration Tests', () => {
     {
       path: 'key',
       tableName: 'test_map_keys_key_text_index_order',
-      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`,
+      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
     },
     {
       path: 'kv',
       tableName: 'test_map_keys_kv_text_index_order',
       indexDefinition: `LogAttributeItems Array(String) MATERIALIZED
           arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1`,
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
     },
   ])(
     'getMapKeys - $path text index ordering',
@@ -1719,4 +1754,92 @@ describe('Metadata Integration Tests', () => {
       });
     },
   );
+
+  describe('getAllKeyValues - values repeated across parts', () => {
+    // mergeTreeTextIndex returns one row per part and token, so a token cap
+    // stops before the later parts' values. Each part holds the same SHARED
+    // values plus OWN values no other part has.
+    const tableName = 'test_text_index_values_across_parts';
+    const PARTS = 25;
+    const SHARED = 1000;
+    const OWN = 100;
+    const dateRange = (): [Date, Date] => [
+      new Date(Date.now() - 60 * 60 * 1000),
+      new Date(Date.now() + 60 * 1000),
+    ];
+    let textIndexSupported = false;
+
+    beforeAll(async () => {
+      const probe = new Metadata(hdxClient, new MetadataCache());
+      textIndexSupported = supportsMergeTreeTextIndex(
+        await probe.getServerVersion({ connectionId: 'test_connection' }),
+      );
+      if (!textIndexSupported) return;
+
+      await client.command({
+        query: `CREATE OR REPLACE TABLE default.${tableName} (
+            Timestamp DateTime64(9),
+            ResourceAttributes Map(LowCardinality(String), String),
+            ResourceAttributeItems Array(String) MATERIALIZED
+              arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))')),
+            Tags Array(String),
+            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1,
+            INDEX idx_tags Tags TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
+          )
+          ENGINE = MergeTree()
+          PARTITION BY toDate(Timestamp)
+          ORDER BY Timestamp
+        `,
+      });
+      await client.command({
+        query: `SYSTEM STOP MERGES default.${tableName}`,
+      });
+      for (let i = 0; i < PARTS; i++) {
+        await client.command({
+          query: `INSERT INTO default.${tableName} (Timestamp, ResourceAttributes, Tags)
+            SELECT now64(9) - INTERVAL 10 MINUTE, map('pod', v), [v]
+            FROM (
+              SELECT if(number < ${SHARED}, concat('shared-', toString(number)), concat('own-${i}-', toString(number))) AS v
+              FROM numbers(${SHARED + OWN})
+            )`,
+        });
+      }
+    });
+
+    afterAll(async () => {
+      await client.command({
+        query: `DROP TABLE IF EXISTS default.${tableName}`,
+      });
+    });
+
+    it.each([
+      ['Map key=value', "ResourceAttributes['pod']"],
+      ['plain-column', 'Tags'],
+    ])(
+      'returns every distinct value from the %s text index for "Load more"',
+      async (_, keyExpression) => {
+        if (!textIndexSupported) {
+          console.warn(
+            'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()',
+          );
+          return;
+        }
+
+        // "Load more" passes null: every part, no token cap.
+        const metadata = new Metadata(hdxClient, new MetadataCache());
+        const result = await metadata.getAllKeyValues({
+          databaseName: 'default',
+          tableName,
+          keyExpressions: [keyExpression],
+          maxValuesPerKey: 10000,
+          mapKeyIndexMaxParts: null,
+          connectionId: 'test_connection',
+          dateRange: dateRange(),
+          timestampValueExpression: 'Timestamp',
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0].value).toHaveLength(SHARED + OWN * PARTS);
+      },
+    );
+  });
 });