From f08606405adbca3e1db55689896a55f34689f55e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 13:06:06 -0400 Subject: [PATCH] feat(app): search across multiple sources, with one source as N=1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The search page had one results table bound to one source. It now takes a list: pick up to 3 log/trace sources and their rows interleave into a single timestamp-ordered timeline, each tagged with the source it came from. One source is not a separate path — it is N=1. Its spec carries the user's own SELECT and ORDER BY, so the table renders the authored columns with no source column, sorts, denoises, and reports errors to the page exactly as before. Several sources project the canonical aliases instead, and the client merges the streams behind the safe frontier. There is no UNION: sources can live on different ClickHouse connections, each keeps its own per-table machinery (Lucene serializer, text-index detection, materialized-column rewrites, query settings), and a source that fails degrades to a status chip instead of failing the search. Selections are shareable via ?sources=. Multi-source is Lucene-only; the histogram, total count, and filters sidebar stay single-source until the next change. --- .changeset/multi-source-search.md | 15 + packages/app/src/DBSearchPage.tsx | 655 ++++++++++++++---- .../DBSearchPage.directTrace.test.tsx | 3 + packages/app/src/components/DBRowTable.tsx | 8 +- .../components/DBSearchPageFilters/hooks.ts | 5 +- .../components/DBSqlRowTableWithSidebar.tsx | 4 +- .../app/src/components/MultiSourceBadge.tsx | 13 +- .../components/MultiSourceColumnPicker.tsx | 71 ++ .../app/src/components/SearchResultsTable.tsx | 618 +++++++++++++++++ packages/app/src/defaults.ts | 8 + .../__tests__/useMultiSourceSearch.test.ts | 87 +++ .../app/src/hooks/useMultiSourceSearch.ts | 148 ++++ .../app/src/hooks/useResolvedSourcesParam.ts | 58 ++ packages/app/src/hooks/useSourceSlots.ts | 32 + 14 files changed, 1592 insertions(+), 133 deletions(-) create mode 100644 .changeset/multi-source-search.md create mode 100644 packages/app/src/components/MultiSourceColumnPicker.tsx create mode 100644 packages/app/src/components/SearchResultsTable.tsx create mode 100644 packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts create mode 100644 packages/app/src/hooks/useMultiSourceSearch.ts create mode 100644 packages/app/src/hooks/useResolvedSourcesParam.ts create mode 100644 packages/app/src/hooks/useSourceSlots.ts diff --git a/.changeset/multi-source-search.md b/.changeset/multi-source-search.md new file mode 100644 index 0000000000..9d4868f8f4 --- /dev/null +++ b/.changeset/multi-source-search.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/app': minor +'@hyperdx/common-utils': minor +--- + +Search across multiple sources at once. The search page's source selector can +now expand into a multi-select (up to 3 log/trace sources): results interleave +into one timestamp-ordered timeline with a per-row source badge, normalized +columns (Timestamp, Source, Service, Level, Message, and Duration when traces +are included), a histogram stacked by source, and an add-column picker over the +union of the selected sources' columns. Each source runs its own query +pipeline — sources on different connections work, and a failing source shows a +status chip instead of failing the whole search. Multi-source mode is +Lucene-only and shareable via URL; saved searches and alerts remain +single-source for now. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index cdae127d1e..c11ff3efe1 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -31,7 +31,10 @@ import { ColumnMeta, } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; -import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + buildMultiSourceSearchConfig, + buildSearchChartConfig, +} from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { aliasMapToWithClauses, isBrowser, @@ -95,21 +98,31 @@ import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { FavoriteButton } from '@/components/FavoriteButton'; import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover'; import { InputControlled } from '@/components/InputControlled'; +import MultiSourceColumnPicker from '@/components/MultiSourceColumnPicker'; import OnboardingModal from '@/components/OnboardingModal'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; import SearchPageActionBar from '@/components/SearchPageActionBar'; +import SearchResultsTable from '@/components/SearchResultsTable'; import SearchTotalCountChart from '@/components/SearchTotalCountChart'; +import { SourceMultiSelectControlled } from '@/components/SourceMultiSelect'; import { TableSourceForm } from '@/components/Sources/SourceForm'; import { SourceSelectControlled } from '@/components/SourceSelect'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; import { Tags } from '@/components/Tags'; import { TimePicker } from '@/components/TimePicker'; import { IS_LOCAL_MODE } from '@/config'; +import { MAX_SEARCH_SOURCES } from '@/defaults'; import { useAliasMapFromChartConfig } from '@/hooks/useChartConfig'; import { useExplainQuery } from '@/hooks/useExplainQuery'; +import { + resolveExtraColumnsForSource, + unresolvedFilterColumns, + useMultiSourceColumns, +} from '@/hooks/useMultiSourceSearch'; import { useResolvedSourceParam } from '@/hooks/useResolvedSourceParam'; +import { useResolvedSourcesParam } from '@/hooks/useResolvedSourcesParam'; import { withAppNav } from '@/layout'; import { useCreateSavedSearch, @@ -133,7 +146,6 @@ import { } from '@/utils'; import ChartSQLPreview, { SQLPreview } from './components/ChartSQLPreview'; -import DBSqlRowTableWithSideBar from './components/DBSqlRowTableWithSidebar'; import PatternTable from './components/PatternTable'; import { DBSearchHeatmapChart } from './components/Search/DBSearchHeatmapChart'; import DirectTraceSidePanel from './components/Search/DirectTraceSidePanel'; @@ -181,6 +193,10 @@ const ALLOWED_SOURCE_KINDS = [SourceKind.Log, SourceKind.Trace]; const SearchConfigSchema = z.object({ select: z.string(), source: z.string(), + // Multi-source search: the full selection (2+ engages multi mode). The + // single `source` field stays the primary (= sources[0]) so every + // single-source code path keeps working unchanged. + sources: z.array(z.string()), where: z.string(), whereLanguage: z.enum(['sql', 'lucene']), orderBy: z.string(), @@ -865,6 +881,25 @@ function optimizeDefaultOrderBy( : `${orderByArr[0]} DESC`; } +/** + * Per-source default ORDER BY for multi-source search. Same resolution as + * useDefaultOrderBy minus the table sorting-key optimization (which needs a + * metadata query per source): the source's explicit orderByExpression, else + * its timestamp expression(s) DESC. Time-window pagination requires the first + * term to be the source's timestamp, which this guarantees. + */ +function multiSourceDefaultOrderBy(source: TSource): string { + const isEventSource = + source.kind === SourceKind.Log || source.kind === SourceKind.Trace; + const explicit = isEventSource ? source.orderByExpression?.trim() : undefined; + if (explicit) return explicit; + return optimizeDefaultOrderBy( + source.timestampValueExpression ?? '', + isEventSource ? source.displayedTimestampValueExpression : undefined, + undefined, + ); +} + export function useDefaultOrderBy(sourceID: string | undefined | null) { const { data: source } = useSource({ id: sourceID, @@ -895,6 +930,9 @@ function formatDroppedFiltersMessage(count: number): string { // This is outside as it needs to be a stable reference const queryStateMap = { source: parseAsString, + // JSON-encoded (not comma-separated) because source names may contain + // commas; `source` is always written alongside it as the primary. + sources: parseAsJsonEncoded(), where: parseAsStringEncoded, select: parseAsStringEncoded, whereLanguage: parseAsStringEnum<'sql' | 'lucene'>(['sql', 'lucene']), @@ -1116,6 +1154,7 @@ export function DBSearchPage() { (savedSearchId || directTraceId || rawSearchedConfig.source ? '' : defaultSourceId), + sources: searchedConfig.sources ?? [], filters: searchedConfig.filters ?? [], orderBy: searchedConfig.orderBy ?? '', }, @@ -1184,6 +1223,7 @@ export function DBSearchPage() { whereLanguage: searchedConfig?.whereLanguage ?? getStoredLanguage() ?? 'lucene', source: searchedConfig?.source ?? undefined, + sources: searchedConfig?.sources ?? [], filters: searchedConfig?.filters ?? [], orderBy: searchedConfig?.orderBy ?? '', }); @@ -1198,6 +1238,7 @@ export function DBSearchPage() { // to an existing source. const isSearchConfigEmpty = !rawSearchedConfig.source && + !rawSearchedConfig.sources?.length && !where && !select && !whereLanguage && @@ -1240,6 +1281,7 @@ export function DBSearchPage() { savedSearch, searchedConfig, rawSearchedConfig.source, + rawSearchedConfig.sources, setSearchedConfig, savedSearchId, defaultSourceId, @@ -1268,12 +1310,15 @@ export function DBSearchPage() { const onSubmit = useCallback(() => { onSearch(displayedTimeInputValue); handleSubmit( - ({ select, where, whereLanguage, source, filters, orderBy }) => { + ({ select, where, whereLanguage, source, sources, filters, orderBy }) => { setSearchedConfig({ select, where, whereLanguage, source, + // Writer discipline: only 2+ selections persist the list; a single + // selection clears it so old-style URLs stay canonical. + sources: sources.length > 1 ? sources : null, filters, orderBy, }); @@ -1301,8 +1346,123 @@ export function DBSearchPage() { [debouncedSubmit, setValue], ); + const watchedSource = useWatch({ + control, + name: 'source', + // Watch will reset when changing saved search, so we need to default to the URL + defaultValue: searchedConfig.source ?? undefined, + }); + + // --- Multi-source search: selection & schema state ------------------------ + // 2+ resolved sources in the ?sources= param engage multi mode: one + // independent query pipeline per source, merged client-side. The single + // `source` (primary) keeps every existing code path working; multi mode + // only swaps what gets rendered below. Declared before the filter-state + // hooks so they can work against the union of the selected schemas. + const { sources: searchedMultiSources } = useResolvedSourcesParam( + rawSearchedConfig.sources, + { kinds: ALLOWED_SOURCE_KINDS }, + ); + const isMultiSource = searchedMultiSources.length > 1; + // Delta/pattern analyses are per-source; multi mode pins the results view. + const effectiveAnalysisMode = isMultiSource ? 'results' : analysisMode; + // Raw SQL WHERE names concrete columns of a concrete table — reinterpreting + // it per source risks silently-wrong results, so multi mode requires Lucene. + const isMultiSourceSqlBlocked = + isMultiSource && + (searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene') === + 'sql' && + !!searchedConfig.where; + + // A hand-authored URL may carry only ?sources=; backfill the primary so the + // single-source machinery (form, chart config) has one. + useEffect(() => { + if (!rawSearchedConfig.source && searchedMultiSources.length > 0) { + setSearchedConfig({ source: searchedMultiSources[0].id }); + } + }, [rawSearchedConfig.source, searchedMultiSources, setSearchedConfig]); + + const watchedSources = useWatch({ control, name: 'sources' }); + const formSourceCount = watchedSources?.length ?? 0; + // The multi-select UI stays visible while the user is composing a selection + // (even before a second source is added). + const [multiPickerOpen, setMultiPickerOpen] = useState(false); + const isMultiSelectUI = multiPickerOpen || formSourceCount > 1; + const formIsMulti = formSourceCount > 1; + + const enterMultiSourceSelect = useCallback(() => { + setValue('sources', watchedSource ? [watchedSource] : []); + setMultiPickerOpen(true); + }, [setValue, watchedSource]); + + // Keep the primary `source` field in sync with the selection and re-run the + // search when the selection changes. + const prevWatchedSourcesRef = useRef(null); + useEffect(() => { + const current = watchedSources ?? []; + const prev = prevWatchedSourcesRef.current; + if (prev != null && JSON.stringify(prev) === JSON.stringify(current)) { + return; + } + prevWatchedSourcesRef.current = current; + if (prev == null) { + // Initial hydration from the URL — nothing changed. + return; + } + if (current.length > 0 && current[0] !== watchedSource) { + setValue('source', current[0]); + } + if ((prev?.length ?? 0) <= 1 && current.length > 1) { + // Entering multi mode: the single-source SELECT/ORDER BY strings don't + // translate to the canonical multi-source shape. + setValue('select', ''); + setValue('orderBy', ''); + } + debouncedSubmit(); + }, [watchedSources, watchedSource, setValue, debouncedSubmit]); + + // Collapse the picker back to the single-source select when the user + // reduces a real multi selection to one source. Keyed on the >1 → ≤1 + // transition so it can't fire in the just-opened composing state (picker + // open, one source selected, second not yet picked). + const prevFormSourceCountRef = useRef(formSourceCount); + useEffect(() => { + const prev = prevFormSourceCountRef.current; + prevFormSourceCountRef.current = formSourceCount; + if (prev > 1 && formSourceCount <= 1) { + setMultiPickerOpen(false); + } + }, [formSourceCount]); + + // Per-source top-level columns: powers the add-column picker, the + // per-source `column vs NULL` projection for user-picked extras, and + // per-source filter resolvability. Driven by the form's draft selection + // while composing (so the picker has options before the search is + // submitted), falling back to the searched selection. + const formMultiSources = useMemo( + () => + (watchedSources ?? []) + .map(id => inputSourceObjs?.find(s => s.id === id)) + .filter((s): s is TSource => s != null), + [watchedSources, inputSourceObjs], + ); + const { + columnsBySourceId, + unionColumns, + dateTimeColumns: multiDateTimeColumns, + } = useMultiSourceColumns( + formMultiSources.length > 1 + ? formMultiSources + : isMultiSource + ? searchedMultiSources + : [], + ); + // --- End multi-source selection & schema state ----------------------------- + // Top-level column names for the active source, used to quote - // filter keys that contain special characters. + // filter keys that contain special characters. In multi mode this is the + // union across the selected sources, so filter keys from any of them + // escape correctly. const { data: inputSourceColumns } = useColumns( { databaseName: inputSourceObj?.from?.databaseName ?? '', @@ -1311,20 +1471,19 @@ export function DBSearchPage() { }, { enabled: !!inputSourceObj }, ); - const knownColumns = useMemo( - () => - inputSourceColumns - ? new Set(inputSourceColumns.map(c => c.name)) - : new Set(), - [inputSourceColumns], - ); + const knownColumns = useMemo(() => { + if (isMultiSource) { + const union = new Set(); + for (const names of columnsBySourceId.values()) { + for (const name of names) union.add(name); + } + return union; + } + return inputSourceColumns + ? new Set(inputSourceColumns.map(c => c.name)) + : new Set(); + }, [inputSourceColumns, isMultiSource, columnsBySourceId]); - const watchedSource = useWatch({ - control, - name: 'source', - // Watch will reset when changing saved search, so we need to default to the URL - defaultValue: searchedConfig.source ?? undefined, - }); const prevSourceRef = useRef(watchedSource); // Set when the user switches sources via the dropdown. The follow-up // effect waits for the new source's columns to load and then drops any @@ -1347,11 +1506,21 @@ export function DBSearchPage() { const { dateTimeColumns, onResolvedColumnsChange } = useResolvedDateTimeColumns(inputSourceColumns); + // In multi mode, date/time-typed filter keys may come from any selected + // source's schema. + const effectiveDateTimeColumns = useMemo( + () => + isMultiSource && multiDateTimeColumns.size > 0 + ? new Map([...dateTimeColumns, ...multiDateTimeColumns]) + : dateTimeColumns, + [isMultiSource, dateTimeColumns, multiDateTimeColumns], + ); + const filters = useWatch({ name: 'filters', control }); const searchFilters = useSearchPageFilterState({ searchQuery: filters ?? undefined, onFilterChange: handleSetFilters, - dateTimeColumns, + dateTimeColumns: effectiveDateTimeColumns, knownColumns, }); @@ -1489,6 +1658,135 @@ export function DBSearchPage() { const { data: chartConfig, isLoading: isChartConfigLoading } = useSearchedConfigToChartConfig(chartSearchConfig, defaultSearchConfig); + // --- Multi-source search: query specs ------------------------------------- + // In multi mode the `select` param holds the extra column names picked by + // the user (the canonical columns are always projected). + const multiExtraColumnNames = useMemo( + () => + isMultiSource ? splitAndTrimWithBracket(searchedConfig.select ?? '') : [], + [isMultiSource, searchedConfig.select], + ); + + // The add-column picker edits the form's draft select (like the SELECT + // editor it replaces), then auto-submits. + const inputSelect = useWatch({ name: 'select', control }); + const multiPickerValue = useMemo( + () => (formIsMulti ? splitAndTrimWithBracket(inputSelect ?? '') : []), + [formIsMulti, inputSelect], + ); + const onMultiColumnsChange = useCallback( + (columns: string[]) => { + setValue('select', columns.join(', ')); + debouncedSubmit(); + }, + [setValue, debouncedSubmit], + ); + + // Sidebar filters apply per source. A source whose table lacks a filtered + // column can't answer the filtered search — it's excluded entirely (with a + // visible reason on its status chip) rather than silently returning rows + // that ignore the filter. + const multiSourceFilters = useMemo( + () => (isMultiSource ? (searchedConfig.filters ?? []) : []), + [isMultiSource, searchedConfig.filters], + ); + const multiDisabledReasons = useMemo(() => { + const reasons = new Map(); + if (!isMultiSource || multiSourceFilters.length === 0) return reasons; + for (const source of searchedMultiSources) { + const missing = unresolvedFilterColumns( + multiSourceFilters, + columnsBySourceId.get(source.id), + ); + if (missing.length > 0) { + reasons.set( + source.id, + `${source.name} is excluded: the active filter uses ${missing.join( + ', ', + )}, which it doesn't have`, + ); + } + } + return reasons; + }, [ + isMultiSource, + multiSourceFilters, + searchedMultiSources, + columnsBySourceId, + ]); + + // The single-source chart config, pinned to the searched time range. + const dbSqlRowTableConfig = useMemo(() => { + if (chartConfig == null) { + return undefined; + } + + return { + ...chartConfig, + dateRange: searchedTimeRange, + }; + }, [chartConfig, searchedTimeRange]); + + // The search's query plan: one spec per selected source. A single source is + // just N=1 — its spec carries the user's own SELECT/ORDER BY, so the results + // table renders exactly what the user asked for. The canonical aliases only + // come into play when there is more than one source to reconcile. + const searchStreamSpecs = useMemo(() => { + if (!isMultiSource) { + if (dbSqlRowTableConfig == null || searchedSource == null) return []; + return [{ source: searchedSource, config: dbSqlRowTableConfig }]; + } + if (isMultiSourceSqlBlocked) return []; + // Extra columns and filters both need each source's DESCRIBE (to resolve + // column-vs-NULL and filter resolvability); hold the row queries until + // they've loaded so we don't fire throwaway or erroring queries. + if ( + (multiExtraColumnNames.length > 0 || multiSourceFilters.length > 0) && + columnsBySourceId.size < searchedMultiSources.length + ) { + return []; + } + const includeDuration = searchedMultiSources.some(isTraceSource); + const where = searchedConfig.where ?? ''; + return searchedMultiSources.map(source => ({ + source, + disabledReason: multiDisabledReasons.get(source.id), + config: { + ...buildMultiSourceSearchConfig( + source, + { + where, + whereLanguage: 'lucene', + filters: multiSourceFilters, + orderBy: multiSourceDefaultOrderBy(source), + }, + { + includeDuration, + extraColumns: resolveExtraColumnsForSource( + multiExtraColumnNames, + columnsBySourceId.get(source.id), + ), + }, + ), + dateRange: searchedTimeRange, + }, + })); + }, [ + isMultiSource, + isMultiSourceSqlBlocked, + searchedMultiSources, + searchedSource, + dbSqlRowTableConfig, + searchedConfig.where, + multiExtraColumnNames, + multiSourceFilters, + multiDisabledReasons, + columnsBySourceId, + searchedTimeRange, + ]); + + // --- End multi-source search --------------------------------------------- + // query error handling const { hasQueryError, queryError } = useMemo(() => { const hasQueryError = Object.values(_queryErrors).length > 0; @@ -1650,24 +1948,16 @@ export function DBSearchPage() { setTimeout(() => setCollapseAllRows(false), 100); }, [interval, updateRelativeTimeInputValue, setIsLive]); - const dbSqlRowTableConfig = useMemo(() => { - if (chartConfig == null) { - return undefined; - } - - return { - ...chartConfig, - dateRange: searchedTimeRange, - }; - }, [chartConfig, searchedTimeRange]); - // Stable key for persisting column widths in localStorage. Scoped per saved - // search when one is loaded, else per source for ad-hoc searches. + // search when one is loaded, else per source (or source set) for ad-hoc + // searches. const columnSizeTableId = savedSearchId ? `db-search-saved-${savedSearchId}` - : searchedConfig.source - ? `db-search-source-${searchedConfig.source}` - : undefined; + : isMultiSource + ? `db-search-multi-${searchedMultiSources.map(s => s.id).join('-')}` + : searchedConfig.source + ? `db-search-source-${searchedConfig.source}` + : undefined; const displayedColumns = useMemo(() => { // `select` is typed as `string | DerivedColumn[]` upstream, but in the @@ -1969,6 +2259,28 @@ export function DBSearchPage() { ], ); + // Multi-source rows span schemas, so the single-source column toggles are + // omitted — the side panel hides them. Property-add-to-filter IS wired: + // filters resolve per source, and a source that lacks the column is + // excluded with a visible reason. Passing no `source` is required: with a + // null context source, deriveRowSidePanelContextForSource treats every row + // as same-source, which is exactly the cross-source semantics filters now + // have. + const multiRowTableContext = useMemo( + () => ({ + onPropertyAddClick: searchFilters.setFilterValue, + generateSearchUrl, + isChildModalOpen: isDrawerChildModalOpen, + setChildModalOpen: setDrawerChildModalOpen, + }), + [ + searchFilters.setFilterValue, + generateSearchUrl, + isDrawerChildModalOpen, + setDrawerChildModalOpen, + ], + ); + const inputSourceTableConnection = useMemo( () => tcFromSource(inputSourceObj), [inputSourceObj], @@ -2220,22 +2532,49 @@ export function DBSearchPage() { > {/* */} - setIsSourceSchemaPreviewOpen(true)} - isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( - inputSourceObj, - )} - allowedSourceKinds={ALLOWED_SOURCE_KINDS} - data-testid="source-selector" - style={{ minWidth: 150 }} - /> + {isMultiSelectUI ? ( + + ) : ( + <> + setIsSourceSchemaPreviewOpen(true)} + isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( + inputSourceObj, + )} + allowedSourceKinds={ALLOWED_SOURCE_KINDS} + data-testid="source-selector" + style={{ minWidth: 150 }} + /> + + + + + + + )} setIsSourceSchemaPreviewOpen(false)} /> - - - - + {formIsMulti ? ( + + ) : ( + + )} + {!formIsMulti && ( + + + + )} <> {!savedSearchId ? ( - + + ) : ( + + )} @@ -2346,7 +2710,7 @@ export function DBSearchPage() { setInputValue={setDisplayedTimeInputValue} onSearch={onTimePickerSearch} onRelativeSearch={onTimePickerRelativeSearch} - showLive={analysisMode === 'results'} + showLive={effectiveAnalysisMode === 'results'} isLiveMode={isLive} // Default to relative time mode if the user has made changes to interval and reloaded. defaultRelativeTimeMode={ @@ -2381,7 +2745,7 @@ export function DBSearchPage() { @@ -2425,7 +2789,7 @@ export function DBSearchPage() { height: '100%', }} > - {!isFilterSidebarCollapsed && ( + {!isFilterSidebarCollapsed && !isMultiSource && ( )} - {analysisMode === 'pattern' && + {effectiveAnalysisMode === 'pattern' && histogramTimeChartConfig != null && ( @@ -2523,7 +2887,7 @@ export function DBSearchPage() { )} - {analysisMode === 'delta' && + {effectiveAnalysisMode === 'delta' && searchedSource != null && isTraceSource(searchedSource) && ( )} - {analysisMode === 'results' && ( + {effectiveAnalysisMode === 'results' && isMultiSource && ( + + {isMultiSourceSqlBlocked ? ( + + + SQL search isn't supported across multiple sources + + + A SQL WHERE clause references the columns of one + specific table. Switch the search language to Lucene to + search across sources, or go back to a single source. + + + ) : ( + <> + {/* The histogram, total count, and filters sidebar come + with the next change; searching several sources + returns the merged results table on its own. */} + + + + + )} + + )} + {effectiveAnalysisMode === 'results' && !isMultiSource && ( {chartConfig && histogramTimeChartConfig && ( <> @@ -2725,32 +3134,26 @@ export function DBSearchPage() { px="sm" data-testid="search-results-panel" > - {chartConfig && - searchedConfig.source && - dbSqlRowTableConfig && ( - - )} + )} diff --git a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx index d3f10d02a5..330a024fbd 100644 --- a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx +++ b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx @@ -211,6 +211,9 @@ jest.mock('../components/ChartSQLPreview', () => ({ SQLPreview: () =>
, })); jest.mock('../components/DBSqlRowTableWithSidebar', () => () =>
); +// Multi-source components pull in DBRowSidePanel (and its deep import graph), +// which this test isolates away just like DBSqlRowTableWithSidebar above. +jest.mock('../components/SearchResultsTable', () => () =>
); jest.mock('../components/PatternTable', () => () =>
); jest.mock('../components/Search/DBSearchHeatmapChart', () => ({ DBSearchHeatmapChart: () =>
, diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 21834391a0..59bdf5398d 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1419,7 +1419,7 @@ export function appendSelectWithAdditionalKeys( } } -function getSelectLength(select: SelectList): number { +export function getSelectLength(select: SelectList): number { if (typeof select === 'string') { return select.split(',').filter(s => s.trim().length > 0).length; } else { @@ -1512,7 +1512,7 @@ export function useConfigWithAdditionalSelect( * (e.g. `SELECT *`). Exported for SearchResultsTable, which resolves the same * columns when rendering a single source's own SELECT. */ -function selectColumnMapWithoutAdditionalKeys( +export function selectColumnMapWithoutAdditionalKeys( selectMeta: ColumnMetaType[] | undefined, additionalKeysLength: number | undefined, ): Map< @@ -1547,7 +1547,7 @@ export type DBRowTableVariant = 'default' | 'muted'; * implementation. Denoising is inherently single-source: it mines patterns * from one table's body column against that source's severity expression. */ -function useDenoisedRows({ +export function useDenoisedRows({ config, sourceId, processedRows, @@ -1653,7 +1653,7 @@ function useDenoisedRows({ } /** The "Removed Noisy Event Patterns" summary shown above denoised results. */ -function DenoisedPatternsSummary({ +export function DenoisedPatternsSummary({ noisyPatterns, hasNoisyPatterns, }: { diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index 8c4b924bb8..a9d676102a 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -254,6 +254,7 @@ export function useFetchFacets({ filterState, showMoreFields, disableValues, + enabled = true, }: { chartConfig: BuilderChartConfigWithDateRange; sourceId: string | null; @@ -262,6 +263,8 @@ export function useFetchFacets({ filterState?: FilterState; showMoreFields?: boolean; disableValues?: boolean; + /** Disable all data fetching (e.g. an unused multi-source hook slot). */ + enabled?: boolean; }) { const facetsQuery = useFacets({ chartConfig, @@ -270,7 +273,7 @@ export function useFetchFacets({ dateRange, filterState, showMoreFields, - enabled: true, + enabled, disableValues, }); diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index bd5824e372..b500296ede 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -157,7 +157,9 @@ enum InlineTab { ColumnValues = 'columnValues', } -function RowOverviewPanelWrapper({ +// Exported for MultiSourceRowTable, which renders the same expanded-row +// overview but resolves the source per row instead of once per table. +export function RowOverviewPanelWrapper({ source, rowId, aliasWith, diff --git a/packages/app/src/components/MultiSourceBadge.tsx b/packages/app/src/components/MultiSourceBadge.tsx index b536e9219b..7d9d75798f 100644 --- a/packages/app/src/components/MultiSourceBadge.tsx +++ b/packages/app/src/components/MultiSourceBadge.tsx @@ -1,4 +1,15 @@ -import React from 'react'; +import { COLORS } from '@/utils'; + +/** + * Stable color for the Nth selected source in a multi-source search. Indexed + * by position in the selection (not hashed) so the ≤MAX_SEARCH_SOURCES badges + * never collide; the same assignment is used by the results table badge, the + * histogram series, and the per-source status chips so a source reads as one + * color everywhere on the page. + */ +export function getMultiSourceColor(index: number): string { + return COLORS[index % COLORS.length]; +} /** Colored-dot source label used in the merged results table. */ export function SourceBadge({ name, color }: { name: string; color?: string }) { diff --git a/packages/app/src/components/MultiSourceColumnPicker.tsx b/packages/app/src/components/MultiSourceColumnPicker.tsx new file mode 100644 index 0000000000..55230dd479 --- /dev/null +++ b/packages/app/src/components/MultiSourceColumnPicker.tsx @@ -0,0 +1,71 @@ +import { useCallback, useMemo } from 'react'; +import { Group, MultiSelect, Text } from '@mantine/core'; + +import { MultiSourceColumnOption } from '@/hooks/useMultiSourceSearch'; + +/** + * Multi-source replacement for the free-text SELECT editor: pick extra + * columns from the union of the selected sources' top-level columns. Columns + * missing from a source render as blank cells for that source's rows. + */ +export default function MultiSourceColumnPicker({ + unionColumns, + totalSources, + value, + onChange, +}: { + unionColumns: MultiSourceColumnOption[]; + totalSources: number; + /** Currently selected extra column names. */ + value: string[]; + onChange: (columns: string[]) => void; +}) { + const availabilityByName = useMemo( + () => new Map(unionColumns.map(c => [c.name, c.availableCount])), + [unionColumns], + ); + + const data = useMemo( + () => + unionColumns.map(c => ({ + value: c.name, + label: c.name, + })), + [unionColumns], + ); + + const renderOption = useCallback( + ({ option }: { option: { value: string; label: string } }) => { + const available = availabilityByName.get(option.value) ?? 0; + return ( + + + {option.label} + + {available < totalSources && ( + + {available}/{totalSources} sources + + )} + + ); + }, + [availabilityByName, totalSources], + ); + + return ( + + ); +} diff --git a/packages/app/src/components/SearchResultsTable.tsx b/packages/app/src/components/SearchResultsTable.tsx new file mode 100644 index 0000000000..4129828b92 --- /dev/null +++ b/packages/app/src/components/SearchResultsTable.tsx @@ -0,0 +1,618 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useQueryState } from 'nuqs'; +import { + chSqlToAliasMap, + ClickHouseQueryError, + ColumnMetaType, + convertCHDataTypeToJSType, + isJSDataTypeJSONStringifiable, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { MULTI_SOURCE_ALIASES } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + BuilderChartConfigWithDateRange, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Flex, Group, Loader, Text, Tooltip } from '@mantine/core'; +import { IconAlertTriangle, IconFilterOff } from '@tabler/icons-react'; +import { SortingState } from '@tanstack/react-table'; + +import api from '@/api'; +import { searchChartConfigDefaults } from '@/defaults'; +import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; +import useRowWhere, { RowWhereResult, WithClause } from '@/hooks/useRowWhere'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; +import { + mergeStreams, + MULTI_SOURCE_ROW_FIELDS, + StreamSnapshot, +} from '@/utils/multiSourceMerge'; +import { parseAsStringEncoded } from '@/utils/queryParsers'; + +import ChartErrorState from './charts/ChartErrorState'; +import DBRowSidePanel, { + RowSidePanelContext, + RowSidePanelContextProps, +} from './DBRowSidePanel'; +import { + DenoisedPatternsSummary, + getSelectLength, + RawLogTable, + selectColumnMapWithoutAdditionalKeys, + useConfigWithAdditionalSelect, + useDenoisedRows, +} from './DBRowTable'; +import { RowOverviewPanelWrapper } from './DBSqlRowTableWithSidebar'; +import { getMultiSourceColor, SourceBadge } from './MultiSourceBadge'; + +/** + * One selected source plus its fully-built chart config. + * + * With a single source the config carries that source's own SELECT (the user + * authored it); with several, each config projects the canonical + * MULTI_SOURCE_ALIASES so the merged rows share one shape. + */ +export type SearchStreamSpec = { + source: TSource; + config: BuilderChartConfigWithDateRange; + /** + * When set, the source doesn't run at all (e.g. an active filter references + * a column its table lacks); shown on the source's status chip. + */ + disabledReason?: string; +}; + +// Placeholder config for unused hook slots. The metadata hooks inside +// useConfigWithAdditionalSelect self-disable on empty table names, and the +// paginated query slot is explicitly disabled, so this never reaches +// ClickHouse. +const STUB_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +const EMPTY_CHSQL = { sql: '', params: {} }; +const EMPTY_EXTRA_COLUMNS: string[] = []; + +type SourceStream = { + spec: SearchStreamSpec | undefined; + data: ReturnType['data']; + fetchNextPage: ReturnType['fetchNextPage']; + hasNextPage: boolean; + isFetching: boolean; + isError: boolean; + error: Error | ClickHouseQueryError | null; + getRowWhere: (row: Record) => RowWhereResult; + /** Row-identity columns appended to the SELECT, trimmed off for display. */ + additionalKeysLength: number | undefined; +}; + +/** + * One source's independent query pipeline: the same + * defaults → additional-key SELECT merge → windowed offset pagination → + * row-WHERE machinery as the single-source DBSqlRowTable, packaged as a + * useMultiSourceSlots slot hook. Unused slots get a stub config and stay + * disabled. + */ +function useSourceStream( + spec: SearchStreamSpec | undefined, + { + enabled, + isLive, + enableSmallFirstWindow, + queryKeyPrefix, + }: { + enabled: boolean; + isLive: boolean; + enableSmallFirstWindow?: boolean; + queryKeyPrefix?: string; + }, +): SourceStream { + const { data: me } = api.useMe(); + + const configWithDefaults = useMemo( + () => ({ + ...searchChartConfigDefaults(me?.team), + ...(spec?.config ?? STUB_CONFIG), + }), + [me, spec?.config], + ); + + const mergedConfig = useConfigWithAdditionalSelect( + configWithDefaults, + spec?.source.id, + ); + + const { data, fetchNextPage, hasNextPage, isFetching, isError, error } = + useOffsetPaginatedQuery(mergedConfig ?? configWithDefaults, { + enabled: + enabled && + spec != null && + spec.disabledReason == null && + mergedConfig != null && + // An empty SELECT renders invalid SQL; wait for one to resolve. + getSelectLength(spec.config.select) > 0, + isLive, + queryKeyPrefix, + enableSmallFirstWindow, + }); + + const aliasMap = useMemo(() => { + const map = chSqlToAliasMap(data?.chSql ?? EMPTY_CHSQL); + // NULL-literal projections (`NULL AS "__hdx_duration_ms"` where a source + // lacks the field) are dropped by the SQL alias parser. Backfill them so + // the row-WHERE clause emits `isNull(NULL)` rather than referencing the + // alias as a (nonexistent) table column. ClickHouse reports NULL literals + // as Nullable(Nothing). + for (const col of data?.meta ?? []) { + if (map[col.name] == null && col.type === 'Nullable(Nothing)') { + map[col.name] = 'NULL'; + } + } + return map; + }, [data]); + + const getRowWhere = useRowWhere({ + meta: data?.meta, + aliasMap, + primaryKeyColumns: mergedConfig?.rowKeyColumns, + }); + + // Stable identity per content change, so downstream merge memos don't + // recompute (and re-sort every fetched row) on unrelated parent renders. + return useMemo( + () => ({ + spec, + data, + fetchNextPage, + hasNextPage: hasNextPage ?? false, + isFetching, + isError, + error: error ?? null, + getRowWhere, + additionalKeysLength: mergedConfig?.additionalKeysLength, + }), + [ + spec, + data, + fetchNextPage, + hasNextPage, + isFetching, + isError, + error, + getRowWhere, + mergedConfig?.additionalKeysLength, + ], + ); +} + +const COLUMN_NAME_MAP: Record = { + [MULTI_SOURCE_ALIASES.timestamp]: 'Timestamp', + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: 'Source', + [MULTI_SOURCE_ALIASES.service]: 'Service', + [MULTI_SOURCE_ALIASES.severity]: 'Level', + [MULTI_SOURCE_ALIASES.durationMs]: 'Duration (ms)', + [MULTI_SOURCE_ALIASES.body]: 'Message', +}; + +function StreamStatusChips({ streams }: { streams: SourceStream[] }) { + return ( + + {streams.map((stream, i) => { + if (stream.spec == null) return null; + const name = stream.spec.source.name; + const disabledReason = stream.spec.disabledReason; + return ( + + + {stream.isFetching && } + {disabledReason != null && ( + + + + + + )} + {stream.isError && ( + + + + + + )} + + ); + })} + + ); +} + +export default function SearchResultsTable({ + sources: specs, + isLive, + enabled = true, + extraColumnNames = EMPTY_EXTRA_COLUMNS, + denoiseResults = false, + sortOrder, + onSortingChange, + onError, + onResolvedColumnsChange, + onScroll, + onSidebarOpen, + onExpandedRowsChange, + collapseAllRows, + enableSmallFirstWindow, + tableId, + context, + keepOpenSelector, + // Row queries are keyed separately from the page's chart/count queries, so + // "is the search fetching?" (live-tail pause, latency telemetry) keeps + // measuring the same thing it always has. + queryKeyPrefix = 'dbSqlRowTable', +}: { + /** 1..MAX_SEARCH_SOURCES selected sources with their built configs. */ + sources: SearchStreamSpec[]; + isLive: boolean; + enabled?: boolean; + /** User-picked extra columns projected into every source's SELECT (N>1). */ + extraColumnNames?: string[]; + /** Drop noisy event patterns from the results (single source only). */ + denoiseResults?: boolean; + /** Current sort, for the single-source case where sorting is supported. */ + sortOrder?: SortingState; + onSortingChange?: (v: SortingState | null) => void; + /** + * Surface a query failure to the page. Only called with a single source — + * with several, a failing source is isolated to its own status chip rather + * than failing the whole search. + */ + onError?: (error: Error | ClickHouseQueryError) => void; + onResolvedColumnsChange?: (meta: ColumnMetaType[]) => void; + onScroll?: (scrollTop: number) => void; + onSidebarOpen?: (rowId: string) => void; + onExpandedRowsChange?: (hasExpandedRows: boolean) => void; + collapseAllRows?: boolean; + enableSmallFirstWindow?: boolean; + tableId?: string; + context?: RowSidePanelContextProps; + keepOpenSelector?: string; + queryKeyPrefix?: string; +}) { + const slots = useMultiSourceSlots(specs, useSourceStream, { + enabled, + isLive, + enableSmallFirstWindow, + queryKeyPrefix, + }); + + const streams = useMemo( + () => + slots.filter( + (s): s is SourceStream & { spec: SearchStreamSpec } => s.spec != null, + ), + [slots], + ); + + // With one source the table shows that source's own SELECT, sorts, and + // denoises — everything the single-source search has always done. The + // canonical aliases, source badges, and cross-source merge only come into + // play once a second source is selected. + const isSingleSource = specs.length === 1; + const singleStream = isSingleSource ? streams[0] : undefined; + + const snapshots: StreamSnapshot[] = useMemo( + () => + streams.map((stream, i) => ({ + sourceId: stream.spec.source.id, + sourceName: stream.spec.source.name, + sourceColor: getMultiSourceColor(i), + rows: stream.data?.data ?? [], + window: stream.data?.window ?? null, + lastPageRowCount: stream.data?.lastPageRowCount ?? null, + hasNextPage: stream.hasNextPage, + isActive: !stream.isError && stream.spec.disabledReason == null, + dateRange: stream.spec.config.dateRange, + })), + [streams], + ); + + // One source needs no merge: its rows already arrive timestamp-ordered from + // its own ORDER BY, and there is no other stream to hold a frontier against. + const merged = useMemo( + () => + isSingleSource + ? null + : mergeStreams(snapshots, 'DESC', MULTI_SOURCE_ALIASES.timestamp), + [isSingleSource, snapshots], + ); + + const columnTypeMap = useMemo(() => { + if (singleStream != null) { + // The user's SELECT columns, positionally trimmed of the row-identity + // columns the query appends (same resolution as DBSqlRowTable). + return selectColumnMapWithoutAdditionalKeys( + singleStream.data?.meta, + singleStream.additionalKeysLength, + ); + } + // Merge column meta across streams by canonical alias name, preferring a + // resolved type over the Nullable(Nothing) a `NULL AS "alias"` projection + // reports. + const map = new Map(); + for (const stream of streams) { + for (const col of stream.data?.meta ?? []) { + const jsType = convertCHDataTypeToJSType(col.type); + const existing = map.get(col.name); + if (existing == null || existing._type == null) { + map.set(col.name, { _type: jsType }); + } + } + } + map.set(MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME, { + _type: JSDataType.String, + }); + return map; + }, [streams, singleStream]); + + const includeDuration = specs.some(s => s.source.kind === SourceKind.Trace); + + const displayedColumns = useMemo(() => { + if (isSingleSource) { + return Array.from(columnTypeMap.keys()); + } + return [ + MULTI_SOURCE_ALIASES.timestamp, + MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME, + MULTI_SOURCE_ALIASES.service, + MULTI_SOURCE_ALIASES.severity, + ...(includeDuration ? [MULTI_SOURCE_ALIASES.durationMs] : []), + ...extraColumnNames, + MULTI_SOURCE_ALIASES.body, + ]; + }, [isSingleSource, columnTypeMap, includeDuration, extraColumnNames]); + + // Stringify object-typed cells (Map/Array/JSON) the same way DBSqlRowTable + // does — both for display and because useRowWhere expects the stringified + // form when rebuilding a row WHERE clause. + const rows = useMemo(() => { + const baseRows = singleStream + ? (singleStream.data?.data ?? []) + : (merged?.rows ?? []); + const objectColumns = [...columnTypeMap.entries()] + .filter(([, v]) => isJSDataTypeJSONStringifiable(v._type)) + .map(([name]) => name); + if (objectColumns.length === 0) { + return baseRows; + } + return baseRows.map(row => { + const newRow = { ...row }; + for (const col of objectColumns) { + if (!(col in newRow) || newRow[col] == null) continue; + if (columnTypeMap.get(col)?._type === JSDataType.JSON) { + newRow[col] = JSON.stringify(newRow[col]).replace(/\//g, '\\/'); + } else { + newRow[col] = JSON.stringify(newRow[col]); + } + } + return newRow; + }); + }, [singleStream, merged?.rows, columnTypeMap]); + + const patternColumn = displayedColumns[displayedColumns.length - 1]; + const denoise = useDenoisedRows({ + config: singleStream?.spec.config ?? STUB_CONFIG, + sourceId: singleStream?.spec.source.id, + processedRows: rows, + patternColumn, + // Denoising mines patterns from one table's body column; it has no + // cross-source meaning, so it only runs with a single source. + denoiseResults: denoiseResults && isSingleSource, + isLive, + }); + + // Row identity dispatches to the row's own stream: each stream has its own + // result meta / alias map / primary-key columns. The client-side source tags + // are stripped first — they aren't real columns. + const generateRowId = useCallback( + (row: Record): RowWhereResult => { + if (singleStream != null) { + return singleStream.getRowWhere(row); + } + const { + [MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]: sourceId, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: _name, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_COLOR]: _color, + ...dbRow + } = row; + const stream = streams.find(s => s.spec.source.id === sourceId); + if (stream == null) { + return { where: '', aliasWith: [] }; + } + return stream.getRowWhere(dbRow); + }, + [streams, singleStream], + ); + + // Advance only the stream(s) holding the frontier back; the leaders keep + // their fetched-but-held rows until the laggards catch up. + const fetchNextPage = useCallback(() => { + if (singleStream != null) { + singleStream.fetchNextPage({ cancelRefetch: false }); + return; + } + for (const sourceId of merged?.laggingSourceIds ?? []) { + const stream = streams.find(s => s.spec.source.id === sourceId); + stream?.fetchNextPage({ cancelRefetch: false }); + } + }, [singleStream, merged?.laggingSourceIds, streams]); + + const hasNextPage = streams.some(s => !s.isError && s.hasNextPage); + const isFetching = streams.some(s => s.isFetching); + const isLoading = denoiseResults + ? isFetching || denoise.isFetching + : isFetching; + const allFailed = streams.length > 0 && streams.every(s => s.isError); + const firstError = streams.find(s => s.error != null)?.error ?? undefined; + + // A single source's failure is the whole search's failure, so the page owns + // the error UI (and drops out of live tail), exactly as before. + useEffect(() => { + if (singleStream?.isError && singleStream.error != null) { + onError?.(singleStream.error); + } + }, [singleStream?.isError, singleStream?.error, onError]); + + const singleMeta = singleStream?.data?.meta; + useEffect(() => { + if (singleMeta != null && singleMeta.length > 0) { + onResolvedColumnsChange?.(singleMeta); + } + }, [singleMeta, onResolvedColumnsChange]); + + // Side panel wiring — the same URL-param contract as the legacy table, + // except the panel's source comes from the clicked row rather than being + // fixed for the page. + const [rowId, setRowId] = useQueryState('rowWhere', parseAsStringEncoded); + const [rowSource, setRowSource] = useQueryState('rowSource'); + const [aliasWith, setAliasWith] = useState([]); + + const onRowDetailsClick = useCallback( + (row: Record) => { + const rowWhere = generateRowId(row); + if (!rowWhere.where) return; + setRowId(rowWhere.where); + setAliasWith(rowWhere.aliasWith); + setRowSource( + row[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID] ?? + singleStream?.spec.source.id ?? + null, + ); + onSidebarOpen?.(rowWhere.where); + }, + [generateRowId, setRowId, setRowSource, onSidebarOpen, singleStream], + ); + + const onCloseSidebar = useCallback(() => { + setRowId(null); + setRowSource(null); + }, [setRowId, setRowSource]); + + const sourceForRow = useCallback( + (id: unknown) => + specs.find(s => s.source.id === id)?.source ?? + // Links predating the rowSource param (and every single-source link) + // carry only rowWhere; there is exactly one source it can belong to. + (isSingleSource ? specs[0]?.source : undefined), + [specs, isSingleSource], + ); + + const panelSource = useMemo( + () => sourceForRow(rowSource), + [sourceForRow, rowSource], + ); + + const renderRowDetails = useCallback( + (r: { id: string; aliasWith?: WithClause[]; [key: string]: unknown }) => { + const source = sourceForRow(r[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]); + if (!source) { + return
Loading...
; + } + return ( + + ); + }, + [sourceForRow], + ); + + const loadingDate = singleStream + ? singleStream.data?.window?.direction === 'ASC' + ? singleStream.data?.window?.endTime + : singleStream.data?.window?.startTime + : merged?.frontier != null && hasNextPage + ? new Date(merged.frontier) + : undefined; + + const firstConfig = streams[0]?.spec.config; + + return ( + + {panelSource != null && ( + + )} + + {/* One source needs no legend: every row came from it. */} + {!isSingleSource && } + {denoiseResults && isSingleSource && ( + + )} + {allFailed && !isSingleSource ? ( + + ) : ( + + )} + + + ); +} diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 5894d8b78d..5b67f68216 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -2,6 +2,14 @@ import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist // Limit defaults export const DEFAULT_SEARCH_ROW_LIMIT = 200; + +// Ceiling on how many sources one search can span. Cost scales linearly with +// the selection: each source runs its own result stream plus histogram/count +// aggregates (~3 ClickHouse queries per source per refresh, re-fired every +// live-tail tick), so 3 keeps the worst case bounded while covering the +// common "app logs + infra logs + traces" setups. Also the hook-slot count in +// useMultiSourceSlots — raising it means adding a slot there too. +export const MAX_SEARCH_SOURCES = 3; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100; export const DEFAULT_SERIES_LIMIT = 100; diff --git a/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts new file mode 100644 index 0000000000..58be8d33d2 --- /dev/null +++ b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts @@ -0,0 +1,87 @@ +import { Filter } from '@hyperdx/common-utils/dist/types'; + +import { + filterRootColumn, + resolveExtraColumnsForSource, + unresolvedFilterColumns, +} from '@/hooks/useMultiSourceSearch'; + +describe('filterRootColumn', () => { + it('extracts a plain column reference', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: 'ServiceName', + right: "'cart'", + }; + expect(filterRootColumn(filter)).toBe('ServiceName'); + }); + + it('extracts the root of a map subscript', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: "LogAttributes['level']", + right: "'error'", + }; + expect(filterRootColumn(filter)).toBe('LogAttributes'); + }); + + it('extracts a backticked identifier', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: '`weird-col`', + right: "'x'", + }; + expect(filterRootColumn(filter)).toBe('weird-col'); + }); + + it('returns null for raw sql and lucene filters', () => { + expect( + filterRootColumn({ type: 'sql', condition: "Foo = 'bar'" }), + ).toBeNull(); + expect( + filterRootColumn({ type: 'lucene', condition: 'foo:bar' }), + ).toBeNull(); + }); +}); + +describe('unresolvedFilterColumns', () => { + const filters: Filter[] = [ + { type: 'sql_ast', operator: '=', left: 'ServiceName', right: "'cart'" }, + { type: 'sql_ast', operator: '=', left: 'StatusCode', right: "'Unset'" }, + { type: 'sql', condition: 'anything' }, + ]; + + it('reports columns the source lacks', () => { + expect( + unresolvedFilterColumns(filters, new Set(['ServiceName', 'Body'])), + ).toEqual(['StatusCode']); + }); + + it('is empty when every attributable column resolves', () => { + expect( + unresolvedFilterColumns(filters, new Set(['ServiceName', 'StatusCode'])), + ).toEqual([]); + }); + + it('is empty (not excluding) while columns are still unknown', () => { + expect(unresolvedFilterColumns(filters, undefined)).toEqual([]); + }); +}); + +describe('resolveExtraColumnsForSource', () => { + it('projects the column where present and NULL where missing', () => { + expect( + resolveExtraColumnsForSource( + ['ServiceName', 'StatusCode', 'weird col'], + new Set(['ServiceName', 'weird col']), + ), + ).toEqual([ + { name: 'ServiceName', expression: 'ServiceName' }, + { name: 'StatusCode', expression: null }, + { name: 'weird col', expression: '`weird col`' }, + ]); + }); +}); diff --git a/packages/app/src/hooks/useMultiSourceSearch.ts b/packages/app/src/hooks/useMultiSourceSearch.ts new file mode 100644 index 0000000000..f4ecdb4da4 --- /dev/null +++ b/packages/app/src/hooks/useMultiSourceSearch.ts @@ -0,0 +1,148 @@ +import { useMemo } from 'react'; +import { + ColumnMeta, + filterColumnMetaByType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { MultiSourceExtraColumn } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { Filter, TSource } from '@hyperdx/common-utils/dist/types'; + +import { useColumns } from '@/hooks/useMetadata'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; + +const EMPTY_SOURCE_PARAMS = { + databaseName: '', + tableName: '', + connectionId: '', +}; + +function columnsParamsFor(source: TSource | undefined) { + if (source == null) return EMPTY_SOURCE_PARAMS; + return { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + connectionId: source.connection, + }; +} + +export type MultiSourceColumnOption = { + name: string; + /** How many of the selected sources have this column. */ + availableCount: number; +}; + +/** Slot hook: DESCRIBE columns for one source. Stable — `.data` is cached. */ +function useSourceColumnsSlot( + source: TSource | undefined, +): ColumnMeta[] | undefined { + return useColumns(columnsParamsFor(source)).data; +} + +/** + * Top-level columns (DESCRIBE) for each selected source of a multi-source + * search, plus the deduped union with per-column availability counts for the + * add-column picker. useColumns self-disables for unused slots. + */ +export function useMultiSourceColumns(sources: TSource[]): { + columnsBySourceId: Map>; + unionColumns: MultiSourceColumnOption[]; + /** Union of Date/DateTime column name → ClickHouse type across sources. */ + dateTimeColumns: Map; +} { + const slotData = useMultiSourceSlots( + sources, + useSourceColumnsSlot, + undefined, + ); + + return useMemo(() => { + const columnsBySourceId = new Map>(); + const availability = new Map(); + const dateTimeColumns = new Map(); + + for (let i = 0; i < sources.length; i++) { + const source = sources[i]; + const columns = slotData[i]; + if (source == null || columns == null) continue; + const names = new Set(columns.map(c => c.name)); + columnsBySourceId.set(source.id, names); + for (const name of names) { + availability.set(name, (availability.get(name) ?? 0) + 1); + } + for (const col of filterColumnMetaByType(columns, [JSDataType.Date]) ?? + []) { + if (!dateTimeColumns.has(col.name)) { + dateTimeColumns.set(col.name, col.type); + } + } + } + + const unionColumns = [...availability.entries()] + .map(([name, availableCount]) => ({ name, availableCount })) + .sort( + (a, b) => + b.availableCount - a.availableCount || a.name.localeCompare(b.name), + ); + + return { columnsBySourceId, unionColumns, dateTimeColumns }; + }, [slotData, sources]); +} + +/** + * Root column a filter references, for per-source resolvability checks. + * sql_ast filters carry the escaped SQL key in `left` (e.g. `ServiceName`, + * a backticked identifier, or `LogAttributes['level']` whose root is + * `LogAttributes`). Other filter types (raw sql/lucene conditions) can't be + * attributed to a single column and return null — callers should apply them + * to every source and rely on per-source error isolation. + */ +export function filterRootColumn(filter: Filter): string | null { + if (filter.type !== 'sql_ast') return null; + const left = filter.left.trim(); + const backticked = left.match(/^`([^`]+)`/); + if (backticked) return backticked[1]; + const plain = left.match(/^[A-Za-z_][A-Za-z0-9_]*/); + return plain ? plain[0] : null; +} + +/** + * For one source: which of the active filters reference a column its table + * doesn't have. A non-empty result means the source can't answer the + * filtered search and should be excluded (with a visible reason). + */ +export function unresolvedFilterColumns( + filters: Filter[], + sourceColumns: Set | undefined, +): string[] { + if (sourceColumns == null) return []; + const missing = new Set(); + for (const filter of filters) { + const root = filterRootColumn(filter); + if (root != null && !sourceColumns.has(root)) { + missing.add(root); + } + } + return [...missing]; +} + +/** Quote a column name as a ClickHouse identifier when it needs it. */ +function quoteIdentifier(name: string): string { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) + ? name + : `\`${name.replace(/`/g, '\\`')}\``; +} + +/** + * Resolve the user-picked extra column names into per-source SELECT + * expressions: the (quoted) column itself where the source's table has it, + * NULL otherwise — so every source still returns the same result shape. + */ +export function resolveExtraColumnsForSource( + extraColumnNames: string[], + sourceColumns: Set | undefined, +): MultiSourceExtraColumn[] { + return extraColumnNames.map(name => ({ + name, + expression: sourceColumns?.has(name) ? quoteIdentifier(name) : null, + })); +} diff --git a/packages/app/src/hooks/useResolvedSourcesParam.ts b/packages/app/src/hooks/useResolvedSourcesParam.ts new file mode 100644 index 0000000000..deb2f449d1 --- /dev/null +++ b/packages/app/src/hooks/useResolvedSourcesParam.ts @@ -0,0 +1,58 @@ +import { useEffect, useMemo } from 'react'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { notifications } from '@mantine/notifications'; + +import { MAX_SEARCH_SOURCES } from '@/defaults'; +import { useSources } from '@/source'; +import { resolveSourcesParam } from '@/utils/sourceParams'; + +const EMPTY_SOURCES: TSource[] = []; + +/** + * Resolves the multi-source search param (a list of source IDs or names) to + * the matching sources, deduped and capped at MAX_SEARCH_SOURCES. + * + * Elements that don't match any usable source are dropped from the selection + * and reported once via a Mantine warning, mirroring useResolvedSourceParam. + */ +export function useResolvedSourcesParam( + paramValues: string[] | null | undefined, + { kinds }: { kinds?: SourceKind[] } = {}, +): { sources: TSource[] } { + const { data: allSources } = useSources(); + + // Key the memo on a serialized `kinds` so callers can pass inline arrays + // without breaking memoization. + const kindsKey = kinds?.join(','); + const { sources, unresolvedKey } = useMemo(() => { + const allKinds = new Set(Object.values(SourceKind)); + const resolvedKinds = kindsKey + ? kindsKey.split(',').filter((k): k is SourceKind => allKinds.has(k)) + : undefined; + const resolution = resolveSourcesParam(paramValues, allSources, { + kinds: resolvedKinds, + max: MAX_SEARCH_SOURCES, + }); + if (resolution.status !== 'resolved') { + return { sources: EMPTY_SOURCES, unresolvedKey: undefined }; + } + return { + sources: resolution.sources.length ? resolution.sources : EMPTY_SOURCES, + unresolvedKey: resolution.unresolved.length + ? resolution.unresolved.join(', ') + : undefined, + }; + }, [paramValues, allSources, kindsKey]); + + useEffect(() => { + if (unresolvedKey == null) return; + notifications.show({ + id: 'sources-param-unresolved-' + unresolvedKey, + color: 'yellow', + title: 'Some sources were not found', + message: `No searchable source matches: ${unresolvedKey}. They may have been renamed or deleted.`, + }); + }, [unresolvedKey]); + + return useMemo(() => ({ sources }), [sources]); +} diff --git a/packages/app/src/hooks/useSourceSlots.ts b/packages/app/src/hooks/useSourceSlots.ts new file mode 100644 index 0000000000..04286e5832 --- /dev/null +++ b/packages/app/src/hooks/useSourceSlots.ts @@ -0,0 +1,32 @@ +import { useMemo } from 'react'; + +import { MAX_SEARCH_SOURCES } from '@/defaults'; + +/** + * Run one instance of a hook per selected source of a search. + * + * The rules of hooks require a constant hook count per component, but search + * needs one pipeline per selected source — and `useQueries` can't cover these + * pipelines (row streams are `useInfiniteQuery`-based, which has no plural + * form, and the chart/facet pipelines compose other hooks). So the hook count + * is pinned at MAX_SEARCH_SOURCES here, in one place: unused slots receive + * `undefined` and every slot hook is expected to self-disable for it. + * + * `useSlot` must be a stable, named hook (the rules-of-hooks lint understands + * `use*`-named parameters) and should return a memoized value, so the array + * this returns is referentially stable and safe to use in dependency lists. + * + * Lives in its own module so both the search hooks and the metadata hooks can + * use it without an import cycle. + */ +export function useMultiSourceSlots( + items: readonly Item[], + useSlot: (item: Item | undefined, opts: Opts) => Result, + opts: Opts, +): Result[] { + const s0 = useSlot(items[0], opts); + const s1 = useSlot(items[1], opts); + const s2 = useSlot(items[2], opts); + const count = Math.min(items.length, MAX_SEARCH_SOURCES); + return useMemo(() => [s0, s1, s2].slice(0, count), [s0, s1, s2, count]); +}