diff --git a/.changeset/multi-source-search.md b/.changeset/multi-source-search.md new file mode 100644 index 0000000000..9d4868f8f4 --- /dev/null +++ b/.changeset/multi-source-search.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/app': minor +'@hyperdx/common-utils': minor +--- + +Search across multiple sources at once. The search page's source selector can +now expand into a multi-select (up to 3 log/trace sources): results interleave +into one timestamp-ordered timeline with a per-row source badge, normalized +columns (Timestamp, Source, Service, Level, Message, and Duration when traces +are included), a histogram stacked by source, and an add-column picker over the +union of the selected sources' columns. Each source runs its own query +pipeline — sources on different connections work, and a failing source shows a +status chip instead of failing the whole search. Multi-source mode is +Lucene-only and shareable via URL; saved searches and alerts remain +single-source for now. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index cdae127d1e..c11ff3efe1 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -31,7 +31,10 @@ import { ColumnMeta, } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; -import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + buildMultiSourceSearchConfig, + buildSearchChartConfig, +} from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { aliasMapToWithClauses, isBrowser, @@ -95,21 +98,31 @@ import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { FavoriteButton } from '@/components/FavoriteButton'; import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover'; import { InputControlled } from '@/components/InputControlled'; +import MultiSourceColumnPicker from '@/components/MultiSourceColumnPicker'; import OnboardingModal from '@/components/OnboardingModal'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; import SearchPageActionBar from '@/components/SearchPageActionBar'; +import SearchResultsTable from '@/components/SearchResultsTable'; import SearchTotalCountChart from '@/components/SearchTotalCountChart'; +import { SourceMultiSelectControlled } from '@/components/SourceMultiSelect'; import { TableSourceForm } from '@/components/Sources/SourceForm'; import { SourceSelectControlled } from '@/components/SourceSelect'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; import { Tags } from '@/components/Tags'; import { TimePicker } from '@/components/TimePicker'; import { IS_LOCAL_MODE } from '@/config'; +import { MAX_SEARCH_SOURCES } from '@/defaults'; import { useAliasMapFromChartConfig } from '@/hooks/useChartConfig'; import { useExplainQuery } from '@/hooks/useExplainQuery'; +import { + resolveExtraColumnsForSource, + unresolvedFilterColumns, + useMultiSourceColumns, +} from '@/hooks/useMultiSourceSearch'; import { useResolvedSourceParam } from '@/hooks/useResolvedSourceParam'; +import { useResolvedSourcesParam } from '@/hooks/useResolvedSourcesParam'; import { withAppNav } from '@/layout'; import { useCreateSavedSearch, @@ -133,7 +146,6 @@ import { } from '@/utils'; import ChartSQLPreview, { SQLPreview } from './components/ChartSQLPreview'; -import DBSqlRowTableWithSideBar from './components/DBSqlRowTableWithSidebar'; import PatternTable from './components/PatternTable'; import { DBSearchHeatmapChart } from './components/Search/DBSearchHeatmapChart'; import DirectTraceSidePanel from './components/Search/DirectTraceSidePanel'; @@ -181,6 +193,10 @@ const ALLOWED_SOURCE_KINDS = [SourceKind.Log, SourceKind.Trace]; const SearchConfigSchema = z.object({ select: z.string(), source: z.string(), + // Multi-source search: the full selection (2+ engages multi mode). The + // single `source` field stays the primary (= sources[0]) so every + // single-source code path keeps working unchanged. + sources: z.array(z.string()), where: z.string(), whereLanguage: z.enum(['sql', 'lucene']), orderBy: z.string(), @@ -865,6 +881,25 @@ function optimizeDefaultOrderBy( : `${orderByArr[0]} DESC`; } +/** + * Per-source default ORDER BY for multi-source search. Same resolution as + * useDefaultOrderBy minus the table sorting-key optimization (which needs a + * metadata query per source): the source's explicit orderByExpression, else + * its timestamp expression(s) DESC. Time-window pagination requires the first + * term to be the source's timestamp, which this guarantees. + */ +function multiSourceDefaultOrderBy(source: TSource): string { + const isEventSource = + source.kind === SourceKind.Log || source.kind === SourceKind.Trace; + const explicit = isEventSource ? source.orderByExpression?.trim() : undefined; + if (explicit) return explicit; + return optimizeDefaultOrderBy( + source.timestampValueExpression ?? '', + isEventSource ? source.displayedTimestampValueExpression : undefined, + undefined, + ); +} + export function useDefaultOrderBy(sourceID: string | undefined | null) { const { data: source } = useSource({ id: sourceID, @@ -895,6 +930,9 @@ function formatDroppedFiltersMessage(count: number): string { // This is outside as it needs to be a stable reference const queryStateMap = { source: parseAsString, + // JSON-encoded (not comma-separated) because source names may contain + // commas; `source` is always written alongside it as the primary. + sources: parseAsJsonEncoded(), where: parseAsStringEncoded, select: parseAsStringEncoded, whereLanguage: parseAsStringEnum<'sql' | 'lucene'>(['sql', 'lucene']), @@ -1116,6 +1154,7 @@ export function DBSearchPage() { (savedSearchId || directTraceId || rawSearchedConfig.source ? '' : defaultSourceId), + sources: searchedConfig.sources ?? [], filters: searchedConfig.filters ?? [], orderBy: searchedConfig.orderBy ?? '', }, @@ -1184,6 +1223,7 @@ export function DBSearchPage() { whereLanguage: searchedConfig?.whereLanguage ?? getStoredLanguage() ?? 'lucene', source: searchedConfig?.source ?? undefined, + sources: searchedConfig?.sources ?? [], filters: searchedConfig?.filters ?? [], orderBy: searchedConfig?.orderBy ?? '', }); @@ -1198,6 +1238,7 @@ export function DBSearchPage() { // to an existing source. const isSearchConfigEmpty = !rawSearchedConfig.source && + !rawSearchedConfig.sources?.length && !where && !select && !whereLanguage && @@ -1240,6 +1281,7 @@ export function DBSearchPage() { savedSearch, searchedConfig, rawSearchedConfig.source, + rawSearchedConfig.sources, setSearchedConfig, savedSearchId, defaultSourceId, @@ -1268,12 +1310,15 @@ export function DBSearchPage() { const onSubmit = useCallback(() => { onSearch(displayedTimeInputValue); handleSubmit( - ({ select, where, whereLanguage, source, filters, orderBy }) => { + ({ select, where, whereLanguage, source, sources, filters, orderBy }) => { setSearchedConfig({ select, where, whereLanguage, source, + // Writer discipline: only 2+ selections persist the list; a single + // selection clears it so old-style URLs stay canonical. + sources: sources.length > 1 ? sources : null, filters, orderBy, }); @@ -1301,8 +1346,123 @@ export function DBSearchPage() { [debouncedSubmit, setValue], ); + const watchedSource = useWatch({ + control, + name: 'source', + // Watch will reset when changing saved search, so we need to default to the URL + defaultValue: searchedConfig.source ?? undefined, + }); + + // --- Multi-source search: selection & schema state ------------------------ + // 2+ resolved sources in the ?sources= param engage multi mode: one + // independent query pipeline per source, merged client-side. The single + // `source` (primary) keeps every existing code path working; multi mode + // only swaps what gets rendered below. Declared before the filter-state + // hooks so they can work against the union of the selected schemas. + const { sources: searchedMultiSources } = useResolvedSourcesParam( + rawSearchedConfig.sources, + { kinds: ALLOWED_SOURCE_KINDS }, + ); + const isMultiSource = searchedMultiSources.length > 1; + // Delta/pattern analyses are per-source; multi mode pins the results view. + const effectiveAnalysisMode = isMultiSource ? 'results' : analysisMode; + // Raw SQL WHERE names concrete columns of a concrete table — reinterpreting + // it per source risks silently-wrong results, so multi mode requires Lucene. + const isMultiSourceSqlBlocked = + isMultiSource && + (searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene') === + 'sql' && + !!searchedConfig.where; + + // A hand-authored URL may carry only ?sources=; backfill the primary so the + // single-source machinery (form, chart config) has one. + useEffect(() => { + if (!rawSearchedConfig.source && searchedMultiSources.length > 0) { + setSearchedConfig({ source: searchedMultiSources[0].id }); + } + }, [rawSearchedConfig.source, searchedMultiSources, setSearchedConfig]); + + const watchedSources = useWatch({ control, name: 'sources' }); + const formSourceCount = watchedSources?.length ?? 0; + // The multi-select UI stays visible while the user is composing a selection + // (even before a second source is added). + const [multiPickerOpen, setMultiPickerOpen] = useState(false); + const isMultiSelectUI = multiPickerOpen || formSourceCount > 1; + const formIsMulti = formSourceCount > 1; + + const enterMultiSourceSelect = useCallback(() => { + setValue('sources', watchedSource ? [watchedSource] : []); + setMultiPickerOpen(true); + }, [setValue, watchedSource]); + + // Keep the primary `source` field in sync with the selection and re-run the + // search when the selection changes. + const prevWatchedSourcesRef = useRef(null); + useEffect(() => { + const current = watchedSources ?? []; + const prev = prevWatchedSourcesRef.current; + if (prev != null && JSON.stringify(prev) === JSON.stringify(current)) { + return; + } + prevWatchedSourcesRef.current = current; + if (prev == null) { + // Initial hydration from the URL — nothing changed. + return; + } + if (current.length > 0 && current[0] !== watchedSource) { + setValue('source', current[0]); + } + if ((prev?.length ?? 0) <= 1 && current.length > 1) { + // Entering multi mode: the single-source SELECT/ORDER BY strings don't + // translate to the canonical multi-source shape. + setValue('select', ''); + setValue('orderBy', ''); + } + debouncedSubmit(); + }, [watchedSources, watchedSource, setValue, debouncedSubmit]); + + // Collapse the picker back to the single-source select when the user + // reduces a real multi selection to one source. Keyed on the >1 → ≤1 + // transition so it can't fire in the just-opened composing state (picker + // open, one source selected, second not yet picked). + const prevFormSourceCountRef = useRef(formSourceCount); + useEffect(() => { + const prev = prevFormSourceCountRef.current; + prevFormSourceCountRef.current = formSourceCount; + if (prev > 1 && formSourceCount <= 1) { + setMultiPickerOpen(false); + } + }, [formSourceCount]); + + // Per-source top-level columns: powers the add-column picker, the + // per-source `column vs NULL` projection for user-picked extras, and + // per-source filter resolvability. Driven by the form's draft selection + // while composing (so the picker has options before the search is + // submitted), falling back to the searched selection. + const formMultiSources = useMemo( + () => + (watchedSources ?? []) + .map(id => inputSourceObjs?.find(s => s.id === id)) + .filter((s): s is TSource => s != null), + [watchedSources, inputSourceObjs], + ); + const { + columnsBySourceId, + unionColumns, + dateTimeColumns: multiDateTimeColumns, + } = useMultiSourceColumns( + formMultiSources.length > 1 + ? formMultiSources + : isMultiSource + ? searchedMultiSources + : [], + ); + // --- End multi-source selection & schema state ----------------------------- + // Top-level column names for the active source, used to quote - // filter keys that contain special characters. + // filter keys that contain special characters. In multi mode this is the + // union across the selected sources, so filter keys from any of them + // escape correctly. const { data: inputSourceColumns } = useColumns( { databaseName: inputSourceObj?.from?.databaseName ?? '', @@ -1311,20 +1471,19 @@ export function DBSearchPage() { }, { enabled: !!inputSourceObj }, ); - const knownColumns = useMemo( - () => - inputSourceColumns - ? new Set(inputSourceColumns.map(c => c.name)) - : new Set(), - [inputSourceColumns], - ); + const knownColumns = useMemo(() => { + if (isMultiSource) { + const union = new Set(); + for (const names of columnsBySourceId.values()) { + for (const name of names) union.add(name); + } + return union; + } + return inputSourceColumns + ? new Set(inputSourceColumns.map(c => c.name)) + : new Set(); + }, [inputSourceColumns, isMultiSource, columnsBySourceId]); - const watchedSource = useWatch({ - control, - name: 'source', - // Watch will reset when changing saved search, so we need to default to the URL - defaultValue: searchedConfig.source ?? undefined, - }); const prevSourceRef = useRef(watchedSource); // Set when the user switches sources via the dropdown. The follow-up // effect waits for the new source's columns to load and then drops any @@ -1347,11 +1506,21 @@ export function DBSearchPage() { const { dateTimeColumns, onResolvedColumnsChange } = useResolvedDateTimeColumns(inputSourceColumns); + // In multi mode, date/time-typed filter keys may come from any selected + // source's schema. + const effectiveDateTimeColumns = useMemo( + () => + isMultiSource && multiDateTimeColumns.size > 0 + ? new Map([...dateTimeColumns, ...multiDateTimeColumns]) + : dateTimeColumns, + [isMultiSource, dateTimeColumns, multiDateTimeColumns], + ); + const filters = useWatch({ name: 'filters', control }); const searchFilters = useSearchPageFilterState({ searchQuery: filters ?? undefined, onFilterChange: handleSetFilters, - dateTimeColumns, + dateTimeColumns: effectiveDateTimeColumns, knownColumns, }); @@ -1489,6 +1658,135 @@ export function DBSearchPage() { const { data: chartConfig, isLoading: isChartConfigLoading } = useSearchedConfigToChartConfig(chartSearchConfig, defaultSearchConfig); + // --- Multi-source search: query specs ------------------------------------- + // In multi mode the `select` param holds the extra column names picked by + // the user (the canonical columns are always projected). + const multiExtraColumnNames = useMemo( + () => + isMultiSource ? splitAndTrimWithBracket(searchedConfig.select ?? '') : [], + [isMultiSource, searchedConfig.select], + ); + + // The add-column picker edits the form's draft select (like the SELECT + // editor it replaces), then auto-submits. + const inputSelect = useWatch({ name: 'select', control }); + const multiPickerValue = useMemo( + () => (formIsMulti ? splitAndTrimWithBracket(inputSelect ?? '') : []), + [formIsMulti, inputSelect], + ); + const onMultiColumnsChange = useCallback( + (columns: string[]) => { + setValue('select', columns.join(', ')); + debouncedSubmit(); + }, + [setValue, debouncedSubmit], + ); + + // Sidebar filters apply per source. A source whose table lacks a filtered + // column can't answer the filtered search — it's excluded entirely (with a + // visible reason on its status chip) rather than silently returning rows + // that ignore the filter. + const multiSourceFilters = useMemo( + () => (isMultiSource ? (searchedConfig.filters ?? []) : []), + [isMultiSource, searchedConfig.filters], + ); + const multiDisabledReasons = useMemo(() => { + const reasons = new Map(); + if (!isMultiSource || multiSourceFilters.length === 0) return reasons; + for (const source of searchedMultiSources) { + const missing = unresolvedFilterColumns( + multiSourceFilters, + columnsBySourceId.get(source.id), + ); + if (missing.length > 0) { + reasons.set( + source.id, + `${source.name} is excluded: the active filter uses ${missing.join( + ', ', + )}, which it doesn't have`, + ); + } + } + return reasons; + }, [ + isMultiSource, + multiSourceFilters, + searchedMultiSources, + columnsBySourceId, + ]); + + // The single-source chart config, pinned to the searched time range. + const dbSqlRowTableConfig = useMemo(() => { + if (chartConfig == null) { + return undefined; + } + + return { + ...chartConfig, + dateRange: searchedTimeRange, + }; + }, [chartConfig, searchedTimeRange]); + + // The search's query plan: one spec per selected source. A single source is + // just N=1 — its spec carries the user's own SELECT/ORDER BY, so the results + // table renders exactly what the user asked for. The canonical aliases only + // come into play when there is more than one source to reconcile. + const searchStreamSpecs = useMemo(() => { + if (!isMultiSource) { + if (dbSqlRowTableConfig == null || searchedSource == null) return []; + return [{ source: searchedSource, config: dbSqlRowTableConfig }]; + } + if (isMultiSourceSqlBlocked) return []; + // Extra columns and filters both need each source's DESCRIBE (to resolve + // column-vs-NULL and filter resolvability); hold the row queries until + // they've loaded so we don't fire throwaway or erroring queries. + if ( + (multiExtraColumnNames.length > 0 || multiSourceFilters.length > 0) && + columnsBySourceId.size < searchedMultiSources.length + ) { + return []; + } + const includeDuration = searchedMultiSources.some(isTraceSource); + const where = searchedConfig.where ?? ''; + return searchedMultiSources.map(source => ({ + source, + disabledReason: multiDisabledReasons.get(source.id), + config: { + ...buildMultiSourceSearchConfig( + source, + { + where, + whereLanguage: 'lucene', + filters: multiSourceFilters, + orderBy: multiSourceDefaultOrderBy(source), + }, + { + includeDuration, + extraColumns: resolveExtraColumnsForSource( + multiExtraColumnNames, + columnsBySourceId.get(source.id), + ), + }, + ), + dateRange: searchedTimeRange, + }, + })); + }, [ + isMultiSource, + isMultiSourceSqlBlocked, + searchedMultiSources, + searchedSource, + dbSqlRowTableConfig, + searchedConfig.where, + multiExtraColumnNames, + multiSourceFilters, + multiDisabledReasons, + columnsBySourceId, + searchedTimeRange, + ]); + + // --- End multi-source search --------------------------------------------- + // query error handling const { hasQueryError, queryError } = useMemo(() => { const hasQueryError = Object.values(_queryErrors).length > 0; @@ -1650,24 +1948,16 @@ export function DBSearchPage() { setTimeout(() => setCollapseAllRows(false), 100); }, [interval, updateRelativeTimeInputValue, setIsLive]); - const dbSqlRowTableConfig = useMemo(() => { - if (chartConfig == null) { - return undefined; - } - - return { - ...chartConfig, - dateRange: searchedTimeRange, - }; - }, [chartConfig, searchedTimeRange]); - // Stable key for persisting column widths in localStorage. Scoped per saved - // search when one is loaded, else per source for ad-hoc searches. + // search when one is loaded, else per source (or source set) for ad-hoc + // searches. const columnSizeTableId = savedSearchId ? `db-search-saved-${savedSearchId}` - : searchedConfig.source - ? `db-search-source-${searchedConfig.source}` - : undefined; + : isMultiSource + ? `db-search-multi-${searchedMultiSources.map(s => s.id).join('-')}` + : searchedConfig.source + ? `db-search-source-${searchedConfig.source}` + : undefined; const displayedColumns = useMemo(() => { // `select` is typed as `string | DerivedColumn[]` upstream, but in the @@ -1969,6 +2259,28 @@ export function DBSearchPage() { ], ); + // Multi-source rows span schemas, so the single-source column toggles are + // omitted — the side panel hides them. Property-add-to-filter IS wired: + // filters resolve per source, and a source that lacks the column is + // excluded with a visible reason. Passing no `source` is required: with a + // null context source, deriveRowSidePanelContextForSource treats every row + // as same-source, which is exactly the cross-source semantics filters now + // have. + const multiRowTableContext = useMemo( + () => ({ + onPropertyAddClick: searchFilters.setFilterValue, + generateSearchUrl, + isChildModalOpen: isDrawerChildModalOpen, + setChildModalOpen: setDrawerChildModalOpen, + }), + [ + searchFilters.setFilterValue, + generateSearchUrl, + isDrawerChildModalOpen, + setDrawerChildModalOpen, + ], + ); + const inputSourceTableConnection = useMemo( () => tcFromSource(inputSourceObj), [inputSourceObj], @@ -2220,22 +2532,49 @@ export function DBSearchPage() { > {/* */} - setIsSourceSchemaPreviewOpen(true)} - isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( - inputSourceObj, - )} - allowedSourceKinds={ALLOWED_SOURCE_KINDS} - data-testid="source-selector" - style={{ minWidth: 150 }} - /> + {isMultiSelectUI ? ( + + ) : ( + <> + setIsSourceSchemaPreviewOpen(true)} + isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( + inputSourceObj, + )} + allowedSourceKinds={ALLOWED_SOURCE_KINDS} + data-testid="source-selector" + style={{ minWidth: 150 }} + /> + + + + + + + )} setIsSourceSchemaPreviewOpen(false)} /> - - - - + {formIsMulti ? ( + + ) : ( + + )} + {!formIsMulti && ( + + + + )} <> {!savedSearchId ? ( - + + ) : ( + + )} @@ -2346,7 +2710,7 @@ export function DBSearchPage() { setInputValue={setDisplayedTimeInputValue} onSearch={onTimePickerSearch} onRelativeSearch={onTimePickerRelativeSearch} - showLive={analysisMode === 'results'} + showLive={effectiveAnalysisMode === 'results'} isLiveMode={isLive} // Default to relative time mode if the user has made changes to interval and reloaded. defaultRelativeTimeMode={ @@ -2381,7 +2745,7 @@ export function DBSearchPage() { @@ -2425,7 +2789,7 @@ export function DBSearchPage() { height: '100%', }} > - {!isFilterSidebarCollapsed && ( + {!isFilterSidebarCollapsed && !isMultiSource && ( )} - {analysisMode === 'pattern' && + {effectiveAnalysisMode === 'pattern' && histogramTimeChartConfig != null && ( @@ -2523,7 +2887,7 @@ export function DBSearchPage() { )} - {analysisMode === 'delta' && + {effectiveAnalysisMode === 'delta' && searchedSource != null && isTraceSource(searchedSource) && ( )} - {analysisMode === 'results' && ( + {effectiveAnalysisMode === 'results' && isMultiSource && ( + + {isMultiSourceSqlBlocked ? ( + + + SQL search isn't supported across multiple sources + + + A SQL WHERE clause references the columns of one + specific table. Switch the search language to Lucene to + search across sources, or go back to a single source. + + + ) : ( + <> + {/* The histogram, total count, and filters sidebar come + with the next change; searching several sources + returns the merged results table on its own. */} + + + + + )} + + )} + {effectiveAnalysisMode === 'results' && !isMultiSource && ( {chartConfig && histogramTimeChartConfig && ( <> @@ -2725,32 +3134,26 @@ export function DBSearchPage() { px="sm" data-testid="search-results-panel" > - {chartConfig && - searchedConfig.source && - dbSqlRowTableConfig && ( - - )} + )} diff --git a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx index d3f10d02a5..330a024fbd 100644 --- a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx +++ b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx @@ -211,6 +211,9 @@ jest.mock('../components/ChartSQLPreview', () => ({ SQLPreview: () =>
, })); jest.mock('../components/DBSqlRowTableWithSidebar', () => () =>
); +// Multi-source components pull in DBRowSidePanel (and its deep import graph), +// which this test isolates away just like DBSqlRowTableWithSidebar above. +jest.mock('../components/SearchResultsTable', () => () =>
); jest.mock('../components/PatternTable', () => () =>
); jest.mock('../components/Search/DBSearchHeatmapChart', () => ({ DBSearchHeatmapChart: () =>
, diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 21834391a0..59bdf5398d 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1419,7 +1419,7 @@ export function appendSelectWithAdditionalKeys( } } -function getSelectLength(select: SelectList): number { +export function getSelectLength(select: SelectList): number { if (typeof select === 'string') { return select.split(',').filter(s => s.trim().length > 0).length; } else { @@ -1512,7 +1512,7 @@ export function useConfigWithAdditionalSelect( * (e.g. `SELECT *`). Exported for SearchResultsTable, which resolves the same * columns when rendering a single source's own SELECT. */ -function selectColumnMapWithoutAdditionalKeys( +export function selectColumnMapWithoutAdditionalKeys( selectMeta: ColumnMetaType[] | undefined, additionalKeysLength: number | undefined, ): Map< @@ -1547,7 +1547,7 @@ export type DBRowTableVariant = 'default' | 'muted'; * implementation. Denoising is inherently single-source: it mines patterns * from one table's body column against that source's severity expression. */ -function useDenoisedRows({ +export function useDenoisedRows({ config, sourceId, processedRows, @@ -1653,7 +1653,7 @@ function useDenoisedRows({ } /** The "Removed Noisy Event Patterns" summary shown above denoised results. */ -function DenoisedPatternsSummary({ +export function DenoisedPatternsSummary({ noisyPatterns, hasNoisyPatterns, }: { diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index 8c4b924bb8..a9d676102a 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -254,6 +254,7 @@ export function useFetchFacets({ filterState, showMoreFields, disableValues, + enabled = true, }: { chartConfig: BuilderChartConfigWithDateRange; sourceId: string | null; @@ -262,6 +263,8 @@ export function useFetchFacets({ filterState?: FilterState; showMoreFields?: boolean; disableValues?: boolean; + /** Disable all data fetching (e.g. an unused multi-source hook slot). */ + enabled?: boolean; }) { const facetsQuery = useFacets({ chartConfig, @@ -270,7 +273,7 @@ export function useFetchFacets({ dateRange, filterState, showMoreFields, - enabled: true, + enabled, disableValues, }); diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index bd5824e372..b500296ede 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -157,7 +157,9 @@ enum InlineTab { ColumnValues = 'columnValues', } -function RowOverviewPanelWrapper({ +// Exported for MultiSourceRowTable, which renders the same expanded-row +// overview but resolves the source per row instead of once per table. +export function RowOverviewPanelWrapper({ source, rowId, aliasWith, diff --git a/packages/app/src/components/MultiSourceBadge.tsx b/packages/app/src/components/MultiSourceBadge.tsx index b536e9219b..7d9d75798f 100644 --- a/packages/app/src/components/MultiSourceBadge.tsx +++ b/packages/app/src/components/MultiSourceBadge.tsx @@ -1,4 +1,15 @@ -import React from 'react'; +import { COLORS } from '@/utils'; + +/** + * Stable color for the Nth selected source in a multi-source search. Indexed + * by position in the selection (not hashed) so the ≤MAX_SEARCH_SOURCES badges + * never collide; the same assignment is used by the results table badge, the + * histogram series, and the per-source status chips so a source reads as one + * color everywhere on the page. + */ +export function getMultiSourceColor(index: number): string { + return COLORS[index % COLORS.length]; +} /** Colored-dot source label used in the merged results table. */ export function SourceBadge({ name, color }: { name: string; color?: string }) { diff --git a/packages/app/src/components/MultiSourceColumnPicker.tsx b/packages/app/src/components/MultiSourceColumnPicker.tsx new file mode 100644 index 0000000000..55230dd479 --- /dev/null +++ b/packages/app/src/components/MultiSourceColumnPicker.tsx @@ -0,0 +1,71 @@ +import { useCallback, useMemo } from 'react'; +import { Group, MultiSelect, Text } from '@mantine/core'; + +import { MultiSourceColumnOption } from '@/hooks/useMultiSourceSearch'; + +/** + * Multi-source replacement for the free-text SELECT editor: pick extra + * columns from the union of the selected sources' top-level columns. Columns + * missing from a source render as blank cells for that source's rows. + */ +export default function MultiSourceColumnPicker({ + unionColumns, + totalSources, + value, + onChange, +}: { + unionColumns: MultiSourceColumnOption[]; + totalSources: number; + /** Currently selected extra column names. */ + value: string[]; + onChange: (columns: string[]) => void; +}) { + const availabilityByName = useMemo( + () => new Map(unionColumns.map(c => [c.name, c.availableCount])), + [unionColumns], + ); + + const data = useMemo( + () => + unionColumns.map(c => ({ + value: c.name, + label: c.name, + })), + [unionColumns], + ); + + const renderOption = useCallback( + ({ option }: { option: { value: string; label: string } }) => { + const available = availabilityByName.get(option.value) ?? 0; + return ( + + + {option.label} + + {available < totalSources && ( + + {available}/{totalSources} sources + + )} + + ); + }, + [availabilityByName, totalSources], + ); + + return ( + + ); +} diff --git a/packages/app/src/components/SearchResultsTable.tsx b/packages/app/src/components/SearchResultsTable.tsx new file mode 100644 index 0000000000..4129828b92 --- /dev/null +++ b/packages/app/src/components/SearchResultsTable.tsx @@ -0,0 +1,618 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useQueryState } from 'nuqs'; +import { + chSqlToAliasMap, + ClickHouseQueryError, + ColumnMetaType, + convertCHDataTypeToJSType, + isJSDataTypeJSONStringifiable, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { MULTI_SOURCE_ALIASES } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + BuilderChartConfigWithDateRange, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Flex, Group, Loader, Text, Tooltip } from '@mantine/core'; +import { IconAlertTriangle, IconFilterOff } from '@tabler/icons-react'; +import { SortingState } from '@tanstack/react-table'; + +import api from '@/api'; +import { searchChartConfigDefaults } from '@/defaults'; +import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; +import useRowWhere, { RowWhereResult, WithClause } from '@/hooks/useRowWhere'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; +import { + mergeStreams, + MULTI_SOURCE_ROW_FIELDS, + StreamSnapshot, +} from '@/utils/multiSourceMerge'; +import { parseAsStringEncoded } from '@/utils/queryParsers'; + +import ChartErrorState from './charts/ChartErrorState'; +import DBRowSidePanel, { + RowSidePanelContext, + RowSidePanelContextProps, +} from './DBRowSidePanel'; +import { + DenoisedPatternsSummary, + getSelectLength, + RawLogTable, + selectColumnMapWithoutAdditionalKeys, + useConfigWithAdditionalSelect, + useDenoisedRows, +} from './DBRowTable'; +import { RowOverviewPanelWrapper } from './DBSqlRowTableWithSidebar'; +import { getMultiSourceColor, SourceBadge } from './MultiSourceBadge'; + +/** + * One selected source plus its fully-built chart config. + * + * With a single source the config carries that source's own SELECT (the user + * authored it); with several, each config projects the canonical + * MULTI_SOURCE_ALIASES so the merged rows share one shape. + */ +export type SearchStreamSpec = { + source: TSource; + config: BuilderChartConfigWithDateRange; + /** + * When set, the source doesn't run at all (e.g. an active filter references + * a column its table lacks); shown on the source's status chip. + */ + disabledReason?: string; +}; + +// Placeholder config for unused hook slots. The metadata hooks inside +// useConfigWithAdditionalSelect self-disable on empty table names, and the +// paginated query slot is explicitly disabled, so this never reaches +// ClickHouse. +const STUB_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +const EMPTY_CHSQL = { sql: '', params: {} }; +const EMPTY_EXTRA_COLUMNS: string[] = []; + +type SourceStream = { + spec: SearchStreamSpec | undefined; + data: ReturnType['data']; + fetchNextPage: ReturnType['fetchNextPage']; + hasNextPage: boolean; + isFetching: boolean; + isError: boolean; + error: Error | ClickHouseQueryError | null; + getRowWhere: (row: Record) => RowWhereResult; + /** Row-identity columns appended to the SELECT, trimmed off for display. */ + additionalKeysLength: number | undefined; +}; + +/** + * One source's independent query pipeline: the same + * defaults → additional-key SELECT merge → windowed offset pagination → + * row-WHERE machinery as the single-source DBSqlRowTable, packaged as a + * useMultiSourceSlots slot hook. Unused slots get a stub config and stay + * disabled. + */ +function useSourceStream( + spec: SearchStreamSpec | undefined, + { + enabled, + isLive, + enableSmallFirstWindow, + queryKeyPrefix, + }: { + enabled: boolean; + isLive: boolean; + enableSmallFirstWindow?: boolean; + queryKeyPrefix?: string; + }, +): SourceStream { + const { data: me } = api.useMe(); + + const configWithDefaults = useMemo( + () => ({ + ...searchChartConfigDefaults(me?.team), + ...(spec?.config ?? STUB_CONFIG), + }), + [me, spec?.config], + ); + + const mergedConfig = useConfigWithAdditionalSelect( + configWithDefaults, + spec?.source.id, + ); + + const { data, fetchNextPage, hasNextPage, isFetching, isError, error } = + useOffsetPaginatedQuery(mergedConfig ?? configWithDefaults, { + enabled: + enabled && + spec != null && + spec.disabledReason == null && + mergedConfig != null && + // An empty SELECT renders invalid SQL; wait for one to resolve. + getSelectLength(spec.config.select) > 0, + isLive, + queryKeyPrefix, + enableSmallFirstWindow, + }); + + const aliasMap = useMemo(() => { + const map = chSqlToAliasMap(data?.chSql ?? EMPTY_CHSQL); + // NULL-literal projections (`NULL AS "__hdx_duration_ms"` where a source + // lacks the field) are dropped by the SQL alias parser. Backfill them so + // the row-WHERE clause emits `isNull(NULL)` rather than referencing the + // alias as a (nonexistent) table column. ClickHouse reports NULL literals + // as Nullable(Nothing). + for (const col of data?.meta ?? []) { + if (map[col.name] == null && col.type === 'Nullable(Nothing)') { + map[col.name] = 'NULL'; + } + } + return map; + }, [data]); + + const getRowWhere = useRowWhere({ + meta: data?.meta, + aliasMap, + primaryKeyColumns: mergedConfig?.rowKeyColumns, + }); + + // Stable identity per content change, so downstream merge memos don't + // recompute (and re-sort every fetched row) on unrelated parent renders. + return useMemo( + () => ({ + spec, + data, + fetchNextPage, + hasNextPage: hasNextPage ?? false, + isFetching, + isError, + error: error ?? null, + getRowWhere, + additionalKeysLength: mergedConfig?.additionalKeysLength, + }), + [ + spec, + data, + fetchNextPage, + hasNextPage, + isFetching, + isError, + error, + getRowWhere, + mergedConfig?.additionalKeysLength, + ], + ); +} + +const COLUMN_NAME_MAP: Record = { + [MULTI_SOURCE_ALIASES.timestamp]: 'Timestamp', + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: 'Source', + [MULTI_SOURCE_ALIASES.service]: 'Service', + [MULTI_SOURCE_ALIASES.severity]: 'Level', + [MULTI_SOURCE_ALIASES.durationMs]: 'Duration (ms)', + [MULTI_SOURCE_ALIASES.body]: 'Message', +}; + +function StreamStatusChips({ streams }: { streams: SourceStream[] }) { + return ( + + {streams.map((stream, i) => { + if (stream.spec == null) return null; + const name = stream.spec.source.name; + const disabledReason = stream.spec.disabledReason; + return ( + + + {stream.isFetching && } + {disabledReason != null && ( + + + + + + )} + {stream.isError && ( + + + + + + )} + + ); + })} + + ); +} + +export default function SearchResultsTable({ + sources: specs, + isLive, + enabled = true, + extraColumnNames = EMPTY_EXTRA_COLUMNS, + denoiseResults = false, + sortOrder, + onSortingChange, + onError, + onResolvedColumnsChange, + onScroll, + onSidebarOpen, + onExpandedRowsChange, + collapseAllRows, + enableSmallFirstWindow, + tableId, + context, + keepOpenSelector, + // Row queries are keyed separately from the page's chart/count queries, so + // "is the search fetching?" (live-tail pause, latency telemetry) keeps + // measuring the same thing it always has. + queryKeyPrefix = 'dbSqlRowTable', +}: { + /** 1..MAX_SEARCH_SOURCES selected sources with their built configs. */ + sources: SearchStreamSpec[]; + isLive: boolean; + enabled?: boolean; + /** User-picked extra columns projected into every source's SELECT (N>1). */ + extraColumnNames?: string[]; + /** Drop noisy event patterns from the results (single source only). */ + denoiseResults?: boolean; + /** Current sort, for the single-source case where sorting is supported. */ + sortOrder?: SortingState; + onSortingChange?: (v: SortingState | null) => void; + /** + * Surface a query failure to the page. Only called with a single source — + * with several, a failing source is isolated to its own status chip rather + * than failing the whole search. + */ + onError?: (error: Error | ClickHouseQueryError) => void; + onResolvedColumnsChange?: (meta: ColumnMetaType[]) => void; + onScroll?: (scrollTop: number) => void; + onSidebarOpen?: (rowId: string) => void; + onExpandedRowsChange?: (hasExpandedRows: boolean) => void; + collapseAllRows?: boolean; + enableSmallFirstWindow?: boolean; + tableId?: string; + context?: RowSidePanelContextProps; + keepOpenSelector?: string; + queryKeyPrefix?: string; +}) { + const slots = useMultiSourceSlots(specs, useSourceStream, { + enabled, + isLive, + enableSmallFirstWindow, + queryKeyPrefix, + }); + + const streams = useMemo( + () => + slots.filter( + (s): s is SourceStream & { spec: SearchStreamSpec } => s.spec != null, + ), + [slots], + ); + + // With one source the table shows that source's own SELECT, sorts, and + // denoises — everything the single-source search has always done. The + // canonical aliases, source badges, and cross-source merge only come into + // play once a second source is selected. + const isSingleSource = specs.length === 1; + const singleStream = isSingleSource ? streams[0] : undefined; + + const snapshots: StreamSnapshot[] = useMemo( + () => + streams.map((stream, i) => ({ + sourceId: stream.spec.source.id, + sourceName: stream.spec.source.name, + sourceColor: getMultiSourceColor(i), + rows: stream.data?.data ?? [], + window: stream.data?.window ?? null, + lastPageRowCount: stream.data?.lastPageRowCount ?? null, + hasNextPage: stream.hasNextPage, + isActive: !stream.isError && stream.spec.disabledReason == null, + dateRange: stream.spec.config.dateRange, + })), + [streams], + ); + + // One source needs no merge: its rows already arrive timestamp-ordered from + // its own ORDER BY, and there is no other stream to hold a frontier against. + const merged = useMemo( + () => + isSingleSource + ? null + : mergeStreams(snapshots, 'DESC', MULTI_SOURCE_ALIASES.timestamp), + [isSingleSource, snapshots], + ); + + const columnTypeMap = useMemo(() => { + if (singleStream != null) { + // The user's SELECT columns, positionally trimmed of the row-identity + // columns the query appends (same resolution as DBSqlRowTable). + return selectColumnMapWithoutAdditionalKeys( + singleStream.data?.meta, + singleStream.additionalKeysLength, + ); + } + // Merge column meta across streams by canonical alias name, preferring a + // resolved type over the Nullable(Nothing) a `NULL AS "alias"` projection + // reports. + const map = new Map(); + for (const stream of streams) { + for (const col of stream.data?.meta ?? []) { + const jsType = convertCHDataTypeToJSType(col.type); + const existing = map.get(col.name); + if (existing == null || existing._type == null) { + map.set(col.name, { _type: jsType }); + } + } + } + map.set(MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME, { + _type: JSDataType.String, + }); + return map; + }, [streams, singleStream]); + + const includeDuration = specs.some(s => s.source.kind === SourceKind.Trace); + + const displayedColumns = useMemo(() => { + if (isSingleSource) { + return Array.from(columnTypeMap.keys()); + } + return [ + MULTI_SOURCE_ALIASES.timestamp, + MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME, + MULTI_SOURCE_ALIASES.service, + MULTI_SOURCE_ALIASES.severity, + ...(includeDuration ? [MULTI_SOURCE_ALIASES.durationMs] : []), + ...extraColumnNames, + MULTI_SOURCE_ALIASES.body, + ]; + }, [isSingleSource, columnTypeMap, includeDuration, extraColumnNames]); + + // Stringify object-typed cells (Map/Array/JSON) the same way DBSqlRowTable + // does — both for display and because useRowWhere expects the stringified + // form when rebuilding a row WHERE clause. + const rows = useMemo(() => { + const baseRows = singleStream + ? (singleStream.data?.data ?? []) + : (merged?.rows ?? []); + const objectColumns = [...columnTypeMap.entries()] + .filter(([, v]) => isJSDataTypeJSONStringifiable(v._type)) + .map(([name]) => name); + if (objectColumns.length === 0) { + return baseRows; + } + return baseRows.map(row => { + const newRow = { ...row }; + for (const col of objectColumns) { + if (!(col in newRow) || newRow[col] == null) continue; + if (columnTypeMap.get(col)?._type === JSDataType.JSON) { + newRow[col] = JSON.stringify(newRow[col]).replace(/\//g, '\\/'); + } else { + newRow[col] = JSON.stringify(newRow[col]); + } + } + return newRow; + }); + }, [singleStream, merged?.rows, columnTypeMap]); + + const patternColumn = displayedColumns[displayedColumns.length - 1]; + const denoise = useDenoisedRows({ + config: singleStream?.spec.config ?? STUB_CONFIG, + sourceId: singleStream?.spec.source.id, + processedRows: rows, + patternColumn, + // Denoising mines patterns from one table's body column; it has no + // cross-source meaning, so it only runs with a single source. + denoiseResults: denoiseResults && isSingleSource, + isLive, + }); + + // Row identity dispatches to the row's own stream: each stream has its own + // result meta / alias map / primary-key columns. The client-side source tags + // are stripped first — they aren't real columns. + const generateRowId = useCallback( + (row: Record): RowWhereResult => { + if (singleStream != null) { + return singleStream.getRowWhere(row); + } + const { + [MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]: sourceId, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: _name, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_COLOR]: _color, + ...dbRow + } = row; + const stream = streams.find(s => s.spec.source.id === sourceId); + if (stream == null) { + return { where: '', aliasWith: [] }; + } + return stream.getRowWhere(dbRow); + }, + [streams, singleStream], + ); + + // Advance only the stream(s) holding the frontier back; the leaders keep + // their fetched-but-held rows until the laggards catch up. + const fetchNextPage = useCallback(() => { + if (singleStream != null) { + singleStream.fetchNextPage({ cancelRefetch: false }); + return; + } + for (const sourceId of merged?.laggingSourceIds ?? []) { + const stream = streams.find(s => s.spec.source.id === sourceId); + stream?.fetchNextPage({ cancelRefetch: false }); + } + }, [singleStream, merged?.laggingSourceIds, streams]); + + const hasNextPage = streams.some(s => !s.isError && s.hasNextPage); + const isFetching = streams.some(s => s.isFetching); + const isLoading = denoiseResults + ? isFetching || denoise.isFetching + : isFetching; + const allFailed = streams.length > 0 && streams.every(s => s.isError); + const firstError = streams.find(s => s.error != null)?.error ?? undefined; + + // A single source's failure is the whole search's failure, so the page owns + // the error UI (and drops out of live tail), exactly as before. + useEffect(() => { + if (singleStream?.isError && singleStream.error != null) { + onError?.(singleStream.error); + } + }, [singleStream?.isError, singleStream?.error, onError]); + + const singleMeta = singleStream?.data?.meta; + useEffect(() => { + if (singleMeta != null && singleMeta.length > 0) { + onResolvedColumnsChange?.(singleMeta); + } + }, [singleMeta, onResolvedColumnsChange]); + + // Side panel wiring — the same URL-param contract as the legacy table, + // except the panel's source comes from the clicked row rather than being + // fixed for the page. + const [rowId, setRowId] = useQueryState('rowWhere', parseAsStringEncoded); + const [rowSource, setRowSource] = useQueryState('rowSource'); + const [aliasWith, setAliasWith] = useState([]); + + const onRowDetailsClick = useCallback( + (row: Record) => { + const rowWhere = generateRowId(row); + if (!rowWhere.where) return; + setRowId(rowWhere.where); + setAliasWith(rowWhere.aliasWith); + setRowSource( + row[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID] ?? + singleStream?.spec.source.id ?? + null, + ); + onSidebarOpen?.(rowWhere.where); + }, + [generateRowId, setRowId, setRowSource, onSidebarOpen, singleStream], + ); + + const onCloseSidebar = useCallback(() => { + setRowId(null); + setRowSource(null); + }, [setRowId, setRowSource]); + + const sourceForRow = useCallback( + (id: unknown) => + specs.find(s => s.source.id === id)?.source ?? + // Links predating the rowSource param (and every single-source link) + // carry only rowWhere; there is exactly one source it can belong to. + (isSingleSource ? specs[0]?.source : undefined), + [specs, isSingleSource], + ); + + const panelSource = useMemo( + () => sourceForRow(rowSource), + [sourceForRow, rowSource], + ); + + const renderRowDetails = useCallback( + (r: { id: string; aliasWith?: WithClause[]; [key: string]: unknown }) => { + const source = sourceForRow(r[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]); + if (!source) { + return
Loading...
; + } + return ( + + ); + }, + [sourceForRow], + ); + + const loadingDate = singleStream + ? singleStream.data?.window?.direction === 'ASC' + ? singleStream.data?.window?.endTime + : singleStream.data?.window?.startTime + : merged?.frontier != null && hasNextPage + ? new Date(merged.frontier) + : undefined; + + const firstConfig = streams[0]?.spec.config; + + return ( + + {panelSource != null && ( + + )} + + {/* One source needs no legend: every row came from it. */} + {!isSingleSource && } + {denoiseResults && isSingleSource && ( + + )} + {allFailed && !isSingleSource ? ( + + ) : ( + + )} + + + ); +} diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 5894d8b78d..5b67f68216 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -2,6 +2,14 @@ import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist // Limit defaults export const DEFAULT_SEARCH_ROW_LIMIT = 200; + +// Ceiling on how many sources one search can span. Cost scales linearly with +// the selection: each source runs its own result stream plus histogram/count +// aggregates (~3 ClickHouse queries per source per refresh, re-fired every +// live-tail tick), so 3 keeps the worst case bounded while covering the +// common "app logs + infra logs + traces" setups. Also the hook-slot count in +// useMultiSourceSlots — raising it means adding a slot there too. +export const MAX_SEARCH_SOURCES = 3; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100; export const DEFAULT_SERIES_LIMIT = 100; diff --git a/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts new file mode 100644 index 0000000000..58be8d33d2 --- /dev/null +++ b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts @@ -0,0 +1,87 @@ +import { Filter } from '@hyperdx/common-utils/dist/types'; + +import { + filterRootColumn, + resolveExtraColumnsForSource, + unresolvedFilterColumns, +} from '@/hooks/useMultiSourceSearch'; + +describe('filterRootColumn', () => { + it('extracts a plain column reference', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: 'ServiceName', + right: "'cart'", + }; + expect(filterRootColumn(filter)).toBe('ServiceName'); + }); + + it('extracts the root of a map subscript', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: "LogAttributes['level']", + right: "'error'", + }; + expect(filterRootColumn(filter)).toBe('LogAttributes'); + }); + + it('extracts a backticked identifier', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: '`weird-col`', + right: "'x'", + }; + expect(filterRootColumn(filter)).toBe('weird-col'); + }); + + it('returns null for raw sql and lucene filters', () => { + expect( + filterRootColumn({ type: 'sql', condition: "Foo = 'bar'" }), + ).toBeNull(); + expect( + filterRootColumn({ type: 'lucene', condition: 'foo:bar' }), + ).toBeNull(); + }); +}); + +describe('unresolvedFilterColumns', () => { + const filters: Filter[] = [ + { type: 'sql_ast', operator: '=', left: 'ServiceName', right: "'cart'" }, + { type: 'sql_ast', operator: '=', left: 'StatusCode', right: "'Unset'" }, + { type: 'sql', condition: 'anything' }, + ]; + + it('reports columns the source lacks', () => { + expect( + unresolvedFilterColumns(filters, new Set(['ServiceName', 'Body'])), + ).toEqual(['StatusCode']); + }); + + it('is empty when every attributable column resolves', () => { + expect( + unresolvedFilterColumns(filters, new Set(['ServiceName', 'StatusCode'])), + ).toEqual([]); + }); + + it('is empty (not excluding) while columns are still unknown', () => { + expect(unresolvedFilterColumns(filters, undefined)).toEqual([]); + }); +}); + +describe('resolveExtraColumnsForSource', () => { + it('projects the column where present and NULL where missing', () => { + expect( + resolveExtraColumnsForSource( + ['ServiceName', 'StatusCode', 'weird col'], + new Set(['ServiceName', 'weird col']), + ), + ).toEqual([ + { name: 'ServiceName', expression: 'ServiceName' }, + { name: 'StatusCode', expression: null }, + { name: 'weird col', expression: '`weird col`' }, + ]); + }); +}); diff --git a/packages/app/src/hooks/useMultiSourceSearch.ts b/packages/app/src/hooks/useMultiSourceSearch.ts new file mode 100644 index 0000000000..f4ecdb4da4 --- /dev/null +++ b/packages/app/src/hooks/useMultiSourceSearch.ts @@ -0,0 +1,148 @@ +import { useMemo } from 'react'; +import { + ColumnMeta, + filterColumnMetaByType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { MultiSourceExtraColumn } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { Filter, TSource } from '@hyperdx/common-utils/dist/types'; + +import { useColumns } from '@/hooks/useMetadata'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; + +const EMPTY_SOURCE_PARAMS = { + databaseName: '', + tableName: '', + connectionId: '', +}; + +function columnsParamsFor(source: TSource | undefined) { + if (source == null) return EMPTY_SOURCE_PARAMS; + return { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + connectionId: source.connection, + }; +} + +export type MultiSourceColumnOption = { + name: string; + /** How many of the selected sources have this column. */ + availableCount: number; +}; + +/** Slot hook: DESCRIBE columns for one source. Stable — `.data` is cached. */ +function useSourceColumnsSlot( + source: TSource | undefined, +): ColumnMeta[] | undefined { + return useColumns(columnsParamsFor(source)).data; +} + +/** + * Top-level columns (DESCRIBE) for each selected source of a multi-source + * search, plus the deduped union with per-column availability counts for the + * add-column picker. useColumns self-disables for unused slots. + */ +export function useMultiSourceColumns(sources: TSource[]): { + columnsBySourceId: Map>; + unionColumns: MultiSourceColumnOption[]; + /** Union of Date/DateTime column name → ClickHouse type across sources. */ + dateTimeColumns: Map; +} { + const slotData = useMultiSourceSlots( + sources, + useSourceColumnsSlot, + undefined, + ); + + return useMemo(() => { + const columnsBySourceId = new Map>(); + const availability = new Map(); + const dateTimeColumns = new Map(); + + for (let i = 0; i < sources.length; i++) { + const source = sources[i]; + const columns = slotData[i]; + if (source == null || columns == null) continue; + const names = new Set(columns.map(c => c.name)); + columnsBySourceId.set(source.id, names); + for (const name of names) { + availability.set(name, (availability.get(name) ?? 0) + 1); + } + for (const col of filterColumnMetaByType(columns, [JSDataType.Date]) ?? + []) { + if (!dateTimeColumns.has(col.name)) { + dateTimeColumns.set(col.name, col.type); + } + } + } + + const unionColumns = [...availability.entries()] + .map(([name, availableCount]) => ({ name, availableCount })) + .sort( + (a, b) => + b.availableCount - a.availableCount || a.name.localeCompare(b.name), + ); + + return { columnsBySourceId, unionColumns, dateTimeColumns }; + }, [slotData, sources]); +} + +/** + * Root column a filter references, for per-source resolvability checks. + * sql_ast filters carry the escaped SQL key in `left` (e.g. `ServiceName`, + * a backticked identifier, or `LogAttributes['level']` whose root is + * `LogAttributes`). Other filter types (raw sql/lucene conditions) can't be + * attributed to a single column and return null — callers should apply them + * to every source and rely on per-source error isolation. + */ +export function filterRootColumn(filter: Filter): string | null { + if (filter.type !== 'sql_ast') return null; + const left = filter.left.trim(); + const backticked = left.match(/^`([^`]+)`/); + if (backticked) return backticked[1]; + const plain = left.match(/^[A-Za-z_][A-Za-z0-9_]*/); + return plain ? plain[0] : null; +} + +/** + * For one source: which of the active filters reference a column its table + * doesn't have. A non-empty result means the source can't answer the + * filtered search and should be excluded (with a visible reason). + */ +export function unresolvedFilterColumns( + filters: Filter[], + sourceColumns: Set | undefined, +): string[] { + if (sourceColumns == null) return []; + const missing = new Set(); + for (const filter of filters) { + const root = filterRootColumn(filter); + if (root != null && !sourceColumns.has(root)) { + missing.add(root); + } + } + return [...missing]; +} + +/** Quote a column name as a ClickHouse identifier when it needs it. */ +function quoteIdentifier(name: string): string { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) + ? name + : `\`${name.replace(/`/g, '\\`')}\``; +} + +/** + * Resolve the user-picked extra column names into per-source SELECT + * expressions: the (quoted) column itself where the source's table has it, + * NULL otherwise — so every source still returns the same result shape. + */ +export function resolveExtraColumnsForSource( + extraColumnNames: string[], + sourceColumns: Set | undefined, +): MultiSourceExtraColumn[] { + return extraColumnNames.map(name => ({ + name, + expression: sourceColumns?.has(name) ? quoteIdentifier(name) : null, + })); +} diff --git a/packages/app/src/hooks/useResolvedSourcesParam.ts b/packages/app/src/hooks/useResolvedSourcesParam.ts new file mode 100644 index 0000000000..deb2f449d1 --- /dev/null +++ b/packages/app/src/hooks/useResolvedSourcesParam.ts @@ -0,0 +1,58 @@ +import { useEffect, useMemo } from 'react'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { notifications } from '@mantine/notifications'; + +import { MAX_SEARCH_SOURCES } from '@/defaults'; +import { useSources } from '@/source'; +import { resolveSourcesParam } from '@/utils/sourceParams'; + +const EMPTY_SOURCES: TSource[] = []; + +/** + * Resolves the multi-source search param (a list of source IDs or names) to + * the matching sources, deduped and capped at MAX_SEARCH_SOURCES. + * + * Elements that don't match any usable source are dropped from the selection + * and reported once via a Mantine warning, mirroring useResolvedSourceParam. + */ +export function useResolvedSourcesParam( + paramValues: string[] | null | undefined, + { kinds }: { kinds?: SourceKind[] } = {}, +): { sources: TSource[] } { + const { data: allSources } = useSources(); + + // Key the memo on a serialized `kinds` so callers can pass inline arrays + // without breaking memoization. + const kindsKey = kinds?.join(','); + const { sources, unresolvedKey } = useMemo(() => { + const allKinds = new Set(Object.values(SourceKind)); + const resolvedKinds = kindsKey + ? kindsKey.split(',').filter((k): k is SourceKind => allKinds.has(k)) + : undefined; + const resolution = resolveSourcesParam(paramValues, allSources, { + kinds: resolvedKinds, + max: MAX_SEARCH_SOURCES, + }); + if (resolution.status !== 'resolved') { + return { sources: EMPTY_SOURCES, unresolvedKey: undefined }; + } + return { + sources: resolution.sources.length ? resolution.sources : EMPTY_SOURCES, + unresolvedKey: resolution.unresolved.length + ? resolution.unresolved.join(', ') + : undefined, + }; + }, [paramValues, allSources, kindsKey]); + + useEffect(() => { + if (unresolvedKey == null) return; + notifications.show({ + id: 'sources-param-unresolved-' + unresolvedKey, + color: 'yellow', + title: 'Some sources were not found', + message: `No searchable source matches: ${unresolvedKey}. They may have been renamed or deleted.`, + }); + }, [unresolvedKey]); + + return useMemo(() => ({ sources }), [sources]); +} diff --git a/packages/app/src/hooks/useSourceSlots.ts b/packages/app/src/hooks/useSourceSlots.ts new file mode 100644 index 0000000000..04286e5832 --- /dev/null +++ b/packages/app/src/hooks/useSourceSlots.ts @@ -0,0 +1,32 @@ +import { useMemo } from 'react'; + +import { MAX_SEARCH_SOURCES } from '@/defaults'; + +/** + * Run one instance of a hook per selected source of a search. + * + * The rules of hooks require a constant hook count per component, but search + * needs one pipeline per selected source — and `useQueries` can't cover these + * pipelines (row streams are `useInfiniteQuery`-based, which has no plural + * form, and the chart/facet pipelines compose other hooks). So the hook count + * is pinned at MAX_SEARCH_SOURCES here, in one place: unused slots receive + * `undefined` and every slot hook is expected to self-disable for it. + * + * `useSlot` must be a stable, named hook (the rules-of-hooks lint understands + * `use*`-named parameters) and should return a memoized value, so the array + * this returns is referentially stable and safe to use in dependency lists. + * + * Lives in its own module so both the search hooks and the metadata hooks can + * use it without an import cycle. + */ +export function useMultiSourceSlots( + items: readonly Item[], + useSlot: (item: Item | undefined, opts: Opts) => Result, + opts: Opts, +): Result[] { + const s0 = useSlot(items[0], opts); + const s1 = useSlot(items[1], opts); + const s2 = useSlot(items[2], opts); + const count = Math.min(items.length, MAX_SEARCH_SOURCES); + return useMemo(() => [s0, s1, s2].slice(0, count), [s0, s1, s2, count]); +}