diff --git a/.changeset/source-vars-3.md b/.changeset/source-vars-3.md new file mode 100644 index 0000000000..743757dbd6 --- /dev/null +++ b/.changeset/source-vars-3.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': minor +--- + +feat: add data source variables to dashboard filters and tiles diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index f5000d1a75..ce09acd36d 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -28,11 +28,9 @@ import { } from '@hyperdx/common-utils/dist/core/metadata'; import { convertToDashboardTemplate, - displayTypeSupportsBuilderAlerts, - displayTypeSupportsPromQLAlerts, - displayTypeSupportsRawSqlAlerts, Granularity, isTimeSeriesDisplayType, + savedChartConfigSupportsAlerts, } from '@hyperdx/common-utils/dist/core/utils'; import { getAdhocFilterTargetsForTile, @@ -56,6 +54,11 @@ import { isImportableDashboard, } from '@hyperdx/common-utils/dist/iac'; import { isMissingFiltersMacro } from '@hyperdx/common-utils/dist/macros'; +import { + getConfigReferenceVariables, + resolveConfigReferences, + restoreConfigReferences, +} from '@hyperdx/common-utils/dist/referenceVariables'; import { AlertState, BroadcastFilter, @@ -455,6 +458,7 @@ const Tile = ({ filters, variables, unsatisfiedRequiredFilters, + referenceVariableNames, showAlertAnnotations, showReleaseAnnotations, isLive, @@ -489,6 +493,12 @@ const Tile = ({ variables?: ChartVariable[]; /** The dashboard's required filters that have nothing selected. */ unsatisfiedRequiredFilters?: DashboardFilter[]; + /** + * The data source and connection variables the tile's stored config + * references. `chart` carries their selections, or the references while + * unselected. + */ + referenceVariableNames?: string[]; // When true, draw alert firing/recovery annotations on this tile's chart. showAlertAnnotations?: boolean; // When true, draw release markers on this tile's chart. @@ -604,8 +614,17 @@ const Tile = ({ id: chart.config.source, }); + const usesReferenceVariable = !!referenceVariableNames?.length; + const isSourceMissing = !!chart.config.source && isSourceFetched && source == null; + // A data source or connection variable can select a source on another + // connection than the one a raw SQL tile queries. + const isSourceOnOtherConnection = + usesReferenceVariable && + source != null && + isRawSqlSavedChartConfig(chart.config) && + source.connection !== chart.config.connection; const isSourceUnset = !!chart.config && isBuilderSavedChartConfig(chart.config) && @@ -655,9 +674,15 @@ const Tile = ({ sourceId: chart.config.source, unsatisfiedRequiredFilters, referencedVariables: tileVariables, + referenceVariableNames, }), ), - [unsatisfiedRequiredFilters, chart.config, tileVariables], + [ + unsatisfiedRequiredFilters, + chart.config, + tileVariables, + referenceVariableNames, + ], ); const missingRequiredFilterNames = useMemo( () => JSON.parse(serializedMissingRequiredFilterNames), @@ -929,16 +954,13 @@ const Tile = ({ return url && addTimeZone(url); }, [dateRange, queriedConfig, source, addTimeZone]); + // An alert must query one fixed source and connection + const supportsAlerts = + !usesReferenceVariable && savedChartConfigSupportsAlerts(chart.config); + const hoverToolbar = useMemo(() => { if (readOnly) return null; - const isRawSql = isRawSqlSavedChartConfig(chart.config); - const isPromQL = isPromqlSavedChartConfig(chart.config); - const displayTypeSupportsAlerts = isRawSql - ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) - : isPromQL - ? displayTypeSupportsPromQLAlerts(chart.config.displayType) - : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; return ( @@ -968,7 +990,7 @@ const Tile = ({ )} - {displayTypeSupportsAlerts && + {supportsAlerts && (alert ? ( // Existing alert: bell with a colored status dot indicator. { if (readOnly) return null; - const isRawSql = isRawSqlSavedChartConfig(chart.config); - const isPromQL = isPromqlSavedChartConfig(chart.config); - const showAlerts = isRawSql - ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) - : isPromQL - ? displayTypeSupportsPromQLAlerts(chart.config.displayType) - : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; return ( @@ -1113,7 +1128,7 @@ const Tile = ({ Replay search )} - {showAlerts && ( + {supportsAlerts && ( <> - {`Missing required filters: ${missingRequiredFilterNames.join( + {`Missing required filters or variables: ${missingRequiredFilterNames.join( ', ', )}. Select a value for each to load this tile.`} @@ -1269,6 +1284,11 @@ const Tile = ({ The data source for this tile is not set. Edit the tile to select a data source. + ) : isSourceOnOtherConnection ? ( + + The selected data source is on a different connection than this + tile's query. Select a source on the same connection. + ) : ( <> {effectiveQueriedConfig && @@ -1501,6 +1521,7 @@ const Tile = ({ filterWarning, isSourceMissing, isSourceUnset, + isSourceOnOtherConnection, isBlockedByRequiredFilters, missingRequiredFilterNames, hasBeenVisible, @@ -1900,28 +1921,6 @@ function DBDashboardPage({ const { data: allDashboards } = useDashboards(); const [highlightedTileId] = useQueryState('highlightedTileId'); - const tableConnections = useMemo(() => { - if (!dashboard) return []; - const tc: TableConnection[] = []; - - for (const { config } of dashboard.tiles) { - if (!isBuilderSavedChartConfig(config)) continue; - const source = sources?.find(v => v.id === config.source); - if (!source || source.disabled) continue; - // TODO: will need to update this when we allow for multiple metrics per chart - const firstSelect = config.select[0]; - const metricType = - typeof firstSelect !== 'string' ? firstSelect?.metricType : undefined; - const tableName = getMetricTableName(source, metricType); - if (!tableName) continue; - tc.push({ - ...tcFromSource(source), - tableName, - }); - } - - return tc; - }, [dashboard, sources]); const [granularity, setGranularity] = useQueryState( 'granularity', @@ -1966,8 +1965,60 @@ function DBDashboardPage({ ignoredVariableNames, getFilterQueriesForSource, variables, + referenceVariableValues, + resolvedFilters, + unresolvedReferenceVariablesByFilterId, unsatisfiedRequiredFilters, - } = useDashboardFilters(filters); + } = useDashboardFilters(filters, { sources }); + // Tiles as they query, by ID: source and connection variable references + // replaced by the selections. Edits and saves read `dashboard.tiles`, so the + // stored references survive. + const resolvedTiles = useMemo( + () => + new Map( + (dashboard?.tiles ?? []).map(tile => { + const config = resolveConfigReferences( + tile.config, + referenceVariableValues, + ); + return [tile.id, config === tile.config ? tile : { ...tile, config }]; + }), + ), + [dashboard?.tiles, referenceVariableValues], + ); + // Kept apart from `resolvedTiles` so a tile's list only changes with its config + const referenceVariableNamesByTileId = useMemo( + () => + new Map( + (dashboard?.tiles ?? []).map(tile => [ + tile.id, + getConfigReferenceVariables(tile.config), + ]), + ), + [dashboard?.tiles], + ); + + const tableConnections = useMemo(() => { + const tc: TableConnection[] = []; + + for (const { config } of resolvedTiles.values()) { + if (!isBuilderSavedChartConfig(config)) continue; + const source = sources?.find(v => v.id === config.source); + if (!source || source.disabled) continue; + // TODO: will need to update this when we allow for multiple metrics per chart + const firstSelect = config.select[0]; + const metricType = + typeof firstSelect !== 'string' ? firstSelect?.metricType : undefined; + const tableName = getMetricTableName(source, metricType); + if (!tableName) continue; + tc.push({ + ...tcFromSource(source), + tableName, + }); + } + + return tc; + }, [resolvedTiles, sources]); const dashboardReady = !!dashboard?.id && @@ -2406,10 +2457,10 @@ function DBDashboardPage({ // The filter bar is hidden in kiosk mode, so a click would add conditions // the viewer can't see or remove. if (isKioskMode) return handlers; - for (const tile of dashboard?.tiles ?? []) { + for (const tile of resolvedTiles.values()) { const sourceId = 'source' in tile.config ? tile.config.source : undefined; const targetIds = getAdhocFilterTargetsForTile( - filters, + resolvedFilters, tile.config, sourceId, ).map(filter => filter.id); @@ -2420,19 +2471,23 @@ function DBDashboardPage({ } } return handlers; - }, [isKioskMode, dashboard?.tiles, filters, applyAdhocAction]); + }, [isKioskMode, resolvedTiles, resolvedFilters, applyAdhocAction]); const renderTileComponent = useCallback( (chart: Tile) => { + const resolvedChart = resolvedTiles.get(chart.id) ?? chart; // Resolve the tile's source ID so per-source-scoped filters can be // narrowed to only the tiles they target. Builder and RawSQL configs // both carry a `source` field; markdown / other configs don't. const tileSourceId = - 'source' in chart.config ? chart.config.source : undefined; + 'source' in resolvedChart.config + ? resolvedChart.config.source + : undefined; return ( setEditedTile(chart)} readOnly={isKioskMode} @@ -2459,7 +2514,14 @@ function DBDashboardPage({ c => c.id === chart.id, ); if (chartIndex === -1) return; - draft.tiles[chartIndex] = newChart; + // Keep the source as stored, not as resolved for display + draft.tiles[chartIndex] = { + ...newChart, + config: restoreConfigReferences( + newChart.config, + chart.config, + ), + }; }), ); } @@ -2532,6 +2594,8 @@ function DBDashboardPage({ ); }, [ + resolvedTiles, + referenceVariableNamesByTileId, dashboard, searchedTimeRange, isRefreshEnabled, @@ -3324,7 +3388,7 @@ function DBDashboardPage({ )} {!isKioskMode && ( )} {/* Selection indicator */} diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx index 5c15cdf9a5..1ef6ffdbfc 100644 --- a/packages/app/src/DashboardFilters.tsx +++ b/packages/app/src/DashboardFilters.tsx @@ -5,12 +5,14 @@ import { getFilterVariableName, getPendingFilterValuesVariables, isAdhocFilter, + isDataSourceFilter, isFilterGlobalRequirement, isFilterRequired, isFilterVariableEnabled, isQueryExpressionFilter, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; +import { ReferenceVariableValues } from '@hyperdx/common-utils/dist/referenceVariables'; import { AdhocFilterCondition, ChartVariable, @@ -28,6 +30,7 @@ import { getPendingVariablesTooltip, } from './components/DashboardFilterLabel'; import { FilterLinkToggle } from './components/FilterLinkToggle'; +import { DataSourceDashboardFilter } from './components/ReferenceVariableDashboardFilter'; import { VirtualMultiSelect } from './components/VirtualMultiSelect/VirtualMultiSelect'; import { useDashboardFilterValues } from './hooks/useDashboardFilterValues'; @@ -173,6 +176,13 @@ interface DashboardFilterProps { variables?: ChartVariable[]; /** Required filters with nothing selected, which can block ad hoc filters. */ unsatisfiedRequiredFilters?: DashboardFilter[]; + /** The selected ID of each data source and connection variable, by name. */ + referenceVariableValues?: ReferenceVariableValues; + /** + * The unselected reference variables each filter's fields reference, by + * filter ID. `filters` are otherwise expected with their references resolved. + */ + unresolvedReferenceVariablesByFilterId?: ReadonlyMap; } const DashboardFilters = ({ @@ -184,6 +194,8 @@ const DashboardFilters = ({ onSetAdhocConditions, variables, unsatisfiedRequiredFilters, + referenceVariableValues, + unresolvedReferenceVariablesByFilterId, }: DashboardFilterProps) => { // "Link" mode (opt-in, off by default): each dropdown's values are narrowed by // the others' selections. Off by default because contingent value lookups @@ -191,13 +203,22 @@ const DashboardFilters = ({ // on, all of a source's facets are computed in a single groupUniqArrayIf scan. const [linked, setLinked] = useState(false); + // A filter whose data source variable has no selection has nothing to query yet + const queryableFilters = useMemo( + () => + filters.filter( + filter => !unresolvedReferenceVariablesByFilterId?.has(filter.id), + ), + [filters, unresolvedReferenceVariablesByFilterId], + ); + const { data: filterValuesById, erroredFilterIds, filterErrorMessages, isFetching, } = useDashboardFilterValues({ - filters, + filters: queryableFilters, dateRange, variables, // Only narrow by sibling selections when linked. @@ -214,6 +235,25 @@ const DashboardFilters = ({ return ( {filters.map(filter => { + const unresolvedVariables = + unresolvedReferenceVariablesByFilterId?.get(filter.id) ?? []; + if (isDataSourceFilter(filter)) { + return ( + + onSetFilterValue(filter.id, value ? [value] : []) + } + effect={getFilterEffect(filter)} + requiredTooltip={getRequiredFilterTooltip(filter)} + /> + ); + } if (isAdhocFilter(filter)) { return ( variable.name !== ownName); }, [variables, filter]); + // Ad hoc conditions and reference variables have fixed selection rules + const showSelectionOptions = + showRequiredFilterOptions && + formFilterType !== 'ADHOC' && + !isReferenceVariableFilterType(formFilterType); + const isNew = !filter; const showTypeInput = !!showVariableOptions; @@ -236,7 +245,12 @@ export const DashboardFilterEditForm = ({ message="Failed to load filter" showErrorMessage > - {formFilterType === 'STATIC_LIST' ? ( + {formFilterType === 'DATA_SOURCE' ? ( + + ) : formFilterType === 'STATIC_LIST' ? ( - {showRequiredFilterOptions && formFilterType !== 'ADHOC' && ( + {showSelectionOptions && ( IS_PROMQL_ENABLED || kind !== SourceKind.Promql) + .map(kind => ({ value: kind, label: SOURCE_KIND_LABELS[kind] })); + +interface DataSourceFilterEditFormProps { + control: FilterFormControl; + /** Filters other than the one being edited, used to keep variable names unique. */ + otherFilters: DashboardFilter[]; +} + +/** Form for editing a variable whose value is a source. */ +export const DataSourceFilterEditForm = ({ + control, + otherFilters, +}: DataSourceFilterEditFormProps) => { + const { errors } = useFormState({ control }); + + return ( + <> + + + (value?.length ?? 0) > 0 || 'Select at least one source kind', + }} + render={({ field: { onChange, value } }) => ( + ( + + {SOURCE_KIND_ICONS[option.value]} + {option.label} + + )} + /> + )} + /> + + + + + ); +}; diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts index 9f2ea9c142..9b3f63813e 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts @@ -1,4 +1,4 @@ -import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; +import { DashboardFilter, SourceKind } from '@hyperdx/common-utils/dist/types'; import { FilterFormValues, @@ -317,6 +317,74 @@ describe('toSavedFilter', () => { ).toThrow(); }); + it('saves a data source filter as single-select and required', () => { + const saved = toSavedFilter( + formValues({ + type: 'DATA_SOURCE', + name: 'Logs source', + allowedSourceKinds: [SourceKind.Log], + isRequired: false, + isSingleSelect: false, + expression: 'ignored', + source: 'ignored', + }), + ); + + expect(saved).toEqual({ + id: expect.any(String), + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'Logs_source', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }); + expect(toFormValues(saved).allowedSourceKinds).toEqual([SourceKind.Log]); + }); + + it('saves a data source filter limited to a connection variable', () => { + expect( + toSavedFilter( + formValues({ + type: 'DATA_SOURCE', + name: 'Logs source', + allowedSourceKinds: [SourceKind.Log], + connection: '$conn', + }), + ), + ).toMatchObject({ connection: '$conn' }); + }); + + it('saves a connection filter as single-select and required', () => { + const saved = toSavedFilter( + formValues({ + type: 'CONNECTION', + name: 'Cluster', + isSingleSelect: false, + source: 'ignored', + }), + ); + + expect(saved).toEqual({ + id: expect.any(String), + type: 'CONNECTION', + name: 'Cluster', + variableName: 'Cluster', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }); + }); + + it('rejects a data source filter without source kinds', () => { + expect(() => + toSavedFilter(formValues({ type: 'DATA_SOURCE', name: 'Source' })), + ).toThrow(); + }); + it('rejects values the schema cannot accept', () => { expect(() => toSavedFilter(formValues({ type: 'STATIC_LIST', options: [] })), @@ -324,7 +392,10 @@ describe('toSavedFilter', () => { }); }); -/** The minimum a form needs to save, per filter type the form supports. */ +/** + * The minimum a form needs to save, per filter type with configurable + * selection limits. + */ const byType: Record< Exclude, Partial diff --git a/packages/app/src/components/DashboardFiltersModal/filterFormState.ts b/packages/app/src/components/DashboardFiltersModal/filterFormState.ts index 7e450282eb..ca83a13f77 100644 --- a/packages/app/src/components/DashboardFiltersModal/filterFormState.ts +++ b/packages/app/src/components/DashboardFiltersModal/filterFormState.ts @@ -5,11 +5,13 @@ import { isFilterGlobalRequirement, isFilterRequired, isFilterVariableEnabled, + isReferenceVariableFilterType, } from '@hyperdx/common-utils/dist/filters'; import { AdhocDashboardFilter, DashboardFilter, DashboardFilterSchema, + DataSourceDashboardFilter, PromqlLabelDashboardFilter, QueryExpressionDashboardFilter, StaticListDashboardFilter, @@ -53,7 +55,8 @@ export type FilterFormValues = { | 'isGlobalRequirement' | 'maxSelections' > & - Pick; + Pick & + Pick; export type FilterFormControl = Control; @@ -65,6 +68,7 @@ export const toFormValues = ( const staticList = filter?.type === 'STATIC_LIST' ? filter : undefined; const promqlLabel = filter?.type === 'PROMETHEUS_LABEL' ? filter : undefined; const adhoc = filter?.type === 'ADHOC' ? filter : undefined; + const dataSource = filter?.type === 'DATA_SOURCE' ? filter : undefined; return { id: filter?.id ?? crypto.randomUUID(), @@ -100,6 +104,10 @@ export const toFormValues = ( // ADHOC fields sourceType: adhoc?.sourceType ?? 'sql', sources: adhoc?.sources ?? [], + + // DATA_SOURCE fields + allowedSourceKinds: dataSource?.allowedSourceKinds ?? [], + connection: dataSource?.connection, }; }; @@ -140,6 +148,27 @@ export const toSavedFilter = (values: FilterFormValues): DashboardFilter => { }); } + // Reference variables are always single-select and required, so the + // checkboxes are not consulted + if (isReferenceVariableFilterType(values.type)) { + return DashboardFilterSchema.parse({ + id: values.id, + type: values.type, + name: values.name, + ...(values.type === 'DATA_SOURCE' + ? { + allowedSourceKinds: values.allowedSourceKinds, + connection: values.connection || undefined, + } + : {}), + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + variableName: getFilterVariableName(values), + }); + } + const appliesTo = values.appliesToSourceIds?.filter(id => !!id?.length); const trimmedWhere = values.where?.trim() ?? ''; const whereFields = { diff --git a/packages/app/src/components/ReferenceVariableDashboardFilter.tsx b/packages/app/src/components/ReferenceVariableDashboardFilter.tsx new file mode 100644 index 0000000000..173ddf309c --- /dev/null +++ b/packages/app/src/components/ReferenceVariableDashboardFilter.tsx @@ -0,0 +1,145 @@ +import { useCallback, useMemo } from 'react'; +import { DataSourceDashboardFilter as DataSourceFilter } from '@hyperdx/common-utils/dist/types'; +import { ComboboxItem, Group, Select, Stack } from '@mantine/core'; +import { IconStack } from '@tabler/icons-react'; + +import { + getSourceOptionIcon, + sourceSelectFilter, + SourceSelectGroup, + useFilteredSortedSourceItems, + useSourceKindMap, +} from '@/components/sourceSelectUtils'; +import { useSources } from '@/source'; + +import { + DashboardFilterLabel, + FilterCaution, + FilterEffect, + getPendingVariablesTooltip, +} from './DashboardFilterLabel'; + +type ReferenceVariableFilterProps = { + value: string | undefined; + onChange: (id: string | undefined) => void; + effect: FilterEffect; + requiredTooltip: string; +}; + +/** + * The filter bar control for a reference variable: a single picker over the + * IDs the variable can select. + */ +function ReferenceVariableSelect({ + name, + items, + value, + onChange, + effect, + requiredTooltip, + pendingVariables, + placeholder, + renderIcon, +}: ReferenceVariableFilterProps & { + name: string; + items: ComboboxItem[] | SourceSelectGroup[]; + pendingVariables?: string[]; + placeholder: string; + renderIcon: (value: string | null) => React.ReactNode; +}) { + // A selection that is no longer offered (deleted, disabled, or no longer + // allowed by the variable) shows as empty, so it reads as unselected. + const isOffered = useMemo( + () => + items.some(item => + 'group' in item + ? item.items.some(i => i.value === value) + : item.value === value, + ), + [items, value], + ); + const selected = value && isOffered ? value : null; + + const renderOption = useCallback( + ({ option }: { option: ComboboxItem }) => ( + + {renderIcon(option.value)} + {option.label} + + ), + [renderIcon], + ); + + return ( + + + {selected == null && ( + + )} + {!!pendingVariables?.length && ( + + )} + +