Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/source-vars-4.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@hyperdx/app': minor
---

feat: let tiles and filters use a data source variable as their source
60 changes: 46 additions & 14 deletions packages/app/src/DBDashboardImportPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import {
displayTypeRequiresSource,
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables';
import {
type DashboardFilter,
type DashboardTemplate,
Expand Down Expand Up @@ -390,11 +391,25 @@ function resolveAppliesToSources(
return { ids: slots.filter(Boolean), slots };
}

/**
* A data source variable reference (`$name`) is portable as written, so
* import neither maps it nor asks the user to.
*/
const isVariableRef = (name: string | undefined) =>
parseVariableReference(name) != null;

const sourceVariableRefs = (names: string[] | undefined) =>
names?.filter(isVariableRef) ?? [];

const mappableSourceNames = (names: string[] | undefined) =>
names?.filter(name => !isVariableRef(name));

/** Whether the filter names a single source that import must map. */
const isSourceBackedTemplateFilter = (
filter: DashboardFilter,
): filter is QueryExpressionDashboardFilter | PromqlLabelDashboardFilter =>
isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter);
(isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter)) &&
!isVariableRef(filter.source);

/** Returns the filter if it names a source, or `undefined` if it doesn't. */
const sourceBackedTemplateFilter = (filter: DashboardFilter | undefined) =>
Expand Down Expand Up @@ -487,6 +502,7 @@ export function buildMappingFormSchema(
// meaningful source even when the template names one.
if (
config.source &&
!isVariableRef(config.source) &&
displayTypeRequiresSource(config.displayType) &&
!data.tileSourceMappings[idx]
) {
Expand Down Expand Up @@ -523,7 +539,10 @@ export function buildMappingFormSchema(
const mappedSourceIds = data.filterAdhocSourceMappings?.[idx]?.filter(
id => !!id,
);
if (!mappedSourceIds?.length) {
if (
!mappedSourceIds?.length &&
sourceVariableRefs(filter.sources).length === 0
) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['filterAdhocSourceMappings', idx],
Expand Down Expand Up @@ -624,7 +643,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) {

const tileSourceMappings = input.tiles.map(tile => {
const config = tile.config as SavedChartConfig;
if (!config.source) return '';
if (!config.source || isVariableRef(config.source)) return '';
const match = sources.find(
source => source.name.toLowerCase() === config.source!.toLowerCase(),
);
Expand Down Expand Up @@ -1042,15 +1061,24 @@ export function Mapping({ input }: { input: DashboardTemplate }) {

// Zip the source mappings with the input filters
const zippedFilters = input.filters?.map((filter, idx) => {
const appliesTo = data.filterAppliesToSourceMappings?.[idx]?.filter(
id => !!id?.length,
);
const appliesToSourceIds = appliesTo?.length ? appliesTo : undefined;
// Data source variable references carry over unmapped
const appliesTo = [
...sourceVariableRefs(
appliesToTemplateFilter(filter)?.appliesToSourceIds,
),
...(data.filterAppliesToSourceMappings?.[idx]?.filter(
id => !!id?.length,
) ?? []),
];
const appliesToSourceIds = appliesTo.length ? appliesTo : undefined;
if (isAdhocFilter(filter)) {
return {
...filter,
sources:
data.filterAdhocSourceMappings?.[idx]?.filter(id => !!id) ?? [],
sources: [
...sourceVariableRefs(filter.sources),
...(data.filterAdhocSourceMappings?.[idx]?.filter(id => !!id) ??
[]),
],
appliesToSourceIds,
};
}
Expand Down Expand Up @@ -1145,7 +1173,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
return (
<Fragment key={tile.id}>
{/** Mapping for the tile's source, if one exists (they're optional for raw sql tiles) */}
{tile.config.source && (
{tile.config.source && !isVariableRef(tile.config.source) && (
<Table.Tr>
<Table.Td>{tile.config.name}</Table.Td>
<Table.Td>Data source</Table.Td>
Expand Down Expand Up @@ -1228,8 +1256,12 @@ export function Mapping({ input }: { input: DashboardTemplate }) {

{/** Map filter sources */}
{input.filters?.map((filter, i) => {
const appliesToSourceIds =
appliesToTemplateFilter(filter)?.appliesToSourceIds;
const appliesToSourceIds = mappableSourceNames(
appliesToTemplateFilter(filter)?.appliesToSourceIds,
);
const adhocSourceNames = isAdhocFilter(filter)
? mappableSourceNames(filter.sources)
: undefined;
return (
<Fragment key={filter.id}>
{isSourceBackedTemplateFilter(filter) && (
Expand All @@ -1256,11 +1288,11 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
<Table.Td />
</Table.Tr>
)}
{isAdhocFilter(filter) && (
{isAdhocFilter(filter) && !!adhocSourceNames?.length && (
<Table.Tr>
<Table.Td>{filter.name} (Filter)</Table.Td>
<Table.Td>Data sources</Table.Td>
<Table.Td>{filter.sources.join(', ')}</Table.Td>
<Table.Td>{adhocSourceNames.join(', ')}</Table.Td>
<Table.Td>
<SourceMultiSelectControlled
control={control}
Expand Down
8 changes: 6 additions & 2 deletions packages/app/src/DBDashboardPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover'
import { InlineNameInput } from '@/components/InlineNameInput/InlineNameInput';
import { PageHeader } from '@/components/PageHeader';
import { PageLayout } from '@/components/PageLayout';
import { ReferenceVariablesProvider } from '@/components/sourceSelectUtils';
import { SqlVariablesProvider } from '@/components/SQLEditor/variableCompletions';
import { TimePicker } from '@/components/TimePicker';
import {
Expand Down Expand Up @@ -3288,7 +3289,10 @@ function DBDashboardPage({
// `<PageLayout>` return shallow and prevents the tile-grid tree below
// from being wrapped in an extra indentation level.
const dashboardBody = (
<>
<ReferenceVariablesProvider
filters={filters}
values={referenceVariableValues}
>
<Head>
<title>{title}</title>
</Head>
Expand Down Expand Up @@ -3580,7 +3584,7 @@ function DBDashboardPage({
variables={variables}
/>
)}
</>
</ReferenceVariablesProvider>
);

return (
Expand Down
54 changes: 54 additions & 0 deletions packages/app/src/__tests__/DBDashboardImportPage.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -666,3 +666,57 @@ describe('Dashboard import - ad hoc filter sources', () => {
expect(issuesFor(['src-logs'])).toBeUndefined();
});
});

describe('Dashboard import - data source variables', () => {
const sourceVariableTemplate = DashboardTemplateSchema.parse({
version: '0.1.0',
name: 'Source Variables',
tiles: [builderTile('tile-line', 'line', '$logs', 0)],
filters: [
{
id: 'ds',
type: 'DATA_SOURCE',
name: 'Logs source',
variableName: 'logs',
allowedSourceKinds: [SourceKind.Log],
minSelections: 1,
maxSelections: 1,
isBroadcastEnabled: false,
isVariableEnabled: true,
},
{
id: 'svc',
type: 'QUERY_EXPRESSION',
name: 'Service',
expression: 'ServiceName',
source: '$logs',
},
{
id: 'adhoc',
type: 'ADHOC',
name: 'Conditions',
sourceType: 'sql',
sources: ['$logs', 'Traces'],
isVariableEnabled: true,
},
],
});

it('keeps references as written without asking for a mapping', async () => {
renderWithMantine(<Mapping input={sourceVariableTemplate} />);

const finish = await screen.findByRole('button', {
name: /finish import/i,
});
expect(screen.queryByText('$logs')).not.toBeInTheDocument();
await act(async () => {
fireEvent.click(finish);
});

await waitFor(() => expect(mockMutateAsync).toHaveBeenCalledTimes(1));
const payload = mockMutateAsync.mock.calls[0][0];
expect(payload.tiles[0].config.source).toBe('$logs');
expect(payload.filters[1].source).toBe('$logs');
expect(payload.filters[2].sources).toEqual(['$logs', 'src-traces']);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { IconCirclePlus } from '@tabler/icons-react';

import { HeatmapModeControl } from '@/components/DBEditTimeChartForm/HeatmapModeControl';
import { SourceSelectControlled } from '@/components/SourceSelect';
import { useResolvedReference } from '@/components/sourceSelectUtils';
import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata';
import { useSource } from '@/source';

Expand Down Expand Up @@ -80,7 +81,7 @@ export default function PromqlChartEditor({
);
}, []);

const sourceId = useWatch({ control, name: 'source' });
const sourceId = useResolvedReference(useWatch({ control, name: 'source' }));
const displayType = useWatch({ control, name: 'displayType' });
const { data: source } = useSource({ id: sourceId });
// The form can still hold a non-PromQL source right after switching a tile
Expand All @@ -107,6 +108,7 @@ export default function PromqlChartEditor({
name="source"
data-testid="source-selector"
allowedSourceKinds={allowedSourceKinds}
allowSourceVariables
/>
</Group>
{displayType === DisplayType.Heatmap && (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import SourceSchemaPreview, {
isSourceSchemaPreviewEnabled,
} from '@/components/SourceSchemaPreview';
import { SourceSelectControlled } from '@/components/SourceSelect';
import { useResolvedReference } from '@/components/sourceSelectUtils';
import { SQLEditorControlled } from '@/components/SQLEditor/SQLEditor';
import { type SQLCompletion } from '@/components/SQLEditor/utils';
import { IS_LOCAL_MODE } from '@/config';
Expand Down Expand Up @@ -148,7 +149,9 @@ export default function RawSqlChartEditor({

const displayType = useWatch({ control, name: 'displayType' });
const connection = useWatch({ control, name: 'connection' });
const source = useWatch({ control, name: 'source' });
// A data source variable reference syncs and expands macros as the source
// it currently selects.
const source = useResolvedReference(useWatch({ control, name: 'source' }));
const sqlTemplate = useWatch({ control, name: 'sqlTemplate' });
const chartName = useWatch({ control, name: 'name' });
const sourceObject = sources?.find(s => s.id === source);
Expand Down Expand Up @@ -293,6 +296,7 @@ export default function RawSqlChartEditor({
name="source"
data-testid="source-selector"
allowedSourceKinds={allowedSourceKinds}
allowSourceVariables
connectionId={connection}
size="xs"
clearable
Expand Down
84 changes: 84 additions & 0 deletions packages/app/src/components/ChartEditor/__tests__/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
convertFormStateToSavedChartConfig,
convertSavedChartConfigToFormState,
getAllowedSourceKinds,
getInterchangeableSourceKind,
promqlExpressionPlaceholder,
validateChartForm,
} from '@/components/ChartEditor/utils';
Expand Down Expand Up @@ -2705,3 +2706,86 @@ describe('promqlExpressionPlaceholder', () => {
);
});
});

describe('data source variable references', () => {
it('saves the reference rather than the source it resolves to', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
source: '$logs',
series: [seriesItem],
};
expect(convertFormStateToSavedChartConfig(form, logSource)).toMatchObject({
source: '$logs',
});
});

it('queries the resolved source for a raw SQL chart', () => {
const form: ChartEditorFormState = {
configType: 'sql',
displayType: DisplayType.Table,
source: '$logs',
connection: 'conn-1',
sqlTemplate: 'SELECT 1',
series: [],
};
expect(convertFormStateToSavedChartConfig(form, logSource)).toMatchObject({
source: '$logs',
});
expect(
convertFormStateToChartConfig(form, dateRange, logSource),
).toMatchObject({ source: logSource.id });
});

it('saves a reference with no selected source when its kind is known', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
source: '$logs',
series: [
{ ...seriesItem, metricName: 'cpu', metricType: MetricsDataType.Gauge },
],
};
const saved = convertFormStateToSavedChartConfig(
form,
undefined,
SourceKind.Log,
);
expect(saved).toMatchObject({ source: '$logs' });
expect(saved).toMatchObject({
select: [expect.not.objectContaining({ metricName: 'cpu' })],
});
});

it('validates a reference with no selected source by its kind', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
source: '$logs',
series: [seriesItem],
};
expect(
validateChartForm(form, undefined, jest.fn(), {
sourceKind: SourceKind.Log,
}),
).toEqual([]);
expect(
validateChartForm(form, undefined, jest.fn(), {
missingSourceMessage: 'Select a source for $logs',
}),
).toEqual([{ path: 'source', message: 'Select a source for $logs' }]);
});
});

describe('getInterchangeableSourceKind', () => {
it.each<[SourceKind[], SourceKind | undefined]>([
[[SourceKind.Log], SourceKind.Log],
[[SourceKind.Log, SourceKind.Trace], SourceKind.Log],
[[SourceKind.Log, SourceKind.Metric], undefined],
[[SourceKind.Log, SourceKind.Session], undefined],
[[SourceKind.Session, SourceKind.Promql], undefined],
[[], undefined],
])(
'returns a stand-in for %p only when charts treat them alike',
(kinds, expected) => {
expect(getInterchangeableSourceKind(kinds)).toBe(expected);
},
);
});
Loading
Loading