feat: adjust alert template title and body to reflect alert state

.changeset/spotty-yaks-sniff.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": minor
+---
+
+feat: adjust alert template title and body to reflect alert state

packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts

@@ -43,6 +43,7 @@ import {
   buildAlertMessageTemplateHdxLink,
   buildAlertMessageTemplateTitle,
   getDefaultExternalAction,
+  isAlertResolved,
   renderAlertTemplate,
   translateExternalActionsToInternal,
 } from '@/tasks/checkAlerts/template';
@@ -229,16 +230,70 @@ describe('checkAlerts', () => {
         buildAlertMessageTemplateTitle({
           view: defaultSearchView,
         }),
-      ).toMatchInlineSnapshot(`"Alert for \\"My Search\\" - 10 lines found"`);
+      ).toMatchInlineSnapshot(
+        `"🚨 Alert for \\"My Search\\" - 10 lines found"`,
+      );
       expect(
         buildAlertMessageTemplateTitle({
           view: defaultChartView,
         }),
       ).toMatchInlineSnapshot(
-        `"Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
+        `"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
       );
     });
 
+    it('buildAlertMessageTemplateTitle with state parameter', () => {
+      // Test ALERT state (should have 🚨 emoji)
+      expect(
+        buildAlertMessageTemplateTitle({
+          view: defaultSearchView,
+          state: AlertState.ALERT,
+        }),
+      ).toMatchInlineSnapshot(
+        `"🚨 Alert for \\"My Search\\" - 10 lines found"`,
+      );
+      expect(
+        buildAlertMessageTemplateTitle({
+          view: defaultChartView,
+          state: AlertState.ALERT,
+        }),
+      ).toMatchInlineSnapshot(
+        `"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
+      );
+
+      // Test OK state (should have ✅ emoji)
+      expect(
+        buildAlertMessageTemplateTitle({
+          view: defaultSearchView,
+          state: AlertState.OK,
+        }),
+      ).toMatchInlineSnapshot(
+        `"✅ Alert for \\"My Search\\" - 10 lines found"`,
+      );
+      expect(
+        buildAlertMessageTemplateTitle({
+          view: defaultChartView,
+          state: AlertState.OK,
+        }),
+      ).toMatchInlineSnapshot(
+        `"✅ Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
+      );
+    });
+
+    it('isAlertResolved', () => {
+      // Test OK state returns true
+      expect(isAlertResolved(AlertState.OK)).toBe(true);
+
+      // Test ALERT state returns false
+      expect(isAlertResolved(AlertState.ALERT)).toBe(false);
+
+      // Test INSUFFICIENT_DATA state returns false
+      expect(isAlertResolved(AlertState.INSUFFICIENT_DATA)).toBe(false);
+
+      // Test DISABLED state returns false
+      expect(isAlertResolved(AlertState.DISABLED)).toBe(false);
+    });
+
     it('getDefaultExternalAction', () => {
       expect(
         getDefaultExternalAction({
@@ -372,7 +427,7 @@ describe('checkAlerts', () => {
             },
           },
         },
-        title: 'Alert for "My Search" - 10 lines found',
+        title: '🚨 Alert for "My Search" - 10 lines found',
         teamWebhooksById: new Map<string, typeof webhook>([
           [webhook._id.toString(), webhook],
         ]),
@@ -382,12 +437,12 @@ describe('checkAlerts', () => {
         1,
         'https://hooks.slack.com/services/123',
         {
-          text: 'Alert for "My Search" - 10 lines found',
+          text: '🚨 Alert for "My Search" - 10 lines found',
           blocks: [
             {
               text: {
                 text: [
-                  '*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
+                  '*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
                   'Group: "http"',
                   '10 lines found, expected less than 1 lines',
                   'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
@@ -432,7 +487,7 @@ describe('checkAlerts', () => {
             webhookName: 'My_Webhook',
           },
         },
-        title: 'Alert for "My Search" - 10 lines found',
+        title: '🚨 Alert for "My Search" - 10 lines found',
         teamWebhooksById: new Map<string, typeof webhook>([
           [webhook._id.toString(), webhook],
         ]),
@@ -442,12 +497,12 @@ describe('checkAlerts', () => {
         1,
         'https://hooks.slack.com/services/123',
         {
-          text: 'Alert for "My Search" - 10 lines found',
+          text: '🚨 Alert for "My Search" - 10 lines found',
           blocks: [
             {
               text: {
                 text: [
-                  '*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
+                  '*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
                   'Group: "http"',
                   '10 lines found, expected less than 1 lines',
                   'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
@@ -517,7 +572,7 @@ describe('checkAlerts', () => {
             },
           },
         },
-        title: 'Alert for "My Search" - 10 lines found',
+        title: '🚨 Alert for "My Search" - 10 lines found',
         teamWebhooksById,
       });
 
@@ -541,20 +596,20 @@ describe('checkAlerts', () => {
             host: 'web2',
           },
         },
-        title: 'Alert for "My Search" - 10 lines found',
+        title: '🚨 Alert for "My Search" - 10 lines found',
         teamWebhooksById,
       });
 
       expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2);
       expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
         'https://hooks.slack.com/services/123',
         {
-          text: 'Alert for "My Search" - 10 lines found',
+          text: '🚨 Alert for "My Search" - 10 lines found',
           blocks: [
             {
               text: {
                 text: [
-                  '*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
+                  '*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
                   'Group: "http"',
                   '10 lines found, expected less than 1 lines',
                   'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
@@ -578,12 +633,12 @@ describe('checkAlerts', () => {
       expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
         'https://hooks.slack.com/services/456',
         {
-          text: 'Alert for "My Search" - 10 lines found',
+          text: '🚨 Alert for "My Search" - 10 lines found',
           blocks: [
             {
               text: {
                 text: [
-                  '*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
+                  '*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
                   'Group: "http"',
                   '10 lines found, expected less than 1 lines',
                   'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
@@ -605,6 +660,63 @@ describe('checkAlerts', () => {
         },
       );
     });
+
+    it('renderAlertTemplate - resolved alert with simplified message', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      const webhook = await new Webhook({
+        team: team._id,
+        service: 'slack',
+        url: 'https://hooks.slack.com/services/123',
+        name: 'My_Webhook',
+      }).save();
+
+      await renderAlertTemplate({
+        alertProvider,
+        clickhouseClient: {} as any,
+        metadata: {} as any,
+        state: AlertState.OK, // Resolved state
+        template: '@webhook-My_Webhook',
+        view: {
+          ...defaultSearchView,
+          alert: {
+            ...defaultSearchView.alert,
+            channel: {
+              type: null, // using template instead
+            },
+          },
+        },
+        title: '✅ Alert for "My Search" - 10 lines found',
+        teamWebhooksById: new Map<string, typeof webhook>([
+          [webhook._id.toString(), webhook],
+        ]),
+      });
+
+      expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
+      expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
+        'https://hooks.slack.com/services/123',
+        {
+          text: '✅ Alert for "My Search" - 10 lines found',
+          blocks: [
+            {
+              text: {
+                text: expect.stringContaining('The alert has been resolved'),
+                type: 'mrkdwn',
+              },
+              type: 'section',
+            },
+          ],
+        },
+      );
+
+      // Verify the message includes the time range but not detailed logs
+      const callArgs = (slack.postMessageToWebhook as any).mock.calls[0][1];
+      const messageText = callArgs.blocks[0].text.text;
+      expect(messageText).toContain('The alert has been resolved');
+      expect(messageText).toContain('Time Range (UTC):');
+      expect(messageText).toContain('Group: "http"');
+      // Should NOT contain detailed log data
+      expect(messageText).not.toContain('lines found, expected');
+    });
   });
 
   describe('processAlert', () => {
@@ -915,7 +1027,7 @@ describe('checkAlerts', () => {
         1,
         'https://hooks.slack.com/services/123',
         {
-          text: 'Alert for "My Search" - 3 lines found',
+          text: '🚨 Alert for "My Search" - 3 lines found',
           blocks: [
             {
               text: expect.any(Object),
@@ -928,7 +1040,7 @@ describe('checkAlerts', () => {
         2,
         'https://hooks.slack.com/services/123',
         {
-          text: 'Alert for "My Search" - 1 lines found',
+          text: '🚨 Alert for "My Search" - 1 lines found',
           blocks: [
             {
               text: expect.any(Object),
@@ -1089,12 +1201,12 @@ describe('checkAlerts', () => {
         1,
         'https://hooks.slack.com/services/123',
         {
-          text: 'Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1',
+          text: '🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1',
           blocks: [
             {
               text: {
                 text: [
-                  `*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1>*`,
+                  `*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1>*`,
                   '',
                   '3 exceeds 1',
                   'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)',
@@ -1281,7 +1393,7 @@ describe('checkAlerts', () => {
       expect(fetchMock).toHaveBeenCalledWith('https://webhook.site/123', {
         method: 'POST',
         body: JSON.stringify({
-          text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1`,
+          text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1`,
         }),
         headers: {
           'Content-Type': 'application/json',
@@ -2087,12 +2199,12 @@ describe('checkAlerts', () => {
         1,
         'https://hooks.slack.com/services/123',
         {
-          text: 'Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1',
+          text: '🚨 Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1',
           blocks: [
             {
               text: {
                 text: [
-                  `*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1>*`,
+                  `*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1>*`,
                   '',
                   '6.25 exceeds 1',
                   'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)',

packages/api/src/tasks/checkAlerts/index.ts

@@ -147,6 +147,7 @@ const fireChannelEvent = async ({
     title: buildAlertMessageTemplateTitle({
       template: alert.name,
       view: templateView,
+      state,
     }),
     template: alert.message,
     view: templateView,

packages/api/src/tasks/checkAlerts/template.ts

@@ -83,6 +83,10 @@ interface Message {
   eventId: string;
 }
 
+export const isAlertResolved = (state?: AlertState): boolean => {
+  return state === AlertState.OK;
+};
+
 export const notifyChannel = async ({
   channel,
   message,
@@ -304,20 +308,27 @@ export const buildAlertMessageTemplateHdxLink = (
 export const buildAlertMessageTemplateTitle = ({
   template,
   view,
+  state,
 }: {
   template?: string | null;
   view: AlertMessageTemplateDefaultView;
+  state?: AlertState;
 }) => {
   const { alert, dashboard, savedSearch, value } = view;
   const handlebars = createHandlebarsWithHelpers();
+
+  // Add emoji prefix based on alert state
+  const emoji = isAlertResolved(state) ? '✅ ' : '🚨 ';
+
   if (alert.source === AlertSource.SAVED_SEARCH) {
     if (savedSearch == null) {
       throw new Error(`Source is ${alert.source}  but savedSearch is null`);
     }
     // TODO: using template engine to render the title
-    return template
+    const baseTitle = template
       ? handlebars.compile(template)(view)
       : `Alert for "${savedSearch.name}" - ${value} lines found`;
+    return `${emoji}${baseTitle}`;
   } else if (alert.source === AlertSource.TILE) {
     if (dashboard == null) {
       throw new Error(`Source is ${alert.source} but dashboard is null`);
@@ -328,7 +339,7 @@ export const buildAlertMessageTemplateTitle = ({
         `Tile with id ${alert.tileId} not found in dashboard ${dashboard.name}`,
       );
     }
-    return template
+    const baseTitle = template
       ? handlebars.compile(template)(view)
       : `Alert for "${tile.config.name}" in "${dashboard.name}" - ${value} ${
           doesExceedThreshold(alert.thresholdType, alert.threshold, value)
@@ -339,6 +350,7 @@ export const buildAlertMessageTemplateTitle = ({
               ? 'falls below'
               : 'exceeds'
         } ${alert.threshold}`;
+    return `${emoji}${baseTitle}`;
   }
 
   throw new Error(`Unsupported alert source: ${(alert as any).source}`);
@@ -527,9 +539,14 @@ export const renderAlertTemplate = async ({
   })})`;
   let rawTemplateBody;
 
+  // For resolved alerts, use a simple message instead of fetching data
+  if (isAlertResolved(state)) {
+    rawTemplateBody = `${group ? `Group: "${group}" - ` : ''}The alert has been resolved.\n${timeRangeMessage}
+${targetTemplate}`;
+  }
   // TODO: support advanced routing with template engine
   // users should be able to use '@' syntax to trigger alerts
-  if (alert.source === AlertSource.SAVED_SEARCH) {
+  else if (alert.source === AlertSource.SAVED_SEARCH) {
     if (savedSearch == null) {
       throw new Error(`Source is ${alert.source} but savedSearch is null`);
     }