feat(deployments): stream build server logs (#2663)

* Use read-only project-scoped s2 tokens for streaming deployment logs

* Add http2 to remix polyfills

Needed for using s2 client-side.

* Stream build-server logs in the deployment details page

* Disable 12-hour format in the DateTime component

* Enable collapsing the logs panel

* Auto-collapse logs for succesful/timedout/queued deployments

* Make S2 env vars optional

* Show the logs section only for gh-triggered deployments

* Cache s2 access tokens in redis

* Reset streaming state

* Expose 12h format as a param for the Datetime components

Author: myftija

apps/webapp/app/components/primitives/DateTime.tsx

@@ -13,7 +13,9 @@ type DateTimeProps = {
   includeTime?: boolean;
   showTimezone?: boolean;
   showTooltip?: boolean;
+  hideDate?: boolean;
   previousDate?: Date | string | null; // Add optional previous date for comparison
+  hour12?: boolean;
 };
 
 export const DateTime = ({
@@ -23,6 +25,7 @@ export const DateTime = ({
   includeTime = true,
   showTimezone = false,
   showTooltip = true,
+  hour12 = true,
 }: DateTimeProps) => {
   const locales = useLocales();
   const [localTimeZone, setLocalTimeZone] = useState<string>("UTC");
@@ -50,7 +53,8 @@ export const DateTime = ({
         timeZone ?? localTimeZone,
         locales,
         includeSeconds,
-        includeTime
+        includeTime,
+        hour12
       ).replace(/\s/g, String.fromCharCode(32))}
       {showTimezone ? ` (${timeZone ?? "UTC"})` : null}
     </Fragment>
@@ -66,7 +70,8 @@ export function formatDateTime(
   timeZone: string,
   locales: string[],
   includeSeconds: boolean,
-  includeTime: boolean
+  includeTime: boolean,
+  hour12: boolean = true
 ): string {
   return new Intl.DateTimeFormat(locales, {
     year: "numeric",
@@ -76,6 +81,7 @@ export function formatDateTime(
     minute: includeTime ? "numeric" : undefined,
     second: includeTime && includeSeconds ? "numeric" : undefined,
     timeZone,
+    hour12,
   }).format(date);
 }
 
@@ -122,7 +128,7 @@ export function formatDateTimeISO(date: Date, timeZone: string): string {
 }
 
 // New component that only shows date when it changes
-export const SmartDateTime = ({ date, previousDate = null, timeZone = "UTC" }: DateTimeProps) => {
+export const SmartDateTime = ({ date, previousDate = null, timeZone = "UTC", hour12 = true }: DateTimeProps) => {
   const locales = useLocales();
   const realDate = typeof date === "string" ? new Date(date) : date;
   const realPrevDate = previousDate
@@ -132,8 +138,8 @@ export const SmartDateTime = ({ date, previousDate = null, timeZone = "UTC" }: D
     : null;
 
   // Initial formatted values
-  const initialTimeOnly = formatTimeOnly(realDate, timeZone, locales);
-  const initialWithDate = formatSmartDateTime(realDate, timeZone, locales);
+  const initialTimeOnly = formatTimeOnly(realDate, timeZone, locales, hour12);
+  const initialWithDate = formatSmartDateTime(realDate, timeZone, locales, hour12);
 
   // State for the formatted time
   const [formattedDateTime, setFormattedDateTime] = useState<string>(
@@ -150,10 +156,10 @@ export const SmartDateTime = ({ date, previousDate = null, timeZone = "UTC" }: D
     // Format with appropriate function
     setFormattedDateTime(
       showDatePart
-        ? formatSmartDateTime(realDate, userTimeZone, locales)
-        : formatTimeOnly(realDate, userTimeZone, locales)
+        ? formatSmartDateTime(realDate, userTimeZone, locales, hour12)
+        : formatTimeOnly(realDate, userTimeZone, locales, hour12)
     );
-  }, [locales, realDate, realPrevDate]);
+  }, [locales, realDate, realPrevDate, hour12]);
 
   return <Fragment>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</Fragment>;
 };
@@ -168,7 +174,7 @@ function isSameDay(date1: Date, date2: Date): boolean {
 }
 
 // Format with date and time
-function formatSmartDateTime(date: Date, timeZone: string, locales: string[]): string {
+function formatSmartDateTime(date: Date, timeZone: string, locales: string[], hour12: boolean = true): string {
   return new Intl.DateTimeFormat(locales, {
     month: "short",
     day: "numeric",
@@ -178,18 +184,20 @@ function formatSmartDateTime(date: Date, timeZone: string, locales: string[]): s
     timeZone,
     // @ts-ignore fractionalSecondDigits works in most modern browsers
     fractionalSecondDigits: 3,
+    hour12,
   }).format(date);
 }
 
 // Format time only
-function formatTimeOnly(date: Date, timeZone: string, locales: string[]): string {
+function formatTimeOnly(date: Date, timeZone: string, locales: string[], hour12: boolean = true): string {
   return new Intl.DateTimeFormat(locales, {
-    hour: "numeric",
+    hour: "2-digit",
     minute: "numeric",
     second: "numeric",
     timeZone,
     // @ts-ignore fractionalSecondDigits works in most modern browsers
     fractionalSecondDigits: 3,
+    hour12,
   }).format(date);
 }
 
@@ -198,6 +206,8 @@ export const DateTimeAccurate = ({
   timeZone = "UTC",
   previousDate = null,
   showTooltip = true,
+  hideDate = false,
+  hour12 = true,
 }: DateTimeProps) => {
   const locales = useLocales();
   const [localTimeZone, setLocalTimeZone] = useState<string>("UTC");
@@ -214,11 +224,13 @@ export const DateTimeAccurate = ({
   }, []);
 
   // Smart formatting based on whether date changed
-  const formattedDateTime = realPrevDate
+  const formattedDateTime = hideDate
+    ? formatTimeOnly(realDate, localTimeZone, locales, hour12)
+    : realPrevDate
     ? isSameDay(realDate, realPrevDate)
-      ? formatTimeOnly(realDate, localTimeZone, locales)
-      : formatDateTimeAccurate(realDate, localTimeZone, locales)
-    : formatDateTimeAccurate(realDate, localTimeZone, locales);
+      ? formatTimeOnly(realDate, localTimeZone, locales, hour12)
+      : formatDateTimeAccurate(realDate, localTimeZone, locales, hour12)
+    : formatDateTimeAccurate(realDate, localTimeZone, locales, hour12);
 
   if (!showTooltip)
     return <Fragment>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</Fragment>;
@@ -241,7 +253,7 @@ export const DateTimeAccurate = ({
   );
 };
 
-function formatDateTimeAccurate(date: Date, timeZone: string, locales: string[]): string {
+function formatDateTimeAccurate(date: Date, timeZone: string, locales: string[], hour12: boolean = true): string {
   const formattedDateTime = new Intl.DateTimeFormat(locales, {
     month: "short",
     day: "numeric",
@@ -251,33 +263,35 @@ function formatDateTimeAccurate(date: Date, timeZone: string, locales: string[])
     timeZone,
     // @ts-ignore fractionalSecondDigits works in most modern browsers
     fractionalSecondDigits: 3,
+    hour12,
   }).format(date);
 
   return formattedDateTime;
 }
 
-export const DateTimeShort = ({ date, timeZone = "UTC" }: DateTimeProps) => {
+export const DateTimeShort = ({ date, timeZone = "UTC", hour12 = true }: DateTimeProps) => {
   const locales = useLocales();
   const realDate = typeof date === "string" ? new Date(date) : date;
-  const initialFormattedDateTime = formatDateTimeShort(realDate, timeZone, locales);
+  const initialFormattedDateTime = formatDateTimeShort(realDate, timeZone, locales, hour12);
   const [formattedDateTime, setFormattedDateTime] = useState<string>(initialFormattedDateTime);
 
   useEffect(() => {
     const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
-    setFormattedDateTime(formatDateTimeShort(realDate, resolvedOptions.timeZone, locales));
-  }, [locales, realDate]);
+    setFormattedDateTime(formatDateTimeShort(realDate, resolvedOptions.timeZone, locales, hour12));
+  }, [locales, realDate, hour12]);
 
   return <Fragment>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</Fragment>;
 };
 
-function formatDateTimeShort(date: Date, timeZone: string, locales: string[]): string {
+function formatDateTimeShort(date: Date, timeZone: string, locales: string[], hour12: boolean = true): string {
   const formattedDateTime = new Intl.DateTimeFormat(locales, {
     hour: "numeric",
     minute: "numeric",
     second: "numeric",
     timeZone,
     // @ts-ignore fractionalSecondDigits works in most modern browsers
     fractionalSecondDigits: 3,
+    hour12,
   }).format(date);
 
   return formattedDateTime;

apps/webapp/app/components/primitives/Paragraph.tsx

@@ -17,6 +17,10 @@ const paragraphVariants = {
     text: "font-sans text-sm font-normal text-text-bright",
     spacing: "mb-2",
   },
+  "small/dimmed": {
+    text: "font-sans text-sm font-normal text-text-dimmed",
+    spacing: "mb-2",
+  },
   "extra-small": {
     text: "font-sans text-xs font-normal text-text-dimmed",
     spacing: "mb-1.5",
@@ -25,6 +29,14 @@ const paragraphVariants = {
     text: "font-sans text-xs font-normal text-text-bright",
     spacing: "mb-1.5",
   },
+  "extra-small/dimmed": {
+    text: "font-sans text-xs font-normal text-text-dimmed",
+    spacing: "mb-1.5",
+  },
+  "extra-small/dimmed/mono": {
+    text: "font-mono text-xs font-normal text-text-dimmed",
+    spacing: "mb-1.5",
+  },
   "extra-small/mono": {
     text: "font-mono text-xs font-normal text-text-dimmed",
     spacing: "mb-1.5",

apps/webapp/app/env.server.ts

@@ -25,6 +25,27 @@ const GithubAppEnvSchema = z.preprocess(
   ])
 );
 
+// eventually we can make all S2 env vars required once the S2 OSS version is out
+const S2EnvSchema = z.preprocess(
+  (val) => {
+    const obj = val as any;
+    if (!obj || !obj.S2_ENABLED) {
+      return { ...obj, S2_ENABLED: "0" };
+    }
+    return obj;
+  },
+  z.discriminatedUnion("S2_ENABLED", [
+    z.object({
+      S2_ENABLED: z.literal("1"),
+      S2_ACCESS_TOKEN: z.string(),
+      S2_DEPLOYMENT_LOGS_BASIN_NAME: z.string(),
+    }),
+    z.object({
+      S2_ENABLED: z.literal("0"),
+    }),
+  ])
+);
+
 const EnvironmentSchema = z
   .object({
     NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]),
@@ -1202,7 +1223,8 @@ const EnvironmentSchema = z
 
     VERY_SLOW_QUERY_THRESHOLD_MS: z.coerce.number().int().optional(),
   })
-  .and(GithubAppEnvSchema);
+  .and(GithubAppEnvSchema)
+  .and(S2EnvSchema);
 
 export type Environment = z.infer<typeof EnvironmentSchema>;
 export const env = EnvironmentSchema.parse(process.env);

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -1,16 +1,34 @@
 import {
   DeploymentErrorData,
   ExternalBuildData,
+  logger,
   prepareDeploymentError,
 } from "@trigger.dev/core/v3";
-import { RuntimeEnvironment, type WorkerDeployment } from "@trigger.dev/database";
+import { type RuntimeEnvironment, type WorkerDeployment } from "@trigger.dev/database";
 import { type PrismaClient, prisma } from "~/db.server";
 import { type Organization } from "~/models/organization.server";
 import { type Project } from "~/models/project.server";
 import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
 import { type User } from "~/models/user.server";
 import { getUsername } from "~/utils/username";
 import { processGitMetadata } from "./BranchesPresenter.server";
+import { S2 } from "@s2-dev/streamstore";
+import { env } from "~/env.server";
+import { createRedisClient } from "~/redis.server";
+import { tryCatch } from "@trigger.dev/core";
+
+const S2_TOKEN_KEY_PREFIX = "s2-token:project:";
+
+const s2TokenRedis = createRedisClient("s2-token-cache", {
+  host: env.CACHE_REDIS_HOST,
+  port: env.CACHE_REDIS_PORT,
+  username: env.CACHE_REDIS_USERNAME,
+  password: env.CACHE_REDIS_PASSWORD,
+  tlsDisabled: env.CACHE_REDIS_TLS_DISABLED === "true",
+  clusterMode: env.CACHE_REDIS_CLUSTER_MODE_ENABLED === "1",
+});
+
+const s2 = env.S2_ENABLED === "1" ? new S2({ accessToken: env.S2_ACCESS_TOKEN }) : undefined;
 
 export type ErrorData = {
   name: string;
@@ -43,6 +61,7 @@ export class DeploymentPresenter {
       select: {
         id: true,
         organizationId: true,
+        externalRef: true,
       },
       where: {
         slug: projectSlug,
@@ -138,11 +157,29 @@ export class DeploymentPresenter {
       },
     });
 
+    const gitMetadata = processGitMetadata(deployment.git);
+
     const externalBuildData = deployment.externalBuildData
       ? ExternalBuildData.safeParse(deployment.externalBuildData)
       : undefined;
 
+    let s2Logs = undefined;
+    if (env.S2_ENABLED === "1" && gitMetadata?.source === "trigger_github_app") {
+      const [error, accessToken] = await tryCatch(this.getS2AccessToken(project.externalRef));
+
+      if (error) {
+        logger.error("Failed getting S2 access token", { error });
+      } else {
+        s2Logs = {
+          basin: env.S2_DEPLOYMENT_LOGS_BASIN_NAME,
+          stream: `projects/${project.externalRef}/deployments/${deployment.shortCode}`,
+          accessToken,
+        };
+      }
+    }
+
     return {
+      s2Logs,
       deployment: {
         id: deployment.id,
         shortCode: deployment.shortCode,
@@ -178,11 +215,46 @@ export class DeploymentPresenter {
         errorData: DeploymentPresenter.prepareErrorData(deployment.errorData),
         isBuilt: !!deployment.builtAt,
         type: deployment.type,
-        git: processGitMetadata(deployment.git),
+        git: gitMetadata,
       },
     };
   }
 
+  private async getS2AccessToken(projectRef: string): Promise<string> {
+    if (env.S2_ENABLED !== "1" || !s2) {
+      throw new Error("Failed getting S2 access token: S2 is not enabled");
+    }
+
+    const redisKey = `${S2_TOKEN_KEY_PREFIX}${projectRef}`;
+    const cachedToken = await s2TokenRedis.get(redisKey);
+
+    if (cachedToken) {
+      return cachedToken;
+    }
+
+    const { access_token: accessToken } = await s2.accessTokens.issue({
+      id: `${projectRef}-${new Date().getTime()}`,
+      expires_at: new Date(Date.now() + 60 * 60 * 1000).toISOString(), // 1 hour
+      scope: {
+        ops: ["read"],
+        basins: {
+          exact: env.S2_DEPLOYMENT_LOGS_BASIN_NAME,
+        },
+        streams: {
+          prefix: `projects/${projectRef}/deployments/`,
+        },
+      },
+    });
+
+    await s2TokenRedis.setex(
+      redisKey,
+      59 * 60, // slightly shorter than the token validity period
+      accessToken
+    );
+
+    return accessToken;
+  }
+
   public static prepareErrorData(errorData: WorkerDeployment["errorData"]): ErrorData | undefined {
     if (!errorData) {
       return;