feat: Enable custom cron schedule (#2809)

Resolves #2786
Fixes #2796

## Changes
- Enables setting and viewing custom cron schedules.
- Custom cron option with support for cron macros @hourly, @daily, @Weekly, @monthly, @Yearly
- Updated documentation for crontab, with links to crontab.guru

Author: SuaYoo

chart/app-templates/crawl_cron_job.yaml

@@ -30,5 +30,5 @@ spec:
           restartPolicy: Never
           containers:
             - name: noop
-              image: "docker.io/tianon/true"
+              image: "docker.io/tianon/true:multiarch"
               imagePullPolicy: IfNotPresent

frontend/docs/docs/user-guide/workflow-setup.md

@@ -318,9 +318,6 @@ Automatically start crawls periodically on a daily, weekly, or monthly schedule.
 
 ### Crawl Schedule Type
 
-#### Run Immediately on Save
-:   When selected, the crawl will run immediately as configured. It will not run again unless manually instructed.
-
 #### Run on a Recurring Basis
 :   When selected, additional configuration options for instructing the system when to run the crawl will be shown. If a crawl is already running when the schedule is set to activate it, the scheduled crawl will not run.
 
@@ -331,6 +328,26 @@ Automatically start crawls periodically on a daily, weekly, or monthly schedule.
 
 Set how often a scheduled crawl will run.
 
+#### Options
+
+All option support specifying the specific hour and minute the crawl should run.
+
+##### Daily
+
+Run crawl once every day.
+
+##### Weekly
+
+Run crawl once every week.
+
+##### Monthly
+
+Run crawl once every month.
+
+##### Custom
+
+Run crawl at a custom interval, such as hourly or yearly. See [Cron Schedule](#cron-schedule) for details.
+
 ### Day
 
 Sets the day of the week for which crawls scheduled with a `Weekly` _Frequency_ will run.
@@ -343,6 +360,33 @@ Sets the date of the month for which crawls scheduled with a `Monthly` _Frequenc
 
 Sets the time that the scheduled crawl will start according to your current timezone.
 
+### Cron Schedule
+
+When using a `Custom` _Frequency_, a custom schedule can be specified by using a Cron expression or supported macros.
+
+Cron expressions should follow the Unix Cron format:
+
+| Position | * | * | * | * | * |
+| - | - | - | - | - | - |
+| **Description** | minute | hour | day of the month | month | day of the week |
+| **Possible Values** | 0 - 59 | 0 - 23 | 1 - 31 | 1 - 12 | 0 - 6<br/>or `sun`, `mon`, `tue`, `wed`, `thu`, `fri`, `sat` |
+
+For example, `0 0 31 12 *` would run a crawl on December 31st every year and `0 0 * * fri` would run a crawl every Friday at midnight.
+
+Additionally, the following macros are supported:
+
+| Value | Description |
+| - | - |
+| `@yearly` | Run once a year at midnight of 1 January |
+| `@monthly` | 	Run once a month at midnight of the first day of the month |
+| `@weekly` | Run once a week at midnight on Sunday |
+| `@daily` | Run once a day at midnight |
+| `@hourly` | Run once an hour at the beginning of the hour |
+
+You can use a tool like [crontab.guru](https://crontab.guru/) to check Cron syntax validity and view [common expressions](https://crontab.guru/examples.html).
+
+Cron schedules are always in [UTC](https://en.wikipedia.org/wiki/Coordinated_Universal_Time).
+
 ## Metadata
 
 Describe and organize your crawl workflow and the resulting archived items.

frontend/package.json

@@ -35,6 +35,7 @@
     "color": "^4.0.1",
     "construct-style-sheets-polyfill": "^3.1.0",
     "copy-webpack-plugin": "^12.0.2",
+    "cronstrue": "^3.2.0",
     "css-loader": "^6.3.0",
     "css-selector-parser": "^3.0.5",
     "date-fns": "^3.6.0",

frontend/src/features/crawl-workflows/workflow-editor.ts

@@ -2,6 +2,7 @@ import { consume } from "@lit/context";
 import { localized, msg, str } from "@lit/localize";
 import type {
   SlBlurEvent,
+  SlChangeEvent,
   SlCheckbox,
   SlDetails,
   SlHideEvent,
@@ -105,6 +106,7 @@ import {
   getUTCSchedule,
   humanizeNextDate,
   humanizeSchedule,
+  validateCron,
 } from "@/utils/cron";
 import { makeCurrentTargetHandler, stopProp } from "@/utils/events";
 import { formValidator, maxLengthValidator } from "@/utils/form";
@@ -267,7 +269,12 @@ export class WorkflowEditor extends BtrixElement {
   @property({ type: String })
   initialScopeType?: FormState["scopeType"];
 
-  @property({ type: Object })
+  @property({
+    type: Object,
+    // Fixes values being reset on navigation events,
+    // such as when the user guide is open
+    hasChanged: (a, b) => !isEqual(a, b),
+  })
   initialWorkflow?: WorkflowParams;
 
   private updatingScopeType = false;
@@ -344,7 +351,7 @@ export class WorkflowEditor extends BtrixElement {
 
   private get utcSchedule() {
     if (!this.formState.scheduleFrequency) {
-      return "";
+      return this.formState.scheduleCustom;
     }
     return getUTCSchedule({
       interval: this.formState.scheduleFrequency,
@@ -2047,19 +2054,48 @@ https://archiveweb.page/images/${"logo.svg"}`}
   }
 
   private readonly renderScheduleCron = () => {
-    const utcSchedule = this.utcSchedule;
+    const scheduledDate = (schedule?: string, opts?: { utc: boolean }) => {
+      if (!schedule) return nothing;
+
+      const humanized = humanizeSchedule(schedule);
+
+      if (!humanized) return nothing;
+
+      return html`
+        <div class="mt-3 text-xs text-neutral-500">
+          <p class="mb-1">
+            ${msg("Schedule:")}
+            <span class="text-blue-500">${humanized}</span>
+          </p>
+          <p>
+            ${msg("Next scheduled run:")}
+            <span>${humanizeNextDate(schedule, opts)}</span>
+          </p>
+        </div>
+      `;
+    };
+
+    const hourly_macro_code = html`<code>@hourly</code>`;
+    const yearly_macro_code = html`<code>@yearly</code>`;
+
     return html`
       ${this.renderSectionHeading(msg("Set Schedule"))}
       ${inputCol(html`
         <sl-select
           name="scheduleFrequency"
           label=${msg("Frequency")}
           value=${this.formState.scheduleFrequency}
-          @sl-change=${(e: Event) =>
+          @sl-change=${(e: Event) => {
+            const scheduleFrequency = (e.target as HTMLSelectElement)
+              .value as FormState["scheduleFrequency"];
+
             this.updateFormState({
-              scheduleFrequency: (e.target as HTMLSelectElement)
-                .value as FormState["scheduleFrequency"],
-            })}
+              scheduleFrequency,
+              scheduleCustom: scheduleFrequency
+                ? ""
+                : this.formState.scheduleCustom || "",
+            });
+          }}
         >
           <sl-option value="daily"
             >${this.scheduleFrequencyLabels["daily"]}</sl-option
@@ -2070,6 +2106,8 @@ https://archiveweb.page/images/${"logo.svg"}`}
           <sl-option value="monthly"
             >${this.scheduleFrequencyLabels["monthly"]}</sl-option
           >
+          <sl-divider></sl-divider>
+          <sl-option value="">${msg("Custom")}</sl-option>
         </sl-select>
       `)}
       ${this.renderHelpTextCol(
@@ -2122,44 +2160,72 @@ https://archiveweb.page/images/${"logo.svg"}`}
           )}
         `,
       )}
-      ${inputCol(html`
-        <btrix-time-input
-          hour=${ifDefined(this.formState.scheduleTime?.hour)}
-          minute=${ifDefined(this.formState.scheduleTime?.minute)}
-          period=${ifDefined(this.formState.scheduleTime?.period)}
-          @time-change=${(e: TimeInputChangeEvent) => {
-            this.updateFormState({
-              scheduleTime: e.detail,
-            });
-          }}
-        >
-          <span slot="label">${msg("Start Time")}</span>
-        </btrix-time-input>
-        <div class="mt-3 text-xs text-neutral-500">
-          <p class="mb-1">
-            ${msg(
-              html`Schedule:
-                <span class="text-blue-500"
-                  >${utcSchedule
-                    ? humanizeSchedule(utcSchedule)
-                    : msg("Invalid date")}</span
-                >.`,
-            )}
-          </p>
-          <p>
+      ${when(
+        this.formState.scheduleFrequency,
+        () =>
+          html`${inputCol(html`
+            <btrix-time-input
+              hour=${ifDefined(this.formState.scheduleTime?.hour)}
+              minute=${ifDefined(this.formState.scheduleTime?.minute)}
+              period=${ifDefined(this.formState.scheduleTime?.period)}
+              @time-change=${(e: TimeInputChangeEvent) => {
+                this.updateFormState({
+                  scheduleTime: e.detail,
+                });
+              }}
+            >
+              <span slot="label">${msg("Start Time")}</span>
+            </btrix-time-input>
+            ${scheduledDate(this.utcSchedule)}
+          `)}
+          ${this.renderHelpTextCol(
+            msg(`A crawl will run at this time in your current timezone.`),
+          )}`,
+        () => html`
+          ${inputCol(html`
+            <sl-input
+              name="scheduleCustom"
+              label=${msg("Cron Schedule")}
+              class="part-[input]:font-mono"
+              placeholder="@hourly"
+              value=${ifDefined(this.formState.scheduleCustom)}
+              minlength="6"
+              @sl-change=${(e: SlChangeEvent) => {
+                const input = e.target as SlInput;
+                const value = (e.target as SlInput).value;
+
+                if (!value) return;
+
+                const { valid, error } = validateCron(value);
+
+                if (valid) {
+                  input.helpText = "";
+                  input.setCustomValidity("");
+                } else {
+                  const errorMessage =
+                    error ?? msg("Please fix invalid Cron expression syntax.");
+
+                  input.helpText = errorMessage;
+                  input.setCustomValidity(errorMessage);
+                }
+              }}
+              required
+            >
+            </sl-input>
+            ${scheduledDate(this.formState.scheduleCustom)}
+          `)}
+          ${this.renderHelpTextCol(html`
+            ${msg("Specify a schedule in Cron format.")}
             ${msg(
-              html`Next scheduled run:
-                <span
-                  >${utcSchedule
-                    ? humanizeNextDate(utcSchedule)
-                    : msg("Invalid date")}</span
-                >.`,
+              html`Supports Unix cron syntax and certain macros like
+              ${hourly_macro_code} and ${yearly_macro_code}.`,
             )}
-          </p>
-        </div>
-      `)}
-      ${this.renderHelpTextCol(
-        msg(`A crawl will run at this time in your current timezone.`),
+            ${this.renderUserGuideLink({
+              hash: "cron-schedule",
+              content: msg("More details"),
+            })}
+          `)}
+        `,
       )}
     `;
   };
@@ -3017,7 +3083,12 @@ https://archiveweb.page/images/${"logo.svg"}`}
       description: this.formState.description,
       browserWindows: this.formState.browserWindows,
       profileid: this.formState.browserProfile?.id || "",
-      schedule: this.formState.scheduleType === "cron" ? this.utcSchedule : "",
+      schedule:
+        this.formState.scheduleType === "none"
+          ? ""
+          : (this.formState.scheduleFrequency
+              ? this.utcSchedule
+              : this.formState.scheduleCustom) || "",
       crawlTimeout: this.formState.crawlTimeoutMinutes * 60,
       maxCrawlSize: this.formState.maxCrawlSizeGB * BYTES_PER_GB,
       tags: this.formState.tags,

frontend/src/utils/cron.test.ts

@@ -0,0 +1,72 @@
+import { expect } from "@open-wc/testing";
+
+import { getScheduleInterval, humanizeSchedule } from "./cron";
+
+describe("cron utils", () => {
+  describe("getScheduleInterval()", () => {
+    it("handles daily intervals", () => {
+      expect(getScheduleInterval("1 1 * * *")).to.equal("daily");
+    });
+
+    it("handles weekly intervals", () => {
+      expect(getScheduleInterval("1 1 * * 1")).to.equal("weekly");
+      expect(getScheduleInterval("1 1 * * FRI")).to.equal("weekly");
+    });
+
+    it("handles monthly intervals", () => {
+      expect(getScheduleInterval("1 1 1 * *")).to.equal("monthly");
+    });
+
+    it("returns null if not daily, weekly, or monthly", () => {
+      // Every minute:
+      expect(getScheduleInterval("* * * * *")).to.equal(null);
+      expect(getScheduleInterval("* 1 * * *")).to.equal(null);
+      expect(getScheduleInterval("* * 1 * *")).to.equal(null);
+      expect(getScheduleInterval("* * * 1 *")).to.equal(null);
+      expect(getScheduleInterval("* * * * 1")).to.equal(null);
+      expect(getScheduleInterval("*/5 * * * *")).to.equal(null);
+      // Hourly:
+      expect(getScheduleInterval("1 * * * *")).to.equal(null);
+      expect(getScheduleInterval("0 */5 * * *")).to.equal(null);
+      // Yearly:
+      expect(getScheduleInterval("1 1 1 JAN *")).to.equal(null);
+      expect(getScheduleInterval("1 1 1 1 *")).to.equal(null);
+      expect(getScheduleInterval("1 1 1 1 1")).to.equal(null);
+    });
+
+    it("returns null for macros", () => {
+      expect(getScheduleInterval("@yearly")).to.equal(null);
+      expect(getScheduleInterval("@monthly")).to.equal(null);
+      expect(getScheduleInterval("@weekly")).to.equal(null);
+      expect(getScheduleInterval("@daily")).to.equal(null);
+      expect(getScheduleInterval("@hourly")).to.equal(null);
+    });
+  });
+
+  describe("humanizeSchedule()", () => {
+    it("humanizes daily schedule", () => {
+      expect(humanizeSchedule("30 1 * * *")).to.equal(
+        "Every day at 7:30 PM GMT-6",
+      );
+    });
+
+    it("humanizes weekly schedule", () => {
+      expect(humanizeSchedule("30 1 * * 1")).to.equal(
+        "Every Sunday at 8:30 PM GMT-5",
+      );
+    });
+
+    it("humanizes monthly schedule", () => {
+      expect(humanizeSchedule("30 1 1 * *")).to.equal(
+        "On day 30 of the month at 8:30 PM GMT-5",
+      );
+    });
+
+    it("humanizes schedule without a known interval", () => {
+      expect(humanizeSchedule("* * * * *")).to.equal("Every minute (UTC)");
+      expect(humanizeSchedule("30 * 1 * *")).to.equal(
+        "At 30 minutes past the hour, on day 1 of the month (UTC)",
+      );
+    });
+  });
+});