From 403d37789b16effc2577c8d9c6c2508c9743d140 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 7 Nov 2024 13:16:46 +0800 Subject: [PATCH 01/29] Document the job header and its contents --- dashboard/dashboard.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index a0304387..28187761 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -417,6 +417,9 @@

This page shows all of the crawls that ArchiveBot is currently running.

+

+ Each job has a header with the URL, note, request/queue stats, options and an identifier. +

To show or hide a job, click anywhere on its stats line. From 51cd24db5dc7a9732bda29e496fbdd0089335b4b Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:09:59 +0800 Subject: [PATCH 02/29] Refer to the job header rather than stats line --- dashboard/dashboard.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 28187761..077585bc 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -421,9 +421,9 @@ Each job has a header with the URL, note, request/queue stats, options and an identifier.

- To show or hide a job, click anywhere on its stats line. + To show or hide a job, click anywhere on its header. - The color coding for the job stats line is: + The color coding for the job header is: in progress, finished normally, finished with abort, From 9efff36bd65ed2675e69d9033109c10b8ecafc64 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:11:29 +0800 Subject: [PATCH 03/29] Clarify that the filter operates on job logs and is a regex --- dashboard/dashboard.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 077585bc..31bf1fbf 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -398,7 +398,7 @@ ArchiveBot tracking 0 crawls. See also pipeline or job reports. - Show: + Logs @@ -442,12 +442,12 @@ Keyboard shortcuts:

From 8208ce1080ec7d822c0545ec443999d1b0ff10fe Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:50:27 +0800 Subject: [PATCH 04/29] Document the title URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 31bf1fbf..83593486 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -467,6 +467,7 @@

  • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
  • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
    • +
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From 164f75ad0581e43e0a0d0f41affb128c70272098 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:50:52 +0800 Subject: [PATCH 05/29] Document the host URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 83593486..9516842c 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -468,6 +468,7 @@

  • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
    • +
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From 7cbec614f8472881e6985bf73142b39a3412855a Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:51:20 +0800 Subject: [PATCH 06/29] Document the dumpMax URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 9516842c..42585ff6 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -469,6 +469,7 @@

  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
    • +
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From 49a514a678e6693bb05c7bd2ec30a871417890c7 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:51:34 +0800 Subject: [PATCH 07/29] Document the debug URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 42585ff6..c562c306 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -470,6 +470,7 @@

  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
    • +
  • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From b023aecfc54ac88d817834ab667dd466250d2c05 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:23:53 +0800 Subject: [PATCH 08/29] Add job notes to job URL title Makes it easier to access the notes when not in aligned mode. --- dashboard/assets/scripts/dashboard.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index f570f438..86d020df 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -225,10 +225,11 @@ class JobsTracker { } class JobRenderInfo { - constructor(logWindow, logSegment, statsElements, jobNote, lineCountWindow, lineCountSegments) { + constructor(logWindow, logSegment, statsElements, jobUrl, jobNote, lineCountWindow, lineCountSegments) { this.logWindow = logWindow; this.logSegment = logSegment; this.statsElements = statsElements; + this.jobUrl = jobUrl; this.jobNote = jobNote; this.lineCountWindow = lineCountWindow; this.lineCountSegments = lineCountSegments; @@ -411,13 +412,14 @@ class JobsRenderer { ], ), ]); + const jobUrl = statsElements.jobInfo.querySelector(".job-url"); const logWindow = h("div", logWindowAttrs, logSegment); const div = h("div", { className: "log-container", id: `log-container-${ident}` }, [ h("div", { className: "job-header" }, [statsElements.jobInfo, h("span", { className: "job-ident" }, ident)]), logWindow, ]); - this.renderInfo[ident] = new JobRenderInfo(logWindow, logSegment, statsElements, jobNote, 0, [0]); + this.renderInfo[ident] = new JobRenderInfo(logWindow, logSegment, statsElements, jobUrl, jobNote, 0, [0]); this.container.insertBefore(div, beforeElement); // Filter hasn't changed, but we might need to filter out the new job, or // add/remove log-window-expanded class @@ -580,6 +582,11 @@ class JobsRenderer { // Update note info.jobNote.textContent = isBlank(jobData.note) ? "" : ` (${jobData.note})`; + if (isBlank(jobData.note)) { + info.jobUrl.removeAttribute("title"); + } else { + info.jobUrl.title = jobData.note; + } info.lineCountWindow += linesRendered; info.lineCountSegments[info.lineCountSegments.length - 1] += linesRendered; From b8391361c86c2ef8f5be150bc6acbc9296b2932a Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:48:57 +0800 Subject: [PATCH 09/29] Document the mouseover info for the URL and queue count --- dashboard/dashboard.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index c562c306..55eb8389 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -430,7 +430,7 @@ finished with fatal exception.

    - Mouse over the job start date or the response count for additional information. + Mouse over the job URL, start date, response count or the queue count for additional information.

    To pause scrolling, move your mouse inside a log window. From 77747d8d80be1e3a8d27c2681b2a837e8a2b61a6 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Wed, 16 Oct 2024 13:02:38 +0800 Subject: [PATCH 10/29] Compile the filter regex only once when filtering jobs Previously it was compiled once for each job being filtered. Suggested-by: JustAnotherArchivist Fixes: commit 990b70d2d62b6528e1762b91e8baf6fbc0066807 --- dashboard/assets/scripts/dashboard.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 86d020df..0cdbe656 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -628,14 +628,14 @@ class JobsRenderer { } applyFilter() { - const query = this.filterBox.value; + const query = RegExp(this.filterBox.value); let matches = 0; const matchedWindows = []; const unmatchedWindows = []; this.firstFilterMatch = null; for (const job of this.jobs.sorted) { const w = this.renderInfo[job.ident].logWindow; - if (!RegExp(query).test(job.url)) { + if (!query.test(job.url)) { w.classList.add("log-window-hidden"); unmatchedWindows.push(w); From 6374329f4bacc3b1540356745db026862db1b9a8 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:13:28 +0800 Subject: [PATCH 11/29] Add identifiers for the filter modification buttons --- dashboard/dashboard.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 55eb8389..aafe75bc 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -399,8 +399,8 @@ tracking 0 crawls. See also pipeline or job reports. Logs - - + +

    😊
    From 25b8c0f736ad2f4dda7de3ca2d56de9459bffd9f Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:21:58 +0800 Subject: [PATCH 12/29] Add a button and key to revert to the initial filter When the initial filter is your nick, this makes it easier to switch between monitoring only your personal jobs and monitoring all jobs or other jobs. --- dashboard/assets/scripts/dashboard.js | 18 ++++++++++++++++-- dashboard/dashboard.html | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 0cdbe656..f508402b 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1027,7 +1027,7 @@ class Dashboard { const batchMaxItems = args.batchMaxItems ? Number(args.batchMaxItems) : 250; const showNicks = args.showNicks ? Boolean(Number(args.showNicks)) : false; const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; - const initialFilter = args.initialFilter ?? "^$"; + this.initialFilter = args.initialFilter ?? "^$"; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1076,7 +1076,19 @@ class Dashboard { addPageStyles(".job-nick-aligned { width: 0; }"); } - this.setFilter(initialFilter); + if (args.initialFilter != null) { + byId("set-filter-none").after( + h("input", { + className: "button", + type: "button", + id: "set-filter-initial", + onclick: () => { ds.setFilter(ds.initialFilter) }, + value: "Initial", + }) + ); + byId("set-filter-none").after("\n"); + } + this.setFilter(this.initialFilter); const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1198,6 +1210,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ev.preventDefault(); byId("filter-box").focus(); byId("filter-box").select(); + } else if (ev.which === 105 /* i */) { + ds.setFilter(ds.initialFilter); } else if (ev.which === 118 /* v */) { window.open(this.jobsRenderer.firstFilterMatch.url); } diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index aafe75bc..a9592db0 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -447,6 +447,7 @@
  • a - show all job log
  • n - hide all job log
  • f - move focus to filter box +
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log
  • ? - show/hide help text From cc9975f6d74fad20090a796991a82481deded689 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:31:37 +0800 Subject: [PATCH 13/29] Allow hiding the job headers for hidden job logs This saves a lot of space when a lot of job logs are hidden. --- dashboard/assets/scripts/dashboard.js | 10 ++++++++++ dashboard/dashboard.html | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index f508402b..4d11f0bd 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1028,6 +1028,7 @@ class Dashboard { const showNicks = args.showNicks ? Boolean(Number(args.showNicks)) : false; const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; this.initialFilter = args.initialFilter ?? "^$"; + const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1090,6 +1091,8 @@ class Dashboard { } this.setFilter(this.initialFilter); + this.showAllHeaders(showAllHeaders); + const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1214,6 +1217,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ds.setFilter(ds.initialFilter); } else if (ev.which === 118 /* v */) { window.open(this.jobsRenderer.firstFilterMatch.url); + } else if (ev.which === 104 /* h */) { + ds.showAllHeaders(!byId("show-all-headers").checked); } } @@ -1270,6 +1275,11 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId("filter-box").value = value; byId("filter-box").onchange(); } + + showAllHeaders(value) { + byId('show-all-headers').checked = value; + byId('hide-headers').sheet.disabled = value; + } } const ds = new Dashboard(); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index a9592db0..5ff08c1b 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -391,6 +391,12 @@ top: 0; } +
    @@ -401,6 +407,8 @@ Logs + +
    😊
    @@ -449,6 +457,7 @@
  • f - move focus to filter box
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log +
  • h - show/hide headers for hidden job logs
  • ? - show/hide help text

    @@ -465,6 +474,7 @@

    • To specify an initial filter, add ?initialFilter=TEXT to the dashboard URL. The default is ^$.
      • +
    • To initially hide headers for hidden job logs, add ?showAllHeaders=0 to the dashboard URL. The default is to show them.
    • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
    • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
      • From 15058ffb142f57fd43d7ae5e7b8b524b4be68987 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:33:01 +0800 Subject: [PATCH 14/29] Add an option to specify the port for the recent logs Allows using archivebot-dashboard-repeater on another port on localhost. See-also: https://github.com/iakat/archivebot-dashboard-repeater --- dashboard/assets/scripts/dashboard.js | 4 +++- dashboard/dashboard.html | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 4d11f0bd..037ed60e 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1038,6 +1038,8 @@ class Dashboard { } this.host = args.host ? args.host : location.hostname; + this.port = args.port ? `:${Number(args.port)}` : ''; + this.dumpTraffic = args.dumpMax && Number(args.dumpMax) > 0; if (this.dumpTraffic) { this.dumpMax = Number(args.dumpMax); @@ -1180,7 +1182,7 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; const size_mb = Math.round((100 * ev.total) / 1e6) / 100; byId("meta-info").textContent = `Recent data: ${percent}% (${size_mb}MB)`; }; - xhr.open("GET", `//${this.host}/logs/recent?cb=${Date.now()}${Math.random()}`); + xhr.open("GET", `//${this.host}${this.port}/logs/recent?cb=${Date.now()}${Math.random()}`); xhr.setRequestHeader("Accept", "application/json"); xhr.send(""); }); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 5ff08c1b..58bcb2f4 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -480,6 +480,7 @@
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
    • To append some text to the page title, add ?title=TEXT to the dashboard URL.
    • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
      • +
    • To override the port of the recent logs server, add ?port=TEXT to the dashboard URL.
    • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
    • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.
    From 5a105d37b24592e28dde99f7dcdf6e902c58469e Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:54:36 +0800 Subject: [PATCH 15/29] Add an option to specify the URL for the logs stream Allows using archivebot-dashboard-repeater on localhost. See-also: https://github.com/iakat/archivebot-dashboard-repeater --- dashboard/assets/scripts/dashboard.js | 6 +++--- dashboard/dashboard.html | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 037ed60e..3f5fdee5 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1039,6 +1039,8 @@ class Dashboard { this.host = args.host ? args.host : location.hostname; this.port = args.port ? `:${Number(args.port)}` : ''; + const wsproto = window.location.protocol === "https:" ? "wss:" : "ws:"; + this.websocketUrl = args.websocketUrl ?? `${wsproto}//${this.host}:4568/stream`; this.dumpTraffic = args.dumpMax && Number(args.dumpMax) > 0; if (this.dumpTraffic) { @@ -1233,9 +1235,7 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; } connectWebSocket() { - const wsproto = window.location.protocol === "https:" ? "wss:" : "ws:"; - - this.ws = new WebSocket(`${wsproto}//${this.host}:4568/stream`); + this.ws = new WebSocket(this.websocketUrl); this.ws.onmessage = (ev) => { this.newItemsReceived += 1; diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 58bcb2f4..adfcca52 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -481,6 +481,7 @@
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
  • To override the port of the recent logs server, add ?port=TEXT to the dashboard URL.
    • +
  • To override the URL of the logs stream, add ?websocketUrl=ws://localhost:4568/stream to the dashboard URL.
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
  • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.
    • From 709789ebf0b424a2721254499e9f71163e99dec9 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:28:02 +0800 Subject: [PATCH 16/29] Add a slight delay when typing into the job log filter Avoids applying the filter multiple times while typing, since that can be too slow and block the UI sometimes. --- dashboard/assets/scripts/dashboard.js | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 3f5fdee5..18197f38 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -122,14 +122,6 @@ function regExpEscape(s) { return escaped; } -function addAnyChangeListener(elem, func) { - // DOM0 handler for convenient use by Clear button - elem.onchange = func; - elem.addEventListener("keydown", func, false); - elem.addEventListener("paste", func, false); - elem.addEventListener("input", func, false); -} - function scrollToBottom(elem) { // Scroll to the bottom. To avoid serious performance problems in Firefox, // use a big number instead of elem.scrollHeight. @@ -284,7 +276,25 @@ class JobsRenderer { constructor(container, filterBox, historyLines, showNicks, contextMenuRenderer) { this.container = container; this.filterBox = filterBox; - addAnyChangeListener(this.filterBox, () => this.applyFilter()); + this.filterTimeout = null; + this.filterBox.onchange = (e) => { + const repeats = [ + "insertText", + "deleteContent", + "deleteContentForward", + "deleteContentBackward", + ]; + let ms = e && e.inputType && repeats.includes(e.inputType) ? 100 : 0; + ms = !this.filterBox.value ? 0 : ms; + clearTimeout(this.filterTimeout); + this.filterTimeout = setTimeout(() => { + if (this.filterBox.value !== this.filterBox.old) { + this.applyFilter(); + this.filterBox.old = this.filterBox.value; + } + }, ms); + }; + this.filterBox.oninput = this.filterBox.onchange; this.filterBox.onkeypress = (ev) => { // Don't let `j` or `k` in the filter box cause the job window to switch ev.stopPropagation(); From 83712778d13cb4ffe9058ec95e20ca7d89dc9146 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:42:52 +0800 Subject: [PATCH 17/29] Allow hiding jobs based on their status Hiding finished jobs can save space when there are a lot of them. --- dashboard/assets/scripts/dashboard.js | 37 ++++++++++++++++++ dashboard/dashboard.html | 54 +++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 18197f38..ec45ff7e 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1039,6 +1039,10 @@ class Dashboard { const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; this.initialFilter = args.initialFilter ?? "^$"; const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; + const showRunningJobs = args.showRunningJobs ? Boolean(Number(args.showRunningJobs)) : true; + const showFinishedJobs = args.showFinishedJobs ? Boolean(Number(args.showFinishedJobs)) : true; + const showFatalJobs = args.showFatalJobs ? Boolean(Number(args.showFatalJobs)) : true; + const showAbortedJobs = args.showAbortedJobs ? Boolean(Number(args.showAbortedJobs)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1107,6 +1111,11 @@ class Dashboard { this.showAllHeaders(showAllHeaders); + this.showRunningJobs(showRunningJobs); + this.showFinishedJobs(showFinishedJobs); + this.showFatalJobs(showFatalJobs); + this.showAbortedJobs(showAbortedJobs); + const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1233,6 +1242,14 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; window.open(this.jobsRenderer.firstFilterMatch.url); } else if (ev.which === 104 /* h */) { ds.showAllHeaders(!byId("show-all-headers").checked); + } else if (ev.which === 114 /* r */) { + ds.showRunningJobs(!byId("show-running-jobs").checked); + } else if (ev.which === 100 /* d */) { + ds.showFinishedJobs(!byId("show-finished-jobs").checked); + } else if (ev.which === 99 /* c */) { + ds.showFatalJobs(!byId("show-fatal-jobs").checked); + } else if (ev.which === 115 /* s */) { + ds.showAbortedJobs(!byId("show-aborted-jobs").checked); } } @@ -1292,6 +1309,26 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId('show-all-headers').checked = value; byId('hide-headers').sheet.disabled = value; } + + showRunningJobs(value) { + byId('show-running-jobs').checked = value; + byId('hide-running').sheet.disabled = value; + } + + showFinishedJobs(value) { + byId('show-finished-jobs').checked = value; + byId('hide-done').sheet.disabled = value; + } + + showFatalJobs(value) { + byId('show-fatal-jobs').checked = value; + byId('hide-fatal').sheet.disabled = value; + } + + showAbortedJobs(value) { + byId('show-aborted-jobs').checked = value; + byId('hide-aborted').sheet.disabled = value; + } } const ds = new Dashboard(); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index adfcca52..e6ec18a6 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -57,6 +57,11 @@ font-size: 18px; } +.drop-down { + display: inline flex; + flex-direction: column; +} + .padded-page { padding: 20px 27px 20px 27px; } @@ -397,6 +402,34 @@ content-visibility: hidden; } + + + +
    @@ -409,6 +442,17 @@ +
    + Jobs + +
    + +
    + +
    + +
    +
    😊
    @@ -458,6 +502,10 @@
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log
  • h - show/hide headers for hidden job logs +
  • r - show/hide header+log for running jobs +
  • d - show/hide header+log for finished jobs +
  • c - show/hide header+log for fatal jobs +
  • s - show/hide header+log for aborted jobs
  • ? - show/hide help text

    @@ -475,6 +523,12 @@

    • To specify an initial filter, add ?initialFilter=TEXT to the dashboard URL. The default is ^$.
    • To initially hide headers for hidden job logs, add ?showAllHeaders=0 to the dashboard URL. The default is to show them.
      • +
    • To initially hide different job types, add these to the dashboard URL. The default is to show them. + ?showRunningJobs=0 + ?showFinishedJobs=0 + ?showFatalJobs=0 + ?showAbortedJobs=0 +
    • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
    • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
      • From 1d64338a9f8101a2cd1b27fd977496d3b6689321 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 16:00:37 +0800 Subject: [PATCH 18/29] Fix detection of finished jobs Fixes: commit e179725059e7752f6d97a5e72f4556256e78fa12 --- dashboard/assets/scripts/dashboard.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index ec45ff7e..45509c4c 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -492,10 +492,12 @@ class JobsRenderer { logSegment.appendChild(h("div", Reusable.obj_className_line_stdout, line)); renderedLines += 1; - // Check for 'Finished RsyncUpload for Item' - // instead of 'Starting MarkItemAsDone for Item' - // because the latter is often missing - if (/^Finished RsyncUpload for Item/.test(line)) { + // Check for several completion messages + // because some of them are often missing + // Ignore error jobs as they get done messages. + if (!info.statsElements.jobInfo.classList.contains("job-info-fatal") && + !info.statsElements.jobInfo.classList.contains("job-info-aborted") && + /^ *[1-9][0-9]* bytes\.$|^Starting (RelabelIfAborted|MarkItemAsDone) for Item$|^Finished (WgetDownload|MoveFiles|StopHeartbeat) for Item$/.test(line)) { info.statsElements.jobInfo.classList.add("job-info-done"); this.jobs.markFinished(ident); } else if ( From ac8a80f59747280674228d0614d39698ea6abd61 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:47:53 +0800 Subject: [PATCH 19/29] Detect jobs that failed and allow filtering them out --- dashboard/assets/scripts/dashboard.js | 13 +++++++++++++ dashboard/dashboard.html | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 45509c4c..a9ceed08 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -497,9 +497,12 @@ class JobsRenderer { // Ignore error jobs as they get done messages. if (!info.statsElements.jobInfo.classList.contains("job-info-fatal") && !info.statsElements.jobInfo.classList.contains("job-info-aborted") && + !info.statsElements.jobInfo.classList.contains("job-info-failed") && /^ *[1-9][0-9]* bytes\.$|^Starting (RelabelIfAborted|MarkItemAsDone) for Item$|^Finished (WgetDownload|MoveFiles|StopHeartbeat) for Item$/.test(line)) { info.statsElements.jobInfo.classList.add("job-info-done"); this.jobs.markFinished(ident); + } else if (/^ *0 bytes\.$/.test(line)) { + info.statsElements.jobInfo.classList.add("job-info-failed"); } else if ( /^CRITICAL (Sorry|Please report)|^ERROR Fatal exception|No space left on device|^Fatal Python error:|^(Thread|Current thread) 0x/.test( line, @@ -520,6 +523,7 @@ class JobsRenderer { } else if (/^Received item /.test(line)) { // Clear other statuses if a job restarts with the same job ID info.statsElements.jobInfo.classList.remove("job-info-done"); + info.statsElements.jobInfo.classList.remove("job-info-failed"); info.statsElements.jobInfo.classList.remove("job-info-fatal"); info.statsElements.jobInfo.classList.remove("job-info-aborted"); this.jobs.markUnfinished(ident); @@ -1043,6 +1047,7 @@ class Dashboard { const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; const showRunningJobs = args.showRunningJobs ? Boolean(Number(args.showRunningJobs)) : true; const showFinishedJobs = args.showFinishedJobs ? Boolean(Number(args.showFinishedJobs)) : true; + const showFailedJobs = args.showFailedJobs ? Boolean(Number(args.showFailedJobs)) : true; const showFatalJobs = args.showFatalJobs ? Boolean(Number(args.showFatalJobs)) : true; const showAbortedJobs = args.showAbortedJobs ? Boolean(Number(args.showAbortedJobs)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; @@ -1115,6 +1120,7 @@ class Dashboard { this.showRunningJobs(showRunningJobs); this.showFinishedJobs(showFinishedJobs); + this.showFailedJobs(showFailedJobs); this.showFatalJobs(showFatalJobs); this.showAbortedJobs(showAbortedJobs); @@ -1248,6 +1254,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ds.showRunningJobs(!byId("show-running-jobs").checked); } else if (ev.which === 100 /* d */) { ds.showFinishedJobs(!byId("show-finished-jobs").checked); + } else if (ev.which === 98 /* b */) { + ds.showFailedJobs(!byId("show-failed-jobs").checked); } else if (ev.which === 99 /* c */) { ds.showFatalJobs(!byId("show-fatal-jobs").checked); } else if (ev.which === 115 /* s */) { @@ -1322,6 +1330,11 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId('hide-done').sheet.disabled = value; } + showFailedJobs(value) { + byId('show-failed-jobs').checked = value; + byId('hide-failed').sheet.disabled = value; + } + showFatalJobs(value) { byId('show-fatal-jobs').checked = value; byId('hide-fatal').sheet.disabled = value; diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index e6ec18a6..1d61d99b 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -133,6 +133,10 @@ color: #DD0000 !important; } +.job-info-failed { + color: #CC7676 !important; +} + .inline-stat { /* Needed for 'Align!' feature */ display: inline-block; @@ -405,6 +409,7 @@ +