Yo buddy! You're gonna need JavaScript (ES5+) and WebSocket -> TCP:4568 in order to view this fancy dashboard.
-+ This page shows all of the crawls that ArchiveBot is currently running. +
++ To clear all finished jobs, reload the page. +
++ If your adblocker is enabled for this domain, you will see slower performance, and some URLs will not be displayed. +
++ To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. +
++ These ignore sets are available for crawls. The global ignore set automatically applies to all crawls. +
++ GitHub: ArchiveBot. +
++
- This page shows all of the crawls that ArchiveBot is currently running. -
-- To clear all finished jobs, reload the page. -
-- If your adblocker is enabled for this domain, you will see slower performance, and some URLs will not be displayed. -
-- To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. -
-- These ignore sets are available for crawls. The global ignore set automatically applies to all crawls. -
-- GitHub: ArchiveBot. -
--
| Ident | ||
|---|---|---|
| !{{ :: job.depth == 'shallow' ? 'ao' : 'a' }} |
Date: Tue, 9 Sep 2025 01:16:04 +0100
Subject: [PATCH 17/27] dashboard3: use clusterize.js to display logs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
*insane* boost in performance with this commit 🎉
note to self: cannot copy log lines when it's clusterized
---
dashboard/assets/dashboard3/Dashboard.hx | 121 ++++---
dashboard/assets/dashboard3/dashboard3.js | 387 ++++------------------
dashboard/dashboard3.html | 8 +-
3 files changed, 146 insertions(+), 370 deletions(-)
diff --git a/dashboard/assets/dashboard3/Dashboard.hx b/dashboard/assets/dashboard3/Dashboard.hx
index 58fecb77..96432c3b 100644
--- a/dashboard/assets/dashboard3/Dashboard.hx
+++ b/dashboard/assets/dashboard3/Dashboard.hx
@@ -28,6 +28,9 @@ class LogLine {
}
class Job {
+ public var clusterize:Dynamic;
+ private var clusterizeRows:Array ' + text + ' ';
+ } else {
+ text = '' + text + ' ';
+ }
- var numToTrim = logElement.childElementCount - logLines.length;
+ clusterizeRows.push(text);
+ }
- if (numToTrim > 0) {
- for (dummy in 0...numToTrim) {
- var child = logElement.firstChild;
- if (child != null) {
- logElement.removeChild(child);
- }
- }
+ if (clusterizeRows.length > maxScrollback) {
+ clusterizeRows = clusterizeRows.slice(clusterizeRows.length - maxScrollback);
}
- logElement.setAttribute("data-autoscroll-dirty", "true");
+ untyped clusterize.update(clusterizeRows);
pendingLogLines = 0;
}
+ public function enforceScrollback(maxScrollback:Int) {
+ if (logLines.length > maxScrollback) {
+ logLines = logLines.slice(logLines.length - maxScrollback);
+ }
+ if (clusterizeRows.length > maxScrollback) {
+ clusterizeRows = clusterizeRows.slice(clusterizeRows.length - maxScrollback);
+ if (clusterize != null) {
+ untyped clusterize.update(clusterizeRows);
+ }
+ }
+ }
+
public function attachAntiScroll() {
var logWindow = Browser.document.getElementById('job-log-${ident}');
@@ -354,7 +376,7 @@ class Dashboard {
scope.showNicks = showNicks;
scope.drawInterval = drawInterval;
scope.currentPage = 1;
- scope.pageSize = 10;
+ scope.pageSize = 20;
scope.totalPages = 1;
dashboardControllerScopeApply = Reflect.field(scope, "$apply").bind(scope);
scope.filterOperator = function (job:Job) {
@@ -381,6 +403,9 @@ class Dashboard {
maxScrollback = 50;
}
changeMaxScrollback(maxScrollback);
+ for (job in jobs) {
+ job.enforceScrollback(this.maxScrollback);
+ }
// pagination
var filtered = scope.jobs.filter(scope.filterOperator);
@@ -561,7 +586,7 @@ class Dashboard {
for (job in jobs) {
if (!job.logPaused) {
- job.drawPendingLogLines();
+ job.drawPendingLogLines(maxScrollback);
}
}
@@ -569,17 +594,13 @@ class Dashboard {
}
private function scrollLogsToBottom() {
- var nodes = Browser.document.querySelectorAll("[data-autoscroll-dirty].autoscroll");
- var pending = new Array" + text + " ";
+ } else {
+ text = "" + text + " ";
}
+ this.clusterizeRows.push(text);
+ }
+ if(this.clusterizeRows.length > maxScrollback) {
+ this.clusterizeRows = this.clusterizeRows.slice(this.clusterizeRows.length - maxScrollback);
}
- logElement.setAttribute("data-autoscroll-dirty","true");
+ this.clusterize.update(this.clusterizeRows);
this.pendingLogLines = 0;
}
+ enforceScrollback(maxScrollback) {
+ if(this.logLines.length > maxScrollback) {
+ this.logLines = this.logLines.slice(this.logLines.length - maxScrollback);
+ }
+ if(this.clusterizeRows.length > maxScrollback) {
+ this.clusterizeRows = this.clusterizeRows.slice(this.clusterizeRows.length - maxScrollback);
+ if(this.clusterize != null) {
+ this.clusterize.update(this.clusterizeRows);
+ }
+ }
+ }
static parseInt(thing) {
if(thing != null) {
try {
@@ -186,10 +193,6 @@ class Job {
}
}
}
-Job.__name__ = true;
-Object.assign(Job.prototype, {
- __class__: Job
-});
class Dashboard {
constructor(hostname,maxScrollback,showNicks,drawInterval) {
if(drawInterval == null) {
@@ -249,7 +252,7 @@ class Dashboard {
scope.showNicks = showNicks;
scope.drawInterval = drawInterval;
scope.currentPage = 1;
- scope.pageSize = 10;
+ scope.pageSize = 20;
scope.totalPages = 1;
_gthis.dashboardControllerScopeApply = Reflect.field(scope,"$apply").bind(scope);
scope.filterOperator = function(job) {
@@ -277,18 +280,21 @@ class Dashboard {
maxScrollback = 50;
}
_gthis.changeMaxScrollback(maxScrollback);
+ let _g = 0;
+ let _g1 = _gthis.jobs;
+ while(_g < _g1.length) _g1[_g++].enforceScrollback(_gthis.maxScrollback);
let _this = scope.jobs;
let f = scope.filterOperator;
- let _g = [];
- let _g1 = 0;
- while(_g1 < _this.length) {
- let v = _this[_g1];
- ++_g1;
+ let _g2 = [];
+ let _g3 = 0;
+ while(_g3 < _this.length) {
+ let v = _this[_g3];
+ ++_g3;
if(f(v)) {
- _g.push(v);
+ _g2.push(v);
}
}
- scope.totalPages = Math.max(1,Math.ceil(_g.length / scope.pageSize)) | 0;
+ scope.totalPages = Math.max(1,Math.ceil(_g2.length / scope.pageSize)) | 0;
if(oldVals != null && filterQuery != oldVals[0]) {
scope.currentPage = 1;
} else if(scope.currentPage > scope.totalPages) {
@@ -383,7 +389,7 @@ class Dashboard {
job = new Job(ident);
this.jobMap.h[ident] = job;
this.jobs.push(job);
- console.log("Dashboard.hx:540:","Load job " + ident);
+ console.log("Dashboard.hx:565:","Load job " + ident);
} else {
job = this.jobMap.h[ident];
}
@@ -406,22 +412,24 @@ class Dashboard {
let job = _g1[_g];
++_g;
if(!job.logPaused) {
- job.drawPendingLogLines();
+ job.drawPendingLogLines(this.maxScrollback);
}
}
this.scrollLogsToBottom();
}
scrollLogsToBottom() {
- let nodes = window.document.querySelectorAll("[data-autoscroll-dirty].autoscroll");
- let pending = [];
let _g = 0;
- while(_g < nodes.length) {
- let element = js_Boot.__cast(nodes[_g++] , HTMLElement);
- element.removeAttribute("data-autoscroll-dirty");
- pending.push(element);
+ let _g1 = this.jobs;
+ while(_g < _g1.length) {
+ let job = _g1[_g];
+ ++_g;
+ if(!job.logPaused && job.clusterize != null) {
+ let scrollEl = window.document.getElementById("job-log-" + job.ident);
+ if(scrollEl != null) {
+ scrollEl.scrollTop = scrollEl.scrollHeight;
+ }
+ }
}
- let _g1 = 0;
- while(_g1 < pending.length) pending[_g1++].scrollTop = 99999;
}
static getQueryArgs() {
let query = $global.location.search;
@@ -446,11 +454,6 @@ class Dashboard {
new Dashboard(hostname,50,showNicks).run();
}
}
-Dashboard.__name__ = true;
-Object.assign(Dashboard.prototype, {
- __class__: Dashboard
-});
-Math.__name__ = true;
class Reflect {
static field(o,field) {
try {
@@ -460,11 +463,7 @@ class Reflect {
}
}
}
-Reflect.__name__ = true;
class Std {
- static string(s) {
- return js_Boot.__string_rec(s,"");
- }
static parseInt(x) {
let v = parseInt(x);
if(isNaN(v)) {
@@ -473,64 +472,16 @@ class Std {
return v;
}
}
-Std.__name__ = true;
class StringTools {
static replace(s,sub,by) {
return s.split(sub).join(by);
}
}
-StringTools.__name__ = true;
-class haxe_IMap {
-}
-haxe_IMap.__name__ = true;
-haxe_IMap.__isInterface__ = true;
-class haxe_Exception extends Error {
- constructor(message,previous,native) {
- super(message);
- this.message = message;
- this.__previousException = previous;
- this.__nativeException = native != null ? native : this;
- }
- get_native() {
- return this.__nativeException;
- }
- static thrown(value) {
- if(((value) instanceof haxe_Exception)) {
- return value.get_native();
- } else if(((value) instanceof Error)) {
- return value;
- } else {
- let e = new haxe_ValueException(value);
- return e;
- }
- }
-}
-haxe_Exception.__name__ = true;
-haxe_Exception.__super__ = Error;
-Object.assign(haxe_Exception.prototype, {
- __class__: haxe_Exception
-});
-class haxe_ValueException extends haxe_Exception {
- constructor(value,previous,native) {
- super(String(value),previous,native);
- this.value = value;
- }
-}
-haxe_ValueException.__name__ = true;
-haxe_ValueException.__super__ = haxe_Exception;
-Object.assign(haxe_ValueException.prototype, {
- __class__: haxe_ValueException
-});
class haxe_ds_StringMap {
constructor() {
this.h = Object.create(null);
}
}
-haxe_ds_StringMap.__name__ = true;
-haxe_ds_StringMap.__interfaces__ = [haxe_IMap];
-Object.assign(haxe_ds_StringMap.prototype, {
- __class__: haxe_ds_StringMap
-});
class haxe_iterators_ArrayIterator {
constructor(array) {
this.current = 0;
@@ -543,205 +494,7 @@ class haxe_iterators_ArrayIterator {
return this.array[this.current++];
}
}
-haxe_iterators_ArrayIterator.__name__ = true;
-Object.assign(haxe_iterators_ArrayIterator.prototype, {
- __class__: haxe_iterators_ArrayIterator
-});
-class js_Boot {
- static getClass(o) {
- if(o == null) {
- return null;
- } else if(((o) instanceof Array)) {
- return Array;
- } else {
- let cl = o.__class__;
- if(cl != null) {
- return cl;
- }
- let name = js_Boot.__nativeClassName(o);
- if(name != null) {
- return js_Boot.__resolveNativeClass(name);
- }
- return null;
- }
- }
- static __string_rec(o,s) {
- if(o == null) {
- return "null";
- }
- if(s.length >= 5) {
- return "<...>";
- }
- let t = typeof(o);
- if(t == "function" && (o.__name__ || o.__ename__)) {
- t = "object";
- }
- switch(t) {
- case "function":
- return "
From 01a61a91f6f9e5cd8b0d4aff6fce3bd41320def6 Mon Sep 17 00:00:00 2001
From: exurd <7785022+exurd@users.noreply.github.com>
Date: Tue, 9 Sep 2025 02:37:48 +0100
Subject: [PATCH 20/27] dashboard3: add minify option in Makefile
also:
- add .gitignore for dashboard3.min.* files so they don't get committed
---
dashboard/.gitignore | 1 +
dashboard/assets/dashboard3/Makefile | 8 +++++++-
2 files changed, 8 insertions(+), 1 deletion(-)
create mode 100644 dashboard/.gitignore
diff --git a/dashboard/.gitignore b/dashboard/.gitignore
new file mode 100644
index 00000000..52f6d3d0
--- /dev/null
+++ b/dashboard/.gitignore
@@ -0,0 +1 @@
+dashboard3.min.*
diff --git a/dashboard/assets/dashboard3/Makefile b/dashboard/assets/dashboard3/Makefile
index cd6d80db..11a32051 100644
--- a/dashboard/assets/dashboard3/Makefile
+++ b/dashboard/assets/dashboard3/Makefile
@@ -1,4 +1,10 @@
-all: dashboard
+all: dashboard minify
dashboard: Dashboard.hx
haxe -main Dashboard -D analyzer-optimize -dce full -D js-es=6 -js dashboard3.js
+
+minify: dashboard3.js dashboard3.css ../../dashboard3.html
+# npm i -g terser csso-cli html-minifier-next
+ terser dashboard3.js -c -m --ecma 2015 -o dashboard3.min.js
+ csso dashboard3.css -o dashboard3.min.css
+ html-minifier-next --collapse-whitespace --minify-js true ../../dashboard3.html -o ../../dashboard3.min.html
From ca5a571d8ee970ecee470ceb68a3a22992739be1 Mon Sep 17 00:00:00 2001
From: exurd <7785022+exurd@users.noreply.github.com>
Date: Tue, 9 Sep 2025 02:47:15 +0100
Subject: [PATCH 21/27] dashboard3: fix dashboard.css not overriding correctly
amendment: tactically move around the css links to be more optimal
---
dashboard/dashboard3.html | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/dashboard/dashboard3.html b/dashboard/dashboard3.html
index 629b1adf..7d8c4459 100644
--- a/dashboard/dashboard3.html
+++ b/dashboard/dashboard3.html
@@ -10,14 +10,14 @@
+
+
-
-
+
+
+
+
with @@ -238,16 +242,6 @@ {{:: job.attachAntiScroll() }}
-
-
-
-
-
-
-
- |
|
|
-
+
-
+
+
+ |