From 32f8afac065e2d83283a0dc98ed1d2a216bf1ac2 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 01:19:19 +0000 Subject: [PATCH] c0mpute edge: https for Moshpit names pointed at c0mpute sites bin/moshpit-c0mpute-edge.ts terminates TLS for names whose holders ran `moshcode name c0mpute `, with a leaf the Moshpit CA signs for the edge account on first use (key + CSR by openssl, kept on disk, renewed inside the last ten days), and maps https:///path to /s//path on the fastest healthy c0mpute gateway, failing over on refused connections, timeouts and 5xx. Gateways come from C0MPUTE_GATEWAYS seeds and the public node list (`gateway` field), health-checked every minute; the last good list survives a failed discovery. An SNI that is not a c0mpute name never reaches the CA, and a Host that disagrees with the SNI gets 421. Plain http serves the same sites. Tests run real sockets: a fake registry signing with an openssl CA, a broken and a working gateway, and an https client trusting only that CA. Co-Authored-By: Claude Opus 5.5 --- README.md | 53 +++ bin/moshpit-c0mpute-edge.ts | 47 +++ lib/c0mpute-edge.ts | 514 +++++++++++++++++++++++++++ package.json | 6 +- systemd/moshpit-c0mpute-edge.service | 29 ++ tests/c0mpute-edge.test.ts | 312 ++++++++++++++++ 6 files changed, 959 insertions(+), 2 deletions(-) create mode 100644 bin/moshpit-c0mpute-edge.ts create mode 100644 lib/c0mpute-edge.ts create mode 100644 systemd/moshpit-c0mpute-edge.service create mode 100644 tests/c0mpute-edge.test.ts diff --git a/README.md b/README.md index 43b167a..f1f45a9 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,59 @@ The pin is `SHA-256(SubjectPublicKeyInfo)`, base64 — the RFC 7469 format, so `openssl x509 -pubkey | openssl pkey -pubin -outform der | openssl dgst -sha256` prints the same string. `tests/spki.test.ts` asserts we agree with openssl. +## c0mpute edge + +`bin/moshpit-c0mpute-edge.ts` serves Moshpit names whose holders pointed them at +a static site on [c0mpute](https://c0mpute.com), so `https://blue.eggs` works +with no server of the holder's anywhere: + +```sh +c0mpute hosting publish ./public --name blog # prints the site id +moshcode name c0mpute blue.eggs # one registry write +``` + +A c0mpute gateway serves any site at `http://:7777/s//`, but +it speaks plain HTTP and holds no certificate for a Moshpit name. The edge is +the one hop that does: + +``` +browser --TLS (Moshpit CA leaf)--> edge --HTTP--> any healthy c0mpute gateway --libp2p--> hosting nodes +``` + +- The registry resolves a name with a c0mpute site to the edge's hostname + (`MOSHPIT_C0MPUTE_EDGE` on the registry), so every resolver already deployed + (the bridge, `dns.moshcode.sh`, TronBrowser) reaches it unchanged. +- On the first handshake for a name the edge asks the registry which site it + is, generates a key, and asks the Moshpit CA for a leaf with its own API key. + The registry signs for the edge account (`MOSHPIT_C0MPUTE_EDGE_USERS`) only + while the holder has the name pointed at c0mpute. An SNI that is not a c0mpute + name never reaches the CA. Leaves are kept in `EDGE_CERT_DIR` and renewed in + the background inside their last ten days. +- `https://blue.eggs/path` becomes `/s//path` on the fastest healthy + gateway, failing over to the next on a refused connection, a timeout or a 5xx. + Gateways come from `C0MPUTE_GATEWAYS` and from the public node list (nodes + that announce a gateway URL), re-checked every minute; the last good list is + kept when discovery fails. No c0mpute server is on the path: any gateway can + serve any site. +- Plain `http://` serves the same sites rather than redirecting, for the reason + `nginx/moshpit-origin.conf` gives: a visitor without the Moshpit root gets a + page instead of a certificate error. + +| Variable | Default | | +|---|---|---| +| `MOSHPIT_API_KEY` | | the edge account's key; required to issue leaves | +| `MOSHPIT_REGISTRY_BASE` | `https://pit.moshcode.sh` | | +| `C0MPUTE_GATEWAYS` | | seed gateways, comma-separated (`http://host:7777`) | +| `C0MPUTE_NODES_URL` | `https://c0mpute.com/api/nodes` | gateway discovery; empty disables | +| `EDGE_CERT_DIR` | `/var/lib/moshpit/c0mpute-edge` | | +| `EDGE_HTTPS_PORT` / `EDGE_HTTP_PORT` | `443` / `80` | `EDGE_HTTP_PORT=0` disables http | + +It needs 80 and 443 to itself: run it on its own box or address, not behind the +origin nginx (which would answer with its own default certificate). +`systemd/moshpit-c0mpute-edge.service` runs it with its variables in +`/etc/moshpit/c0mpute-edge.env`. Several edges behind one `MOSHPIT_C0MPUTE_EDGE` +hostname (several A/AAAA records) is how it gets redundancy. + ## Gateway `nginx/moshpit-gateway.conf` — there is no `ssl_certificate` directive anywhere diff --git a/bin/moshpit-c0mpute-edge.ts b/bin/moshpit-c0mpute-edge.ts new file mode 100644 index 0000000..413205e --- /dev/null +++ b/bin/moshpit-c0mpute-edge.ts @@ -0,0 +1,47 @@ +#!/usr/bin/env node +// The c0mpute edge: serves Moshpit names pointed at c0mpute sites over https. +// See lib/c0mpute-edge.ts for what it does and why, and the README's +// "c0mpute edge" section for how to run one. + +import { realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { CertStore, GatewayPool, SiteLookup, startEdge } from "../lib/c0mpute-edge.ts"; + +export function edgeConfig(env: Record = process.env) { + const int = (v: string | undefined, d: number) => (v !== undefined && /^\d+$/.test(v) ? Number(v) : d); + return { + host: env.EDGE_HOST || "::", + httpsPort: int(env.EDGE_HTTPS_PORT, 443), + httpPort: int(env.EDGE_HTTP_PORT, 80), + registryBase: env.MOSHPIT_REGISTRY_BASE || "https://pit.moshcode.sh", + apiKey: env.MOSHPIT_API_KEY || "", + certDir: env.EDGE_CERT_DIR || "/var/lib/moshpit/c0mpute-edge", + seeds: (env.C0MPUTE_GATEWAYS || "").split(/[\s,]+/).filter(Boolean), + // The public node list, for gateways that announce a URL. Empty disables + // discovery and the edge uses the seeds alone. + nodesUrl: env.C0MPUTE_NODES_URL === undefined ? "https://c0mpute.com/api/nodes" : env.C0MPUTE_NODES_URL || null, + refreshMs: int(env.C0MPUTE_REFRESH_SECONDS, 60) * 1000, + }; +} + +async function main() { + const cfg = edgeConfig(); + const log = (line: string) => console.log(line); + if (!cfg.apiKey) log("[edge] warning: MOSHPIT_API_KEY is not set; only certificates already on disk can be served"); + + const gateways = new GatewayPool({ seeds: cfg.seeds, nodesUrl: cfg.nodesUrl }); + await gateways.refresh(); + if (!gateways.size) log("[edge] warning: no c0mpute gateways known yet (set C0MPUTE_GATEWAYS)"); + const timer = setInterval(() => { void gateways.refresh(); }, cfg.refreshMs); + timer.unref(); + + const sites = new SiteLookup({ registryBase: cfg.registryBase }); + const certs = new CertStore({ dir: cfg.certDir, registryBase: cfg.registryBase, apiKey: cfg.apiKey, log }); + await startEdge({ sites, gateways, certs, host: cfg.host, httpsPort: cfg.httpsPort, httpPort: cfg.httpPort, log }); + log(`[edge] serving c0mpute sites on https :${cfg.httpsPort}${cfg.httpPort ? ` and http :${cfg.httpPort}` : ""}; ${gateways.size} gateway(s) known`); +} + +const invoked = (() => { try { return realpathSync(process.argv[1] ?? "") === fileURLToPath(import.meta.url); } catch { return false; } })(); +if (invoked) { + main().catch((e) => { console.error(`[edge] ${e.message}`); process.exit(1); }); +} diff --git a/lib/c0mpute-edge.ts b/lib/c0mpute-edge.ts new file mode 100644 index 0000000..04f3a32 --- /dev/null +++ b/lib/c0mpute-edge.ts @@ -0,0 +1,514 @@ +// The c0mpute edge: https:// for a site that lives on c0mpute. +// +// c0mpute keeps static sites alive peer to peer. Hosting nodes hold copies, +// and every c0mpute gateway serves any site at http://:7777/s//, +// fetching chunks from whichever holder answers and checking each against its +// hash. What a gateway cannot do is answer https://blue.eggs: it speaks plain +// HTTP and holds no certificate for a Moshpit name. +// +// This is the piece that does. The registry resolves a name whose holder set +// `c0mpute=` to this edge's hostname. The edge: +// +// 1. reads the SNI, asks the registry which site the name is, and presents a +// leaf the Moshpit CA signs for the name (asked for on first use with the +// edge's API key, which the registry honours only while the holder has the +// name pointed at c0mpute); +// 2. maps https://blue.eggs/path to /s//path on a healthy c0mpute +// gateway, fastest first, failing over to the next on a refused +// connection, a timeout or a 5xx. +// +// Nothing on the c0mpute side is special about this: any gateway serves any +// site, so there is no c0mpute server on the path to lose. Gateways are +// discovered from seeds and from the public node list (nodes that announce a +// gateway URL), and the last good list is kept when discovery fails. + +import { execFile } from "node:child_process"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { createServer as createHttpServer } from "node:http"; +import type { IncomingMessage, Server as HttpServer, ServerResponse } from "node:http"; +import { createServer as createHttpsServer } from "node:https"; +import type { Server as HttpsServer } from "node:https"; +import { X509Certificate } from "node:crypto"; +import { createSecureContext } from "node:tls"; +import type { SecureContext, TLSSocket } from "node:tls"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const run = promisify(execFile); + +/** A c0mpute site id: 20 lowercase hex. */ +export const SITE_RE = /^[0-9a-f]{20}$/; + +const NAME_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?\.[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + +/** `label.tld`, lowercased, or null. Only the shape the registry holds. */ +export function normalizeHost(input: string | undefined | null): string | null { + const raw = String(input ?? "").trim().toLowerCase().replace(/\.$/, ""); + const host = raw.replace(/:\d+$/, ""); + return NAME_RE.test(host) ? host : null; +} + +/* ------------------------------------------------------------- gateways */ + +/** An http(s) origin with no path, or null. */ +export function normalizeGateway(input: unknown): string | null { + if (typeof input !== "string" || input.length > 300) return null; + let url: URL; + try { url = new URL(input.trim()); } catch { return null; } + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username || url.password) return null; + return url.origin; +} + +/** + * Gateway origins out of a c0mpute node list (`/api/nodes` or an aggregator's + * `/nodes`): every node that announces one in `gateway`. + */ +export function gatewaysFromNodes(payload: unknown): string[] { + const nodes = (payload as { nodes?: unknown })?.nodes; + if (!Array.isArray(nodes)) return []; + const out = new Set(); + for (const node of nodes) { + const url = normalizeGateway((node as { gateway?: unknown })?.gateway); + if (url) out.add(url); + } + return [...out]; +} + +type GatewayState = { url: string; healthy: boolean; rttMs: number; failures: number; downUntil: number }; + +export interface GatewayPoolOptions { + seeds?: string[]; + /** A node list to discover gateways from; null disables discovery. */ + nodesUrl?: string | null; + fetchImpl?: typeof fetch; + timeoutMs?: number; + now?: () => number; +} + +/** + * The gateways worth sending a request to, best first. + * + * Health is checked (`/healthz`) on refresh and learned from real traffic in + * between: a gateway that fails a request sits out for a while, longer each + * time, and comes back on the next good answer. Among equally fast gateways + * the order rotates, so one does not take all the load. + */ +export class GatewayPool { + private states = new Map(); + private seeds: string[]; + private nodesUrl: string | null; + private fetchImpl: typeof fetch; + private timeoutMs: number; + private now: () => number; + private turn = 0; + + constructor(opts: GatewayPoolOptions = {}) { + this.seeds = (opts.seeds ?? []).map(normalizeGateway).filter((u): u is string => Boolean(u)); + this.nodesUrl = opts.nodesUrl ?? null; + this.fetchImpl = opts.fetchImpl ?? fetch; + this.timeoutMs = opts.timeoutMs ?? 3000; + this.now = opts.now ?? Date.now; + for (const url of this.seeds) this.ensure(url); + } + + private ensure(url: string): GatewayState { + let s = this.states.get(url); + if (!s) { + s = { url, healthy: true, rttMs: Number.MAX_SAFE_INTEGER, failures: 0, downUntil: 0 }; + this.states.set(url, s); + } + return s; + } + + get size(): number { return this.states.size; } + + /** Rediscover and re-check every gateway. Never throws; a failed discovery keeps the old list. */ + async refresh(): Promise { + if (this.nodesUrl) { + try { + const res = await this.fetchImpl(this.nodesUrl, { signal: AbortSignal.timeout(this.timeoutMs) }); + if (res.ok) for (const url of gatewaysFromNodes(await res.json())) this.ensure(url); + } catch { /* keep what we had */ } + } + await Promise.all([...this.states.values()].map((s) => this.check(s))); + } + + private async check(s: GatewayState): Promise { + const started = this.now(); + try { + const res = await this.fetchImpl(`${s.url}/healthz`, { signal: AbortSignal.timeout(this.timeoutMs) }); + await res.body?.cancel(); + if (!res.ok) throw new Error(String(res.status)); + s.rttMs = Math.max(0, this.now() - started); + this.succeeded(s.url); + } catch { + this.failed(s.url); + } + } + + succeeded(url: string): void { + const s = this.states.get(url); + if (!s) return; + s.healthy = true; + s.failures = 0; + s.downUntil = 0; + } + + failed(url: string): void { + const s = this.states.get(url); + if (!s) return; + s.failures += 1; + s.healthy = false; + // 10 s, 20 s, 40 s ... capped at 5 minutes. + s.downUntil = this.now() + Math.min(300_000, 10_000 * 2 ** (s.failures - 1)); + } + + /** + * Up to `n` gateways to try, in order. Healthy ones first (fastest, rotating + * within the same 50 ms band), then any that are sitting out, so a request + * still has somewhere to go when every gateway has had a bad minute. + */ + pick(n = 3): string[] { + const now = this.now(); + const all = [...this.states.values()]; + const up = all.filter((s) => s.healthy || s.downUntil <= now); + const down = all.filter((s) => !up.includes(s)).sort((a, b) => a.downUntil - b.downUntil); + const band = (s: GatewayState) => Math.floor(Math.min(s.rttMs, 3_600_000) / 50); + up.sort((a, b) => band(a) - band(b) || a.failures - b.failures); + if (up.length > 1) { + const lead = up.filter((s) => band(s) === band(up[0])); + const k = this.turn++ % lead.length; + up.splice(0, lead.length, ...lead.slice(k), ...lead.slice(0, k)); + } + return [...up, ...down].slice(0, n).map((s) => s.url); + } +} + +/* ---------------------------------------------------------------- sites */ + +export interface SiteLookupOptions { + registryBase?: string; + fetchImpl?: typeof fetch; + ttlMs?: number; + missTtlMs?: number; + now?: () => number; +} + +/** Which c0mpute site a name is, from the registry, cached briefly. */ +export class SiteLookup { + private cache = new Map(); + private registryBase: string; + private fetchImpl: typeof fetch; + private ttlMs: number; + private missTtlMs: number; + private now: () => number; + + constructor(opts: SiteLookupOptions = {}) { + this.registryBase = (opts.registryBase ?? "https://pit.moshcode.sh").replace(/\/+$/, ""); + this.fetchImpl = opts.fetchImpl ?? fetch; + this.ttlMs = opts.ttlMs ?? 60_000; + this.missTtlMs = opts.missTtlMs ?? 30_000; + this.now = opts.now ?? Date.now; + } + + async site(name: string): Promise { + const host = normalizeHost(name); + if (!host) return null; + const hit = this.cache.get(host); + if (hit && hit.until > this.now()) return hit.site; + let site: string | null = null; + try { + const res = await this.fetchImpl(`${this.registryBase}/api/moshpit/resolve?name=${encodeURIComponent(host)}`, + { signal: AbortSignal.timeout(4000) }); + if (res.ok) { + const json = (await res.json()) as { c0mpute?: unknown }; + site = typeof json.c0mpute === "string" && SITE_RE.test(json.c0mpute) ? json.c0mpute : null; + } else { + await res.body?.cancel(); + } + } catch { + // A registry outage keeps serving what was known rather than dropping + // every site at once. + if (hit) return hit.site; + return null; + } + this.cache.set(host, { site, until: this.now() + (site ? this.ttlMs : this.missTtlMs) }); + if (this.cache.size > 10_000) this.cache.delete(this.cache.keys().next().value as string); + return site; + } +} + +/* --------------------------------------------------------- certificates */ + +export interface CertStoreOptions { + dir: string; + registryBase?: string; + apiKey?: string; + fetchImpl?: typeof fetch; + /** Renew when fewer than this many ms remain. */ + renewBeforeMs?: number; + now?: () => number; + log?: (line: string) => void; +} + +type Loaded = { context: SecureContext; notAfter: number }; + +/** + * Leaves for the names this edge serves, kept on disk and asked of the + * registry's CA on first use. One request in flight per name; a renewal runs + * in the background while the current leaf keeps being served. + */ +export class CertStore { + private mem = new Map(); + private pending = new Map>(); + private opts: Required> & { apiKey: string }; + + constructor(opts: CertStoreOptions) { + this.opts = { + registryBase: "https://pit.moshcode.sh", + fetchImpl: fetch, + renewBeforeMs: 10 * 24 * 3600 * 1000, + now: Date.now, + log: () => {}, + apiKey: "", + ...opts, + }; + this.opts.registryBase = this.opts.registryBase.replace(/\/+$/, ""); + } + + private paths(name: string) { + return { cert: join(this.opts.dir, `${name}.crt`), key: join(this.opts.dir, `${name}.key`) }; + } + + /** A context for `name`, issuing one if needed. Null when none can be had. */ + async context(name: string): Promise { + const now = this.opts.now(); + const have = this.mem.get(name) ?? (await this.fromDisk(name)); + if (have && have.notAfter > now) { + if (have.notAfter - now < this.opts.renewBeforeMs) void this.issueOnce(name); + return have.context; + } + return (await this.issueOnce(name))?.context ?? null; + } + + private async fromDisk(name: string): Promise { + const p = this.paths(name); + try { + const [cert, key] = await Promise.all([readFile(p.cert, "utf8"), readFile(p.key, "utf8")]); + const loaded = { context: createSecureContext({ cert, key }), notAfter: Date.parse(new X509Certificate(cert).validTo) }; + this.mem.set(name, loaded); + return loaded; + } catch { + return null; + } + } + + private issueOnce(name: string): Promise { + let p = this.pending.get(name); + if (!p) { + p = this.issue(name).catch((e: Error) => { this.opts.log(`[edge] cert ${name}: ${e.message}`); return null; }) + .finally(() => this.pending.delete(name)); + this.pending.set(name, p); + } + return p; + } + + private async issue(name: string): Promise { + if (!this.opts.apiKey) throw new Error("no MOSHPIT_API_KEY to ask the registry with"); + await mkdir(this.opts.dir, { recursive: true, mode: 0o700 }); + const { key, csr } = await makeCsr(name); + const [label, tld] = name.split("."); + const res = await this.opts.fetchImpl(`${this.opts.registryBase}/api/moshpit/tlds/${encodeURIComponent(tld)}/certs`, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${this.opts.apiKey}` }, + body: JSON.stringify({ label, csr }), + signal: AbortSignal.timeout(15_000), + }); + const body = (await res.json().catch(() => null)) as { chain?: string; error?: string } | null; + if (res.status !== 201 || !body?.chain) throw new Error(`registry said ${res.status}${body?.error ? `: ${body.error}` : ""}`); + const p = this.paths(name); + // Key first, then the chain, each by rename, so a crash never leaves a + // chain on disk next to a key it does not match. + await writeFile(`${p.key}.tmp`, key, { mode: 0o600 }); + await writeFile(`${p.cert}.tmp`, body.chain, { mode: 0o644 }); + await rename(`${p.key}.tmp`, p.key); + await rename(`${p.cert}.tmp`, p.cert); + const loaded = { context: createSecureContext({ cert: body.chain, key }), notAfter: Date.parse(new X509Certificate(body.chain).validTo) }; + this.mem.set(name, loaded); + this.opts.log(`[edge] issued ${name} until ${new Date(loaded.notAfter).toISOString()}`); + return loaded; + } +} + +/** A fresh P-256 key and a CSR for `name`, by openssl (what setup-origin.sh uses). */ +export async function makeCsr(name: string): Promise<{ key: string; csr: string }> { + if (!normalizeHost(name)) throw new Error(`not a name: ${name}`); + // Key and request both to stdout in one call: no temp file holding a + // private key, and no /dev/stdin, which is not there for every process. + const { stdout } = await run("openssl", ["req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", + "-nodes", "-keyout", "-", "-out", "-", "-subj", `/CN=${name}`]); + const key = stdout.match(/-----BEGIN PRIVATE KEY-----[\s\S]+?-----END PRIVATE KEY-----\n?/)?.[0]; + const csr = stdout.match(/-----BEGIN CERTIFICATE REQUEST-----[\s\S]+?-----END CERTIFICATE REQUEST-----\n?/)?.[0]; + if (!key || !csr) throw new Error("openssl did not produce a key and a request"); + return { key, csr }; +} + +/* ------------------------------------------------------------- serving */ + +/** Response headers worth passing back from a gateway. */ +const PASS_HEADERS = ["content-type", "content-length", "etag", "last-modified", "cache-control", "content-encoding"]; + +export interface EdgeHandlerOptions { + sites: Pick; + gateways: Pick; + fetchImpl?: typeof fetch; + timeoutMs?: number; + attempts?: number; + log?: (line: string) => void; +} + +/** The gateway URL for a request path on a site. */ +export function upstreamUrl(gateway: string, site: string, path: string): string { + const rel = path.startsWith("/") ? path : `/${path}`; + return `${gateway}/s/${site}${rel}`; +} + +/** An HTTP handler serving c0mpute sites by Host, for both the http and https servers. */ +export function createEdgeHandler(opts: EdgeHandlerOptions) { + const fetchImpl = opts.fetchImpl ?? fetch; + const timeoutMs = opts.timeoutMs ?? 10_000; + const attempts = opts.attempts ?? 3; + const log = opts.log ?? (() => {}); + + return async function handle(req: IncomingMessage, res: ServerResponse): Promise { + const send = (status: number, text: string) => { + res.writeHead(status, { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" }); + res.end(req.method === "HEAD" ? undefined : `${text}\n`); + }; + + const host = normalizeHost(req.headers.host); + if (!host) return send(400, "this edge serves Moshpit names pointed at c0mpute sites"); + + // Over TLS, the name asked for in the handshake and the Host must agree: + // the certificate presented was for the SNI, and serving another name's + // site under it would be serving a page the certificate does not vouch for. + const sni = (req.socket as TLSSocket).servername; + if (typeof sni === "string" && sni && normalizeHost(sni) !== host) return send(421, "misdirected request"); + + if (req.method !== "GET" && req.method !== "HEAD") { + res.setHeader("allow", "GET, HEAD"); + return send(405, "static site: GET and HEAD only"); + } + + const site = await opts.sites.site(host); + if (!site) return send(404, `${host} does not point at a c0mpute site`); + + const path = (req.url || "/").startsWith("/") ? (req.url || "/") : "/"; + for (const gw of opts.gateways.pick(attempts)) { + let upstream: Response; + try { + upstream = await fetchImpl(upstreamUrl(gw, site, path), { + method: req.method, + headers: { + ...(req.headers["if-none-match"] ? { "if-none-match": String(req.headers["if-none-match"]) } : {}), + ...(req.headers["accept-encoding"] ? { "accept-encoding": String(req.headers["accept-encoding"]) } : {}), + }, + redirect: "manual", + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (e) { + opts.gateways.failed(gw); + log(`[edge] ${host} via ${gw}: ${(e as Error).message}`); + continue; + } + if (upstream.status >= 500) { + await upstream.body?.cancel(); + opts.gateways.failed(gw); + log(`[edge] ${host} via ${gw}: ${upstream.status}`); + continue; + } + opts.gateways.succeeded(gw); + const headers: Record = { "x-c0mpute-gateway": new URL(gw).host }; + for (const h of PASS_HEADERS) { + const v = upstream.headers.get(h); + if (v !== null) headers[h] = v; + } + res.writeHead(upstream.status, headers); + if (req.method === "HEAD" || !upstream.body) { + await upstream.body?.cancel(); + res.end(); + return; + } + const reader = upstream.body.getReader(); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (!res.write(value)) await new Promise((r) => res.once("drain", r)); + } + } catch { + res.destroy(); + return; + } + res.end(); + return; + } + send(502, "no c0mpute gateway answered for this site; try again shortly"); + }; +} + +/* --------------------------------------------------------------- servers */ + +export interface EdgeServerOptions extends EdgeHandlerOptions { + certs: Pick; + host?: string; + httpsPort?: number; + /** 0 disables plain HTTP. */ + httpPort?: number; +} + +/** + * Start the edge: https (SNI picks the leaf) and, unless disabled, plain http + * serving the same sites. Plain http is served rather than redirected, for the + * reason moshpit-origin.conf gives: a visitor who has not installed the Moshpit + * root gets a working page over http instead of a certificate error. + */ +export async function startEdge(opts: EdgeServerOptions): Promise<{ https: HttpsServer; http: HttpServer | null; close(): Promise }> { + const handler = createEdgeHandler(opts); + const onRequest = (req: IncomingMessage, res: ServerResponse) => { + handler(req, res).catch(() => { try { res.destroy(); } catch { /* gone */ } }); + }; + + const https = createHttpsServer({ + SNICallback: (servername, cb) => { + const name = normalizeHost(servername); + if (!name) return cb(new Error("not a Moshpit name")); + // Only names the registry says are c0mpute sites: an arbitrary SNI must + // not make this edge ask the CA for anything. + opts.sites.site(name) + .then((site) => (site ? opts.certs.context(name) : null)) + .then((ctx) => (ctx ? cb(null, ctx) : cb(new Error(`no certificate for ${name}`)))) + .catch((e: Error) => cb(e)); + }, + }, onRequest); + + const listen = (server: HttpServer | HttpsServer, port: number) => new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, opts.host ?? "::", () => { server.off("error", reject); resolve(); }); + }); + + await listen(https, opts.httpsPort ?? 443); + let http: HttpServer | null = null; + if (opts.httpPort !== 0) { + http = createHttpServer(onRequest); + await listen(http, opts.httpPort ?? 80); + } + return { + https, + http, + close: async () => { + await Promise.all([https, http].filter(Boolean).map((s) => new Promise((r) => s!.close(() => r())))); + }, + }; +} diff --git a/package.json b/package.json index 7cdf8e4..df1f487 100644 --- a/package.json +++ b/package.json @@ -8,14 +8,16 @@ "bin": { "moshpit-proxy": "bin/moshpit-proxy.ts", "moshpit-pin": "bin/moshpit-pin.ts", - "moshpit-trust": "bin/moshpit-trust.ts" + "moshpit-trust": "bin/moshpit-trust.ts", + "moshpit-c0mpute-edge": "bin/moshpit-c0mpute-edge.ts" }, "scripts": { "start": "node bin/moshpit-proxy.ts", "pin": "node bin/moshpit-pin.ts", "trust": "node bin/moshpit-trust.ts", "upstreams": "node scripts/gen-upstreams.ts", - "test": "node --test tests/*.test.ts" + "test": "node --test tests/*.test.ts", + "edge": "node bin/moshpit-c0mpute-edge.ts" }, "devDependencies": { "@types/node": "^22.10.0", diff --git a/systemd/moshpit-c0mpute-edge.service b/systemd/moshpit-c0mpute-edge.service new file mode 100644 index 0000000..a7d9da5 --- /dev/null +++ b/systemd/moshpit-c0mpute-edge.service @@ -0,0 +1,29 @@ +[Unit] +# Serves Moshpit names whose holders pointed them at a c0mpute site +# (`moshcode name c0mpute `): TLS with a leaf the registry signs +# for each name, content from whichever c0mpute gateways are healthy. +Description=Moshpit c0mpute edge +Documentation=https://github.com/profullstack/moshpit-proxy#c0mpute-edge +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +# MOSHPIT_API_KEY (the edge account's key; the registry must list that account +# in MOSHPIT_C0MPUTE_EDGE_USERS), optional C0MPUTE_GATEWAYS seeds and the rest +# of the EDGE_* variables live here, root-only. +EnvironmentFile=/etc/moshpit/c0mpute-edge.env +ExecStart=/usr/bin/env node /opt/moshpit/bin/moshpit-c0mpute-edge.ts +StateDirectory=moshpit/c0mpute-edge +Environment=EDGE_CERT_DIR=/var/lib/moshpit/c0mpute-edge +AmbientCapabilities=CAP_NET_BIND_SERVICE +DynamicUser=yes +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/tests/c0mpute-edge.test.ts b/tests/c0mpute-edge.test.ts new file mode 100644 index 0000000..6023540 --- /dev/null +++ b/tests/c0mpute-edge.test.ts @@ -0,0 +1,312 @@ +// The c0mpute edge, end to end against real sockets: a fake registry that +// signs with a real (openssl) CA, fake c0mpute gateways, and an https client +// that trusts only that CA, the way a browser with the Moshpit root does. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createServer } from "node:http"; +import type { IncomingMessage, Server, ServerResponse } from "node:http"; +import { request as httpsRequest } from "node:https"; +import { request as httpRequest } from "node:http"; +import type { AddressInfo } from "node:net"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { + CertStore, + GatewayPool, + SiteLookup, + createEdgeHandler, + gatewaysFromNodes, + normalizeGateway, + normalizeHost, + startEdge, + upstreamUrl, +} from "../lib/c0mpute-edge.ts"; +import { edgeConfig } from "../bin/moshpit-c0mpute-edge.ts"; +import { tempDir } from "./helpers.ts"; + +const run = promisify(execFile); +const SITE = "0123456789abcdef0123"; + +function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise<{ server: Server; url: string }> { + const server = createServer(handler); + return new Promise((resolve) => server.listen(0, "127.0.0.1", () => { + resolve({ server, url: `http://127.0.0.1:${(server.address() as AddressInfo).port}` }); + })); +} + +function body(req: IncomingMessage): Promise { + return new Promise((resolve) => { let s = ""; req.on("data", (d) => { s += d; }); req.on("end", () => resolve(s)); }); +} + +function fakeFetch(routes: Record Response | Promise>): typeof fetch { + return (async (input: string | URL | Request) => { + const url = String(input); + const route = routes[url]; + if (!route) throw new Error(`connect ECONNREFUSED ${url}`); + return route(); + }) as typeof fetch; +} + +/* ------------------------------------------------------------- pure */ + +test("names, gateways and upstream paths", () => { + assert.equal(normalizeHost("Blue.Eggs"), "blue.eggs"); + assert.equal(normalizeHost("blue.eggs:443"), "blue.eggs"); + assert.equal(normalizeHost("blue.eggs."), "blue.eggs"); + assert.equal(normalizeHost("www.blue.eggs"), null, "the registry holds label.tld only"); + assert.equal(normalizeHost("127.0.0.1"), null); + assert.equal(normalizeHost(undefined), null); + + assert.equal(normalizeGateway("http://1.2.3.4:7777/"), "http://1.2.3.4:7777"); + assert.equal(normalizeGateway("https://gw.example.org/x"), "https://gw.example.org"); + assert.equal(normalizeGateway("ftp://gw.example.org"), null); + assert.equal(normalizeGateway("http://u:p@gw.example.org"), null); + assert.equal(normalizeGateway(42), null); + + assert.deepEqual(gatewaysFromNodes({ nodes: [ + { id: "a", gateway: "http://1.2.3.4:7777" }, + { id: "b" }, + { id: "c", gateway: "javascript:alert(1)" }, + { id: "d", gateway: "http://1.2.3.4:7777/" }, + ] }), ["http://1.2.3.4:7777"]); + assert.deepEqual(gatewaysFromNodes(null), []); + + assert.equal(upstreamUrl("http://g:7777", SITE, "/about?x=1"), `http://g:7777/s/${SITE}/about?x=1`); + assert.equal(upstreamUrl("http://g:7777", SITE, "/"), `http://g:7777/s/${SITE}/`); +}); + +test("edge config reads the environment", () => { + const c = edgeConfig({ C0MPUTE_GATEWAYS: "http://a:7777, http://b:7777", C0MPUTE_NODES_URL: "", EDGE_HTTP_PORT: "0" }); + assert.deepEqual(c.seeds, ["http://a:7777", "http://b:7777"]); + assert.equal(c.nodesUrl, null, "empty disables discovery"); + assert.equal(c.httpPort, 0); + assert.equal(edgeConfig({}).nodesUrl, "https://c0mpute.com/api/nodes"); + assert.equal(edgeConfig({}).httpsPort, 443); +}); + +/* ------------------------------------------------------------ gateways */ + +test("the pool discovers, health-checks, and orders gateways", async () => { + let clock = 1_000_000; + const ok = () => new Response("{}", { status: 200 }); + const pool = new GatewayPool({ + seeds: ["http://seed:7777"], + nodesUrl: "https://c0mpute.example/api/nodes", + now: () => clock, + fetchImpl: fakeFetch({ + "https://c0mpute.example/api/nodes": () => Response.json({ nodes: [{ gateway: "http://found:7777" }, { gateway: "http://dead:7777" }] }), + "http://seed:7777/healthz": ok, + "http://found:7777/healthz": ok, + // dead has no route: connection refused + }), + }); + await pool.refresh(); + assert.equal(pool.size, 3); + const picked = pool.pick(3); + assert.equal(picked.length, 3); + assert.equal(picked[2], "http://dead:7777", "a failing gateway goes last, not away"); + assert.deepEqual(new Set(picked.slice(0, 2)), new Set(["http://seed:7777", "http://found:7777"])); + + // Equally fast gateways take turns at the front. + const fronts = new Set([pool.pick(1)[0], pool.pick(1)[0]]); + assert.equal(fronts.size, 2, "rotates among the fastest"); + + // A failure sends a gateway to the back until its back-off runs out. + pool.failed("http://seed:7777"); + assert.notEqual(pool.pick(1)[0], "http://seed:7777"); + clock += 11_000; + assert.ok(pool.pick(2).includes("http://seed:7777"), "back after its back-off"); + pool.succeeded("http://seed:7777"); +}); + +test("a failed discovery keeps the gateways already known", async () => { + const pool = new GatewayPool({ + seeds: ["http://seed:7777"], + nodesUrl: "https://c0mpute.example/api/nodes", + fetchImpl: fakeFetch({ "http://seed:7777/healthz": () => new Response("ok") }), + }); + await pool.refresh(); + assert.deepEqual(pool.pick(), ["http://seed:7777"]); +}); + +/* --------------------------------------------------------------- sites */ + +test("site lookups come from the registry, cached, and survive an outage", async () => { + let calls = 0; + let down = false; + let clock = 0; + const lookup = new SiteLookup({ + registryBase: "https://pit.example", + now: () => clock, + fetchImpl: (async (input: string | URL | Request) => { + calls++; + if (down) throw new Error("ECONNREFUSED"); + const name = new URL(String(input)).searchParams.get("name"); + if (name === "blue.eggs") return Response.json({ name, c0mpute: SITE, target: "edge.example" }); + if (name === "evil.eggs") return Response.json({ name, c0mpute: "../../x" }); + return Response.json({ name, c0mpute: null }); + }) as typeof fetch, + }); + assert.equal(await lookup.site("blue.eggs"), SITE); + assert.equal(await lookup.site("BLUE.eggs"), SITE); + assert.equal(calls, 1, "cached"); + assert.equal(await lookup.site("red.eggs"), null); + assert.equal(await lookup.site("evil.eggs"), null, "only a real site id is believed"); + assert.equal(await lookup.site("not a name"), null); + clock += 120_000; + down = true; + assert.equal(await lookup.site("blue.eggs"), SITE, "an outage keeps the last answer"); +}); + +/* ------------------------------------------------------------- serving */ + +async function makeCa(dir: string) { + const key = join(dir, "ca.key"); + const crt = join(dir, "ca.crt"); + await run("openssl", ["req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256", "-nodes", + "-keyout", key, "-out", crt, "-days", "2", "-subj", "/CN=Test Moshpit Root", + "-addext", "basicConstraints=critical,CA:TRUE", "-addext", "keyUsage=critical,keyCertSign,cRLSign"]); + return { key, crt, pem: await readFile(crt, "utf8") }; +} + +async function sign(dir: string, ca: { key: string; crt: string; pem: string }, name: string, csr: string, days = 30) { + const csrPath = join(dir, `${name}.${Date.now()}.csr`); + const ext = join(dir, `${name}.ext`); + const out = join(dir, `${name}.${Date.now()}.crt`); + await writeFile(csrPath, csr); + await writeFile(ext, `basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:${name},DNS:*.${name}\n`); + await run("openssl", ["x509", "-req", "-in", csrPath, "-CA", ca.crt, "-CAkey", ca.key, "-CAcreateserial", + "-out", out, "-days", String(days), "-extfile", ext]); + return (await readFile(out, "utf8")) + ca.pem; +} + +function get(port: number, name: string, path = "/", { ca, host = name, servername = name }: { ca: string; host?: string; servername?: string }) { + return new Promise<{ status: number; headers: Record; text: string }>((resolve, reject) => { + const req = httpsRequest({ host: "127.0.0.1", port, path, servername, ca, headers: { host } }, (res) => { + let text = ""; + res.on("data", (d) => { text += d; }); + res.on("end", () => resolve({ status: res.statusCode ?? 0, headers: res.headers, text })); + }); + req.on("error", reject); + req.end(); + }); +} + +test("https:// is served from a healthy gateway with a registry-signed leaf", async (t) => { + const dir = await tempDir("moshpit-edge-"); + const ca = await makeCa(dir); + + // Two gateways: one broken (503 on everything), one serving the site. + const broken = await listen((_req, res) => { res.writeHead(503); res.end("down"); }); + const good = await listen((req, res) => { + if (req.url === "/healthz") { res.writeHead(200); return res.end("ok"); } + const prefix = `/s/${SITE}`; + if (!req.url?.startsWith(prefix)) { res.writeHead(404); return res.end("no such site"); } + const path = req.url.slice(prefix.length); + res.writeHead(200, { "content-type": "text/html", etag: '"v1"', "set-cookie": "leak=1" }); + res.end(`

site ${SITE} at ${path}

`); + }); + + // The registry: resolves blue.eggs to the site; signs only names with a site. + const issued: string[] = []; + const registry = await listen(async (req, res) => { + const url = new URL(req.url ?? "/", "http://x"); + if (url.pathname === "/api/moshpit/resolve") { + const name = url.searchParams.get("name"); + return res.end(JSON.stringify({ name, c0mpute: name === "blue.eggs" ? SITE : null })); + } + if (req.method === "POST" && url.pathname === "/api/moshpit/tlds/eggs/certs") { + if (req.headers.authorization !== "Bearer edge-key") { res.writeHead(401); return res.end("{}"); } + const { label, csr } = JSON.parse(await body(req)); + if (label !== "blue") { res.writeHead(403); return res.end(JSON.stringify({ error: "not yours" })); } + issued.push(label); + const chain = await sign(dir, ca, `${label}.eggs`, csr); + res.writeHead(201, { "content-type": "application/json" }); + return res.end(JSON.stringify({ chain })); + } + res.writeHead(404); res.end("{}"); + }); + + const gateways = new GatewayPool({ seeds: [broken.url, good.url] }); + await gateways.refresh(); + const sites = new SiteLookup({ registryBase: registry.url }); + const certs = new CertStore({ dir: join(dir, "certs"), registryBase: registry.url, apiKey: "edge-key" }); + const edge = await startEdge({ sites, gateways, certs, host: "127.0.0.1", httpsPort: 0, httpPort: 0 }); + const port = (edge.https.address() as AddressInfo).port; + t.after(async () => { + await edge.close(); + for (const s of [broken.server, good.server, registry.server]) s.close(); + }); + + await t.test("serves the site, mapping the path under /s//", async () => { + const r = await get(port, "blue.eggs", "/about.html", { ca: ca.pem }); + assert.equal(r.status, 200); + assert.equal(r.text, `

site ${SITE} at /about.html

`); + assert.equal(r.headers.etag, '"v1"'); + assert.equal(r.headers["set-cookie"], undefined, "only static-site headers pass through"); + assert.equal(r.headers["x-c0mpute-gateway"], new URL(good.url).host); + }); + + await t.test("asks the CA once, then serves the leaf from disk and memory", async () => { + await get(port, "blue.eggs", "/", { ca: ca.pem }); + await get(port, "blue.eggs", "/x", { ca: ca.pem }); + assert.deepEqual(issued, ["blue"]); + const onDisk = await readFile(join(dir, "certs", "blue.eggs.crt"), "utf8"); + assert.match(onDisk, /BEGIN CERTIFICATE/); + }); + + await t.test("fails over past a broken gateway every time", async () => { + for (let i = 0; i < 4; i++) { + const r = await get(port, "blue.eggs", `/p${i}`, { ca: ca.pem }); + assert.equal(r.status, 200, `request ${i}`); + } + }); + + await t.test("a name that is not a c0mpute site gets no certificate at all", async () => { + await assert.rejects(get(port, "red.eggs", "/", { ca: ca.pem })); + assert.deepEqual(issued, ["blue"], "and the CA was never asked"); + }); + + await t.test("a Host that disagrees with the SNI is refused", async () => { + const r = await get(port, "blue.eggs", "/", { ca: ca.pem, host: "red.eggs" }); + assert.equal(r.status, 421); + }); +}); + +test("plain http serves the same sites, GET and HEAD only, 502 when no gateway answers", async (t) => { + const sites = { site: async (name: string) => (name === "blue.eggs" ? SITE : null) }; + let failures = 0; + const gateways = { pick: () => ["http://g1:7777", "http://g2:7777"], succeeded: () => {}, failed: () => { failures++; } }; + let up = true; + const handler = createEdgeHandler({ + sites, + gateways, + fetchImpl: (async (input: string | URL | Request) => { + if (!up) throw new Error("ECONNREFUSED"); + if (String(input).startsWith("http://g1")) return new Response("bad", { status: 502 }); + return new Response("hello", { status: 200, headers: { "content-type": "text/plain" } }); + }) as typeof fetch, + }); + const srv = await listen((req, res) => { void handler(req, res); }); + t.after(() => srv.server.close()); + const port = Number(new URL(srv.url).port); + const call = (method: string, host: string) => new Promise<{ status: number; text: string }>((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, method, path: "/", headers: { host } }, (res) => { + let text = ""; res.on("data", (d) => { text += d; }); res.on("end", () => resolve({ status: res.statusCode ?? 0, text })); + }); + req.on("error", reject); req.end(); + }); + + assert.deepEqual(await call("GET", "blue.eggs"), { status: 200, text: "hello" }); + assert.equal(failures, 1, "g1's 502 counted against it"); + assert.equal((await call("HEAD", "blue.eggs")).text, ""); + assert.equal((await call("POST", "blue.eggs")).status, 405); + assert.equal((await call("GET", "red.eggs")).status, 404); + assert.equal((await call("GET", "127.0.0.1")).status, 400); + up = false; + assert.equal((await call("GET", "blue.eggs")).status, 502); +});